A-Team_Digital_Solutions/android_kernel_motorola_sm6375
Watch
2
0
Fork
You've already forked android_kernel_motorola_sm6375
 0
mirror of https://github.com/BobTheBlinker/android_kernel_motorola_sm6375.git synced 2026-10-29 03:01:39 -04:00
Code Issues Projects Releases Packages Wiki Activity

Merge branch 'linus' of git://git.kernel.org/pub/scm/linux/kernel/git/herbert/crypto-2.6

Browse source 
Pull crypto updates from Herbert Xu:
 "API:
   - Fix dcache flushing crash in skcipher.
   - Add hash finup self-tests.
   - Reschedule during speed tests.

  Algorithms:
   - Remove insecure vmac and replace it with vmac64.
   - Add public key verification for DH/ECDH.

  Drivers:
   - Decrease priority of sha-mb on x86.
   - Improve NEON latency/throughput on ARM64.
   - Add md5/sha384/sha512/des/3des to inside-secure.
   - Support eip197d in inside-secure.
   - Only register algorithms supported by the host in virtio.
   - Add cts and remove incompatible cts1 from ccree.
   - Add hisilicon SEC security accelerator driver.
   - Replace msm hwrng driver with qcom pseudo rng driver.

  Misc:
   - Centralize CRC polynomials"

* 'linus' of git://git.kernel.org/pub/scm/linux/kernel/git/herbert/crypto-2.6: (121 commits)
  crypto: arm64/ghash-ce - implement 4-way aggregation
  crypto: arm64/ghash-ce - replace NEON yield check with block limit
  crypto: hisilicon - sec_send_request() can be static
  lib/mpi: remove redundant variable esign
  crypto: arm64/aes-ce-gcm - don't reload key schedule if avoidable
  crypto: arm64/aes-ce-gcm - implement 2-way aggregation
  crypto: arm64/aes-ce-gcm - operate on two input blocks at a time
  crypto: dh - make crypto_dh_encode_key() make robust
  crypto: dh - fix calculating encoded key size
  crypto: ccp - Check for NULL PSP pointer at module unload
  crypto: arm/chacha20 - always use vrev for 16-bit rotates
  crypto: ccree - allow bigger than sector XTS op
  crypto: ccree - zero all of request ctx before use
  crypto: ccree - remove cipher ivgen left overs
  crypto: ccree - drop useless type flag during reg
  crypto: ablkcipher - fix crash flushing dcache in error path
  crypto: blkcipher - fix crash flushing dcache in error path
  crypto: skcipher - fix crash flushing dcache in error path
  crypto: skcipher - remove unnecessary setting of walk->nbytes
  crypto: scatterwalk - remove scatterwalk_samebuf()
  ...
This commit is contained in:
Linus Torvalds 2018-08-15 16:01:47 -07:00
commit dafa5f6577
174 changed files with 6686 additions and 2009 deletions
Download patch file Download diff file Expand all files Collapse all files

15
drivers/crypto/Kconfig
View file

@ -585,6 +585,17 @@ config CRYPTO_DEV_QCE
hardware. To compile this driver as a module, choose M here. The
module will be called qcrypto.
config CRYPTO_DEV_QCOM_RNG
tristate "Qualcomm Random Number Generator Driver"
depends on ARCH_QCOM || COMPILE_TEST
select CRYPTO_RNG
help
This driver provides support for the Random Number
Generator hardware found on Qualcomm SoCs.
To compile this driver as a module, choose M here. The
module will be called qcom-rng. If unsure, say N.
config CRYPTO_DEV_VMX
bool "Support for VMX cryptographic acceleration instructions"
depends on PPC64 && VSX
@ -689,8 +700,10 @@ config CRYPTO_DEV_SAFEXCEL
select CRYPTO_AES
select CRYPTO_AUTHENC
select CRYPTO_BLKCIPHER
select CRYPTO_DES
select CRYPTO_HASH
select CRYPTO_HMAC
select CRYPTO_MD5
select CRYPTO_SHA1
select CRYPTO_SHA256
select CRYPTO_SHA512
@ -746,4 +759,6 @@ config CRYPTO_DEV_CCREE
cryptographic operations on the system REE.
If unsure say Y.
source "drivers/crypto/hisilicon/Kconfig"
endif # CRYPTO_HW

2
drivers/crypto/Makefile
View file

@ -33,6 +33,7 @@ obj-$(CONFIG_CRYPTO_DEV_PICOXCELL) += picoxcell_crypto.o
obj-$(CONFIG_CRYPTO_DEV_PPC4XX) += amcc/
obj-$(CONFIG_CRYPTO_DEV_QAT) += qat/
obj-$(CONFIG_CRYPTO_DEV_QCE) += qce/
obj-$(CONFIG_CRYPTO_DEV_QCOM_RNG) += qcom-rng.o
obj-$(CONFIG_CRYPTO_DEV_ROCKCHIP) += rockchip/
obj-$(CONFIG_CRYPTO_DEV_S5P) += s5p-sss.o
obj-$(CONFIG_CRYPTO_DEV_SAHARA) += sahara.o
@ -45,3 +46,4 @@ obj-$(CONFIG_CRYPTO_DEV_VMX) += vmx/
obj-$(CONFIG_CRYPTO_DEV_BCM_SPU) += bcm/
obj-$(CONFIG_CRYPTO_DEV_SAFEXCEL) += inside-secure/
obj-$(CONFIG_CRYPTO_DEV_ARTPEC6) += axis/
obj-y += hisilicon/

18
drivers/crypto/amcc/crypto4xx_core.c
View file

@ -1132,8 +1132,7 @@ static struct crypto4xx_alg_common crypto4xx_alg[] = {
.cra_name = "cbc(aes)",
.cra_driver_name = "cbc-aes-ppc4xx",
.cra_priority = CRYPTO4XX_CRYPTO_PRIORITY,
.cra_flags = CRYPTO_ALG_TYPE_SKCIPHER |
CRYPTO_ALG_ASYNC |
.cra_flags = CRYPTO_ALG_ASYNC |
CRYPTO_ALG_KERN_DRIVER_ONLY,
.cra_blocksize = AES_BLOCK_SIZE,
.cra_ctxsize = sizeof(struct crypto4xx_ctx),
@ -1153,8 +1152,7 @@ static struct crypto4xx_alg_common crypto4xx_alg[] = {
.cra_name = "cfb(aes)",
.cra_driver_name = "cfb-aes-ppc4xx",
.cra_priority = CRYPTO4XX_CRYPTO_PRIORITY,
.cra_flags = CRYPTO_ALG_TYPE_SKCIPHER |
CRYPTO_ALG_ASYNC |
.cra_flags = CRYPTO_ALG_ASYNC |
CRYPTO_ALG_KERN_DRIVER_ONLY,
.cra_blocksize = AES_BLOCK_SIZE,
.cra_ctxsize = sizeof(struct crypto4xx_ctx),
@ -1174,8 +1172,7 @@ static struct crypto4xx_alg_common crypto4xx_alg[] = {
.cra_name = "ctr(aes)",
.cra_driver_name = "ctr-aes-ppc4xx",
.cra_priority = CRYPTO4XX_CRYPTO_PRIORITY,
.cra_flags = CRYPTO_ALG_TYPE_SKCIPHER |
CRYPTO_ALG_NEED_FALLBACK |
.cra_flags = CRYPTO_ALG_NEED_FALLBACK |
CRYPTO_ALG_ASYNC |
CRYPTO_ALG_KERN_DRIVER_ONLY,
.cra_blocksize = AES_BLOCK_SIZE,
@ -1196,8 +1193,7 @@ static struct crypto4xx_alg_common crypto4xx_alg[] = {
.cra_name = "rfc3686(ctr(aes))",
.cra_driver_name = "rfc3686-ctr-aes-ppc4xx",
.cra_priority = CRYPTO4XX_CRYPTO_PRIORITY,
.cra_flags = CRYPTO_ALG_TYPE_SKCIPHER |
CRYPTO_ALG_ASYNC |
.cra_flags = CRYPTO_ALG_ASYNC |
CRYPTO_ALG_KERN_DRIVER_ONLY,
.cra_blocksize = AES_BLOCK_SIZE,
.cra_ctxsize = sizeof(struct crypto4xx_ctx),
@ -1217,8 +1213,7 @@ static struct crypto4xx_alg_common crypto4xx_alg[] = {
.cra_name = "ecb(aes)",
.cra_driver_name = "ecb-aes-ppc4xx",
.cra_priority = CRYPTO4XX_CRYPTO_PRIORITY,
.cra_flags = CRYPTO_ALG_TYPE_SKCIPHER |
CRYPTO_ALG_ASYNC |
.cra_flags = CRYPTO_ALG_ASYNC |
CRYPTO_ALG_KERN_DRIVER_ONLY,
.cra_blocksize = AES_BLOCK_SIZE,
.cra_ctxsize = sizeof(struct crypto4xx_ctx),
@ -1237,8 +1232,7 @@ static struct crypto4xx_alg_common crypto4xx_alg[] = {
.cra_name = "ofb(aes)",
.cra_driver_name = "ofb-aes-ppc4xx",
.cra_priority = CRYPTO4XX_CRYPTO_PRIORITY,
.cra_flags = CRYPTO_ALG_TYPE_SKCIPHER |
CRYPTO_ALG_ASYNC |
.cra_flags = CRYPTO_ALG_ASYNC |
CRYPTO_ALG_KERN_DRIVER_ONLY,
.cra_blocksize = AES_BLOCK_SIZE,
.cra_ctxsize = sizeof(struct crypto4xx_ctx),

35
drivers/crypto/atmel-ecc.c
View file

@ -186,7 +186,10 @@ static int atmel_ecc_init_ecdh_cmd(struct atmel_ecc_cmd *cmd,
* always be the same. Use a macro for the key size to avoid unnecessary
* computations.
*/
copied = sg_copy_to_buffer(pubkey, 1, cmd->data, ATMEL_ECC_PUBKEY_SIZE);
copied = sg_copy_to_buffer(pubkey,
sg_nents_for_len(pubkey,
ATMEL_ECC_PUBKEY_SIZE),
cmd->data, ATMEL_ECC_PUBKEY_SIZE);
if (copied != ATMEL_ECC_PUBKEY_SIZE)
return -EINVAL;
@ -268,15 +271,17 @@ static void atmel_ecdh_done(struct atmel_ecc_work_data *work_data, void *areq,
struct kpp_request *req = areq;
struct atmel_ecdh_ctx *ctx = work_data->ctx;
struct atmel_ecc_cmd *cmd = &work_data->cmd;
size_t copied;
size_t n_sz = ctx->n_sz;
size_t copied, n_sz;
if (status)
goto free_work_data;
/* might want less than we've got */
n_sz = min_t(size_t, ctx->n_sz, req->dst_len);
/* copy the shared secret */
copied = sg_copy_from_buffer(req->dst, 1, &cmd->data[RSP_DATA_IDX],
n_sz);
copied = sg_copy_from_buffer(req->dst, sg_nents_for_len(req->dst, n_sz),
&cmd->data[RSP_DATA_IDX], n_sz);
if (copied != n_sz)
status = -EINVAL;
@ -440,7 +445,7 @@ static int atmel_ecdh_generate_public_key(struct kpp_request *req)
{
struct crypto_kpp *tfm = crypto_kpp_reqtfm(req);
struct atmel_ecdh_ctx *ctx = kpp_tfm_ctx(tfm);
size_t copied;
size_t copied, nbytes;
int ret = 0;
if (ctx->do_fallback) {
@ -448,10 +453,14 @@ static int atmel_ecdh_generate_public_key(struct kpp_request *req)
return crypto_kpp_generate_public_key(req);
}
/* might want less than we've got */
nbytes = min_t(size_t, ATMEL_ECC_PUBKEY_SIZE, req->dst_len);
/* public key was saved at private key generation */
copied = sg_copy_from_buffer(req->dst, 1, ctx->public_key,
ATMEL_ECC_PUBKEY_SIZE);
if (copied != ATMEL_ECC_PUBKEY_SIZE)
copied = sg_copy_from_buffer(req->dst,
sg_nents_for_len(req->dst, nbytes),
ctx->public_key, nbytes);
if (copied != nbytes)
ret = -EINVAL;
return ret;
@ -470,6 +479,10 @@ static int atmel_ecdh_compute_shared_secret(struct kpp_request *req)
return crypto_kpp_compute_shared_secret(req);
}
/* must have exactly two points to be on the curve */
if (req->src_len != ATMEL_ECC_PUBKEY_SIZE)
return -EINVAL;
gfp = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? GFP_KERNEL :
GFP_ATOMIC;
@ -554,10 +567,6 @@ static int atmel_ecdh_init_tfm(struct crypto_kpp *tfm)
}
crypto_kpp_set_flags(fallback, crypto_kpp_get_flags(tfm));
dev_info(&ctx->client->dev, "Using '%s' as fallback implementation.\n",
crypto_tfm_alg_driver_name(crypto_kpp_tfm(fallback)));
ctx->fallback = fallback;
return 0;

4
drivers/crypto/atmel-sha.c
View file

@ -2316,9 +2316,7 @@ struct atmel_sha_authenc_ctx *atmel_sha_authenc_spawn(unsigned long mode)
goto error;
}
tfm = crypto_alloc_ahash(name,
CRYPTO_ALG_TYPE_AHASH,
CRYPTO_ALG_TYPE_AHASH_MASK);
tfm = crypto_alloc_ahash(name, 0, 0);
if (IS_ERR(tfm)) {
err = PTR_ERR(tfm);
goto error;

28
drivers/crypto/axis/artpec6_crypto.c
View file

@ -2704,7 +2704,7 @@ static struct ahash_alg hash_algos[] = {
.cra_name = "sha1",
.cra_driver_name = "artpec-sha1",
.cra_priority = 300,
.cra_flags = CRYPTO_ALG_TYPE_AHASH | CRYPTO_ALG_ASYNC,
.cra_flags = CRYPTO_ALG_ASYNC,
.cra_blocksize = SHA1_BLOCK_SIZE,
.cra_ctxsize = sizeof(struct artpec6_hashalg_context),
.cra_alignmask = 3,
@ -2727,7 +2727,7 @@ static struct ahash_alg hash_algos[] = {
.cra_name = "sha256",
.cra_driver_name = "artpec-sha256",
.cra_priority = 300,
.cra_flags = CRYPTO_ALG_TYPE_AHASH | CRYPTO_ALG_ASYNC,
.cra_flags = CRYPTO_ALG_ASYNC,
.cra_blocksize = SHA256_BLOCK_SIZE,
.cra_ctxsize = sizeof(struct artpec6_hashalg_context),
.cra_alignmask = 3,
@ -2751,7 +2751,7 @@ static struct ahash_alg hash_algos[] = {
.cra_name = "hmac(sha256)",
.cra_driver_name = "artpec-hmac-sha256",
.cra_priority = 300,
.cra_flags = CRYPTO_ALG_TYPE_AHASH | CRYPTO_ALG_ASYNC,
.cra_flags = CRYPTO_ALG_ASYNC,
.cra_blocksize = SHA256_BLOCK_SIZE,
.cra_ctxsize = sizeof(struct artpec6_hashalg_context),
.cra_alignmask = 3,
@ -2777,7 +2777,7 @@ static struct ahash_alg artpec7_hash_algos[] = {
.cra_name = "sha384",
.cra_driver_name = "artpec-sha384",
.cra_priority = 300,
.cra_flags = CRYPTO_ALG_TYPE_AHASH | CRYPTO_ALG_ASYNC,
.cra_flags = CRYPTO_ALG_ASYNC,
.cra_blocksize = SHA384_BLOCK_SIZE,
.cra_ctxsize = sizeof(struct artpec6_hashalg_context),
.cra_alignmask = 3,
@ -2801,7 +2801,7 @@ static struct ahash_alg artpec7_hash_algos[] = {
.cra_name = "hmac(sha384)",
.cra_driver_name = "artpec-hmac-sha384",
.cra_priority = 300,
.cra_flags = CRYPTO_ALG_TYPE_AHASH | CRYPTO_ALG_ASYNC,
.cra_flags = CRYPTO_ALG_ASYNC,
.cra_blocksize = SHA384_BLOCK_SIZE,
.cra_ctxsize = sizeof(struct artpec6_hashalg_context),
.cra_alignmask = 3,
@ -2824,7 +2824,7 @@ static struct ahash_alg artpec7_hash_algos[] = {
.cra_name = "sha512",
.cra_driver_name = "artpec-sha512",
.cra_priority = 300,
.cra_flags = CRYPTO_ALG_TYPE_AHASH | CRYPTO_ALG_ASYNC,
.cra_flags = CRYPTO_ALG_ASYNC,
.cra_blocksize = SHA512_BLOCK_SIZE,
.cra_ctxsize = sizeof(struct artpec6_hashalg_context),
.cra_alignmask = 3,
@ -2848,7 +2848,7 @@ static struct ahash_alg artpec7_hash_algos[] = {
.cra_name = "hmac(sha512)",
.cra_driver_name = "artpec-hmac-sha512",
.cra_priority = 300,
.cra_flags = CRYPTO_ALG_TYPE_AHASH | CRYPTO_ALG_ASYNC,
.cra_flags = CRYPTO_ALG_ASYNC,
.cra_blocksize = SHA512_BLOCK_SIZE,
.cra_ctxsize = sizeof(struct artpec6_hashalg_context),
.cra_alignmask = 3,
@ -2867,8 +2867,7 @@ static struct skcipher_alg crypto_algos[] = {
.cra_name = "ecb(aes)",
.cra_driver_name = "artpec6-ecb-aes",
.cra_priority = 300,
.cra_flags = CRYPTO_ALG_TYPE_SKCIPHER |
CRYPTO_ALG_ASYNC,
.cra_flags = CRYPTO_ALG_ASYNC,
.cra_blocksize = AES_BLOCK_SIZE,
.cra_ctxsize = sizeof(struct artpec6_cryptotfm_context),
.cra_alignmask = 3,
@ -2888,8 +2887,7 @@ static struct skcipher_alg crypto_algos[] = {
.cra_name = "ctr(aes)",
.cra_driver_name = "artpec6-ctr-aes",
.cra_priority = 300,
.cra_flags = CRYPTO_ALG_TYPE_SKCIPHER |
CRYPTO_ALG_ASYNC |
.cra_flags = CRYPTO_ALG_ASYNC |
CRYPTO_ALG_NEED_FALLBACK,
.cra_blocksize = 1,
.cra_ctxsize = sizeof(struct artpec6_cryptotfm_context),
@ -2911,8 +2909,7 @@ static struct skcipher_alg crypto_algos[] = {
.cra_name = "cbc(aes)",
.cra_driver_name = "artpec6-cbc-aes",
.cra_priority = 300,
.cra_flags = CRYPTO_ALG_TYPE_SKCIPHER |
CRYPTO_ALG_ASYNC,
.cra_flags = CRYPTO_ALG_ASYNC,
.cra_blocksize = AES_BLOCK_SIZE,
.cra_ctxsize = sizeof(struct artpec6_cryptotfm_context),
.cra_alignmask = 3,
@ -2933,8 +2930,7 @@ static struct skcipher_alg crypto_algos[] = {
.cra_name = "xts(aes)",
.cra_driver_name = "artpec6-xts-aes",
.cra_priority = 300,
.cra_flags = CRYPTO_ALG_TYPE_SKCIPHER |
CRYPTO_ALG_ASYNC,
.cra_flags = CRYPTO_ALG_ASYNC,
.cra_blocksize = 1,
.cra_ctxsize = sizeof(struct artpec6_cryptotfm_context),
.cra_alignmask = 3,
@ -2964,7 +2960,7 @@ static struct aead_alg aead_algos[] = {
.cra_name = "gcm(aes)",
.cra_driver_name = "artpec-gcm-aes",
.cra_priority = 300,
.cra_flags = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_ASYNC |
.cra_flags = CRYPTO_ALG_ASYNC |
CRYPTO_ALG_KERN_DRIVER_ONLY,
.cra_blocksize = 1,
.cra_ctxsize = sizeof(struct artpec6_cryptotfm_context),

8
drivers/crypto/bcm/cipher.c
View file

@ -3914,8 +3914,7 @@ static struct iproc_alg_s driver_algs[] = {
.cra_name = "md5",
.cra_driver_name = "md5-iproc",
.cra_blocksize = MD5_BLOCK_WORDS * 4,
.cra_flags = CRYPTO_ALG_TYPE_AHASH |
CRYPTO_ALG_ASYNC,
.cra_flags = CRYPTO_ALG_ASYNC,
}
},
.cipher_info = {
@ -4649,8 +4648,7 @@ static int spu_register_ahash(struct iproc_alg_s *driver_alg)
hash->halg.base.cra_ctxsize = sizeof(struct iproc_ctx_s);
hash->halg.base.cra_init = ahash_cra_init;
hash->halg.base.cra_exit = generic_cra_exit;
hash->halg.base.cra_type = &crypto_ahash_type;
hash->halg.base.cra_flags = CRYPTO_ALG_TYPE_AHASH | CRYPTO_ALG_ASYNC;
hash->halg.base.cra_flags = CRYPTO_ALG_ASYNC;
hash->halg.statesize = sizeof(struct spu_hash_export_s);
if (driver_alg->auth_info.mode != HASH_MODE_HMAC) {
@ -4691,7 +4689,7 @@ static int spu_register_aead(struct iproc_alg_s *driver_alg)
aead->base.cra_ctxsize = sizeof(struct iproc_ctx_s);
INIT_LIST_HEAD(&aead->base.cra_list);
aead->base.cra_flags |= CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_ASYNC;
aead->base.cra_flags |= CRYPTO_ALG_ASYNC;
/* setkey set in alg initialization */
aead->setauthsize = aead_setauthsize;
aead->encrypt = aead_encrypt;

3
drivers/crypto/caam/caamhash.c
View file

@ -1846,8 +1846,7 @@ caam_hash_alloc(struct caam_hash_template *template,
alg->cra_priority = CAAM_CRA_PRIORITY;
alg->cra_blocksize = template->blocksize;
alg->cra_alignmask = 0;
alg->cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_TYPE_AHASH;
alg->cra_type = &crypto_ahash_type;
alg->cra_flags = CRYPTO_ALG_ASYNC;
t_alg->alg_type = template->alg_type;

2
drivers/crypto/cavium/cpt/cptvf_algs.c
View file

@ -351,7 +351,7 @@ static int cvm_enc_dec_init(struct crypto_tfm *tfm)
return 0;
}
struct crypto_alg algs[] = { {
static struct crypto_alg algs[] = { {
.cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
.cra_blocksize = AES_BLOCK_SIZE,
.cra_ctxsize = sizeof(struct cvm_enc_ctx),

2
drivers/crypto/cavium/nitrox/nitrox_lib.c
View file

@ -148,7 +148,7 @@ void *crypto_alloc_context(struct nitrox_device *ndev)
void *vaddr;
dma_addr_t dma;
vaddr = dma_pool_alloc(ndev->ctx_pool, (GFP_ATOMIC | __GFP_ZERO), &dma);
vaddr = dma_pool_alloc(ndev->ctx_pool, (GFP_KERNEL | __GFP_ZERO), &dma);
if (!vaddr)
return NULL;

3
drivers/crypto/ccp/ccp-crypto-aes-cmac.c
View file

@ -399,13 +399,12 @@ int ccp_register_aes_cmac_algs(struct list_head *head)
base = &halg->base;
snprintf(base->cra_name, CRYPTO_MAX_ALG_NAME, "cmac(aes)");
snprintf(base->cra_driver_name, CRYPTO_MAX_ALG_NAME, "cmac-aes-ccp");
base->cra_flags = CRYPTO_ALG_TYPE_AHASH | CRYPTO_ALG_ASYNC |
base->cra_flags = CRYPTO_ALG_ASYNC |
CRYPTO_ALG_KERN_DRIVER_ONLY |
CRYPTO_ALG_NEED_FALLBACK;
base->cra_blocksize = AES_BLOCK_SIZE;
base->cra_ctxsize = sizeof(struct ccp_ctx);
base->cra_priority = CCP_CRA_PRIORITY;
base->cra_type = &crypto_ahash_type;
base->cra_init = ccp_aes_cmac_cra_init;
base->cra_exit = ccp_aes_cmac_cra_exit;
base->cra_module = THIS_MODULE;

3
drivers/crypto/ccp/ccp-crypto-sha.c
View file

@ -497,13 +497,12 @@ static int ccp_register_sha_alg(struct list_head *head,
snprintf(base->cra_name, CRYPTO_MAX_ALG_NAME, "%s", def->name);
snprintf(base->cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s",
def->drv_name);
base->cra_flags = CRYPTO_ALG_TYPE_AHASH | CRYPTO_ALG_ASYNC |
base->cra_flags = CRYPTO_ALG_ASYNC |
CRYPTO_ALG_KERN_DRIVER_ONLY |
CRYPTO_ALG_NEED_FALLBACK;
base->cra_blocksize = def->block_size;
base->cra_ctxsize = sizeof(struct ccp_ctx);
base->cra_priority = CCP_CRA_PRIORITY;
base->cra_type = &crypto_ahash_type;
base->cra_init = ccp_sha_cra_init;
base->cra_exit = ccp_sha_cra_exit;
base->cra_module = THIS_MODULE;

35
drivers/crypto/ccp/psp-dev.c
View file

@ -62,14 +62,14 @@ static irqreturn_t psp_irq_handler(int irq, void *data)
int reg;
/* Read the interrupt status: */
status = ioread32(psp->io_regs + PSP_P2CMSG_INTSTS);
status = ioread32(psp->io_regs + psp->vdata->intsts_reg);
/* Check if it is command completion: */
if (!(status & BIT(PSP_CMD_COMPLETE_REG)))
if (!(status & PSP_CMD_COMPLETE))
goto done;
/* Check if it is SEV command completion: */
reg = ioread32(psp->io_regs + PSP_CMDRESP);
reg = ioread32(psp->io_regs + psp->vdata->cmdresp_reg);
if (reg & PSP_CMDRESP_RESP) {
psp->sev_int_rcvd = 1;
wake_up(&psp->sev_int_queue);
@ -77,17 +77,15 @@ static irqreturn_t psp_irq_handler(int irq, void *data)
done:
/* Clear the interrupt status by writing the same value we read. */
iowrite32(status, psp->io_regs + PSP_P2CMSG_INTSTS);
iowrite32(status, psp->io_regs + psp->vdata->intsts_reg);
return IRQ_HANDLED;
}
static void sev_wait_cmd_ioc(struct psp_device *psp, unsigned int *reg)
{
psp->sev_int_rcvd = 0;
wait_event(psp->sev_int_queue, psp->sev_int_rcvd);
*reg = ioread32(psp->io_regs + PSP_CMDRESP);
*reg = ioread32(psp->io_regs + psp->vdata->cmdresp_reg);
}
static int sev_cmd_buffer_len(int cmd)
@ -145,13 +143,15 @@ static int __sev_do_cmd_locked(int cmd, void *data, int *psp_ret)
print_hex_dump_debug("(in): ", DUMP_PREFIX_OFFSET, 16, 2, data,
sev_cmd_buffer_len(cmd), false);
iowrite32(phys_lsb, psp->io_regs + PSP_CMDBUFF_ADDR_LO);
iowrite32(phys_msb, psp->io_regs + PSP_CMDBUFF_ADDR_HI);
iowrite32(phys_lsb, psp->io_regs + psp->vdata->cmdbuff_addr_lo_reg);
iowrite32(phys_msb, psp->io_regs + psp->vdata->cmdbuff_addr_hi_reg);
psp->sev_int_rcvd = 0;
reg = cmd;
reg <<= PSP_CMDRESP_CMD_SHIFT;
reg |= PSP_CMDRESP_IOC;
iowrite32(reg, psp->io_regs + PSP_CMDRESP);
iowrite32(reg, psp->io_regs + psp->vdata->cmdresp_reg);
/* wait for command completion */
sev_wait_cmd_ioc(psp, &reg);
@ -789,7 +789,7 @@ static int sev_misc_init(struct psp_device *psp)
static int sev_init(struct psp_device *psp)
{
/* Check if device supports SEV feature */
if (!(ioread32(psp->io_regs + PSP_FEATURE_REG) & 1)) {
if (!(ioread32(psp->io_regs + psp->vdata->feature_reg) & 1)) {
dev_dbg(psp->dev, "device does not support SEV\n");
return 1;
}
@ -817,11 +817,11 @@ int psp_dev_init(struct sp_device *sp)
goto e_err;
}
psp->io_regs = sp->io_map + psp->vdata->offset;
psp->io_regs = sp->io_map;
/* Disable and clear interrupts until ready */
iowrite32(0, psp->io_regs + PSP_P2CMSG_INTEN);
iowrite32(-1, psp->io_regs + PSP_P2CMSG_INTSTS);
iowrite32(0, psp->io_regs + psp->vdata->inten_reg);
iowrite32(-1, psp->io_regs + psp->vdata->intsts_reg);
/* Request an irq */
ret = sp_request_psp_irq(psp->sp, psp_irq_handler, psp->name, psp);
@ -838,7 +838,9 @@ int psp_dev_init(struct sp_device *sp)
sp->set_psp_master_device(sp);
/* Enable interrupt */
iowrite32(-1, psp->io_regs + PSP_P2CMSG_INTEN);
iowrite32(-1, psp->io_regs + psp->vdata->inten_reg);
dev_notice(dev, "psp enabled\n");
return 0;
@ -856,6 +858,9 @@ void psp_dev_destroy(struct sp_device *sp)
{
struct psp_device *psp = sp->psp_data;
if (!psp)
return;
if (psp->sev_misc)
kref_put(&misc_dev->refcount, sev_exit);

19
drivers/crypto/ccp/psp-dev.h
View file

@ -30,24 +30,7 @@
#include "sp-dev.h"
#define PSP_C2PMSG(_num) ((_num) << 2)
#define PSP_CMDRESP PSP_C2PMSG(32)
#define PSP_CMDBUFF_ADDR_LO PSP_C2PMSG(56)
#define PSP_CMDBUFF_ADDR_HI PSP_C2PMSG(57)
#define PSP_FEATURE_REG PSP_C2PMSG(63)
#define PSP_P2CMSG(_num) ((_num) << 2)
#define PSP_CMD_COMPLETE_REG 1
#define PSP_CMD_COMPLETE PSP_P2CMSG(PSP_CMD_COMPLETE_REG)
#define PSP_P2CMSG_INTEN 0x0110
#define PSP_P2CMSG_INTSTS 0x0114
#define PSP_C2PMSG_ATTR_0 0x0118
#define PSP_C2PMSG_ATTR_1 0x011c
#define PSP_C2PMSG_ATTR_2 0x0120
#define PSP_C2PMSG_ATTR_3 0x0124
#define PSP_P2CMSG_ATTR_0 0x0128
#define PSP_CMD_COMPLETE BIT(1)
#define PSP_CMDRESP_CMD_SHIFT 16
#define PSP_CMDRESP_IOC BIT(0)

7
drivers/crypto/ccp/sp-dev.h
View file

@ -44,7 +44,12 @@ struct ccp_vdata {
};
struct psp_vdata {
const unsigned int offset;
const unsigned int cmdresp_reg;
const unsigned int cmdbuff_addr_lo_reg;
const unsigned int cmdbuff_addr_hi_reg;
const unsigned int feature_reg;
const unsigned int inten_reg;
const unsigned int intsts_reg;
};
/* Structure to hold SP device data */

36
drivers/crypto/ccp/sp-pci.c
View file

@ -269,31 +269,54 @@ static int sp_pci_resume(struct pci_dev *pdev)
#endif
#ifdef CONFIG_CRYPTO_DEV_SP_PSP
static const struct psp_vdata psp_entry = {
.offset = 0x10500,
static const struct psp_vdata pspv1 = {
.cmdresp_reg = 0x10580,
.cmdbuff_addr_lo_reg = 0x105e0,
.cmdbuff_addr_hi_reg = 0x105e4,
.feature_reg = 0x105fc,
.inten_reg = 0x10610,
.intsts_reg = 0x10614,
};
static const struct psp_vdata pspv2 = {
.cmdresp_reg = 0x10980,
.cmdbuff_addr_lo_reg = 0x109e0,
.cmdbuff_addr_hi_reg = 0x109e4,
.feature_reg = 0x109fc,
.inten_reg = 0x10690,
.intsts_reg = 0x10694,
};
#endif
static const struct sp_dev_vdata dev_vdata[] = {
{
{ /* 0 */
.bar = 2,
#ifdef CONFIG_CRYPTO_DEV_SP_CCP
.ccp_vdata = &ccpv3,
#endif
},
{
{ /* 1 */
.bar = 2,
#ifdef CONFIG_CRYPTO_DEV_SP_CCP
.ccp_vdata = &ccpv5a,
#endif
#ifdef CONFIG_CRYPTO_DEV_SP_PSP
.psp_vdata = &psp_entry
.psp_vdata = &pspv1,
#endif
},
{
{ /* 2 */
.bar = 2,
#ifdef CONFIG_CRYPTO_DEV_SP_CCP
.ccp_vdata = &ccpv5b,
#endif
},
{ /* 3 */
.bar = 2,
#ifdef CONFIG_CRYPTO_DEV_SP_CCP
.ccp_vdata = &ccpv5a,
#endif
#ifdef CONFIG_CRYPTO_DEV_SP_PSP
.psp_vdata = &pspv2,
#endif
},
};
@ -301,6 +324,7 @@ static const struct pci_device_id sp_pci_table[] = {
{ PCI_VDEVICE(AMD, 0x1537), (kernel_ulong_t)&dev_vdata[0] },
{ PCI_VDEVICE(AMD, 0x1456), (kernel_ulong_t)&dev_vdata[1] },
{ PCI_VDEVICE(AMD, 0x1468), (kernel_ulong_t)&dev_vdata[2] },
{ PCI_VDEVICE(AMD, 0x1486), (kernel_ulong_t)&dev_vdata[3] },
/* Last entry must be zero */
{ 0, }
};

16
drivers/crypto/ccree/cc_aead.c
View file

@ -2344,7 +2344,6 @@ static struct cc_alg_template aead_algs[] = {
.name = "authenc(hmac(sha1),cbc(aes))",
.driver_name = "authenc-hmac-sha1-cbc-aes-ccree",
.blocksize = AES_BLOCK_SIZE,
.type = CRYPTO_ALG_TYPE_AEAD,
.template_aead = {
.setkey = cc_aead_setkey,
.setauthsize = cc_aead_setauthsize,
@ -2364,7 +2363,6 @@ static struct cc_alg_template aead_algs[] = {
.name = "authenc(hmac(sha1),cbc(des3_ede))",
.driver_name = "authenc-hmac-sha1-cbc-des3-ccree",
.blocksize = DES3_EDE_BLOCK_SIZE,
.type = CRYPTO_ALG_TYPE_AEAD,
.template_aead = {
.setkey = cc_aead_setkey,
.setauthsize = cc_aead_setauthsize,
@ -2384,7 +2382,6 @@ static struct cc_alg_template aead_algs[] = {
.name = "authenc(hmac(sha256),cbc(aes))",
.driver_name = "authenc-hmac-sha256-cbc-aes-ccree",
.blocksize = AES_BLOCK_SIZE,
.type = CRYPTO_ALG_TYPE_AEAD,
.template_aead = {
.setkey = cc_aead_setkey,
.setauthsize = cc_aead_setauthsize,
@ -2404,7 +2401,6 @@ static struct cc_alg_template aead_algs[] = {
.name = "authenc(hmac(sha256),cbc(des3_ede))",
.driver_name = "authenc-hmac-sha256-cbc-des3-ccree",
.blocksize = DES3_EDE_BLOCK_SIZE,
.type = CRYPTO_ALG_TYPE_AEAD,
.template_aead = {
.setkey = cc_aead_setkey,
.setauthsize = cc_aead_setauthsize,
@ -2424,7 +2420,6 @@ static struct cc_alg_template aead_algs[] = {
.name = "authenc(xcbc(aes),cbc(aes))",
.driver_name = "authenc-xcbc-aes-cbc-aes-ccree",
.blocksize = AES_BLOCK_SIZE,
.type = CRYPTO_ALG_TYPE_AEAD,
.template_aead = {
.setkey = cc_aead_setkey,
.setauthsize = cc_aead_setauthsize,
@ -2444,7 +2439,6 @@ static struct cc_alg_template aead_algs[] = {
.name = "authenc(hmac(sha1),rfc3686(ctr(aes)))",
.driver_name = "authenc-hmac-sha1-rfc3686-ctr-aes-ccree",
.blocksize = 1,
.type = CRYPTO_ALG_TYPE_AEAD,
.template_aead = {
.setkey = cc_aead_setkey,
.setauthsize = cc_aead_setauthsize,
@ -2464,7 +2458,6 @@ static struct cc_alg_template aead_algs[] = {
.name = "authenc(hmac(sha256),rfc3686(ctr(aes)))",
.driver_name = "authenc-hmac-sha256-rfc3686-ctr-aes-ccree",
.blocksize = 1,
.type = CRYPTO_ALG_TYPE_AEAD,
.template_aead = {
.setkey = cc_aead_setkey,
.setauthsize = cc_aead_setauthsize,
@ -2484,7 +2477,6 @@ static struct cc_alg_template aead_algs[] = {
.name = "authenc(xcbc(aes),rfc3686(ctr(aes)))",
.driver_name = "authenc-xcbc-aes-rfc3686-ctr-aes-ccree",
.blocksize = 1,
.type = CRYPTO_ALG_TYPE_AEAD,
.template_aead = {
.setkey = cc_aead_setkey,
.setauthsize = cc_aead_setauthsize,
@ -2504,7 +2496,6 @@ static struct cc_alg_template aead_algs[] = {
.name = "ccm(aes)",
.driver_name = "ccm-aes-ccree",
.blocksize = 1,
.type = CRYPTO_ALG_TYPE_AEAD,
.template_aead = {
.setkey = cc_aead_setkey,
.setauthsize = cc_ccm_setauthsize,
@ -2524,7 +2515,6 @@ static struct cc_alg_template aead_algs[] = {
.name = "rfc4309(ccm(aes))",
.driver_name = "rfc4309-ccm-aes-ccree",
.blocksize = 1,
.type = CRYPTO_ALG_TYPE_AEAD,
.template_aead = {
.setkey = cc_rfc4309_ccm_setkey,
.setauthsize = cc_rfc4309_ccm_setauthsize,
@ -2544,7 +2534,6 @@ static struct cc_alg_template aead_algs[] = {
.name = "gcm(aes)",
.driver_name = "gcm-aes-ccree",
.blocksize = 1,
.type = CRYPTO_ALG_TYPE_AEAD,
.template_aead = {
.setkey = cc_aead_setkey,
.setauthsize = cc_gcm_setauthsize,
@ -2564,7 +2553,6 @@ static struct cc_alg_template aead_algs[] = {
.name = "rfc4106(gcm(aes))",
.driver_name = "rfc4106-gcm-aes-ccree",
.blocksize = 1,
.type = CRYPTO_ALG_TYPE_AEAD,
.template_aead = {
.setkey = cc_rfc4106_gcm_setkey,
.setauthsize = cc_rfc4106_gcm_setauthsize,
@ -2584,7 +2572,6 @@ static struct cc_alg_template aead_algs[] = {
.name = "rfc4543(gcm(aes))",
.driver_name = "rfc4543-gcm-aes-ccree",
.blocksize = 1,
.type = CRYPTO_ALG_TYPE_AEAD,
.template_aead = {
.setkey = cc_rfc4543_gcm_setkey,
.setauthsize = cc_rfc4543_gcm_setauthsize,
@ -2621,8 +2608,7 @@ static struct cc_crypto_alg *cc_create_aead_alg(struct cc_alg_template *tmpl,
alg->base.cra_priority = CC_CRA_PRIO;
alg->base.cra_ctxsize = sizeof(struct cc_aead_ctx);
alg->base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_KERN_DRIVER_ONLY |
tmpl->type;
alg->base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_KERN_DRIVER_ONLY;
alg->init = cc_aead_init;
alg->exit = cc_aead_exit;

8
drivers/crypto/ccree/cc_buffer_mgr.c
View file

@ -454,9 +454,7 @@ void cc_unmap_cipher_request(struct device *dev, void *ctx,
dev_dbg(dev, "Unmapped iv: iv_dma_addr=%pad iv_size=%u\n",
&req_ctx->gen_ctx.iv_dma_addr, ivsize);
dma_unmap_single(dev, req_ctx->gen_ctx.iv_dma_addr,
ivsize,
req_ctx->is_giv ? DMA_BIDIRECTIONAL :
DMA_TO_DEVICE);
ivsize, DMA_TO_DEVICE);
}
/* Release pool */
if (req_ctx->dma_buf_type == CC_DMA_BUF_MLLI &&
@ -498,9 +496,7 @@ int cc_map_cipher_request(struct cc_drvdata *drvdata, void *ctx,
dump_byte_array("iv", (u8 *)info, ivsize);
req_ctx->gen_ctx.iv_dma_addr =
dma_map_single(dev, (void *)info,
ivsize,
req_ctx->is_giv ? DMA_BIDIRECTIONAL :
DMA_TO_DEVICE);
ivsize, DMA_TO_DEVICE);
if (dma_mapping_error(dev, req_ctx->gen_ctx.iv_dma_addr)) {
dev_err(dev, "Mapping iv %u B at va=%pK for DMA failed\n",
ivsize, info);

170
drivers/crypto/ccree/cc_cipher.c
View file

@ -19,8 +19,6 @@
#define template_skcipher template_u.skcipher
#define CC_MIN_AES_XTS_SIZE 0x10
#define CC_MAX_AES_XTS_SIZE 0x2000
struct cc_cipher_handle {
struct list_head alg_list;
};
@ -98,8 +96,7 @@ static int validate_data_size(struct cc_cipher_ctx *ctx_p,
case S_DIN_to_AES:
switch (ctx_p->cipher_mode) {
case DRV_CIPHER_XTS:
if (size >= CC_MIN_AES_XTS_SIZE &&
size <= CC_MAX_AES_XTS_SIZE &&
if (size >= AES_BLOCK_SIZE &&
IS_ALIGNED(size, AES_BLOCK_SIZE))
return 0;
break;
@ -593,34 +590,82 @@ static void cc_setup_cipher_data(struct crypto_tfm *tfm,
}
}
/*
* Update a CTR-AES 128 bit counter
*/
static void cc_update_ctr(u8 *ctr, unsigned int increment)
{
if (IS_ENABLED(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS) ||
IS_ALIGNED((unsigned long)ctr, 8)) {
__be64 *high_be = (__be64 *)ctr;
__be64 *low_be = high_be + 1;
u64 orig_low = __be64_to_cpu(*low_be);
u64 new_low = orig_low + (u64)increment;
*low_be = __cpu_to_be64(new_low);
if (new_low < orig_low)
*high_be = __cpu_to_be64(__be64_to_cpu(*high_be) + 1);
} else {
u8 *pos = (ctr + AES_BLOCK_SIZE);
u8 val;
unsigned int size;
for (; increment; increment--)
for (size = AES_BLOCK_SIZE; size; size--) {
val = *--pos + 1;
*pos = val;
if (val)
break;
}
}
}
static void cc_cipher_complete(struct device *dev, void *cc_req, int err)
{
struct skcipher_request *req = (struct skcipher_request *)cc_req;
struct scatterlist *dst = req->dst;
struct scatterlist *src = req->src;
struct cipher_req_ctx *req_ctx = skcipher_request_ctx(req);
struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
unsigned int ivsize = crypto_skcipher_ivsize(tfm);
struct crypto_skcipher *sk_tfm = crypto_skcipher_reqtfm(req);
struct crypto_tfm *tfm = crypto_skcipher_tfm(sk_tfm);
struct cc_cipher_ctx *ctx_p = crypto_tfm_ctx(tfm);
unsigned int ivsize = crypto_skcipher_ivsize(sk_tfm);
unsigned int len;
switch (ctx_p->cipher_mode) {
case DRV_CIPHER_CBC:
/*
* The crypto API expects us to set the req->iv to the last
* ciphertext block. For encrypt, simply copy from the result.
* For decrypt, we must copy from a saved buffer since this
* could be an in-place decryption operation and the src is
* lost by this point.
*/
if (req_ctx->gen_ctx.op_type == DRV_CRYPTO_DIRECTION_DECRYPT) {
memcpy(req->iv, req_ctx->backup_info, ivsize);
kzfree(req_ctx->backup_info);
} else if (!err) {
len = req->cryptlen - ivsize;
scatterwalk_map_and_copy(req->iv, req->dst, len,
ivsize, 0);
}
break;
case DRV_CIPHER_CTR:
/* Compute the counter of the last block */
len = ALIGN(req->cryptlen, AES_BLOCK_SIZE) / AES_BLOCK_SIZE;
cc_update_ctr((u8 *)req->iv, len);
break;
default:
break;
}
cc_unmap_cipher_request(dev, req_ctx, ivsize, src, dst);
kzfree(req_ctx->iv);
/*
* The crypto API expects us to set the req->iv to the last
* ciphertext block. For encrypt, simply copy from the result.
* For decrypt, we must copy from a saved buffer since this
* could be an in-place decryption operation and the src is
* lost by this point.
*/
if (req_ctx->gen_ctx.op_type == DRV_CRYPTO_DIRECTION_DECRYPT) {
memcpy(req->iv, req_ctx->backup_info, ivsize);
kzfree(req_ctx->backup_info);
} else if (!err) {
scatterwalk_map_and_copy(req->iv, req->dst,
(req->cryptlen - ivsize),
ivsize, 0);
}
skcipher_request_complete(req, err);
}
@ -639,7 +684,7 @@ static int cc_cipher_process(struct skcipher_request *req,
struct device *dev = drvdata_to_dev(ctx_p->drvdata);
struct cc_hw_desc desc[MAX_ABLKCIPHER_SEQ_LEN];
struct cc_crypto_req cc_req = {};
int rc, cts_restore_flag = 0;
int rc;
unsigned int seq_len = 0;
gfp_t flags = cc_gfp_flags(&req->base);
@ -671,23 +716,10 @@ static int cc_cipher_process(struct skcipher_request *req,
goto exit_process;
}
/*For CTS in case of data size aligned to 16 use CBC mode*/
if (((nbytes % AES_BLOCK_SIZE) == 0) &&
ctx_p->cipher_mode == DRV_CIPHER_CBC_CTS) {
ctx_p->cipher_mode = DRV_CIPHER_CBC;
cts_restore_flag = 1;
}
/* Setup request structure */
cc_req.user_cb = (void *)cc_cipher_complete;
cc_req.user_arg = (void *)req;
#ifdef ENABLE_CYCLE_COUNT
cc_req.op_type = (direction == DRV_CRYPTO_DIRECTION_DECRYPT) ?
STAT_OP_TYPE_DECODE : STAT_OP_TYPE_ENCODE;
#endif
/* Setup request context */
req_ctx->gen_ctx.op_type = direction;
@ -708,14 +740,6 @@ static int cc_cipher_process(struct skcipher_request *req,
cc_setup_cipher_data(tfm, req_ctx, dst, src, nbytes, req, desc,
&seq_len);
/* do we need to generate IV? */
if (req_ctx->is_giv) {
cc_req.ivgen_dma_addr[0] = req_ctx->gen_ctx.iv_dma_addr;
cc_req.ivgen_dma_addr_len = 1;
/* set the IV size (8/16 B long)*/
cc_req.ivgen_size = ivsize;
}
/* STAT_PHASE_3: Lock HW and push sequence */
rc = cc_send_request(ctx_p->drvdata, &cc_req, desc, seq_len,
@ -728,9 +752,6 @@ static int cc_cipher_process(struct skcipher_request *req,
}
exit_process:
if (cts_restore_flag)
ctx_p->cipher_mode = DRV_CIPHER_CBC_CTS;
if (rc != -EINPROGRESS && rc != -EBUSY) {
kzfree(req_ctx->backup_info);
kzfree(req_ctx->iv);
@ -743,8 +764,7 @@ static int cc_cipher_encrypt(struct skcipher_request *req)
{
struct cipher_req_ctx *req_ctx = skcipher_request_ctx(req);
req_ctx->is_giv = false;
req_ctx->backup_info = NULL;
memset(req_ctx, 0, sizeof(*req_ctx));
return cc_cipher_process(req, DRV_CRYPTO_DIRECTION_ENCRYPT);
}
@ -752,21 +772,28 @@ static int cc_cipher_encrypt(struct skcipher_request *req)
static int cc_cipher_decrypt(struct skcipher_request *req)
{
struct crypto_skcipher *sk_tfm = crypto_skcipher_reqtfm(req);
struct crypto_tfm *tfm = crypto_skcipher_tfm(sk_tfm);
struct cc_cipher_ctx *ctx_p = crypto_tfm_ctx(tfm);
struct cipher_req_ctx *req_ctx = skcipher_request_ctx(req);
unsigned int ivsize = crypto_skcipher_ivsize(sk_tfm);
gfp_t flags = cc_gfp_flags(&req->base);
unsigned int len;
/*
* Allocate and save the last IV sized bytes of the source, which will
* be lost in case of in-place decryption and might be needed for CTS.
*/
req_ctx->backup_info = kmalloc(ivsize, flags);
if (!req_ctx->backup_info)
return -ENOMEM;
memset(req_ctx, 0, sizeof(*req_ctx));
scatterwalk_map_and_copy(req_ctx->backup_info, req->src,
(req->cryptlen - ivsize), ivsize, 0);
req_ctx->is_giv = false;
if (ctx_p->cipher_mode == DRV_CIPHER_CBC) {
/* Allocate and save the last IV sized bytes of the source,
* which will be lost in case of in-place decryption.
*/
req_ctx->backup_info = kzalloc(ivsize, flags);
if (!req_ctx->backup_info)
return -ENOMEM;
len = req->cryptlen - ivsize;
scatterwalk_map_and_copy(req_ctx->backup_info, req->src, len,
ivsize, 0);
}
return cc_cipher_process(req, DRV_CRYPTO_DIRECTION_DECRYPT);
}
@ -927,7 +954,6 @@ static const struct cc_alg_template skcipher_algs[] = {
.name = "ecb(paes)",
.driver_name = "ecb-paes-ccree",
.blocksize = AES_BLOCK_SIZE,
.type = CRYPTO_ALG_TYPE_ABLKCIPHER,
.template_skcipher = {
.setkey = cc_cipher_sethkey,
.encrypt = cc_cipher_encrypt,
@ -944,7 +970,6 @@ static const struct cc_alg_template skcipher_algs[] = {
.name = "cbc(paes)",
.driver_name = "cbc-paes-ccree",
.blocksize = AES_BLOCK_SIZE,
.type = CRYPTO_ALG_TYPE_ABLKCIPHER,
.template_skcipher = {
.setkey = cc_cipher_sethkey,
.encrypt = cc_cipher_encrypt,
@ -961,7 +986,6 @@ static const struct cc_alg_template skcipher_algs[] = {
.name = "ofb(paes)",
.driver_name = "ofb-paes-ccree",
.blocksize = AES_BLOCK_SIZE,
.type = CRYPTO_ALG_TYPE_ABLKCIPHER,
.template_skcipher = {
.setkey = cc_cipher_sethkey,
.encrypt = cc_cipher_encrypt,
@ -975,10 +999,9 @@ static const struct cc_alg_template skcipher_algs[] = {
.min_hw_rev = CC_HW_REV_712,
},
{
.name = "cts1(cbc(paes))",
.driver_name = "cts1-cbc-paes-ccree",
.name = "cts(cbc(paes))",
.driver_name = "cts-cbc-paes-ccree",
.blocksize = AES_BLOCK_SIZE,
.type = CRYPTO_ALG_TYPE_ABLKCIPHER,
.template_skcipher = {
.setkey = cc_cipher_sethkey,
.encrypt = cc_cipher_encrypt,
@ -995,7 +1018,6 @@ static const struct cc_alg_template skcipher_algs[] = {
.name = "ctr(paes)",
.driver_name = "ctr-paes-ccree",
.blocksize = 1,
.type = CRYPTO_ALG_TYPE_ABLKCIPHER,
.template_skcipher = {
.setkey = cc_cipher_sethkey,
.encrypt = cc_cipher_encrypt,
@ -1162,7 +1184,6 @@ static const struct cc_alg_template skcipher_algs[] = {
.name = "ecb(aes)",
.driver_name = "ecb-aes-ccree",
.blocksize = AES_BLOCK_SIZE,
.type = CRYPTO_ALG_TYPE_ABLKCIPHER,
.template_skcipher = {
.setkey = cc_cipher_setkey,
.encrypt = cc_cipher_encrypt,
@ -1179,7 +1200,6 @@ static const struct cc_alg_template skcipher_algs[] = {
.name = "cbc(aes)",
.driver_name = "cbc-aes-ccree",
.blocksize = AES_BLOCK_SIZE,
.type = CRYPTO_ALG_TYPE_ABLKCIPHER,
.template_skcipher = {
.setkey = cc_cipher_setkey,
.encrypt = cc_cipher_encrypt,
@ -1196,7 +1216,6 @@ static const struct cc_alg_template skcipher_algs[] = {
.name = "ofb(aes)",
.driver_name = "ofb-aes-ccree",
.blocksize = AES_BLOCK_SIZE,
.type = CRYPTO_ALG_TYPE_ABLKCIPHER,
.template_skcipher = {
.setkey = cc_cipher_setkey,
.encrypt = cc_cipher_encrypt,
@ -1210,10 +1229,9 @@ static const struct cc_alg_template skcipher_algs[] = {
.min_hw_rev = CC_HW_REV_630,
},
{
.name = "cts1(cbc(aes))",
.driver_name = "cts1-cbc-aes-ccree",
.name = "cts(cbc(aes))",
.driver_name = "cts-cbc-aes-ccree",
.blocksize = AES_BLOCK_SIZE,
.type = CRYPTO_ALG_TYPE_ABLKCIPHER,
.template_skcipher = {
.setkey = cc_cipher_setkey,
.encrypt = cc_cipher_encrypt,
@ -1230,7 +1248,6 @@ static const struct cc_alg_template skcipher_algs[] = {
.name = "ctr(aes)",
.driver_name = "ctr-aes-ccree",
.blocksize = 1,
.type = CRYPTO_ALG_TYPE_ABLKCIPHER,
.template_skcipher = {
.setkey = cc_cipher_setkey,
.encrypt = cc_cipher_encrypt,
@ -1247,7 +1264,6 @@ static const struct cc_alg_template skcipher_algs[] = {
.name = "cbc(des3_ede)",
.driver_name = "cbc-3des-ccree",
.blocksize = DES3_EDE_BLOCK_SIZE,
.type = CRYPTO_ALG_TYPE_ABLKCIPHER,
.template_skcipher = {
.setkey = cc_cipher_setkey,
.encrypt = cc_cipher_encrypt,
@ -1264,7 +1280,6 @@ static const struct cc_alg_template skcipher_algs[] = {
.name = "ecb(des3_ede)",
.driver_name = "ecb-3des-ccree",
.blocksize = DES3_EDE_BLOCK_SIZE,
.type = CRYPTO_ALG_TYPE_ABLKCIPHER,
.template_skcipher = {
.setkey = cc_cipher_setkey,
.encrypt = cc_cipher_encrypt,
@ -1281,7 +1296,6 @@ static const struct cc_alg_template skcipher_algs[] = {
.name = "cbc(des)",
.driver_name = "cbc-des-ccree",
.blocksize = DES_BLOCK_SIZE,
.type = CRYPTO_ALG_TYPE_ABLKCIPHER,
.template_skcipher = {
.setkey = cc_cipher_setkey,
.encrypt = cc_cipher_encrypt,
@ -1298,7 +1312,6 @@ static const struct cc_alg_template skcipher_algs[] = {
.name = "ecb(des)",
.driver_name = "ecb-des-ccree",
.blocksize = DES_BLOCK_SIZE,
.type = CRYPTO_ALG_TYPE_ABLKCIPHER,
.template_skcipher = {
.setkey = cc_cipher_setkey,
.encrypt = cc_cipher_encrypt,
@ -1338,8 +1351,7 @@ static struct cc_crypto_alg *cc_create_alg(const struct cc_alg_template *tmpl,
alg->base.cra_init = cc_cipher_init;
alg->base.cra_exit = cc_cipher_exit;
alg->base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_KERN_DRIVER_ONLY |
CRYPTO_ALG_TYPE_SKCIPHER;
alg->base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_KERN_DRIVER_ONLY;
t_alg->cipher_mode = tmpl->cipher_mode;
t_alg->flow_mode = tmpl->flow_mode;

1
drivers/crypto/ccree/cc_cipher.h
View file

@ -22,7 +22,6 @@ struct cipher_req_ctx {
u32 out_mlli_nents;
u8 *backup_info; /*store iv for generated IV flow*/
u8 *iv;
bool is_giv;
struct mlli_params mlli_params;
};

4
drivers/crypto/ccree/cc_driver.c
View file

@ -131,8 +131,8 @@ static irqreturn_t cc_isr(int irq, void *dev_id)
}
if (irr) {
dev_dbg(dev, "IRR includes unknown cause bits (0x%08X)\n",
irr);
dev_dbg_ratelimited(dev, "IRR includes unknown cause bits (0x%08X)\n",
irr);
/* Just warning */
}

1
drivers/crypto/ccree/cc_driver.h
View file

@ -148,7 +148,6 @@ struct cc_alg_template {
char name[CRYPTO_MAX_ALG_NAME];
char driver_name[CRYPTO_MAX_ALG_NAME];
unsigned int blocksize;
u32 type;
union {
struct skcipher_alg skcipher;
struct aead_alg aead;

85
drivers/crypto/ccree/cc_hash.c
View file

@ -602,7 +602,7 @@ static int cc_hash_update(struct ahash_request *req)
return rc;
}
static int cc_hash_finup(struct ahash_request *req)
static int cc_do_finup(struct ahash_request *req, bool update)
{
struct ahash_req_ctx *state = ahash_request_ctx(req);
struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
@ -619,15 +619,15 @@ static int cc_hash_finup(struct ahash_request *req)
int rc;
gfp_t flags = cc_gfp_flags(&req->base);
dev_dbg(dev, "===== %s-finup (%d) ====\n", is_hmac ? "hmac" : "hash",
nbytes);
dev_dbg(dev, "===== %s-%s (%d) ====\n", is_hmac ? "hmac" : "hash",
update ? "finup" : "final", nbytes);
if (cc_map_req(dev, state, ctx)) {
dev_err(dev, "map_ahash_source() failed\n");
return -EINVAL;
}
if (cc_map_hash_request_final(ctx->drvdata, state, src, nbytes, 1,
if (cc_map_hash_request_final(ctx->drvdata, state, src, nbytes, update,
flags)) {
dev_err(dev, "map_ahash_request_final() failed\n");
cc_unmap_req(dev, state, ctx);
@ -646,67 +646,7 @@ static int cc_hash_finup(struct ahash_request *req)
idx = cc_restore_hash(desc, ctx, state, idx);
if (is_hmac)
idx = cc_fin_hmac(desc, req, idx);
idx = cc_fin_result(desc, req, idx);
rc = cc_send_request(ctx->drvdata, &cc_req, desc, idx, &req->base);
if (rc != -EINPROGRESS && rc != -EBUSY) {
dev_err(dev, "send_request() failed (rc=%d)\n", rc);
cc_unmap_hash_request(dev, state, src, true);
cc_unmap_result(dev, state, digestsize, result);
cc_unmap_req(dev, state, ctx);
}
return rc;
}
static int cc_hash_final(struct ahash_request *req)
{
struct ahash_req_ctx *state = ahash_request_ctx(req);
struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
struct cc_hash_ctx *ctx = crypto_ahash_ctx(tfm);
u32 digestsize = crypto_ahash_digestsize(tfm);
struct scatterlist *src = req->src;
unsigned int nbytes = req->nbytes;
u8 *result = req->result;
struct device *dev = drvdata_to_dev(ctx->drvdata);
bool is_hmac = ctx->is_hmac;
struct cc_crypto_req cc_req = {};
struct cc_hw_desc desc[CC_MAX_HASH_SEQ_LEN];
unsigned int idx = 0;
int rc;
gfp_t flags = cc_gfp_flags(&req->base);
dev_dbg(dev, "===== %s-final (%d) ====\n", is_hmac ? "hmac" : "hash",
nbytes);
if (cc_map_req(dev, state, ctx)) {
dev_err(dev, "map_ahash_source() failed\n");
return -EINVAL;
}
if (cc_map_hash_request_final(ctx->drvdata, state, src, nbytes, 0,
flags)) {
dev_err(dev, "map_ahash_request_final() failed\n");
cc_unmap_req(dev, state, ctx);
return -ENOMEM;
}
if (cc_map_result(dev, state, digestsize)) {
dev_err(dev, "map_ahash_digest() failed\n");
cc_unmap_hash_request(dev, state, src, true);
cc_unmap_req(dev, state, ctx);
return -ENOMEM;
}
/* Setup request structure */
cc_req.user_cb = cc_hash_complete;
cc_req.user_arg = req;
idx = cc_restore_hash(desc, ctx, state, idx);
/* "DO-PAD" must be enabled only when writing current length to HW */
/* Pad the hash */
hw_desc_init(&desc[idx]);
set_cipher_do(&desc[idx], DO_PAD);
set_cipher_mode(&desc[idx], ctx->hw_mode);
@ -731,6 +671,17 @@ static int cc_hash_final(struct ahash_request *req)
return rc;
}
static int cc_hash_finup(struct ahash_request *req)
{
return cc_do_finup(req, true);
}
static int cc_hash_final(struct ahash_request *req)
{
return cc_do_finup(req, false);
}
static int cc_hash_init(struct ahash_request *req)
{
struct ahash_req_ctx *state = ahash_request_ctx(req);
@ -1813,9 +1764,7 @@ static struct cc_hash_alg *cc_alloc_hash_alg(struct cc_hash_template *template,
alg->cra_exit = cc_cra_exit;
alg->cra_init = cc_cra_init;
alg->cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_TYPE_AHASH |
CRYPTO_ALG_KERN_DRIVER_ONLY;
alg->cra_type = &crypto_ahash_type;
alg->cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_KERN_DRIVER_ONLY;
t_crypto_alg->hash_mode = template->hash_mode;
t_crypto_alg->hw_mode = template->hw_mode;

7
drivers/crypto/chelsio/chcr_algo.c
View file

@ -4203,7 +4203,6 @@ static int chcr_unregister_alg(void)
#define SZ_AHASH_CTX sizeof(struct chcr_context)
#define SZ_AHASH_H_CTX (sizeof(struct chcr_context) + sizeof(struct hmac_ctx))
#define SZ_AHASH_REQ_CTX sizeof(struct chcr_ahash_req_ctx)
#define AHASH_CRA_FLAGS (CRYPTO_ALG_TYPE_AHASH | CRYPTO_ALG_ASYNC)
/*
* chcr_register_alg - Register crypto algorithms with kernel framework.
@ -4237,8 +4236,7 @@ static int chcr_register_alg(void)
break;
case CRYPTO_ALG_TYPE_AEAD:
driver_algs[i].alg.aead.base.cra_flags =
CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_ASYNC |
CRYPTO_ALG_NEED_FALLBACK;
CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK;
driver_algs[i].alg.aead.encrypt = chcr_aead_encrypt;
driver_algs[i].alg.aead.decrypt = chcr_aead_decrypt;
driver_algs[i].alg.aead.init = chcr_aead_cra_init;
@ -4258,10 +4256,9 @@ static int chcr_register_alg(void)
a_hash->halg.statesize = SZ_AHASH_REQ_CTX;
a_hash->halg.base.cra_priority = CHCR_CRA_PRIORITY;
a_hash->halg.base.cra_module = THIS_MODULE;
a_hash->halg.base.cra_flags = AHASH_CRA_FLAGS;
a_hash->halg.base.cra_flags = CRYPTO_ALG_ASYNC;
a_hash->halg.base.cra_alignmask = 0;
a_hash->halg.base.cra_exit = NULL;
a_hash->halg.base.cra_type = &crypto_ahash_type;
if (driver_algs[i].type == CRYPTO_ALG_TYPE_HMAC) {
a_hash->halg.base.cra_init = chcr_hmac_cra_init;

2
drivers/crypto/chelsio/chtls/chtls_hw.c
View file

@ -97,7 +97,7 @@ static int chtls_set_tcb_field(struct sock *sk, u16 word, u64 mask, u64 val)
int chtls_set_tcb_tflag(struct sock *sk, unsigned int bit_pos, int val)
{
return chtls_set_tcb_field(sk, 1, 1ULL << bit_pos,
val << bit_pos);
(u64)val << bit_pos);
}
static int chtls_set_tcb_keyid(struct sock *sk, int keyid)

14
drivers/crypto/hisilicon/Kconfig Normal file
View file

@ -0,0 +1,14 @@
# SPDX-License-Identifier: GPL-2.0
config CRYPTO_DEV_HISI_SEC
tristate "Support for Hisilicon SEC crypto block cipher accelerator"
select CRYPTO_BLKCIPHER
select CRYPTO_ALGAPI
select SG_SPLIT
depends on ARM64 || COMPILE_TEST
depends on HAS_IOMEM
help
Support for Hisilicon SEC Engine in Hip06 and Hip07
To compile this as a module, choose M here: the module
will be called hisi_sec.

2
drivers/crypto/hisilicon/Makefile Normal file
View file

@ -0,0 +1,2 @@
# SPDX-License-Identifier: GPL-2.0
obj-$(CONFIG_CRYPTO_DEV_HISI_SEC) += sec/

3
drivers/crypto/hisilicon/sec/Makefile Normal file
View file

@ -0,0 +1,3 @@
# SPDX-License-Identifier: GPL-2.0
obj-$(CONFIG_CRYPTO_DEV_HISI_SEC) += hisi_sec.o
hisi_sec-y = sec_algs.o sec_drv.o

1122
drivers/crypto/hisilicon/sec/sec_algs.c Normal file
View file

File diff suppressed because it is too large Load diff

1323
drivers/crypto/hisilicon/sec/sec_drv.c Normal file
View file

File diff suppressed because it is too large Load diff

428
drivers/crypto/hisilicon/sec/sec_drv.h Normal file
View file

@ -0,0 +1,428 @@
// SPDX-License-Identifier: GPL-2.0
/* Copyright (c) 2016-2017 Hisilicon Limited. */
#ifndef _SEC_DRV_H_
#define _SEC_DRV_H_
#include <crypto/algapi.h>
#include <linux/kfifo.h>
#define SEC_MAX_SGE_NUM 64
#define SEC_HW_RING_NUM 3
#define SEC_CMD_RING 0
#define SEC_OUTORDER_RING 1
#define SEC_DBG_RING 2
/* A reasonable length to balance memory use against flexibility */
#define SEC_QUEUE_LEN 512
#define SEC_MAX_SGE_NUM 64
struct sec_bd_info {
#define SEC_BD_W0_T_LEN_M GENMASK(4, 0)
#define SEC_BD_W0_T_LEN_S 0
#define SEC_BD_W0_C_WIDTH_M GENMASK(6, 5)
#define SEC_BD_W0_C_WIDTH_S 5
#define SEC_C_WIDTH_AES_128BIT 0
#define SEC_C_WIDTH_AES_8BIT 1
#define SEC_C_WIDTH_AES_1BIT 2
#define SEC_C_WIDTH_DES_64BIT 0
#define SEC_C_WIDTH_DES_8BIT 1
#define SEC_C_WIDTH_DES_1BIT 2
#define SEC_BD_W0_C_MODE_M GENMASK(9, 7)
#define SEC_BD_W0_C_MODE_S 7
#define SEC_C_MODE_ECB 0
#define SEC_C_MODE_CBC 1
#define SEC_C_MODE_CTR 4
#define SEC_C_MODE_CCM 5
#define SEC_C_MODE_GCM 6
#define SEC_C_MODE_XTS 7
#define SEC_BD_W0_SEQ BIT(10)
#define SEC_BD_W0_DE BIT(11)
#define SEC_BD_W0_DAT_SKIP_M GENMASK(13, 12)
#define SEC_BD_W0_DAT_SKIP_S 12
#define SEC_BD_W0_C_GRAN_SIZE_19_16_M GENMASK(17, 14)
#define SEC_BD_W0_C_GRAN_SIZE_19_16_S 14
#define SEC_BD_W0_CIPHER_M GENMASK(19, 18)
#define SEC_BD_W0_CIPHER_S 18
#define SEC_CIPHER_NULL 0
#define SEC_CIPHER_ENCRYPT 1
#define SEC_CIPHER_DECRYPT 2
#define SEC_BD_W0_AUTH_M GENMASK(21, 20)
#define SEC_BD_W0_AUTH_S 20
#define SEC_AUTH_NULL 0
#define SEC_AUTH_MAC 1
#define SEC_AUTH_VERIF 2
#define SEC_BD_W0_AI_GEN BIT(22)
#define SEC_BD_W0_CI_GEN BIT(23)
#define SEC_BD_W0_NO_HPAD BIT(24)
#define SEC_BD_W0_HM_M GENMASK(26, 25)
#define SEC_BD_W0_HM_S 25
#define SEC_BD_W0_ICV_OR_SKEY_EN_M GENMASK(28, 27)
#define SEC_BD_W0_ICV_OR_SKEY_EN_S 27
/* Multi purpose field - gran size bits for send, flag for recv */
#define SEC_BD_W0_FLAG_M GENMASK(30, 29)
#define SEC_BD_W0_C_GRAN_SIZE_21_20_M GENMASK(30, 29)
#define SEC_BD_W0_FLAG_S 29
#define SEC_BD_W0_C_GRAN_SIZE_21_20_S 29
#define SEC_BD_W0_DONE BIT(31)
u32 w0;
#define SEC_BD_W1_AUTH_GRAN_SIZE_M GENMASK(21, 0)
#define SEC_BD_W1_AUTH_GRAN_SIZE_S 0
#define SEC_BD_W1_M_KEY_EN BIT(22)
#define SEC_BD_W1_BD_INVALID BIT(23)
#define SEC_BD_W1_ADDR_TYPE BIT(24)
#define SEC_BD_W1_A_ALG_M GENMASK(28, 25)
#define SEC_BD_W1_A_ALG_S 25
#define SEC_A_ALG_SHA1 0
#define SEC_A_ALG_SHA256 1
#define SEC_A_ALG_MD5 2
#define SEC_A_ALG_SHA224 3
#define SEC_A_ALG_HMAC_SHA1 8
#define SEC_A_ALG_HMAC_SHA224 10
#define SEC_A_ALG_HMAC_SHA256 11
#define SEC_A_ALG_HMAC_MD5 12
#define SEC_A_ALG_AES_XCBC 13
#define SEC_A_ALG_AES_CMAC 14
#define SEC_BD_W1_C_ALG_M GENMASK(31, 29)
#define SEC_BD_W1_C_ALG_S 29
#define SEC_C_ALG_DES 0
#define SEC_C_ALG_3DES 1
#define SEC_C_ALG_AES 2
u32 w1;
#define SEC_BD_W2_C_GRAN_SIZE_15_0_M GENMASK(15, 0)
#define SEC_BD_W2_C_GRAN_SIZE_15_0_S 0
#define SEC_BD_W2_GRAN_NUM_M GENMASK(31, 16)
#define SEC_BD_W2_GRAN_NUM_S 16
u32 w2;
#define SEC_BD_W3_AUTH_LEN_OFFSET_M GENMASK(9, 0)
#define SEC_BD_W3_AUTH_LEN_OFFSET_S 0
#define SEC_BD_W3_CIPHER_LEN_OFFSET_M GENMASK(19, 10)
#define SEC_BD_W3_CIPHER_LEN_OFFSET_S 10
#define SEC_BD_W3_MAC_LEN_M GENMASK(24, 20)
#define SEC_BD_W3_MAC_LEN_S 20
#define SEC_BD_W3_A_KEY_LEN_M GENMASK(29, 25)
#define SEC_BD_W3_A_KEY_LEN_S 25
#define SEC_BD_W3_C_KEY_LEN_M GENMASK(31, 30)
#define SEC_BD_W3_C_KEY_LEN_S 30
#define SEC_KEY_LEN_AES_128 0
#define SEC_KEY_LEN_AES_192 1
#define SEC_KEY_LEN_AES_256 2
#define SEC_KEY_LEN_DES 1
#define SEC_KEY_LEN_3DES_3_KEY 1
#define SEC_KEY_LEN_3DES_2_KEY 3
u32 w3;
/* W4,5 */
union {
u32 authkey_addr_lo;
u32 authiv_addr_lo;
};
union {
u32 authkey_addr_hi;
u32 authiv_addr_hi;
};
/* W6,7 */
u32 cipher_key_addr_lo;
u32 cipher_key_addr_hi;
/* W8,9 */
u32 cipher_iv_addr_lo;
u32 cipher_iv_addr_hi;
/* W10,11 */
u32 data_addr_lo;
u32 data_addr_hi;
/* W12,13 */
u32 mac_addr_lo;
u32 mac_addr_hi;
/* W14,15 */
u32 cipher_destin_addr_lo;
u32 cipher_destin_addr_hi;
};
enum sec_mem_region {
SEC_COMMON = 0,
SEC_SAA,
SEC_NUM_ADDR_REGIONS
};
#define SEC_NAME_SIZE 64
#define SEC_Q_NUM 16
/**
* struct sec_queue_ring_cmd - store information about a SEC HW cmd ring
* @used: Local counter used to cheaply establish if the ring is empty.
* @lock: Protect against simultaneous adjusting of the read and write pointers.
* @vaddr: Virtual address for the ram pages used for the ring.
* @paddr: Physical address of the dma mapped region of ram used for the ring.
* @callback: Callback function called on a ring element completing.
*/
struct sec_queue_ring_cmd {
atomic_t used;
struct mutex lock;
struct sec_bd_info *vaddr;
dma_addr_t paddr;
void (*callback)(struct sec_bd_info *resp, void *ctx);
};
struct sec_debug_bd_info;
struct sec_queue_ring_db {
struct sec_debug_bd_info *vaddr;
dma_addr_t paddr;
};
struct sec_out_bd_info;
struct sec_queue_ring_cq {
struct sec_out_bd_info *vaddr;
dma_addr_t paddr;
};
struct sec_dev_info;
enum sec_cipher_alg {
SEC_C_DES_ECB_64,
SEC_C_DES_CBC_64,
SEC_C_3DES_ECB_192_3KEY,
SEC_C_3DES_ECB_192_2KEY,
SEC_C_3DES_CBC_192_3KEY,
SEC_C_3DES_CBC_192_2KEY,
SEC_C_AES_ECB_128,
SEC_C_AES_ECB_192,
SEC_C_AES_ECB_256,
SEC_C_AES_CBC_128,
SEC_C_AES_CBC_192,
SEC_C_AES_CBC_256,
SEC_C_AES_CTR_128,
SEC_C_AES_CTR_192,
SEC_C_AES_CTR_256,
SEC_C_AES_XTS_128,
SEC_C_AES_XTS_256,
SEC_C_NULL,
};
/**
* struct sec_alg_tfm_ctx - hardware specific tranformation context
* @cipher_alg: Cipher algorithm enabled include encryption mode.
* @key: Key storage if required.
* @pkey: DMA address for the key storage.
* @req_template: Request template to save time on setup.
* @queue: The hardware queue associated with this tfm context.
* @lock: Protect key and pkey to ensure they are consistent
* @auth_buf: Current context buffer for auth operations.
* @backlog: The backlog queue used for cases where our buffers aren't
* large enough.
*/
struct sec_alg_tfm_ctx {
enum sec_cipher_alg cipher_alg;
u8 *key;
dma_addr_t pkey;
struct sec_bd_info req_template;
struct sec_queue *queue;
struct mutex lock;
u8 *auth_buf;
struct list_head backlog;
};
/**
* struct sec_request - data associate with a single crypto request
* @elements: List of subparts of this request (hardware size restriction)
* @num_elements: The number of subparts (used as an optimization)
* @lock: Protect elements of this structure against concurrent change.
* @tfm_ctx: hardware specific context.
* @len_in: length of in sgl from upper layers
* @len_out: length of out sgl from upper layers
* @dma_iv: initialization vector - phsyical address
* @err: store used to track errors across subelements of this request.
* @req_base: pointer to base element of associate crypto context.
* This is needed to allow shared handling skcipher, ahash etc.
* @cb: completion callback.
* @backlog_head: list head to allow backlog maintenance.
*
* The hardware is limited in the maximum size of data that it can
* process from a single BD. Typically this is fairly large (32MB)
* but still requires the complexity of splitting the incoming
* skreq up into a number of elements complete with appropriate
* iv chaining.
*/
struct sec_request {
struct list_head elements;
int num_elements;
struct mutex lock;
struct sec_alg_tfm_ctx *tfm_ctx;
int len_in;
int len_out;
dma_addr_t dma_iv;
int err;
struct crypto_async_request *req_base;
void (*cb)(struct sec_bd_info *resp, struct crypto_async_request *req);
struct list_head backlog_head;
};
/**
* struct sec_request_el - A subpart of a request.
* @head: allow us to attach this to the list in the sec_request
* @req: hardware block descriptor corresponding to this request subpart
* @in: hardware sgl for input - virtual address
* @dma_in: hardware sgl for input - physical address
* @sgl_in: scatterlist for this request subpart
* @out: hardware sgl for output - virtual address
* @dma_out: hardware sgl for output - physical address
* @sgl_out: scatterlist for this request subpart
* @sec_req: The request which this subpart forms a part of
* @el_length: Number of bytes in this subpart. Needed to locate
* last ivsize chunk for iv chaining.
*/
struct sec_request_el {
struct list_head head;
struct sec_bd_info req;
struct sec_hw_sgl *in;
dma_addr_t dma_in;
struct scatterlist *sgl_in;
struct sec_hw_sgl *out;
dma_addr_t dma_out;
struct scatterlist *sgl_out;
struct sec_request *sec_req;
size_t el_length;
};
/**
* struct sec_queue - All the information about a HW queue
* @dev_info: The parent SEC device to which this queue belongs.
* @task_irq: Completion interrupt for the queue.
* @name: Human readable queue description also used as irq name.
* @ring: The several HW rings associated with one queue.
* @regs: The iomapped device registers
* @queue_id: Index of the queue used for naming and resource selection.
* @in_use: Flag to say if the queue is in use.
* @expected: The next expected element to finish assuming we were in order.
* @uprocessed: A bitmap to track which OoO elements are done but not handled.
* @softqueue: A software queue used when chaining requirements prevent direct
* use of the hardware queues.
* @havesoftqueue: A flag to say we have a queues - as we may need one for the
* current mode.
* @queuelock: Protect the soft queue from concurrent changes to avoid some
* potential loss of data races.
* @shadow: Pointers back to the shadow copy of the hardware ring element
* need because we can't store any context reference in the bd element.
*/
struct sec_queue {
struct sec_dev_info *dev_info;
int task_irq;
char name[SEC_NAME_SIZE];
struct sec_queue_ring_cmd ring_cmd;
struct sec_queue_ring_cq ring_cq;
struct sec_queue_ring_db ring_db;
void __iomem *regs;
u32 queue_id;
bool in_use;
int expected;
DECLARE_BITMAP(unprocessed, SEC_QUEUE_LEN);
DECLARE_KFIFO_PTR(softqueue, typeof(struct sec_request_el *));
bool havesoftqueue;
struct mutex queuelock;
void *shadow[SEC_QUEUE_LEN];
};
/**
* struct sec_hw_sge: Track each of the 64 element SEC HW SGL entries
* @buf: The IOV dma address for this entry.
* @len: Length of this IOV.
* @pad: Reserved space.
*/
struct sec_hw_sge {
dma_addr_t buf;
unsigned int len;
unsigned int pad;
};
/**
* struct sec_hw_sgl: One hardware SGL entry.
* @next_sgl: The next entry if we need to chain dma address. Null if last.
* @entry_sum_in_chain: The full count of SGEs - only matters for first SGL.
* @entry_sum_in_sgl: The number of SGEs in this SGL element.
* @flag: Unused in skciphers.
* @serial_num: Unsued in skciphers.
* @cpuid: Currently unused.
* @data_bytes_in_sgl: Count of bytes from all SGEs in this SGL.
* @next: Virtual address used to stash the next sgl - useful in completion.
* @reserved: A reserved field not currently used.
* @sge_entries: The (up to) 64 Scatter Gather Entries, representing IOVs.
* @node: Currently unused.
*/
struct sec_hw_sgl {
dma_addr_t next_sgl;
u16 entry_sum_in_chain;
u16 entry_sum_in_sgl;
u32 flag;
u64 serial_num;
u32 cpuid;
u32 data_bytes_in_sgl;
struct sec_hw_sgl *next;
u64 reserved;
struct sec_hw_sge sge_entries[SEC_MAX_SGE_NUM];
u8 node[16];
};
struct dma_pool;
/**
* struct sec_dev_info: The full SEC unit comprising queues and processors.
* @sec_id: Index used to track which SEC this is when more than one is present.
* @num_saas: The number of backed processors enabled.
* @regs: iomapped register regions shared by whole SEC unit.
* @dev_lock: Protects concurrent queue allocation / freeing for the SEC.
* @queues: The 16 queues that this SEC instance provides.
* @dev: Device pointer.
* @hw_sgl_pool: DMA pool used to mimise mapping for the scatter gather lists.
*/
struct sec_dev_info {
int sec_id;
int num_saas;
void __iomem *regs[SEC_NUM_ADDR_REGIONS];
struct mutex dev_lock;
int queues_in_use;
struct sec_queue queues[SEC_Q_NUM];
struct device *dev;
struct dma_pool *hw_sgl_pool;
};
int sec_queue_send(struct sec_queue *queue, struct sec_bd_info *msg, void *ctx);
bool sec_queue_can_enqueue(struct sec_queue *queue, int num);
int sec_queue_stop_release(struct sec_queue *queue);
struct sec_queue *sec_queue_alloc_start_safe(void);
bool sec_queue_empty(struct sec_queue *queue);
/* Algorithm specific elements from sec_algs.c */
void sec_alg_callback(struct sec_bd_info *resp, void *ctx);
int sec_algs_register(void);
void sec_algs_unregister(void);
#endif /* _SEC_DRV_H_ */

472
drivers/crypto/inside-secure/safexcel.c
View file

@ -1,11 +1,8 @@
// SPDX-License-Identifier: GPL-2.0
/*
* Copyright (C) 2017 Marvell
*
* Antoine Tenart <antoine.tenart@free-electrons.com>
*
* This file is licensed under the terms of the GNU General Public
* License version 2. This program is licensed "as is" without any
* warranty of any kind, whether express or implied.
*/
#include <linux/clk.h>
@ -33,7 +30,19 @@ MODULE_PARM_DESC(max_rings, "Maximum number of rings to use.");
static void eip197_trc_cache_init(struct safexcel_crypto_priv *priv)
{
u32 val, htable_offset;
int i;
int i, cs_rc_max, cs_ht_wc, cs_trc_rec_wc, cs_trc_lg_rec_wc;
if (priv->version == EIP197B) {
cs_rc_max = EIP197B_CS_RC_MAX;
cs_ht_wc = EIP197B_CS_HT_WC;
cs_trc_rec_wc = EIP197B_CS_TRC_REC_WC;
cs_trc_lg_rec_wc = EIP197B_CS_TRC_LG_REC_WC;
} else {
cs_rc_max = EIP197D_CS_RC_MAX;
cs_ht_wc = EIP197D_CS_HT_WC;
cs_trc_rec_wc = EIP197D_CS_TRC_REC_WC;
cs_trc_lg_rec_wc = EIP197D_CS_TRC_LG_REC_WC;
}
/* Enable the record cache memory access */
val = readl(priv->base + EIP197_CS_RAM_CTRL);
@ -54,7 +63,7 @@ static void eip197_trc_cache_init(struct safexcel_crypto_priv *priv)
writel(val, priv->base + EIP197_TRC_PARAMS);
/* Clear all records */
for (i = 0; i < EIP197_CS_RC_MAX; i++) {
for (i = 0; i < cs_rc_max; i++) {
u32 val, offset = EIP197_CLASSIFICATION_RAMS + i * EIP197_CS_RC_SIZE;
writel(EIP197_CS_RC_NEXT(EIP197_RC_NULL) |
@ -64,14 +73,14 @@ static void eip197_trc_cache_init(struct safexcel_crypto_priv *priv)
val = EIP197_CS_RC_NEXT(i+1) | EIP197_CS_RC_PREV(i-1);
if (i == 0)
val |= EIP197_CS_RC_PREV(EIP197_RC_NULL);
else if (i == EIP197_CS_RC_MAX - 1)
else if (i == cs_rc_max - 1)
val |= EIP197_CS_RC_NEXT(EIP197_RC_NULL);
writel(val, priv->base + offset + sizeof(u32));
}
/* Clear the hash table entries */
htable_offset = EIP197_CS_RC_MAX * EIP197_CS_RC_SIZE;
for (i = 0; i < 64; i++)
htable_offset = cs_rc_max * EIP197_CS_RC_SIZE;
for (i = 0; i < cs_ht_wc; i++)
writel(GENMASK(29, 0),
priv->base + EIP197_CLASSIFICATION_RAMS + htable_offset + i * sizeof(u32));
@ -82,23 +91,23 @@ static void eip197_trc_cache_init(struct safexcel_crypto_priv *priv)
/* Write head and tail pointers of the record free chain */
val = EIP197_TRC_FREECHAIN_HEAD_PTR(0) |
EIP197_TRC_FREECHAIN_TAIL_PTR(EIP197_CS_RC_MAX - 1);
EIP197_TRC_FREECHAIN_TAIL_PTR(cs_rc_max - 1);
writel(val, priv->base + EIP197_TRC_FREECHAIN);
/* Configure the record cache #1 */
val = EIP197_TRC_PARAMS2_RC_SZ_SMALL(EIP197_CS_TRC_REC_WC) |
EIP197_TRC_PARAMS2_HTABLE_PTR(EIP197_CS_RC_MAX);
val = EIP197_TRC_PARAMS2_RC_SZ_SMALL(cs_trc_rec_wc) |
EIP197_TRC_PARAMS2_HTABLE_PTR(cs_rc_max);
writel(val, priv->base + EIP197_TRC_PARAMS2);
/* Configure the record cache #2 */
val = EIP197_TRC_PARAMS_RC_SZ_LARGE(EIP197_CS_TRC_LG_REC_WC) |
val = EIP197_TRC_PARAMS_RC_SZ_LARGE(cs_trc_lg_rec_wc) |
EIP197_TRC_PARAMS_BLK_TIMER_SPEED(1) |
EIP197_TRC_PARAMS_HTABLE_SZ(2);
writel(val, priv->base + EIP197_TRC_PARAMS);
}
static void eip197_write_firmware(struct safexcel_crypto_priv *priv,
const struct firmware *fw, u32 ctrl,
const struct firmware *fw, int pe, u32 ctrl,
u32 prog_en)
{
const u32 *data = (const u32 *)fw->data;
@ -112,7 +121,7 @@ static void eip197_write_firmware(struct safexcel_crypto_priv *priv,
EIP197_PE(priv) + ctrl);
/* Enable access to the program memory */
writel(prog_en, EIP197_PE(priv) + EIP197_PE_ICE_RAM_CTRL);
writel(prog_en, EIP197_PE(priv) + EIP197_PE_ICE_RAM_CTRL(pe));
/* Write the firmware */
for (i = 0; i < fw->size / sizeof(u32); i++)
@ -120,7 +129,7 @@ static void eip197_write_firmware(struct safexcel_crypto_priv *priv,
priv->base + EIP197_CLASSIFICATION_RAMS + i * sizeof(u32));
/* Disable access to the program memory */
writel(0, EIP197_PE(priv) + EIP197_PE_ICE_RAM_CTRL);
writel(0, EIP197_PE(priv) + EIP197_PE_ICE_RAM_CTRL(pe));
/* Release engine from reset */
val = readl(EIP197_PE(priv) + ctrl);
@ -132,35 +141,62 @@ static int eip197_load_firmwares(struct safexcel_crypto_priv *priv)
{
const char *fw_name[] = {"ifpp.bin", "ipue.bin"};
const struct firmware *fw[FW_NB];
int i, j, ret = 0;
char fw_path[31], *dir = NULL;
int i, j, ret = 0, pe;
u32 val;
switch (priv->version) {
case EIP197B:
dir = "eip197b";
break;
case EIP197D:
dir = "eip197d";
break;
default:
/* No firmware is required */
return 0;
}
for (i = 0; i < FW_NB; i++) {
ret = request_firmware(&fw[i], fw_name[i], priv->dev);
snprintf(fw_path, 31, "inside-secure/%s/%s", dir, fw_name[i]);
ret = request_firmware(&fw[i], fw_path, priv->dev);
if (ret) {
dev_err(priv->dev,
"Failed to request firmware %s (%d)\n",
fw_name[i], ret);
goto release_fw;
if (priv->version != EIP197B)
goto release_fw;
/* Fallback to the old firmware location for the
* EIP197b.
*/
ret = request_firmware(&fw[i], fw_name[i], priv->dev);
if (ret) {
dev_err(priv->dev,
"Failed to request firmware %s (%d)\n",
fw_name[i], ret);
goto release_fw;
}
}
}
}
/* Clear the scratchpad memory */
val = readl(EIP197_PE(priv) + EIP197_PE_ICE_SCRATCH_CTRL);
val |= EIP197_PE_ICE_SCRATCH_CTRL_CHANGE_TIMER |
EIP197_PE_ICE_SCRATCH_CTRL_TIMER_EN |
EIP197_PE_ICE_SCRATCH_CTRL_SCRATCH_ACCESS |
EIP197_PE_ICE_SCRATCH_CTRL_CHANGE_ACCESS;
writel(val, EIP197_PE(priv) + EIP197_PE_ICE_SCRATCH_CTRL);
for (pe = 0; pe < priv->config.pes; pe++) {
/* Clear the scratchpad memory */
val = readl(EIP197_PE(priv) + EIP197_PE_ICE_SCRATCH_CTRL(pe));
val |= EIP197_PE_ICE_SCRATCH_CTRL_CHANGE_TIMER |
EIP197_PE_ICE_SCRATCH_CTRL_TIMER_EN |
EIP197_PE_ICE_SCRATCH_CTRL_SCRATCH_ACCESS |
EIP197_PE_ICE_SCRATCH_CTRL_CHANGE_ACCESS;
writel(val, EIP197_PE(priv) + EIP197_PE_ICE_SCRATCH_CTRL(pe));
memset_io(EIP197_PE(priv) + EIP197_PE_ICE_SCRATCH_RAM, 0,
EIP197_NUM_OF_SCRATCH_BLOCKS * sizeof(u32));
memset_io(EIP197_PE(priv) + EIP197_PE_ICE_SCRATCH_RAM(pe), 0,
EIP197_NUM_OF_SCRATCH_BLOCKS * sizeof(u32));
eip197_write_firmware(priv, fw[FW_IFPP], EIP197_PE_ICE_FPP_CTRL,
EIP197_PE_ICE_RAM_CTRL_FPP_PROG_EN);
eip197_write_firmware(priv, fw[FW_IFPP], pe,
EIP197_PE_ICE_FPP_CTRL(pe),
EIP197_PE_ICE_RAM_CTRL_FPP_PROG_EN);
eip197_write_firmware(priv, fw[FW_IPUE], EIP197_PE_ICE_PUE_CTRL,
EIP197_PE_ICE_RAM_CTRL_PUE_PROG_EN);
eip197_write_firmware(priv, fw[FW_IPUE], pe,
EIP197_PE_ICE_PUE_CTRL(pe),
EIP197_PE_ICE_RAM_CTRL_PUE_PROG_EN);
}
release_fw:
for (j = 0; j < i; j++)
@ -256,7 +292,7 @@ static int safexcel_hw_setup_rdesc_rings(struct safexcel_crypto_priv *priv)
static int safexcel_hw_init(struct safexcel_crypto_priv *priv)
{
u32 version, val;
int i, ret;
int i, ret, pe;
/* Determine endianess and configure byte swap */
version = readl(EIP197_HIA_AIC(priv) + EIP197_HIA_VERSION);
@ -267,6 +303,10 @@ static int safexcel_hw_init(struct safexcel_crypto_priv *priv)
else if (((version >> 16) & 0xffff) == EIP197_HIA_VERSION_LE)
val |= (EIP197_MST_CTRL_NO_BYTE_SWAP >> 24);
/* For EIP197 set maximum number of TX commands to 2^5 = 32 */
if (priv->version == EIP197B || priv->version == EIP197D)
val |= EIP197_MST_CTRL_TX_MAX_CMD(5);
writel(val, EIP197_HIA_AIC(priv) + EIP197_HIA_MST_CTRL);
/* Configure wr/rd cache values */
@ -282,82 +322,94 @@ static int safexcel_hw_init(struct safexcel_crypto_priv *priv)
/* Clear any pending interrupt */
writel(GENMASK(31, 0), EIP197_HIA_AIC_G(priv) + EIP197_HIA_AIC_G_ACK);
/* Data Fetch Engine configuration */
/* Reset all DFE threads */
writel(EIP197_DxE_THR_CTRL_RESET_PE,
EIP197_HIA_DFE_THR(priv) + EIP197_HIA_DFE_THR_CTRL);
if (priv->version == EIP197) {
/* Reset HIA input interface arbiter */
writel(EIP197_HIA_RA_PE_CTRL_RESET,
EIP197_HIA_AIC(priv) + EIP197_HIA_RA_PE_CTRL);
}
/* DMA transfer size to use */
val = EIP197_HIA_DFE_CFG_DIS_DEBUG;
val |= EIP197_HIA_DxE_CFG_MIN_DATA_SIZE(5) | EIP197_HIA_DxE_CFG_MAX_DATA_SIZE(9);
val |= EIP197_HIA_DxE_CFG_MIN_CTRL_SIZE(5) | EIP197_HIA_DxE_CFG_MAX_CTRL_SIZE(7);
val |= EIP197_HIA_DxE_CFG_DATA_CACHE_CTRL(RD_CACHE_3BITS);
val |= EIP197_HIA_DxE_CFG_CTRL_CACHE_CTRL(RD_CACHE_3BITS);
writel(val, EIP197_HIA_DFE(priv) + EIP197_HIA_DFE_CFG);
/* Leave the DFE threads reset state */
writel(0, EIP197_HIA_DFE_THR(priv) + EIP197_HIA_DFE_THR_CTRL);
/* Configure the procesing engine thresholds */
writel(EIP197_PE_IN_xBUF_THRES_MIN(5) | EIP197_PE_IN_xBUF_THRES_MAX(9),
EIP197_PE(priv) + EIP197_PE_IN_DBUF_THRES);
writel(EIP197_PE_IN_xBUF_THRES_MIN(5) | EIP197_PE_IN_xBUF_THRES_MAX(7),
EIP197_PE(priv) + EIP197_PE_IN_TBUF_THRES);
if (priv->version == EIP197) {
/* enable HIA input interface arbiter and rings */
writel(EIP197_HIA_RA_PE_CTRL_EN |
GENMASK(priv->config.rings - 1, 0),
EIP197_HIA_AIC(priv) + EIP197_HIA_RA_PE_CTRL);
}
/* Data Store Engine configuration */
/* Reset all DSE threads */
writel(EIP197_DxE_THR_CTRL_RESET_PE,
EIP197_HIA_DSE_THR(priv) + EIP197_HIA_DSE_THR_CTRL);
/* Wait for all DSE threads to complete */
while ((readl(EIP197_HIA_DSE_THR(priv) + EIP197_HIA_DSE_THR_STAT) &
GENMASK(15, 12)) != GENMASK(15, 12))
;
/* DMA transfer size to use */
val = EIP197_HIA_DSE_CFG_DIS_DEBUG;
val |= EIP197_HIA_DxE_CFG_MIN_DATA_SIZE(7) | EIP197_HIA_DxE_CFG_MAX_DATA_SIZE(8);
val |= EIP197_HIA_DxE_CFG_DATA_CACHE_CTRL(WR_CACHE_3BITS);
val |= EIP197_HIA_DSE_CFG_ALWAYS_BUFFERABLE;
/* FIXME: instability issues can occur for EIP97 but disabling it impact
* performances.
*/
if (priv->version == EIP197)
val |= EIP197_HIA_DSE_CFG_EN_SINGLE_WR;
writel(val, EIP197_HIA_DSE(priv) + EIP197_HIA_DSE_CFG);
/* Leave the DSE threads reset state */
writel(0, EIP197_HIA_DSE_THR(priv) + EIP197_HIA_DSE_THR_CTRL);
/* Configure the procesing engine thresholds */
writel(EIP197_PE_OUT_DBUF_THRES_MIN(7) | EIP197_PE_OUT_DBUF_THRES_MAX(8),
EIP197_PE(priv) + EIP197_PE_OUT_DBUF_THRES);
/* Processing Engine configuration */
for (pe = 0; pe < priv->config.pes; pe++) {
/* Data Fetch Engine configuration */
/* H/W capabilities selection */
val = EIP197_FUNCTION_RSVD;
val |= EIP197_PROTOCOL_ENCRYPT_ONLY | EIP197_PROTOCOL_HASH_ONLY;
val |= EIP197_PROTOCOL_ENCRYPT_HASH | EIP197_PROTOCOL_HASH_DECRYPT;
val |= EIP197_ALG_AES_ECB | EIP197_ALG_AES_CBC;
val |= EIP197_ALG_SHA1 | EIP197_ALG_HMAC_SHA1;
val |= EIP197_ALG_SHA2 | EIP197_ALG_HMAC_SHA2;
writel(val, EIP197_PE(priv) + EIP197_PE_EIP96_FUNCTION_EN);
/* Reset all DFE threads */
writel(EIP197_DxE_THR_CTRL_RESET_PE,
EIP197_HIA_DFE_THR(priv) + EIP197_HIA_DFE_THR_CTRL(pe));
if (priv->version == EIP197B || priv->version == EIP197D) {
/* Reset HIA input interface arbiter */
writel(EIP197_HIA_RA_PE_CTRL_RESET,
EIP197_HIA_AIC(priv) + EIP197_HIA_RA_PE_CTRL(pe));
}
/* DMA transfer size to use */
val = EIP197_HIA_DFE_CFG_DIS_DEBUG;
val |= EIP197_HIA_DxE_CFG_MIN_DATA_SIZE(6) |
EIP197_HIA_DxE_CFG_MAX_DATA_SIZE(9);
val |= EIP197_HIA_DxE_CFG_MIN_CTRL_SIZE(6) |
EIP197_HIA_DxE_CFG_MAX_CTRL_SIZE(7);
val |= EIP197_HIA_DxE_CFG_DATA_CACHE_CTRL(RD_CACHE_3BITS);
val |= EIP197_HIA_DxE_CFG_CTRL_CACHE_CTRL(RD_CACHE_3BITS);
writel(val, EIP197_HIA_DFE(priv) + EIP197_HIA_DFE_CFG(pe));
/* Leave the DFE threads reset state */
writel(0, EIP197_HIA_DFE_THR(priv) + EIP197_HIA_DFE_THR_CTRL(pe));
/* Configure the processing engine thresholds */
writel(EIP197_PE_IN_xBUF_THRES_MIN(6) |
EIP197_PE_IN_xBUF_THRES_MAX(9),
EIP197_PE(priv) + EIP197_PE_IN_DBUF_THRES(pe));
writel(EIP197_PE_IN_xBUF_THRES_MIN(6) |
EIP197_PE_IN_xBUF_THRES_MAX(7),
EIP197_PE(priv) + EIP197_PE_IN_TBUF_THRES(pe));
if (priv->version == EIP197B || priv->version == EIP197D) {
/* enable HIA input interface arbiter and rings */
writel(EIP197_HIA_RA_PE_CTRL_EN |
GENMASK(priv->config.rings - 1, 0),
EIP197_HIA_AIC(priv) + EIP197_HIA_RA_PE_CTRL(pe));
}
/* Data Store Engine configuration */
/* Reset all DSE threads */
writel(EIP197_DxE_THR_CTRL_RESET_PE,
EIP197_HIA_DSE_THR(priv) + EIP197_HIA_DSE_THR_CTRL(pe));
/* Wait for all DSE threads to complete */
while ((readl(EIP197_HIA_DSE_THR(priv) + EIP197_HIA_DSE_THR_STAT(pe)) &
GENMASK(15, 12)) != GENMASK(15, 12))
;
/* DMA transfer size to use */
val = EIP197_HIA_DSE_CFG_DIS_DEBUG;
val |= EIP197_HIA_DxE_CFG_MIN_DATA_SIZE(7) |
EIP197_HIA_DxE_CFG_MAX_DATA_SIZE(8);
val |= EIP197_HIA_DxE_CFG_DATA_CACHE_CTRL(WR_CACHE_3BITS);
val |= EIP197_HIA_DSE_CFG_ALWAYS_BUFFERABLE;
/* FIXME: instability issues can occur for EIP97 but disabling it impact
* performances.
*/
if (priv->version == EIP197B || priv->version == EIP197D)
val |= EIP197_HIA_DSE_CFG_EN_SINGLE_WR;
writel(val, EIP197_HIA_DSE(priv) + EIP197_HIA_DSE_CFG(pe));
/* Leave the DSE threads reset state */
writel(0, EIP197_HIA_DSE_THR(priv) + EIP197_HIA_DSE_THR_CTRL(pe));
/* Configure the procesing engine thresholds */
writel(EIP197_PE_OUT_DBUF_THRES_MIN(7) |
EIP197_PE_OUT_DBUF_THRES_MAX(8),
EIP197_PE(priv) + EIP197_PE_OUT_DBUF_THRES(pe));
/* Processing Engine configuration */
/* H/W capabilities selection */
val = EIP197_FUNCTION_RSVD;
val |= EIP197_PROTOCOL_ENCRYPT_ONLY | EIP197_PROTOCOL_HASH_ONLY;
val |= EIP197_PROTOCOL_ENCRYPT_HASH | EIP197_PROTOCOL_HASH_DECRYPT;
val |= EIP197_ALG_DES_ECB | EIP197_ALG_DES_CBC;
val |= EIP197_ALG_3DES_ECB | EIP197_ALG_3DES_CBC;
val |= EIP197_ALG_AES_ECB | EIP197_ALG_AES_CBC;
val |= EIP197_ALG_MD5 | EIP197_ALG_HMAC_MD5;
val |= EIP197_ALG_SHA1 | EIP197_ALG_HMAC_SHA1;
val |= EIP197_ALG_SHA2 | EIP197_ALG_HMAC_SHA2;
writel(val, EIP197_PE(priv) + EIP197_PE_EIP96_FUNCTION_EN(pe));
}
/* Command Descriptor Rings prepare */
for (i = 0; i < priv->config.rings; i++) {
@ -408,18 +460,20 @@ static int safexcel_hw_init(struct safexcel_crypto_priv *priv)
EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_RING_SIZE);
}
/* Enable command descriptor rings */
writel(EIP197_DxE_THR_CTRL_EN | GENMASK(priv->config.rings - 1, 0),
EIP197_HIA_DFE_THR(priv) + EIP197_HIA_DFE_THR_CTRL);
for (pe = 0; pe < priv->config.pes; pe++) {
/* Enable command descriptor rings */
writel(EIP197_DxE_THR_CTRL_EN | GENMASK(priv->config.rings - 1, 0),
EIP197_HIA_DFE_THR(priv) + EIP197_HIA_DFE_THR_CTRL(pe));
/* Enable result descriptor rings */
writel(EIP197_DxE_THR_CTRL_EN | GENMASK(priv->config.rings - 1, 0),
EIP197_HIA_DSE_THR(priv) + EIP197_HIA_DSE_THR_CTRL);
/* Enable result descriptor rings */
writel(EIP197_DxE_THR_CTRL_EN | GENMASK(priv->config.rings - 1, 0),
EIP197_HIA_DSE_THR(priv) + EIP197_HIA_DSE_THR_CTRL(pe));
}
/* Clear any HIA interrupt */
writel(GENMASK(30, 20), EIP197_HIA_AIC_G(priv) + EIP197_HIA_AIC_G_ACK);
if (priv->version == EIP197) {
if (priv->version == EIP197B || priv->version == EIP197D) {
eip197_trc_cache_init(priv);
ret = eip197_load_firmwares(priv);
@ -452,7 +506,6 @@ void safexcel_dequeue(struct safexcel_crypto_priv *priv, int ring)
{
struct crypto_async_request *req, *backlog;
struct safexcel_context *ctx;
struct safexcel_request *request;
int ret, nreq = 0, cdesc = 0, rdesc = 0, commands, results;
/* If a request wasn't properly dequeued because of a lack of resources,
@ -476,16 +529,10 @@ void safexcel_dequeue(struct safexcel_crypto_priv *priv, int ring)
}
handle_req:
request = kzalloc(sizeof(*request), EIP197_GFP_FLAGS(*req));
if (!request)
goto request_failed;
ctx = crypto_tfm_ctx(req->tfm);
ret = ctx->send(req, ring, request, &commands, &results);
if (ret) {
kfree(request);
ret = ctx->send(req, ring, &commands, &results);
if (ret)
goto request_failed;
}
if (backlog)
backlog->complete(backlog, -EINPROGRESS);
@ -494,14 +541,8 @@ handle_req:
* to the engine because the input data was cached, continue to
* dequeue other requests as this is valid and not an error.
*/
if (!commands && !results) {
kfree(request);
if (!commands && !results)
continue;
}
spin_lock_bh(&priv->ring[ring].egress_lock);
list_add_tail(&request->list, &priv->ring[ring].list);
spin_unlock_bh(&priv->ring[ring].egress_lock);
cdesc += commands;
rdesc += results;
@ -519,7 +560,7 @@ finalize:
if (!nreq)
return;
spin_lock_bh(&priv->ring[ring].egress_lock);
spin_lock_bh(&priv->ring[ring].lock);
priv->ring[ring].requests += nreq;
@ -528,7 +569,7 @@ finalize:
priv->ring[ring].busy = true;
}
spin_unlock_bh(&priv->ring[ring].egress_lock);
spin_unlock_bh(&priv->ring[ring].lock);
/* let the RDR know we have pending descriptors */
writel((rdesc * priv->config.rd_offset) << 2,
@ -560,6 +601,24 @@ inline int safexcel_rdesc_check_errors(struct safexcel_crypto_priv *priv,
return -EINVAL;
}
inline void safexcel_rdr_req_set(struct safexcel_crypto_priv *priv,
int ring,
struct safexcel_result_desc *rdesc,
struct crypto_async_request *req)
{
int i = safexcel_ring_rdr_rdesc_index(priv, ring, rdesc);
priv->ring[ring].rdr_req[i] = req;
}
inline struct crypto_async_request *
safexcel_rdr_req_get(struct safexcel_crypto_priv *priv, int ring)
{
int i = safexcel_ring_first_rdr_index(priv, ring);
return priv->ring[ring].rdr_req[i];
}
void safexcel_complete(struct safexcel_crypto_priv *priv, int ring)
{
struct safexcel_command_desc *cdesc;
@ -588,21 +647,16 @@ void safexcel_inv_complete(struct crypto_async_request *req, int error)
int safexcel_invalidate_cache(struct crypto_async_request *async,
struct safexcel_crypto_priv *priv,
dma_addr_t ctxr_dma, int ring,
struct safexcel_request *request)
dma_addr_t ctxr_dma, int ring)
{
struct safexcel_command_desc *cdesc;
struct safexcel_result_desc *rdesc;
int ret = 0;
spin_lock_bh(&priv->ring[ring].egress_lock);
/* Prepare command descriptor */
cdesc = safexcel_add_cdesc(priv, ring, true, true, 0, 0, 0, ctxr_dma);
if (IS_ERR(cdesc)) {
ret = PTR_ERR(cdesc);
goto unlock;
}
if (IS_ERR(cdesc))
return PTR_ERR(cdesc);
cdesc->control_data.type = EIP197_TYPE_EXTENDED;
cdesc->control_data.options = 0;
@ -617,21 +671,20 @@ int safexcel_invalidate_cache(struct crypto_async_request *async,
goto cdesc_rollback;
}
request->req = async;
goto unlock;
safexcel_rdr_req_set(priv, ring, rdesc, async);
return ret;
cdesc_rollback:
safexcel_ring_rollback_wptr(priv, &priv->ring[ring].cdr);
unlock:
spin_unlock_bh(&priv->ring[ring].egress_lock);
return ret;
}
static inline void safexcel_handle_result_descriptor(struct safexcel_crypto_priv *priv,
int ring)
{
struct safexcel_request *sreq;
struct crypto_async_request *req;
struct safexcel_context *ctx;
int ret, i, nreq, ndesc, tot_descs, handled = 0;
bool should_complete;
@ -646,28 +699,22 @@ handle_results:
goto requests_left;
for (i = 0; i < nreq; i++) {
spin_lock_bh(&priv->ring[ring].egress_lock);
sreq = list_first_entry(&priv->ring[ring].list,
struct safexcel_request, list);
list_del(&sreq->list);
spin_unlock_bh(&priv->ring[ring].egress_lock);
req = safexcel_rdr_req_get(priv, ring);
ctx = crypto_tfm_ctx(sreq->req->tfm);
ndesc = ctx->handle_result(priv, ring, sreq->req,
ctx = crypto_tfm_ctx(req->tfm);
ndesc = ctx->handle_result(priv, ring, req,
&should_complete, &ret);
if (ndesc < 0) {
kfree(sreq);
dev_err(priv->dev, "failed to handle result (%d)", ndesc);
goto acknowledge;
}
if (should_complete) {
local_bh_disable();
sreq->req->complete(sreq->req, ret);
req->complete(req, ret);
local_bh_enable();
}
kfree(sreq);
tot_descs += ndesc;
handled++;
}
@ -686,7 +733,7 @@ acknowledge:
goto handle_results;
requests_left:
spin_lock_bh(&priv->ring[ring].egress_lock);
spin_lock_bh(&priv->ring[ring].lock);
priv->ring[ring].requests -= handled;
safexcel_try_push_requests(priv, ring);
@ -694,7 +741,7 @@ requests_left:
if (!priv->ring[ring].requests)
priv->ring[ring].busy = false;
spin_unlock_bh(&priv->ring[ring].egress_lock);
spin_unlock_bh(&priv->ring[ring].lock);
}
static void safexcel_dequeue_work(struct work_struct *work)
@ -785,17 +832,29 @@ static int safexcel_request_ring_irq(struct platform_device *pdev, const char *n
}
static struct safexcel_alg_template *safexcel_algs[] = {
&safexcel_alg_ecb_des,
&safexcel_alg_cbc_des,
&safexcel_alg_ecb_des3_ede,
&safexcel_alg_cbc_des3_ede,
&safexcel_alg_ecb_aes,
&safexcel_alg_cbc_aes,
&safexcel_alg_md5,
&safexcel_alg_sha1,
&safexcel_alg_sha224,
&safexcel_alg_sha256,
&safexcel_alg_sha384,
&safexcel_alg_sha512,
&safexcel_alg_hmac_md5,
&safexcel_alg_hmac_sha1,
&safexcel_alg_hmac_sha224,
&safexcel_alg_hmac_sha256,
&safexcel_alg_hmac_sha384,
&safexcel_alg_hmac_sha512,
&safexcel_alg_authenc_hmac_sha1_cbc_aes,
&safexcel_alg_authenc_hmac_sha224_cbc_aes,
&safexcel_alg_authenc_hmac_sha256_cbc_aes,
&safexcel_alg_authenc_hmac_sha384_cbc_aes,
&safexcel_alg_authenc_hmac_sha512_cbc_aes,
};
static int safexcel_register_algorithms(struct safexcel_crypto_priv *priv)
@ -805,6 +864,9 @@ static int safexcel_register_algorithms(struct safexcel_crypto_priv *priv)
for (i = 0; i < ARRAY_SIZE(safexcel_algs); i++) {
safexcel_algs[i]->priv = priv;
if (!(safexcel_algs[i]->engines & priv->version))
continue;
if (safexcel_algs[i]->type == SAFEXCEL_ALG_TYPE_SKCIPHER)
ret = crypto_register_skcipher(&safexcel_algs[i]->alg.skcipher);
else if (safexcel_algs[i]->type == SAFEXCEL_ALG_TYPE_AEAD)
@ -820,6 +882,9 @@ static int safexcel_register_algorithms(struct safexcel_crypto_priv *priv)
fail:
for (j = 0; j < i; j++) {
if (!(safexcel_algs[j]->engines & priv->version))
continue;
if (safexcel_algs[j]->type == SAFEXCEL_ALG_TYPE_SKCIPHER)
crypto_unregister_skcipher(&safexcel_algs[j]->alg.skcipher);
else if (safexcel_algs[j]->type == SAFEXCEL_ALG_TYPE_AEAD)
@ -836,6 +901,9 @@ static void safexcel_unregister_algorithms(struct safexcel_crypto_priv *priv)
int i;
for (i = 0; i < ARRAY_SIZE(safexcel_algs); i++) {
if (!(safexcel_algs[i]->engines & priv->version))
continue;
if (safexcel_algs[i]->type == SAFEXCEL_ALG_TYPE_SKCIPHER)
crypto_unregister_skcipher(&safexcel_algs[i]->alg.skcipher);
else if (safexcel_algs[i]->type == SAFEXCEL_ALG_TYPE_AEAD)
@ -847,9 +915,21 @@ static void safexcel_unregister_algorithms(struct safexcel_crypto_priv *priv)
static void safexcel_configure(struct safexcel_crypto_priv *priv)
{
u32 val, mask;
u32 val, mask = 0;
val = readl(EIP197_HIA_AIC_G(priv) + EIP197_HIA_OPTIONS);
/* Read number of PEs from the engine */
switch (priv->version) {
case EIP197B:
case EIP197D:
mask = EIP197_N_PES_MASK;
break;
default:
mask = EIP97_N_PES_MASK;
}
priv->config.pes = (val >> EIP197_N_PES_OFFSET) & mask;
val = (val & GENMASK(27, 25)) >> 25;
mask = BIT(val) - 1;
@ -867,7 +947,9 @@ static void safexcel_init_register_offsets(struct safexcel_crypto_priv *priv)
{
struct safexcel_register_offsets *offsets = &priv->offsets;
if (priv->version == EIP197) {
switch (priv->version) {
case EIP197B:
case EIP197D:
offsets->hia_aic = EIP197_HIA_AIC_BASE;
offsets->hia_aic_g = EIP197_HIA_AIC_G_BASE;
offsets->hia_aic_r = EIP197_HIA_AIC_R_BASE;
@ -878,7 +960,8 @@ static void safexcel_init_register_offsets(struct safexcel_crypto_priv *priv)
offsets->hia_dse_thr = EIP197_HIA_DSE_THR_BASE;
offsets->hia_gen_cfg = EIP197_HIA_GEN_CFG_BASE;
offsets->pe = EIP197_PE_BASE;
} else {
break;
case EIP97IES:
offsets->hia_aic = EIP97_HIA_AIC_BASE;
offsets->hia_aic_g = EIP97_HIA_AIC_G_BASE;
offsets->hia_aic_r = EIP97_HIA_AIC_R_BASE;
@ -889,6 +972,7 @@ static void safexcel_init_register_offsets(struct safexcel_crypto_priv *priv)
offsets->hia_dse_thr = EIP97_HIA_DSE_THR_BASE;
offsets->hia_gen_cfg = EIP97_HIA_GEN_CFG_BASE;
offsets->pe = EIP97_PE_BASE;
break;
}
}
@ -906,6 +990,9 @@ static int safexcel_probe(struct platform_device *pdev)
priv->dev = dev;
priv->version = (enum safexcel_eip_version)of_device_get_match_data(dev);
if (priv->version == EIP197B || priv->version == EIP197D)
priv->flags |= EIP197_TRC_CACHE;
safexcel_init_register_offsets(priv);
res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
@ -957,6 +1044,13 @@ static int safexcel_probe(struct platform_device *pdev)
safexcel_configure(priv);
priv->ring = devm_kzalloc(dev, priv->config.rings * sizeof(*priv->ring),
GFP_KERNEL);
if (!priv->ring) {
ret = -ENOMEM;
goto err_reg_clk;
}
for (i = 0; i < priv->config.rings; i++) {
char irq_name[6] = {0}; /* "ringX\0" */
char wq_name[9] = {0}; /* "wq_ringX\0" */
@ -969,6 +1063,14 @@ static int safexcel_probe(struct platform_device *pdev)
if (ret)
goto err_reg_clk;
priv->ring[i].rdr_req = devm_kzalloc(dev,
sizeof(priv->ring[i].rdr_req) * EIP197_DEFAULT_RING_SIZE,
GFP_KERNEL);
if (!priv->ring[i].rdr_req) {
ret = -ENOMEM;
goto err_reg_clk;
}
ring_irq = devm_kzalloc(dev, sizeof(*ring_irq), GFP_KERNEL);
if (!ring_irq) {
ret = -ENOMEM;
@ -1004,9 +1106,7 @@ static int safexcel_probe(struct platform_device *pdev)
crypto_init_queue(&priv->ring[i].queue,
EIP197_DEFAULT_RING_SIZE);
INIT_LIST_HEAD(&priv->ring[i].list);
spin_lock_init(&priv->ring[i].lock);
spin_lock_init(&priv->ring[i].egress_lock);
spin_lock_init(&priv->ring[i].queue_lock);
}
@ -1034,6 +1134,24 @@ err_core_clk:
return ret;
}
static void safexcel_hw_reset_rings(struct safexcel_crypto_priv *priv)
{
int i;
for (i = 0; i < priv->config.rings; i++) {
/* clear any pending interrupt */
writel(GENMASK(5, 0), EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_STAT);
writel(GENMASK(7, 0), EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_STAT);
/* Reset the CDR base address */
writel(0, EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_RING_BASE_ADDR_LO);
writel(0, EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_RING_BASE_ADDR_HI);
/* Reset the RDR base address */
writel(0, EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_RING_BASE_ADDR_LO);
writel(0, EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_RING_BASE_ADDR_HI);
}
}
static int safexcel_remove(struct platform_device *pdev)
{
@ -1041,6 +1159,8 @@ static int safexcel_remove(struct platform_device *pdev)
int i;
safexcel_unregister_algorithms(priv);
safexcel_hw_reset_rings(priv);
clk_disable_unprepare(priv->clk);
for (i = 0; i < priv->config.rings; i++)
@ -1051,12 +1171,26 @@ static int safexcel_remove(struct platform_device *pdev)
static const struct of_device_id safexcel_of_match_table[] = {
{
.compatible = "inside-secure,safexcel-eip97",
.data = (void *)EIP97,
.compatible = "inside-secure,safexcel-eip97ies",
.data = (void *)EIP97IES,
},
{
.compatible = "inside-secure,safexcel-eip197b",
.data = (void *)EIP197B,
},
{
.compatible = "inside-secure,safexcel-eip197d",
.data = (void *)EIP197D,
},
{
/* Deprecated. Kept for backward compatibility. */
.compatible = "inside-secure,safexcel-eip97",
.data = (void *)EIP97IES,
},
{
/* Deprecated. Kept for backward compatibility. */
.compatible = "inside-secure,safexcel-eip197",
.data = (void *)EIP197,
.data = (void *)EIP197B,
},
{},
};

201
drivers/crypto/inside-secure/safexcel.h
View file

@ -1,11 +1,8 @@
/* SPDX-License-Identifier: GPL-2.0 */
/*
* Copyright (C) 2017 Marvell
*
* Antoine Tenart <antoine.tenart@free-electrons.com>
*
* This file is licensed under the terms of the GNU General Public
* License version 2. This program is licensed "as is" without any
* warranty of any kind, whether express or implied.
*/
#ifndef __SAFEXCEL_H__
@ -95,13 +92,13 @@
#define EIP197_HIA_xDR_STAT 0x003c
/* register offsets */
#define EIP197_HIA_DFE_CFG 0x0000
#define EIP197_HIA_DFE_THR_CTRL 0x0000
#define EIP197_HIA_DFE_THR_STAT 0x0004
#define EIP197_HIA_DSE_CFG 0x0000
#define EIP197_HIA_DSE_THR_CTRL 0x0000
#define EIP197_HIA_DSE_THR_STAT 0x0004
#define EIP197_HIA_RA_PE_CTRL 0x0010
#define EIP197_HIA_DFE_CFG(n) (0x0000 + (128 * (n)))
#define EIP197_HIA_DFE_THR_CTRL(n) (0x0000 + (128 * (n)))
#define EIP197_HIA_DFE_THR_STAT(n) (0x0004 + (128 * (n)))
#define EIP197_HIA_DSE_CFG(n) (0x0000 + (128 * (n)))
#define EIP197_HIA_DSE_THR_CTRL(n) (0x0000 + (128 * (n)))
#define EIP197_HIA_DSE_THR_STAT(n) (0x0004 + (128 * (n)))
#define EIP197_HIA_RA_PE_CTRL(n) (0x0010 + (8 * (n)))
#define EIP197_HIA_RA_PE_STAT 0x0014
#define EIP197_HIA_AIC_R_OFF(r) ((r) * 0x1000)
#define EIP197_HIA_AIC_R_ENABLE_CTRL(r) (0xe008 - EIP197_HIA_AIC_R_OFF(r))
@ -114,18 +111,18 @@
#define EIP197_HIA_MST_CTRL 0xfff4
#define EIP197_HIA_OPTIONS 0xfff8
#define EIP197_HIA_VERSION 0xfffc
#define EIP197_PE_IN_DBUF_THRES 0x0000
#define EIP197_PE_IN_TBUF_THRES 0x0100
#define EIP197_PE_ICE_SCRATCH_RAM 0x0800
#define EIP197_PE_ICE_PUE_CTRL 0x0c80
#define EIP197_PE_ICE_SCRATCH_CTRL 0x0d04
#define EIP197_PE_ICE_FPP_CTRL 0x0d80
#define EIP197_PE_ICE_RAM_CTRL 0x0ff0
#define EIP197_PE_EIP96_FUNCTION_EN 0x1004
#define EIP197_PE_EIP96_CONTEXT_CTRL 0x1008
#define EIP197_PE_EIP96_CONTEXT_STAT 0x100c
#define EIP197_PE_OUT_DBUF_THRES 0x1c00
#define EIP197_PE_OUT_TBUF_THRES 0x1d00
#define EIP197_PE_IN_DBUF_THRES(n) (0x0000 + (0x2000 * (n)))
#define EIP197_PE_IN_TBUF_THRES(n) (0x0100 + (0x2000 * (n)))
#define EIP197_PE_ICE_SCRATCH_RAM(n) (0x0800 + (0x2000 * (n)))
#define EIP197_PE_ICE_PUE_CTRL(n) (0x0c80 + (0x2000 * (n)))
#define EIP197_PE_ICE_SCRATCH_CTRL(n) (0x0d04 + (0x2000 * (n)))
#define EIP197_PE_ICE_FPP_CTRL(n) (0x0d80 + (0x2000 * (n)))
#define EIP197_PE_ICE_RAM_CTRL(n) (0x0ff0 + (0x2000 * (n)))
#define EIP197_PE_EIP96_FUNCTION_EN(n) (0x1004 + (0x2000 * (n)))
#define EIP197_PE_EIP96_CONTEXT_CTRL(n) (0x1008 + (0x2000 * (n)))
#define EIP197_PE_EIP96_CONTEXT_STAT(n) (0x100c + (0x2000 * (n)))
#define EIP197_PE_OUT_DBUF_THRES(n) (0x1c00 + (0x2000 * (n)))
#define EIP197_PE_OUT_TBUF_THRES(n) (0x1d00 + (0x2000 * (n)))
#define EIP197_MST_CTRL 0xfff4
/* EIP197-specific registers, no indirection */
@ -184,6 +181,11 @@
#define EIP197_HIA_RA_PE_CTRL_RESET BIT(31)
#define EIP197_HIA_RA_PE_CTRL_EN BIT(30)
/* EIP197_HIA_OPTIONS */
#define EIP197_N_PES_OFFSET 4
#define EIP197_N_PES_MASK GENMASK(4, 0)
#define EIP97_N_PES_MASK GENMASK(2, 0)
/* EIP197_HIA_AIC_R_ENABLE_CTRL */
#define EIP197_CDR_IRQ(n) BIT((n) * 2)
#define EIP197_RDR_IRQ(n) BIT((n) * 2 + 1)
@ -217,6 +219,7 @@
#define WR_CACHE_4BITS (WR_CACHE_3BITS << 1 | BIT(0))
#define EIP197_MST_CTRL_RD_CACHE(n) (((n) & 0xf) << 0)
#define EIP197_MST_CTRL_WD_CACHE(n) (((n) & 0xf) << 4)
#define EIP197_MST_CTRL_TX_MAX_CMD(n) (((n) & 0xf) << 20)
#define EIP197_MST_CTRL_BYTE_SWAP BIT(24)
#define EIP197_MST_CTRL_NO_BYTE_SWAP BIT(25)
@ -287,7 +290,7 @@ struct safexcel_context_record {
u32 control0;
u32 control1;
__le32 data[24];
__le32 data[40];
} __packed;
/* control0 */
@ -305,14 +308,19 @@ struct safexcel_context_record {
#define CONTEXT_CONTROL_NO_FINISH_HASH BIT(5)
#define CONTEXT_CONTROL_SIZE(n) ((n) << 8)
#define CONTEXT_CONTROL_KEY_EN BIT(16)
#define CONTEXT_CONTROL_CRYPTO_ALG_DES (0x0 << 17)
#define CONTEXT_CONTROL_CRYPTO_ALG_3DES (0x2 << 17)
#define CONTEXT_CONTROL_CRYPTO_ALG_AES128 (0x5 << 17)
#define CONTEXT_CONTROL_CRYPTO_ALG_AES192 (0x6 << 17)
#define CONTEXT_CONTROL_CRYPTO_ALG_AES256 (0x7 << 17)
#define CONTEXT_CONTROL_DIGEST_PRECOMPUTED (0x1 << 21)
#define CONTEXT_CONTROL_DIGEST_HMAC (0x3 << 21)
#define CONTEXT_CONTROL_CRYPTO_ALG_MD5 (0x0 << 23)
#define CONTEXT_CONTROL_CRYPTO_ALG_SHA1 (0x2 << 23)
#define CONTEXT_CONTROL_CRYPTO_ALG_SHA224 (0x4 << 23)
#define CONTEXT_CONTROL_CRYPTO_ALG_SHA256 (0x3 << 23)
#define CONTEXT_CONTROL_CRYPTO_ALG_SHA384 (0x6 << 23)
#define CONTEXT_CONTROL_CRYPTO_ALG_SHA512 (0x5 << 23)
#define CONTEXT_CONTROL_INV_FR (0x5 << 24)
#define CONTEXT_CONTROL_INV_TR (0x6 << 24)
@ -327,6 +335,11 @@ struct safexcel_context_record {
#define CONTEXT_CONTROL_COUNTER_MODE BIT(10)
#define CONTEXT_CONTROL_HASH_STORE BIT(19)
/* The hash counter given to the engine in the context has a granularity of
* 64 bits.
*/
#define EIP197_COUNTER_BLOCK_SIZE 64
/* EIP197_CS_RAM_CTRL */
#define EIP197_TRC_ENABLE_0 BIT(4)
#define EIP197_TRC_ENABLE_1 BIT(5)
@ -349,13 +362,19 @@ struct safexcel_context_record {
#define EIP197_TRC_PARAMS2_RC_SZ_SMALL(n) ((n) << 18)
/* Cache helpers */
#define EIP197_CS_RC_MAX 52
#define EIP197B_CS_RC_MAX 52
#define EIP197D_CS_RC_MAX 96
#define EIP197_CS_RC_SIZE (4 * sizeof(u32))
#define EIP197_CS_RC_NEXT(x) (x)
#define EIP197_CS_RC_PREV(x) ((x) << 10)
#define EIP197_RC_NULL 0x3ff
#define EIP197_CS_TRC_REC_WC 59
#define EIP197_CS_TRC_LG_REC_WC 73
#define EIP197B_CS_TRC_REC_WC 59
#define EIP197D_CS_TRC_REC_WC 64
#define EIP197B_CS_TRC_LG_REC_WC 73
#define EIP197D_CS_TRC_LG_REC_WC 80
#define EIP197B_CS_HT_WC 64
#define EIP197D_CS_HT_WC 256
/* Result data */
struct result_data_desc {
@ -450,6 +469,7 @@ struct safexcel_control_data_desc {
#define EIP197_OPTION_MAGIC_VALUE BIT(0)
#define EIP197_OPTION_64BIT_CTX BIT(1)
#define EIP197_OPTION_CTX_CTRL_IN_CMD BIT(8)
#define EIP197_OPTION_2_TOKEN_IV_CMD GENMASK(11, 10)
#define EIP197_OPTION_4_TOKEN_IV_CMD GENMASK(11, 9)
#define EIP197_TYPE_EXTENDED 0x3
@ -480,7 +500,7 @@ enum eip197_fw {
FW_NB
};
struct safexcel_ring {
struct safexcel_desc_ring {
void *base;
void *base_end;
dma_addr_t base_dma;
@ -489,8 +509,7 @@ struct safexcel_ring {
void *write;
void *read;
/* number of elements used in the ring */
unsigned nr;
/* descriptor element offset */
unsigned offset;
};
@ -500,12 +519,8 @@ enum safexcel_alg_type {
SAFEXCEL_ALG_TYPE_AHASH,
};
struct safexcel_request {
struct list_head list;
struct crypto_async_request *req;
};
struct safexcel_config {
u32 pes;
u32 rings;
u32 cd_size;
@ -521,9 +536,40 @@ struct safexcel_work_data {
int ring;
};
struct safexcel_ring {
spinlock_t lock;
struct workqueue_struct *workqueue;
struct safexcel_work_data work_data;
/* command/result rings */
struct safexcel_desc_ring cdr;
struct safexcel_desc_ring rdr;
/* result ring crypto API request */
struct crypto_async_request **rdr_req;
/* queue */
struct crypto_queue queue;
spinlock_t queue_lock;
/* Number of requests in the engine. */
int requests;
/* The ring is currently handling at least one request */
bool busy;
/* Store for current requests when bailing out of the dequeueing
* function when no enough resources are available.
*/
struct crypto_async_request *req;
struct crypto_async_request *backlog;
};
enum safexcel_eip_version {
EIP97,
EIP197,
EIP97IES = BIT(0),
EIP197B = BIT(1),
EIP197D = BIT(2),
};
struct safexcel_register_offsets {
@ -539,6 +585,10 @@ struct safexcel_register_offsets {
u32 pe;
};
enum safexcel_flags {
EIP197_TRC_CACHE = BIT(0),
};
struct safexcel_crypto_priv {
void __iomem *base;
struct device *dev;
@ -548,46 +598,19 @@ struct safexcel_crypto_priv {
enum safexcel_eip_version version;
struct safexcel_register_offsets offsets;
u32 flags;
/* context DMA pool */
struct dma_pool *context_pool;
atomic_t ring_used;
struct {
spinlock_t lock;
spinlock_t egress_lock;
struct list_head list;
struct workqueue_struct *workqueue;
struct safexcel_work_data work_data;
/* command/result rings */
struct safexcel_ring cdr;
struct safexcel_ring rdr;
/* queue */
struct crypto_queue queue;
spinlock_t queue_lock;
/* Number of requests in the engine. */
int requests;
/* The ring is currently handling at least one request */
bool busy;
/* Store for current requests when bailing out of the dequeueing
* function when no enough resources are available.
*/
struct crypto_async_request *req;
struct crypto_async_request *backlog;
} ring[EIP197_MAX_RINGS];
struct safexcel_ring *ring;
};
struct safexcel_context {
int (*send)(struct crypto_async_request *req, int ring,
struct safexcel_request *request, int *commands,
int *results);
int *commands, int *results);
int (*handle_result)(struct safexcel_crypto_priv *priv, int ring,
struct crypto_async_request *req, bool *complete,
int *ret);
@ -600,13 +623,13 @@ struct safexcel_context {
};
struct safexcel_ahash_export_state {
u64 len;
u64 processed;
u64 len[2];
u64 processed[2];
u32 digest;
u32 state[SHA256_DIGEST_SIZE / sizeof(u32)];
u8 cache[SHA256_BLOCK_SIZE];
u32 state[SHA512_DIGEST_SIZE / sizeof(u32)];
u8 cache[SHA512_BLOCK_SIZE];
};
/*
@ -617,6 +640,7 @@ struct safexcel_ahash_export_state {
struct safexcel_alg_template {
struct safexcel_crypto_priv *priv;
enum safexcel_alg_type type;
u32 engines;
union {
struct skcipher_alg skcipher;
struct aead_alg aead;
@ -635,16 +659,16 @@ int safexcel_rdesc_check_errors(struct safexcel_crypto_priv *priv,
void safexcel_complete(struct safexcel_crypto_priv *priv, int ring);
int safexcel_invalidate_cache(struct crypto_async_request *async,
struct safexcel_crypto_priv *priv,
dma_addr_t ctxr_dma, int ring,
struct safexcel_request *request);
dma_addr_t ctxr_dma, int ring);
int safexcel_init_ring_descriptors(struct safexcel_crypto_priv *priv,
struct safexcel_ring *cdr,
struct safexcel_ring *rdr);
struct safexcel_desc_ring *cdr,
struct safexcel_desc_ring *rdr);
int safexcel_select_ring(struct safexcel_crypto_priv *priv);
void *safexcel_ring_next_rptr(struct safexcel_crypto_priv *priv,
struct safexcel_ring *ring);
struct safexcel_desc_ring *ring);
void *safexcel_ring_first_rptr(struct safexcel_crypto_priv *priv, int ring);
void safexcel_ring_rollback_wptr(struct safexcel_crypto_priv *priv,
struct safexcel_ring *ring);
struct safexcel_desc_ring *ring);
struct safexcel_command_desc *safexcel_add_cdesc(struct safexcel_crypto_priv *priv,
int ring_id,
bool first, bool last,
@ -655,21 +679,44 @@ struct safexcel_result_desc *safexcel_add_rdesc(struct safexcel_crypto_priv *pri
int ring_id,
bool first, bool last,
dma_addr_t data, u32 len);
int safexcel_ring_first_rdr_index(struct safexcel_crypto_priv *priv,
int ring);
int safexcel_ring_rdr_rdesc_index(struct safexcel_crypto_priv *priv,
int ring,
struct safexcel_result_desc *rdesc);
void safexcel_rdr_req_set(struct safexcel_crypto_priv *priv,
int ring,
struct safexcel_result_desc *rdesc,
struct crypto_async_request *req);
inline struct crypto_async_request *
safexcel_rdr_req_get(struct safexcel_crypto_priv *priv, int ring);
void safexcel_inv_complete(struct crypto_async_request *req, int error);
int safexcel_hmac_setkey(const char *alg, const u8 *key, unsigned int keylen,
void *istate, void *ostate);
/* available algorithms */
extern struct safexcel_alg_template safexcel_alg_ecb_des;
extern struct safexcel_alg_template safexcel_alg_cbc_des;
extern struct safexcel_alg_template safexcel_alg_ecb_des3_ede;
extern struct safexcel_alg_template safexcel_alg_cbc_des3_ede;
extern struct safexcel_alg_template safexcel_alg_ecb_aes;
extern struct safexcel_alg_template safexcel_alg_cbc_aes;
extern struct safexcel_alg_template safexcel_alg_md5;
extern struct safexcel_alg_template safexcel_alg_sha1;
extern struct safexcel_alg_template safexcel_alg_sha224;
extern struct safexcel_alg_template safexcel_alg_sha256;
extern struct safexcel_alg_template safexcel_alg_sha384;
extern struct safexcel_alg_template safexcel_alg_sha512;
extern struct safexcel_alg_template safexcel_alg_hmac_md5;
extern struct safexcel_alg_template safexcel_alg_hmac_sha1;
extern struct safexcel_alg_template safexcel_alg_hmac_sha224;
extern struct safexcel_alg_template safexcel_alg_hmac_sha256;
extern struct safexcel_alg_template safexcel_alg_hmac_sha384;
extern struct safexcel_alg_template safexcel_alg_hmac_sha512;
extern struct safexcel_alg_template safexcel_alg_authenc_hmac_sha1_cbc_aes;
extern struct safexcel_alg_template safexcel_alg_authenc_hmac_sha224_cbc_aes;
extern struct safexcel_alg_template safexcel_alg_authenc_hmac_sha256_cbc_aes;
extern struct safexcel_alg_template safexcel_alg_authenc_hmac_sha384_cbc_aes;
extern struct safexcel_alg_template safexcel_alg_authenc_hmac_sha512_cbc_aes;
#endif

490
drivers/crypto/inside-secure/safexcel_cipher.c
View file

@ -1,11 +1,8 @@
// SPDX-License-Identifier: GPL-2.0
/*
* Copyright (C) 2017 Marvell
*
* Antoine Tenart <antoine.tenart@free-electrons.com>
*
* This file is licensed under the terms of the GNU General Public
* License version 2. This program is licensed "as is" without any
* warranty of any kind, whether express or implied.
*/
#include <linux/device.h>
@ -15,6 +12,7 @@
#include <crypto/aead.h>
#include <crypto/aes.h>
#include <crypto/authenc.h>
#include <crypto/des.h>
#include <crypto/sha.h>
#include <crypto/skcipher.h>
#include <crypto/internal/aead.h>
@ -27,21 +25,28 @@ enum safexcel_cipher_direction {
SAFEXCEL_DECRYPT,
};
enum safexcel_cipher_alg {
SAFEXCEL_DES,
SAFEXCEL_3DES,
SAFEXCEL_AES,
};
struct safexcel_cipher_ctx {
struct safexcel_context base;
struct safexcel_crypto_priv *priv;
u32 mode;
enum safexcel_cipher_alg alg;
bool aead;
__le32 key[8];
unsigned int key_len;
/* All the below is AEAD specific */
u32 alg;
u32 hash_alg;
u32 state_sz;
u32 ipad[SHA256_DIGEST_SIZE / sizeof(u32)];
u32 opad[SHA256_DIGEST_SIZE / sizeof(u32)];
u32 ipad[SHA512_DIGEST_SIZE / sizeof(u32)];
u32 opad[SHA512_DIGEST_SIZE / sizeof(u32)];
};
struct safexcel_cipher_req {
@ -57,10 +62,24 @@ static void safexcel_skcipher_token(struct safexcel_cipher_ctx *ctx, u8 *iv,
unsigned offset = 0;
if (ctx->mode == CONTEXT_CONTROL_CRYPTO_MODE_CBC) {
offset = AES_BLOCK_SIZE / sizeof(u32);
memcpy(cdesc->control_data.token, iv, AES_BLOCK_SIZE);
switch (ctx->alg) {
case SAFEXCEL_DES:
offset = DES_BLOCK_SIZE / sizeof(u32);
memcpy(cdesc->control_data.token, iv, DES_BLOCK_SIZE);
cdesc->control_data.options |= EIP197_OPTION_2_TOKEN_IV_CMD;
break;
case SAFEXCEL_3DES:
offset = DES3_EDE_BLOCK_SIZE / sizeof(u32);
memcpy(cdesc->control_data.token, iv, DES3_EDE_BLOCK_SIZE);
cdesc->control_data.options |= EIP197_OPTION_2_TOKEN_IV_CMD;
break;
cdesc->control_data.options |= EIP197_OPTION_4_TOKEN_IV_CMD;
case SAFEXCEL_AES:
offset = AES_BLOCK_SIZE / sizeof(u32);
memcpy(cdesc->control_data.token, iv, AES_BLOCK_SIZE);
cdesc->control_data.options |= EIP197_OPTION_4_TOKEN_IV_CMD;
break;
}
}
token = (struct safexcel_token *)(cdesc->control_data.token + offset);
@ -145,7 +164,7 @@ static int safexcel_skcipher_aes_setkey(struct crypto_skcipher *ctfm,
return ret;
}
if (priv->version == EIP197 && ctx->base.ctxr_dma) {
if (priv->flags & EIP197_TRC_CACHE && ctx->base.ctxr_dma) {
for (i = 0; i < len / sizeof(u32); i++) {
if (ctx->key[i] != cpu_to_le32(aes.key_enc[i])) {
ctx->base.needs_inv = true;
@ -179,12 +198,12 @@ static int safexcel_aead_aes_setkey(struct crypto_aead *ctfm, const u8 *key,
goto badkey;
/* Encryption key */
if (priv->version == EIP197 && ctx->base.ctxr_dma &&
if (priv->flags & EIP197_TRC_CACHE && ctx->base.ctxr_dma &&
memcmp(ctx->key, keys.enckey, keys.enckeylen))
ctx->base.needs_inv = true;
/* Auth key */
switch (ctx->alg) {
switch (ctx->hash_alg) {
case CONTEXT_CONTROL_CRYPTO_ALG_SHA1:
if (safexcel_hmac_setkey("safexcel-sha1", keys.authkey,
keys.authkeylen, &istate, &ostate))
@ -200,6 +219,16 @@ static int safexcel_aead_aes_setkey(struct crypto_aead *ctfm, const u8 *key,
keys.authkeylen, &istate, &ostate))
goto badkey;
break;
case CONTEXT_CONTROL_CRYPTO_ALG_SHA384:
if (safexcel_hmac_setkey("safexcel-sha384", keys.authkey,
keys.authkeylen, &istate, &ostate))
goto badkey;
break;
case CONTEXT_CONTROL_CRYPTO_ALG_SHA512:
if (safexcel_hmac_setkey("safexcel-sha512", keys.authkey,
keys.authkeylen, &istate, &ostate))
goto badkey;
break;
default:
dev_err(priv->dev, "aead: unsupported hash algorithm\n");
goto badkey;
@ -208,7 +237,7 @@ static int safexcel_aead_aes_setkey(struct crypto_aead *ctfm, const u8 *key,
crypto_aead_set_flags(ctfm, crypto_aead_get_flags(ctfm) &
CRYPTO_TFM_RES_MASK);
if (priv->version == EIP197 && ctx->base.ctxr_dma &&
if (priv->flags & EIP197_TRC_CACHE && ctx->base.ctxr_dma &&
(memcmp(ctx->ipad, istate.state, ctx->state_sz) ||
memcmp(ctx->opad, ostate.state, ctx->state_sz)))
ctx->base.needs_inv = true;
@ -258,22 +287,28 @@ static int safexcel_context_control(struct safexcel_cipher_ctx *ctx,
if (ctx->aead)
cdesc->control_data.control0 |= CONTEXT_CONTROL_DIGEST_HMAC |
ctx->alg;
ctx->hash_alg;
switch (ctx->key_len) {
case AES_KEYSIZE_128:
cdesc->control_data.control0 |= CONTEXT_CONTROL_CRYPTO_ALG_AES128;
break;
case AES_KEYSIZE_192:
cdesc->control_data.control0 |= CONTEXT_CONTROL_CRYPTO_ALG_AES192;
break;
case AES_KEYSIZE_256:
cdesc->control_data.control0 |= CONTEXT_CONTROL_CRYPTO_ALG_AES256;
break;
default:
dev_err(priv->dev, "aes keysize not supported: %u\n",
ctx->key_len);
return -EINVAL;
if (ctx->alg == SAFEXCEL_DES) {
cdesc->control_data.control0 |= CONTEXT_CONTROL_CRYPTO_ALG_DES;
} else if (ctx->alg == SAFEXCEL_3DES) {
cdesc->control_data.control0 |= CONTEXT_CONTROL_CRYPTO_ALG_3DES;
} else if (ctx->alg == SAFEXCEL_AES) {
switch (ctx->key_len) {
case AES_KEYSIZE_128:
cdesc->control_data.control0 |= CONTEXT_CONTROL_CRYPTO_ALG_AES128;
break;
case AES_KEYSIZE_192:
cdesc->control_data.control0 |= CONTEXT_CONTROL_CRYPTO_ALG_AES192;
break;
case AES_KEYSIZE_256:
cdesc->control_data.control0 |= CONTEXT_CONTROL_CRYPTO_ALG_AES256;
break;
default:
dev_err(priv->dev, "aes keysize not supported: %u\n",
ctx->key_len);
return -EINVAL;
}
}
ctrl_size = ctx->key_len / sizeof(u32);
@ -298,7 +333,6 @@ static int safexcel_handle_req_result(struct safexcel_crypto_priv *priv, int rin
*ret = 0;
spin_lock_bh(&priv->ring[ring].egress_lock);
do {
rdesc = safexcel_ring_next_rptr(priv, &priv->ring[ring].rdr);
if (IS_ERR(rdesc)) {
@ -315,7 +349,6 @@ static int safexcel_handle_req_result(struct safexcel_crypto_priv *priv, int rin
} while (!rdesc->last_seg);
safexcel_complete(priv, ring);
spin_unlock_bh(&priv->ring[ring].egress_lock);
if (src == dst) {
dma_unmap_sg(priv->dev, src,
@ -335,8 +368,7 @@ static int safexcel_handle_req_result(struct safexcel_crypto_priv *priv, int rin
return ndesc;
}
static int safexcel_aes_send(struct crypto_async_request *base, int ring,
struct safexcel_request *request,
static int safexcel_send_req(struct crypto_async_request *base, int ring,
struct safexcel_cipher_req *sreq,
struct scatterlist *src, struct scatterlist *dst,
unsigned int cryptlen, unsigned int assoclen,
@ -346,7 +378,7 @@ static int safexcel_aes_send(struct crypto_async_request *base, int ring,
struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(base->tfm);
struct safexcel_crypto_priv *priv = ctx->priv;
struct safexcel_command_desc *cdesc;
struct safexcel_result_desc *rdesc;
struct safexcel_result_desc *rdesc, *first_rdesc = NULL;
struct scatterlist *sg;
unsigned int totlen = cryptlen + assoclen;
int nr_src, nr_dst, n_cdesc = 0, n_rdesc = 0, queued = totlen;
@ -386,8 +418,6 @@ static int safexcel_aes_send(struct crypto_async_request *base, int ring,
ctx->opad, ctx->state_sz);
}
spin_lock_bh(&priv->ring[ring].egress_lock);
/* command descriptors */
for_each_sg(src, sg, nr_src, i) {
int len = sg_dma_len(sg);
@ -434,12 +464,12 @@ static int safexcel_aes_send(struct crypto_async_request *base, int ring,
ret = PTR_ERR(rdesc);
goto rdesc_rollback;
}
if (first)
first_rdesc = rdesc;
n_rdesc++;
}
spin_unlock_bh(&priv->ring[ring].egress_lock);
request->req = base;
safexcel_rdr_req_set(priv, ring, first_rdesc, base);
*commands = n_cdesc;
*results = n_rdesc;
@ -452,8 +482,6 @@ cdesc_rollback:
for (i = 0; i < n_cdesc; i++)
safexcel_ring_rollback_wptr(priv, &priv->ring[ring].cdr);
spin_unlock_bh(&priv->ring[ring].egress_lock);
if (src == dst) {
dma_unmap_sg(priv->dev, src,
sg_nents_for_len(src, totlen),
@ -481,7 +509,6 @@ static int safexcel_handle_inv_result(struct safexcel_crypto_priv *priv,
*ret = 0;
spin_lock_bh(&priv->ring[ring].egress_lock);
do {
rdesc = safexcel_ring_next_rptr(priv, &priv->ring[ring].rdr);
if (IS_ERR(rdesc)) {
@ -491,17 +518,13 @@ static int safexcel_handle_inv_result(struct safexcel_crypto_priv *priv,
break;
}
if (rdesc->result_data.error_code) {
dev_err(priv->dev, "cipher: invalidate: result descriptor error (%d)\n",
rdesc->result_data.error_code);
*ret = -EIO;
}
if (likely(!*ret))
*ret = safexcel_rdesc_check_errors(priv, rdesc);
ndesc++;
} while (!rdesc->last_seg);
safexcel_complete(priv, ring);
spin_unlock_bh(&priv->ring[ring].egress_lock);
if (ctx->base.exit_inv) {
dma_pool_free(priv->context_pool, ctx->base.ctxr,
@ -577,15 +600,13 @@ static int safexcel_aead_handle_result(struct safexcel_crypto_priv *priv,
}
static int safexcel_cipher_send_inv(struct crypto_async_request *base,
int ring, struct safexcel_request *request,
int *commands, int *results)
int ring, int *commands, int *results)
{
struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(base->tfm);
struct safexcel_crypto_priv *priv = ctx->priv;
int ret;
ret = safexcel_invalidate_cache(base, priv, ctx->base.ctxr_dma, ring,
request);
ret = safexcel_invalidate_cache(base, priv, ctx->base.ctxr_dma, ring);
if (unlikely(ret))
return ret;
@ -596,7 +617,6 @@ static int safexcel_cipher_send_inv(struct crypto_async_request *base,
}
static int safexcel_skcipher_send(struct crypto_async_request *async, int ring,
struct safexcel_request *request,
int *commands, int *results)
{
struct skcipher_request *req = skcipher_request_cast(async);
@ -605,21 +625,19 @@ static int safexcel_skcipher_send(struct crypto_async_request *async, int ring,
struct safexcel_crypto_priv *priv = ctx->priv;
int ret;
BUG_ON(priv->version == EIP97 && sreq->needs_inv);
BUG_ON(!(priv->flags & EIP197_TRC_CACHE) && sreq->needs_inv);
if (sreq->needs_inv)
ret = safexcel_cipher_send_inv(async, ring, request, commands,
results);
ret = safexcel_cipher_send_inv(async, ring, commands, results);
else
ret = safexcel_aes_send(async, ring, request, sreq, req->src,
ret = safexcel_send_req(async, ring, sreq, req->src,
req->dst, req->cryptlen, 0, 0, req->iv,
commands, results);
return ret;
}
static int safexcel_aead_send(struct crypto_async_request *async, int ring,
struct safexcel_request *request, int *commands,
int *results)
int *commands, int *results)
{
struct aead_request *req = aead_request_cast(async);
struct crypto_aead *tfm = crypto_aead_reqtfm(req);
@ -628,14 +646,13 @@ static int safexcel_aead_send(struct crypto_async_request *async, int ring,
struct safexcel_crypto_priv *priv = ctx->priv;
int ret;
BUG_ON(priv->version == EIP97 && sreq->needs_inv);
BUG_ON(!(priv->flags & EIP197_TRC_CACHE) && sreq->needs_inv);
if (sreq->needs_inv)
ret = safexcel_cipher_send_inv(async, ring, request, commands,
results);
ret = safexcel_cipher_send_inv(async, ring, commands, results);
else
ret = safexcel_aes_send(async, ring, request, sreq, req->src,
req->dst, req->cryptlen, req->assoclen,
ret = safexcel_send_req(async, ring, sreq, req->src, req->dst,
req->cryptlen, req->assoclen,
crypto_aead_authsize(tfm), req->iv,
commands, results);
return ret;
@ -705,9 +722,10 @@ static int safexcel_aead_exit_inv(struct crypto_tfm *tfm)
return safexcel_cipher_exit_inv(tfm, &req->base, sreq, &result);
}
static int safexcel_aes(struct crypto_async_request *base,
static int safexcel_queue_req(struct crypto_async_request *base,
struct safexcel_cipher_req *sreq,
enum safexcel_cipher_direction dir, u32 mode)
enum safexcel_cipher_direction dir, u32 mode,
enum safexcel_cipher_alg alg)
{
struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(base->tfm);
struct safexcel_crypto_priv *priv = ctx->priv;
@ -715,10 +733,11 @@ static int safexcel_aes(struct crypto_async_request *base,
sreq->needs_inv = false;
sreq->direction = dir;
ctx->alg = alg;
ctx->mode = mode;
if (ctx->base.ctxr) {
if (priv->version == EIP197 && ctx->base.needs_inv) {
if (priv->flags & EIP197_TRC_CACHE && ctx->base.needs_inv) {
sreq->needs_inv = true;
ctx->base.needs_inv = false;
}
@ -745,14 +764,16 @@ static int safexcel_aes(struct crypto_async_request *base,
static int safexcel_ecb_aes_encrypt(struct skcipher_request *req)
{
return safexcel_aes(&req->base, skcipher_request_ctx(req),
SAFEXCEL_ENCRYPT, CONTEXT_CONTROL_CRYPTO_MODE_ECB);
return safexcel_queue_req(&req->base, skcipher_request_ctx(req),
SAFEXCEL_ENCRYPT, CONTEXT_CONTROL_CRYPTO_MODE_ECB,
SAFEXCEL_AES);
}
static int safexcel_ecb_aes_decrypt(struct skcipher_request *req)
{
return safexcel_aes(&req->base, skcipher_request_ctx(req),
SAFEXCEL_DECRYPT, CONTEXT_CONTROL_CRYPTO_MODE_ECB);
return safexcel_queue_req(&req->base, skcipher_request_ctx(req),
SAFEXCEL_DECRYPT, CONTEXT_CONTROL_CRYPTO_MODE_ECB,
SAFEXCEL_AES);
}
static int safexcel_skcipher_cra_init(struct crypto_tfm *tfm)
@ -795,7 +816,7 @@ static void safexcel_skcipher_cra_exit(struct crypto_tfm *tfm)
if (safexcel_cipher_cra_exit(tfm))
return;
if (priv->version == EIP197) {
if (priv->flags & EIP197_TRC_CACHE) {
ret = safexcel_skcipher_exit_inv(tfm);
if (ret)
dev_warn(priv->dev, "skcipher: invalidation error %d\n",
@ -815,7 +836,7 @@ static void safexcel_aead_cra_exit(struct crypto_tfm *tfm)
if (safexcel_cipher_cra_exit(tfm))
return;
if (priv->version == EIP197) {
if (priv->flags & EIP197_TRC_CACHE) {
ret = safexcel_aead_exit_inv(tfm);
if (ret)
dev_warn(priv->dev, "aead: invalidation error %d\n",
@ -828,6 +849,7 @@ static void safexcel_aead_cra_exit(struct crypto_tfm *tfm)
struct safexcel_alg_template safexcel_alg_ecb_aes = {
.type = SAFEXCEL_ALG_TYPE_SKCIPHER,
.engines = EIP97IES | EIP197B | EIP197D,
.alg.skcipher = {
.setkey = safexcel_skcipher_aes_setkey,
.encrypt = safexcel_ecb_aes_encrypt,
@ -838,7 +860,7 @@ struct safexcel_alg_template safexcel_alg_ecb_aes = {
.cra_name = "ecb(aes)",
.cra_driver_name = "safexcel-ecb-aes",
.cra_priority = 300,
.cra_flags = CRYPTO_ALG_TYPE_SKCIPHER | CRYPTO_ALG_ASYNC |
.cra_flags = CRYPTO_ALG_ASYNC |
CRYPTO_ALG_KERN_DRIVER_ONLY,
.cra_blocksize = AES_BLOCK_SIZE,
.cra_ctxsize = sizeof(struct safexcel_cipher_ctx),
@ -852,18 +874,21 @@ struct safexcel_alg_template safexcel_alg_ecb_aes = {
static int safexcel_cbc_aes_encrypt(struct skcipher_request *req)
{
return safexcel_aes(&req->base, skcipher_request_ctx(req),
SAFEXCEL_ENCRYPT, CONTEXT_CONTROL_CRYPTO_MODE_CBC);
return safexcel_queue_req(&req->base, skcipher_request_ctx(req),
SAFEXCEL_ENCRYPT, CONTEXT_CONTROL_CRYPTO_MODE_CBC,
SAFEXCEL_AES);
}
static int safexcel_cbc_aes_decrypt(struct skcipher_request *req)
{
return safexcel_aes(&req->base, skcipher_request_ctx(req),
SAFEXCEL_DECRYPT, CONTEXT_CONTROL_CRYPTO_MODE_CBC);
return safexcel_queue_req(&req->base, skcipher_request_ctx(req),
SAFEXCEL_DECRYPT, CONTEXT_CONTROL_CRYPTO_MODE_CBC,
SAFEXCEL_AES);
}
struct safexcel_alg_template safexcel_alg_cbc_aes = {
.type = SAFEXCEL_ALG_TYPE_SKCIPHER,
.engines = EIP97IES | EIP197B | EIP197D,
.alg.skcipher = {
.setkey = safexcel_skcipher_aes_setkey,
.encrypt = safexcel_cbc_aes_encrypt,
@ -875,7 +900,7 @@ struct safexcel_alg_template safexcel_alg_cbc_aes = {
.cra_name = "cbc(aes)",
.cra_driver_name = "safexcel-cbc-aes",
.cra_priority = 300,
.cra_flags = CRYPTO_ALG_TYPE_SKCIPHER | CRYPTO_ALG_ASYNC |
.cra_flags = CRYPTO_ALG_ASYNC |
CRYPTO_ALG_KERN_DRIVER_ONLY,
.cra_blocksize = AES_BLOCK_SIZE,
.cra_ctxsize = sizeof(struct safexcel_cipher_ctx),
@ -887,20 +912,234 @@ struct safexcel_alg_template safexcel_alg_cbc_aes = {
},
};
static int safexcel_cbc_des_encrypt(struct skcipher_request *req)
{
return safexcel_queue_req(&req->base, skcipher_request_ctx(req),
SAFEXCEL_ENCRYPT, CONTEXT_CONTROL_CRYPTO_MODE_CBC,
SAFEXCEL_DES);
}
static int safexcel_cbc_des_decrypt(struct skcipher_request *req)
{
return safexcel_queue_req(&req->base, skcipher_request_ctx(req),
SAFEXCEL_DECRYPT, CONTEXT_CONTROL_CRYPTO_MODE_CBC,
SAFEXCEL_DES);
}
static int safexcel_des_setkey(struct crypto_skcipher *ctfm, const u8 *key,
unsigned int len)
{
struct crypto_tfm *tfm = crypto_skcipher_tfm(ctfm);
struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
u32 tmp[DES_EXPKEY_WORDS];
int ret;
if (len != DES_KEY_SIZE) {
crypto_skcipher_set_flags(ctfm, CRYPTO_TFM_RES_BAD_KEY_LEN);
return -EINVAL;
}
ret = des_ekey(tmp, key);
if (!ret && (tfm->crt_flags & CRYPTO_TFM_REQ_WEAK_KEY)) {
tfm->crt_flags |= CRYPTO_TFM_RES_WEAK_KEY;
return -EINVAL;
}
/* if context exits and key changed, need to invalidate it */
if (ctx->base.ctxr_dma)
if (memcmp(ctx->key, key, len))
ctx->base.needs_inv = true;
memcpy(ctx->key, key, len);
ctx->key_len = len;
return 0;
}
struct safexcel_alg_template safexcel_alg_cbc_des = {
.type = SAFEXCEL_ALG_TYPE_SKCIPHER,
.engines = EIP97IES | EIP197B | EIP197D,
.alg.skcipher = {
.setkey = safexcel_des_setkey,
.encrypt = safexcel_cbc_des_encrypt,
.decrypt = safexcel_cbc_des_decrypt,
.min_keysize = DES_KEY_SIZE,
.max_keysize = DES_KEY_SIZE,
.ivsize = DES_BLOCK_SIZE,
.base = {
.cra_name = "cbc(des)",
.cra_driver_name = "safexcel-cbc-des",
.cra_priority = 300,
.cra_flags = CRYPTO_ALG_TYPE_SKCIPHER | CRYPTO_ALG_ASYNC |
CRYPTO_ALG_KERN_DRIVER_ONLY,
.cra_blocksize = DES_BLOCK_SIZE,
.cra_ctxsize = sizeof(struct safexcel_cipher_ctx),
.cra_alignmask = 0,
.cra_init = safexcel_skcipher_cra_init,
.cra_exit = safexcel_skcipher_cra_exit,
.cra_module = THIS_MODULE,
},
},
};
static int safexcel_ecb_des_encrypt(struct skcipher_request *req)
{
return safexcel_queue_req(&req->base, skcipher_request_ctx(req),
SAFEXCEL_ENCRYPT, CONTEXT_CONTROL_CRYPTO_MODE_ECB,
SAFEXCEL_DES);
}
static int safexcel_ecb_des_decrypt(struct skcipher_request *req)
{
return safexcel_queue_req(&req->base, skcipher_request_ctx(req),
SAFEXCEL_DECRYPT, CONTEXT_CONTROL_CRYPTO_MODE_ECB,
SAFEXCEL_DES);
}
struct safexcel_alg_template safexcel_alg_ecb_des = {
.type = SAFEXCEL_ALG_TYPE_SKCIPHER,
.engines = EIP97IES | EIP197B | EIP197D,
.alg.skcipher = {
.setkey = safexcel_des_setkey,
.encrypt = safexcel_ecb_des_encrypt,
.decrypt = safexcel_ecb_des_decrypt,
.min_keysize = DES_KEY_SIZE,
.max_keysize = DES_KEY_SIZE,
.ivsize = DES_BLOCK_SIZE,
.base = {
.cra_name = "ecb(des)",
.cra_driver_name = "safexcel-ecb-des",
.cra_priority = 300,
.cra_flags = CRYPTO_ALG_TYPE_SKCIPHER | CRYPTO_ALG_ASYNC |
CRYPTO_ALG_KERN_DRIVER_ONLY,
.cra_blocksize = DES_BLOCK_SIZE,
.cra_ctxsize = sizeof(struct safexcel_cipher_ctx),
.cra_alignmask = 0,
.cra_init = safexcel_skcipher_cra_init,
.cra_exit = safexcel_skcipher_cra_exit,
.cra_module = THIS_MODULE,
},
},
};
static int safexcel_cbc_des3_ede_encrypt(struct skcipher_request *req)
{
return safexcel_queue_req(&req->base, skcipher_request_ctx(req),
SAFEXCEL_ENCRYPT, CONTEXT_CONTROL_CRYPTO_MODE_CBC,
SAFEXCEL_3DES);
}
static int safexcel_cbc_des3_ede_decrypt(struct skcipher_request *req)
{
return safexcel_queue_req(&req->base, skcipher_request_ctx(req),
SAFEXCEL_DECRYPT, CONTEXT_CONTROL_CRYPTO_MODE_CBC,
SAFEXCEL_3DES);
}
static int safexcel_des3_ede_setkey(struct crypto_skcipher *ctfm,
const u8 *key, unsigned int len)
{
struct crypto_tfm *tfm = crypto_skcipher_tfm(ctfm);
struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
if (len != DES3_EDE_KEY_SIZE) {
crypto_skcipher_set_flags(ctfm, CRYPTO_TFM_RES_BAD_KEY_LEN);
return -EINVAL;
}
/* if context exits and key changed, need to invalidate it */
if (ctx->base.ctxr_dma) {
if (memcmp(ctx->key, key, len))
ctx->base.needs_inv = true;
}
memcpy(ctx->key, key, len);
ctx->key_len = len;
return 0;
}
struct safexcel_alg_template safexcel_alg_cbc_des3_ede = {
.type = SAFEXCEL_ALG_TYPE_SKCIPHER,
.engines = EIP97IES | EIP197B | EIP197D,
.alg.skcipher = {
.setkey = safexcel_des3_ede_setkey,
.encrypt = safexcel_cbc_des3_ede_encrypt,
.decrypt = safexcel_cbc_des3_ede_decrypt,
.min_keysize = DES3_EDE_KEY_SIZE,
.max_keysize = DES3_EDE_KEY_SIZE,
.ivsize = DES3_EDE_BLOCK_SIZE,
.base = {
.cra_name = "cbc(des3_ede)",
.cra_driver_name = "safexcel-cbc-des3_ede",
.cra_priority = 300,
.cra_flags = CRYPTO_ALG_TYPE_SKCIPHER | CRYPTO_ALG_ASYNC |
CRYPTO_ALG_KERN_DRIVER_ONLY,
.cra_blocksize = DES3_EDE_BLOCK_SIZE,
.cra_ctxsize = sizeof(struct safexcel_cipher_ctx),
.cra_alignmask = 0,
.cra_init = safexcel_skcipher_cra_init,
.cra_exit = safexcel_skcipher_cra_exit,
.cra_module = THIS_MODULE,
},
},
};
static int safexcel_ecb_des3_ede_encrypt(struct skcipher_request *req)
{
return safexcel_queue_req(&req->base, skcipher_request_ctx(req),
SAFEXCEL_ENCRYPT, CONTEXT_CONTROL_CRYPTO_MODE_ECB,
SAFEXCEL_3DES);
}
static int safexcel_ecb_des3_ede_decrypt(struct skcipher_request *req)
{
return safexcel_queue_req(&req->base, skcipher_request_ctx(req),
SAFEXCEL_DECRYPT, CONTEXT_CONTROL_CRYPTO_MODE_ECB,
SAFEXCEL_3DES);
}
struct safexcel_alg_template safexcel_alg_ecb_des3_ede = {
.type = SAFEXCEL_ALG_TYPE_SKCIPHER,
.engines = EIP97IES | EIP197B | EIP197D,
.alg.skcipher = {
.setkey = safexcel_des3_ede_setkey,
.encrypt = safexcel_ecb_des3_ede_encrypt,
.decrypt = safexcel_ecb_des3_ede_decrypt,
.min_keysize = DES3_EDE_KEY_SIZE,
.max_keysize = DES3_EDE_KEY_SIZE,
.ivsize = DES3_EDE_BLOCK_SIZE,
.base = {
.cra_name = "ecb(des3_ede)",
.cra_driver_name = "safexcel-ecb-des3_ede",
.cra_priority = 300,
.cra_flags = CRYPTO_ALG_TYPE_SKCIPHER | CRYPTO_ALG_ASYNC |
CRYPTO_ALG_KERN_DRIVER_ONLY,
.cra_blocksize = DES3_EDE_BLOCK_SIZE,
.cra_ctxsize = sizeof(struct safexcel_cipher_ctx),
.cra_alignmask = 0,
.cra_init = safexcel_skcipher_cra_init,
.cra_exit = safexcel_skcipher_cra_exit,
.cra_module = THIS_MODULE,
},
},
};
static int safexcel_aead_encrypt(struct aead_request *req)
{
struct safexcel_cipher_req *creq = aead_request_ctx(req);
return safexcel_aes(&req->base, creq, SAFEXCEL_ENCRYPT,
CONTEXT_CONTROL_CRYPTO_MODE_CBC);
return safexcel_queue_req(&req->base, creq, SAFEXCEL_ENCRYPT,
CONTEXT_CONTROL_CRYPTO_MODE_CBC, SAFEXCEL_AES);
}
static int safexcel_aead_decrypt(struct aead_request *req)
{
struct safexcel_cipher_req *creq = aead_request_ctx(req);
return safexcel_aes(&req->base, creq, SAFEXCEL_DECRYPT,
CONTEXT_CONTROL_CRYPTO_MODE_CBC);
return safexcel_queue_req(&req->base, creq, SAFEXCEL_DECRYPT,
CONTEXT_CONTROL_CRYPTO_MODE_CBC, SAFEXCEL_AES);
}
static int safexcel_aead_cra_init(struct crypto_tfm *tfm)
@ -926,13 +1165,14 @@ static int safexcel_aead_sha1_cra_init(struct crypto_tfm *tfm)
struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
safexcel_aead_cra_init(tfm);
ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA1;
ctx->hash_alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA1;
ctx->state_sz = SHA1_DIGEST_SIZE;
return 0;
}
struct safexcel_alg_template safexcel_alg_authenc_hmac_sha1_cbc_aes = {
.type = SAFEXCEL_ALG_TYPE_AEAD,
.engines = EIP97IES | EIP197B | EIP197D,
.alg.aead = {
.setkey = safexcel_aead_aes_setkey,
.encrypt = safexcel_aead_encrypt,
@ -943,7 +1183,7 @@ struct safexcel_alg_template safexcel_alg_authenc_hmac_sha1_cbc_aes = {
.cra_name = "authenc(hmac(sha1),cbc(aes))",
.cra_driver_name = "safexcel-authenc-hmac-sha1-cbc-aes",
.cra_priority = 300,
.cra_flags = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_ASYNC |
.cra_flags = CRYPTO_ALG_ASYNC |
CRYPTO_ALG_KERN_DRIVER_ONLY,
.cra_blocksize = AES_BLOCK_SIZE,
.cra_ctxsize = sizeof(struct safexcel_cipher_ctx),
@ -960,13 +1200,14 @@ static int safexcel_aead_sha256_cra_init(struct crypto_tfm *tfm)
struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
safexcel_aead_cra_init(tfm);
ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA256;
ctx->hash_alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA256;
ctx->state_sz = SHA256_DIGEST_SIZE;
return 0;
}
struct safexcel_alg_template safexcel_alg_authenc_hmac_sha256_cbc_aes = {
.type = SAFEXCEL_ALG_TYPE_AEAD,
.engines = EIP97IES | EIP197B | EIP197D,
.alg.aead = {
.setkey = safexcel_aead_aes_setkey,
.encrypt = safexcel_aead_encrypt,
@ -977,7 +1218,7 @@ struct safexcel_alg_template safexcel_alg_authenc_hmac_sha256_cbc_aes = {
.cra_name = "authenc(hmac(sha256),cbc(aes))",
.cra_driver_name = "safexcel-authenc-hmac-sha256-cbc-aes",
.cra_priority = 300,
.cra_flags = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_ASYNC |
.cra_flags = CRYPTO_ALG_ASYNC |
CRYPTO_ALG_KERN_DRIVER_ONLY,
.cra_blocksize = AES_BLOCK_SIZE,
.cra_ctxsize = sizeof(struct safexcel_cipher_ctx),
@ -994,13 +1235,14 @@ static int safexcel_aead_sha224_cra_init(struct crypto_tfm *tfm)
struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
safexcel_aead_cra_init(tfm);
ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA224;
ctx->hash_alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA224;
ctx->state_sz = SHA256_DIGEST_SIZE;
return 0;
}
struct safexcel_alg_template safexcel_alg_authenc_hmac_sha224_cbc_aes = {
.type = SAFEXCEL_ALG_TYPE_AEAD,
.engines = EIP97IES | EIP197B | EIP197D,
.alg.aead = {
.setkey = safexcel_aead_aes_setkey,
.encrypt = safexcel_aead_encrypt,
@ -1011,7 +1253,7 @@ struct safexcel_alg_template safexcel_alg_authenc_hmac_sha224_cbc_aes = {
.cra_name = "authenc(hmac(sha224),cbc(aes))",
.cra_driver_name = "safexcel-authenc-hmac-sha224-cbc-aes",
.cra_priority = 300,
.cra_flags = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_ASYNC |
.cra_flags = CRYPTO_ALG_ASYNC |
CRYPTO_ALG_KERN_DRIVER_ONLY,
.cra_blocksize = AES_BLOCK_SIZE,
.cra_ctxsize = sizeof(struct safexcel_cipher_ctx),
@ -1022,3 +1264,73 @@ struct safexcel_alg_template safexcel_alg_authenc_hmac_sha224_cbc_aes = {
},
},
};
static int safexcel_aead_sha512_cra_init(struct crypto_tfm *tfm)
{
struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
safexcel_aead_cra_init(tfm);
ctx->hash_alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA512;
ctx->state_sz = SHA512_DIGEST_SIZE;
return 0;
}
struct safexcel_alg_template safexcel_alg_authenc_hmac_sha512_cbc_aes = {
.type = SAFEXCEL_ALG_TYPE_AEAD,
.engines = EIP97IES | EIP197B | EIP197D,
.alg.aead = {
.setkey = safexcel_aead_aes_setkey,
.encrypt = safexcel_aead_encrypt,
.decrypt = safexcel_aead_decrypt,
.ivsize = AES_BLOCK_SIZE,
.maxauthsize = SHA512_DIGEST_SIZE,
.base = {
.cra_name = "authenc(hmac(sha512),cbc(aes))",
.cra_driver_name = "safexcel-authenc-hmac-sha512-cbc-aes",
.cra_priority = 300,
.cra_flags = CRYPTO_ALG_ASYNC |
CRYPTO_ALG_KERN_DRIVER_ONLY,
.cra_blocksize = AES_BLOCK_SIZE,
.cra_ctxsize = sizeof(struct safexcel_cipher_ctx),
.cra_alignmask = 0,
.cra_init = safexcel_aead_sha512_cra_init,
.cra_exit = safexcel_aead_cra_exit,
.cra_module = THIS_MODULE,
},
},
};
static int safexcel_aead_sha384_cra_init(struct crypto_tfm *tfm)
{
struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
safexcel_aead_cra_init(tfm);
ctx->hash_alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA384;
ctx->state_sz = SHA512_DIGEST_SIZE;
return 0;
}
struct safexcel_alg_template safexcel_alg_authenc_hmac_sha384_cbc_aes = {
.type = SAFEXCEL_ALG_TYPE_AEAD,
.engines = EIP97IES | EIP197B | EIP197D,
.alg.aead = {
.setkey = safexcel_aead_aes_setkey,
.encrypt = safexcel_aead_encrypt,
.decrypt = safexcel_aead_decrypt,
.ivsize = AES_BLOCK_SIZE,
.maxauthsize = SHA384_DIGEST_SIZE,
.base = {
.cra_name = "authenc(hmac(sha384),cbc(aes))",
.cra_driver_name = "safexcel-authenc-hmac-sha384-cbc-aes",
.cra_priority = 300,
.cra_flags = CRYPTO_ALG_ASYNC |
CRYPTO_ALG_KERN_DRIVER_ONLY,
.cra_blocksize = AES_BLOCK_SIZE,
.cra_ctxsize = sizeof(struct safexcel_cipher_ctx),
.cra_alignmask = 0,
.cra_init = safexcel_aead_sha384_cra_init,
.cra_exit = safexcel_aead_cra_exit,
.cra_module = THIS_MODULE,
},
},
};

560
drivers/crypto/inside-secure/safexcel_hash.c
View file

@ -1,14 +1,12 @@
// SPDX-License-Identifier: GPL-2.0
/*
* Copyright (C) 2017 Marvell
*
* Antoine Tenart <antoine.tenart@free-electrons.com>
*
* This file is licensed under the terms of the GNU General Public
* License version 2. This program is licensed "as is" without any
* warranty of any kind, whether express or implied.
*/
#include <crypto/hmac.h>
#include <crypto/md5.h>
#include <crypto/sha.h>
#include <linux/device.h>
#include <linux/dma-mapping.h>
@ -22,8 +20,8 @@ struct safexcel_ahash_ctx {
u32 alg;
u32 ipad[SHA256_DIGEST_SIZE / sizeof(u32)];
u32 opad[SHA256_DIGEST_SIZE / sizeof(u32)];
u32 ipad[SHA512_DIGEST_SIZE / sizeof(u32)];
u32 opad[SHA512_DIGEST_SIZE / sizeof(u32)];
};
struct safexcel_ahash_req {
@ -38,18 +36,26 @@ struct safexcel_ahash_req {
u32 digest;
u8 state_sz; /* expected sate size, only set once */
u32 state[SHA256_DIGEST_SIZE / sizeof(u32)] __aligned(sizeof(u32));
u32 state[SHA512_DIGEST_SIZE / sizeof(u32)] __aligned(sizeof(u32));
u64 len;
u64 processed;
u64 len[2];
u64 processed[2];
u8 cache[SHA256_BLOCK_SIZE] __aligned(sizeof(u32));
u8 cache[SHA512_BLOCK_SIZE] __aligned(sizeof(u32));
dma_addr_t cache_dma;
unsigned int cache_sz;
u8 cache_next[SHA256_BLOCK_SIZE] __aligned(sizeof(u32));
u8 cache_next[SHA512_BLOCK_SIZE] __aligned(sizeof(u32));
};
static inline u64 safexcel_queued_len(struct safexcel_ahash_req *req)
{
if (req->len[1] > req->processed[1])
return 0xffffffff - (req->len[0] - req->processed[0]);
return req->len[0] - req->processed[0];
}
static void safexcel_hash_token(struct safexcel_command_desc *cdesc,
u32 input_length, u32 result_length)
{
@ -72,9 +78,9 @@ static void safexcel_hash_token(struct safexcel_command_desc *cdesc,
static void safexcel_context_control(struct safexcel_ahash_ctx *ctx,
struct safexcel_ahash_req *req,
struct safexcel_command_desc *cdesc,
unsigned int digestsize,
unsigned int blocksize)
unsigned int digestsize)
{
struct safexcel_crypto_priv *priv = ctx->priv;
int i;
cdesc->control_data.control0 |= CONTEXT_CONTROL_TYPE_HASH_OUT;
@ -82,12 +88,17 @@ static void safexcel_context_control(struct safexcel_ahash_ctx *ctx,
cdesc->control_data.control0 |= req->digest;
if (req->digest == CONTEXT_CONTROL_DIGEST_PRECOMPUTED) {
if (req->processed) {
if (ctx->alg == CONTEXT_CONTROL_CRYPTO_ALG_SHA1)
if (req->processed[0] || req->processed[1]) {
if (ctx->alg == CONTEXT_CONTROL_CRYPTO_ALG_MD5)
cdesc->control_data.control0 |= CONTEXT_CONTROL_SIZE(5);
else if (ctx->alg == CONTEXT_CONTROL_CRYPTO_ALG_SHA1)
cdesc->control_data.control0 |= CONTEXT_CONTROL_SIZE(6);
else if (ctx->alg == CONTEXT_CONTROL_CRYPTO_ALG_SHA224 ||
ctx->alg == CONTEXT_CONTROL_CRYPTO_ALG_SHA256)
cdesc->control_data.control0 |= CONTEXT_CONTROL_SIZE(9);
else if (ctx->alg == CONTEXT_CONTROL_CRYPTO_ALG_SHA384 ||
ctx->alg == CONTEXT_CONTROL_CRYPTO_ALG_SHA512)
cdesc->control_data.control0 |= CONTEXT_CONTROL_SIZE(17);
cdesc->control_data.control1 |= CONTEXT_CONTROL_DIGEST_CNT;
} else {
@ -102,12 +113,28 @@ static void safexcel_context_control(struct safexcel_ahash_ctx *ctx,
* fields. Do this now as we need it to setup the first command
* descriptor.
*/
if (req->processed) {
if (req->processed[0] || req->processed[1]) {
for (i = 0; i < digestsize / sizeof(u32); i++)
ctx->base.ctxr->data[i] = cpu_to_le32(req->state[i]);
if (req->finish)
ctx->base.ctxr->data[i] = cpu_to_le32(req->processed / blocksize);
if (req->finish) {
u64 count = req->processed[0] / EIP197_COUNTER_BLOCK_SIZE;
count += ((0xffffffff / EIP197_COUNTER_BLOCK_SIZE) *
req->processed[1]);
/* This is a haredware limitation, as the
* counter must fit into an u32. This represents
* a farily big amount of input data, so we
* shouldn't see this.
*/
if (unlikely(count & 0xffff0000)) {
dev_warn(priv->dev,
"Input data is too big\n");
return;
}
ctx->base.ctxr->data[i] = cpu_to_le32(count);
}
}
} else if (req->digest == CONTEXT_CONTROL_DIGEST_HMAC) {
cdesc->control_data.control0 |= CONTEXT_CONTROL_SIZE(2 * req->state_sz / sizeof(u32));
@ -126,11 +153,10 @@ static int safexcel_handle_req_result(struct safexcel_crypto_priv *priv, int rin
struct ahash_request *areq = ahash_request_cast(async);
struct crypto_ahash *ahash = crypto_ahash_reqtfm(areq);
struct safexcel_ahash_req *sreq = ahash_request_ctx(areq);
int cache_len;
u64 cache_len;
*ret = 0;
spin_lock_bh(&priv->ring[ring].egress_lock);
rdesc = safexcel_ring_next_rptr(priv, &priv->ring[ring].rdr);
if (IS_ERR(rdesc)) {
dev_err(priv->dev,
@ -141,7 +167,6 @@ static int safexcel_handle_req_result(struct safexcel_crypto_priv *priv, int rin
}
safexcel_complete(priv, ring);
spin_unlock_bh(&priv->ring[ring].egress_lock);
if (sreq->nents) {
dma_unmap_sg(priv->dev, areq->src, sreq->nents, DMA_TO_DEVICE);
@ -164,7 +189,7 @@ static int safexcel_handle_req_result(struct safexcel_crypto_priv *priv, int rin
memcpy(areq->result, sreq->state,
crypto_ahash_digestsize(ahash));
cache_len = sreq->len - sreq->processed;
cache_len = safexcel_queued_len(sreq);
if (cache_len)
memcpy(sreq->cache, sreq->cache_next, cache_len);
@ -174,7 +199,6 @@ static int safexcel_handle_req_result(struct safexcel_crypto_priv *priv, int rin
}
static int safexcel_ahash_send_req(struct crypto_async_request *async, int ring,
struct safexcel_request *request,
int *commands, int *results)
{
struct ahash_request *areq = ahash_request_cast(async);
@ -185,9 +209,10 @@ static int safexcel_ahash_send_req(struct crypto_async_request *async, int ring,
struct safexcel_command_desc *cdesc, *first_cdesc = NULL;
struct safexcel_result_desc *rdesc;
struct scatterlist *sg;
int i, queued, len, cache_len, extra, n_cdesc = 0, ret = 0;
int i, extra, n_cdesc = 0, ret = 0;
u64 queued, len, cache_len;
queued = len = req->len - req->processed;
queued = len = safexcel_queued_len(req);
if (queued <= crypto_ahash_blocksize(ahash))
cache_len = queued;
else
@ -220,16 +245,12 @@ static int safexcel_ahash_send_req(struct crypto_async_request *async, int ring,
}
}
spin_lock_bh(&priv->ring[ring].egress_lock);
/* Add a command descriptor for the cached data, if any */
if (cache_len) {
req->cache_dma = dma_map_single(priv->dev, req->cache,
cache_len, DMA_TO_DEVICE);
if (dma_mapping_error(priv->dev, req->cache_dma)) {
spin_unlock_bh(&priv->ring[ring].egress_lock);
if (dma_mapping_error(priv->dev, req->cache_dma))
return -EINVAL;
}
req->cache_sz = cache_len;
first_cdesc = safexcel_add_cdesc(priv, ring, 1,
@ -260,7 +281,7 @@ static int safexcel_ahash_send_req(struct crypto_async_request *async, int ring,
int sglen = sg_dma_len(sg);
/* Do not overflow the request */
if (queued - sglen < 0)
if (queued < sglen)
sglen = queued;
cdesc = safexcel_add_cdesc(priv, ring, !n_cdesc,
@ -282,8 +303,7 @@ static int safexcel_ahash_send_req(struct crypto_async_request *async, int ring,
send_command:
/* Setup the context options */
safexcel_context_control(ctx, req, first_cdesc, req->state_sz,
crypto_ahash_blocksize(ahash));
safexcel_context_control(ctx, req, first_cdesc, req->state_sz);
/* Add the token */
safexcel_hash_token(first_cdesc, len, req->state_sz);
@ -303,10 +323,11 @@ send_command:
goto unmap_result;
}
spin_unlock_bh(&priv->ring[ring].egress_lock);
safexcel_rdr_req_set(priv, ring, rdesc, &areq->base);
req->processed += len;
request->req = &areq->base;
req->processed[0] += len;
if (req->processed[0] < len)
req->processed[1]++;
*commands = n_cdesc;
*results = 1;
@ -327,7 +348,6 @@ unmap_cache:
req->cache_sz = 0;
}
spin_unlock_bh(&priv->ring[ring].egress_lock);
return ret;
}
@ -335,16 +355,18 @@ static inline bool safexcel_ahash_needs_inv_get(struct ahash_request *areq)
{
struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));
struct safexcel_ahash_req *req = ahash_request_ctx(areq);
struct crypto_ahash *ahash = crypto_ahash_reqtfm(areq);
unsigned int state_w_sz = req->state_sz / sizeof(u32);
u64 processed;
int i;
processed = req->processed[0] / EIP197_COUNTER_BLOCK_SIZE;
processed += (0xffffffff / EIP197_COUNTER_BLOCK_SIZE) * req->processed[1];
for (i = 0; i < state_w_sz; i++)
if (ctx->base.ctxr->data[i] != cpu_to_le32(req->state[i]))
return true;
if (ctx->base.ctxr->data[state_w_sz] !=
cpu_to_le32(req->processed / crypto_ahash_blocksize(ahash)))
if (ctx->base.ctxr->data[state_w_sz] != cpu_to_le32(processed))
return true;
return false;
@ -363,21 +385,16 @@ static int safexcel_handle_inv_result(struct safexcel_crypto_priv *priv,
*ret = 0;
spin_lock_bh(&priv->ring[ring].egress_lock);
rdesc = safexcel_ring_next_rptr(priv, &priv->ring[ring].rdr);
if (IS_ERR(rdesc)) {
dev_err(priv->dev,
"hash: invalidate: could not retrieve the result descriptor\n");
*ret = PTR_ERR(rdesc);
} else if (rdesc->result_data.error_code) {
dev_err(priv->dev,
"hash: invalidate: result descriptor error (%d)\n",
rdesc->result_data.error_code);
*ret = -EINVAL;
} else {
*ret = safexcel_rdesc_check_errors(priv, rdesc);
}
safexcel_complete(priv, ring);
spin_unlock_bh(&priv->ring[ring].egress_lock);
if (ctx->base.exit_inv) {
dma_pool_free(priv->context_pool, ctx->base.ctxr,
@ -413,7 +430,7 @@ static int safexcel_handle_result(struct safexcel_crypto_priv *priv, int ring,
struct safexcel_ahash_req *req = ahash_request_ctx(areq);
int err;
BUG_ON(priv->version == EIP97 && req->needs_inv);
BUG_ON(!(priv->flags & EIP197_TRC_CACHE) && req->needs_inv);
if (req->needs_inv) {
req->needs_inv = false;
@ -428,15 +445,14 @@ static int safexcel_handle_result(struct safexcel_crypto_priv *priv, int ring,
}
static int safexcel_ahash_send_inv(struct crypto_async_request *async,
int ring, struct safexcel_request *request,
int *commands, int *results)
int ring, int *commands, int *results)
{
struct ahash_request *areq = ahash_request_cast(async);
struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));
int ret;
ret = safexcel_invalidate_cache(async, ctx->priv,
ctx->base.ctxr_dma, ring, request);
ctx->base.ctxr_dma, ring);
if (unlikely(ret))
return ret;
@ -447,19 +463,17 @@ static int safexcel_ahash_send_inv(struct crypto_async_request *async,
}
static int safexcel_ahash_send(struct crypto_async_request *async,
int ring, struct safexcel_request *request,
int *commands, int *results)
int ring, int *commands, int *results)
{
struct ahash_request *areq = ahash_request_cast(async);
struct safexcel_ahash_req *req = ahash_request_ctx(areq);
int ret;
if (req->needs_inv)
ret = safexcel_ahash_send_inv(async, ring, request,
commands, results);
ret = safexcel_ahash_send_inv(async, ring, commands, results);
else
ret = safexcel_ahash_send_req(async, ring, request,
commands, results);
ret = safexcel_ahash_send_req(async, ring, commands, results);
return ret;
}
@ -509,17 +523,17 @@ static int safexcel_ahash_cache(struct ahash_request *areq)
{
struct safexcel_ahash_req *req = ahash_request_ctx(areq);
struct crypto_ahash *ahash = crypto_ahash_reqtfm(areq);
int queued, cache_len;
u64 queued, cache_len;
/* cache_len: everyting accepted by the driver but not sent yet,
* tot sz handled by update() - last req sz - tot sz handled by send()
*/
cache_len = req->len - areq->nbytes - req->processed;
/* queued: everything accepted by the driver which will be handled by
* the next send() calls.
* tot sz handled by update() - tot sz handled by send()
*/
queued = req->len - req->processed;
queued = safexcel_queued_len(req);
/* cache_len: everything accepted by the driver but not sent yet,
* tot sz handled by update() - last req sz - tot sz handled by send()
*/
cache_len = queued - areq->nbytes;
/*
* In case there isn't enough bytes to proceed (less than a
@ -546,8 +560,8 @@ static int safexcel_ahash_enqueue(struct ahash_request *areq)
req->needs_inv = false;
if (ctx->base.ctxr) {
if (priv->version == EIP197 &&
!ctx->base.needs_inv && req->processed &&
if (priv->flags & EIP197_TRC_CACHE && !ctx->base.needs_inv &&
(req->processed[0] || req->processed[1]) &&
req->digest == CONTEXT_CONTROL_DIGEST_PRECOMPUTED)
/* We're still setting needs_inv here, even though it is
* cleared right away, because the needs_inv flag can be
@ -590,7 +604,9 @@ static int safexcel_ahash_update(struct ahash_request *areq)
if (!areq->nbytes)
return 0;
req->len += areq->nbytes;
req->len[0] += areq->nbytes;
if (req->len[0] < areq->nbytes)
req->len[1]++;
safexcel_ahash_cache(areq);
@ -605,7 +621,7 @@ static int safexcel_ahash_update(struct ahash_request *areq)
return safexcel_ahash_enqueue(areq);
if (!req->last_req &&
req->len - req->processed > crypto_ahash_blocksize(ahash))
safexcel_queued_len(req) > crypto_ahash_blocksize(ahash))
return safexcel_ahash_enqueue(areq);
return 0;
@ -620,8 +636,11 @@ static int safexcel_ahash_final(struct ahash_request *areq)
req->finish = true;
/* If we have an overall 0 length request */
if (!(req->len + areq->nbytes)) {
if (ctx->alg == CONTEXT_CONTROL_CRYPTO_ALG_SHA1)
if (!req->len[0] && !req->len[1] && !areq->nbytes) {
if (ctx->alg == CONTEXT_CONTROL_CRYPTO_ALG_MD5)
memcpy(areq->result, md5_zero_message_hash,
MD5_DIGEST_SIZE);
else if (ctx->alg == CONTEXT_CONTROL_CRYPTO_ALG_SHA1)
memcpy(areq->result, sha1_zero_message_hash,
SHA1_DIGEST_SIZE);
else if (ctx->alg == CONTEXT_CONTROL_CRYPTO_ALG_SHA224)
@ -630,6 +649,12 @@ static int safexcel_ahash_final(struct ahash_request *areq)
else if (ctx->alg == CONTEXT_CONTROL_CRYPTO_ALG_SHA256)
memcpy(areq->result, sha256_zero_message_hash,
SHA256_DIGEST_SIZE);
else if (ctx->alg == CONTEXT_CONTROL_CRYPTO_ALG_SHA384)
memcpy(areq->result, sha384_zero_message_hash,
SHA384_DIGEST_SIZE);
else if (ctx->alg == CONTEXT_CONTROL_CRYPTO_ALG_SHA512)
memcpy(areq->result, sha512_zero_message_hash,
SHA512_DIGEST_SIZE);
return 0;
}
@ -654,8 +679,10 @@ static int safexcel_ahash_export(struct ahash_request *areq, void *out)
struct safexcel_ahash_req *req = ahash_request_ctx(areq);
struct safexcel_ahash_export_state *export = out;
export->len = req->len;
export->processed = req->processed;
export->len[0] = req->len[0];
export->len[1] = req->len[1];
export->processed[0] = req->processed[0];
export->processed[1] = req->processed[1];
export->digest = req->digest;
@ -676,8 +703,10 @@ static int safexcel_ahash_import(struct ahash_request *areq, const void *in)
if (ret)
return ret;
req->len = export->len;
req->processed = export->processed;
req->len[0] = export->len[0];
req->len[1] = export->len[1];
req->processed[0] = export->processed[0];
req->processed[1] = export->processed[1];
req->digest = export->digest;
@ -743,7 +772,7 @@ static void safexcel_ahash_cra_exit(struct crypto_tfm *tfm)
if (!ctx->base.ctxr)
return;
if (priv->version == EIP197) {
if (priv->flags & EIP197_TRC_CACHE) {
ret = safexcel_ahash_exit_inv(tfm);
if (ret)
dev_warn(priv->dev, "hash: invalidation error %d\n", ret);
@ -755,6 +784,7 @@ static void safexcel_ahash_cra_exit(struct crypto_tfm *tfm)
struct safexcel_alg_template safexcel_alg_sha1 = {
.type = SAFEXCEL_ALG_TYPE_AHASH,
.engines = EIP97IES | EIP197B | EIP197D,
.alg.ahash = {
.init = safexcel_sha1_init,
.update = safexcel_ahash_update,
@ -908,8 +938,7 @@ int safexcel_hmac_setkey(const char *alg, const u8 *key, unsigned int keylen,
u8 *ipad, *opad;
int ret;
tfm = crypto_alloc_ahash(alg, CRYPTO_ALG_TYPE_AHASH,
CRYPTO_ALG_TYPE_AHASH_MASK);
tfm = crypto_alloc_ahash(alg, 0, 0);
if (IS_ERR(tfm))
return PTR_ERR(tfm);
@ -963,7 +992,7 @@ static int safexcel_hmac_alg_setkey(struct crypto_ahash *tfm, const u8 *key,
if (ret)
return ret;
if (priv->version == EIP197 && ctx->base.ctxr) {
if (priv->flags & EIP197_TRC_CACHE && ctx->base.ctxr) {
for (i = 0; i < state_sz / sizeof(u32); i++) {
if (ctx->ipad[i] != le32_to_cpu(istate.state[i]) ||
ctx->opad[i] != le32_to_cpu(ostate.state[i])) {
@ -988,6 +1017,7 @@ static int safexcel_hmac_sha1_setkey(struct crypto_ahash *tfm, const u8 *key,
struct safexcel_alg_template safexcel_alg_hmac_sha1 = {
.type = SAFEXCEL_ALG_TYPE_AHASH,
.engines = EIP97IES | EIP197B | EIP197D,
.alg.ahash = {
.init = safexcel_hmac_sha1_init,
.update = safexcel_ahash_update,
@ -1051,6 +1081,7 @@ static int safexcel_sha256_digest(struct ahash_request *areq)
struct safexcel_alg_template safexcel_alg_sha256 = {
.type = SAFEXCEL_ALG_TYPE_AHASH,
.engines = EIP97IES | EIP197B | EIP197D,
.alg.ahash = {
.init = safexcel_sha256_init,
.update = safexcel_ahash_update,
@ -1113,6 +1144,7 @@ static int safexcel_sha224_digest(struct ahash_request *areq)
struct safexcel_alg_template safexcel_alg_sha224 = {
.type = SAFEXCEL_ALG_TYPE_AHASH,
.engines = EIP97IES | EIP197B | EIP197D,
.alg.ahash = {
.init = safexcel_sha224_init,
.update = safexcel_ahash_update,
@ -1168,6 +1200,7 @@ static int safexcel_hmac_sha224_digest(struct ahash_request *areq)
struct safexcel_alg_template safexcel_alg_hmac_sha224 = {
.type = SAFEXCEL_ALG_TYPE_AHASH,
.engines = EIP97IES | EIP197B | EIP197D,
.alg.ahash = {
.init = safexcel_hmac_sha224_init,
.update = safexcel_ahash_update,
@ -1224,6 +1257,7 @@ static int safexcel_hmac_sha256_digest(struct ahash_request *areq)
struct safexcel_alg_template safexcel_alg_hmac_sha256 = {
.type = SAFEXCEL_ALG_TYPE_AHASH,
.engines = EIP97IES | EIP197B | EIP197D,
.alg.ahash = {
.init = safexcel_hmac_sha256_init,
.update = safexcel_ahash_update,
@ -1251,3 +1285,375 @@ struct safexcel_alg_template safexcel_alg_hmac_sha256 = {
},
},
};
static int safexcel_sha512_init(struct ahash_request *areq)
{
struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));
struct safexcel_ahash_req *req = ahash_request_ctx(areq);
memset(req, 0, sizeof(*req));
req->state[0] = lower_32_bits(SHA512_H0);
req->state[1] = upper_32_bits(SHA512_H0);
req->state[2] = lower_32_bits(SHA512_H1);
req->state[3] = upper_32_bits(SHA512_H1);
req->state[4] = lower_32_bits(SHA512_H2);
req->state[5] = upper_32_bits(SHA512_H2);
req->state[6] = lower_32_bits(SHA512_H3);
req->state[7] = upper_32_bits(SHA512_H3);
req->state[8] = lower_32_bits(SHA512_H4);
req->state[9] = upper_32_bits(SHA512_H4);
req->state[10] = lower_32_bits(SHA512_H5);
req->state[11] = upper_32_bits(SHA512_H5);
req->state[12] = lower_32_bits(SHA512_H6);
req->state[13] = upper_32_bits(SHA512_H6);
req->state[14] = lower_32_bits(SHA512_H7);
req->state[15] = upper_32_bits(SHA512_H7);
ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA512;
req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED;
req->state_sz = SHA512_DIGEST_SIZE;
return 0;
}
static int safexcel_sha512_digest(struct ahash_request *areq)
{
int ret = safexcel_sha512_init(areq);
if (ret)
return ret;
return safexcel_ahash_finup(areq);
}
struct safexcel_alg_template safexcel_alg_sha512 = {
.type = SAFEXCEL_ALG_TYPE_AHASH,
.engines = EIP97IES | EIP197B | EIP197D,
.alg.ahash = {
.init = safexcel_sha512_init,
.update = safexcel_ahash_update,
.final = safexcel_ahash_final,
.finup = safexcel_ahash_finup,
.digest = safexcel_sha512_digest,
.export = safexcel_ahash_export,
.import = safexcel_ahash_import,
.halg = {
.digestsize = SHA512_DIGEST_SIZE,
.statesize = sizeof(struct safexcel_ahash_export_state),
.base = {
.cra_name = "sha512",
.cra_driver_name = "safexcel-sha512",
.cra_priority = 300,
.cra_flags = CRYPTO_ALG_ASYNC |
CRYPTO_ALG_KERN_DRIVER_ONLY,
.cra_blocksize = SHA512_BLOCK_SIZE,
.cra_ctxsize = sizeof(struct safexcel_ahash_ctx),
.cra_init = safexcel_ahash_cra_init,
.cra_exit = safexcel_ahash_cra_exit,
.cra_module = THIS_MODULE,
},
},
},
};
static int safexcel_sha384_init(struct ahash_request *areq)
{
struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));
struct safexcel_ahash_req *req = ahash_request_ctx(areq);
memset(req, 0, sizeof(*req));
req->state[0] = lower_32_bits(SHA384_H0);
req->state[1] = upper_32_bits(SHA384_H0);
req->state[2] = lower_32_bits(SHA384_H1);
req->state[3] = upper_32_bits(SHA384_H1);
req->state[4] = lower_32_bits(SHA384_H2);
req->state[5] = upper_32_bits(SHA384_H2);
req->state[6] = lower_32_bits(SHA384_H3);
req->state[7] = upper_32_bits(SHA384_H3);
req->state[8] = lower_32_bits(SHA384_H4);
req->state[9] = upper_32_bits(SHA384_H4);
req->state[10] = lower_32_bits(SHA384_H5);
req->state[11] = upper_32_bits(SHA384_H5);
req->state[12] = lower_32_bits(SHA384_H6);
req->state[13] = upper_32_bits(SHA384_H6);
req->state[14] = lower_32_bits(SHA384_H7);
req->state[15] = upper_32_bits(SHA384_H7);
ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA384;
req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED;
req->state_sz = SHA512_DIGEST_SIZE;
return 0;
}
static int safexcel_sha384_digest(struct ahash_request *areq)
{
int ret = safexcel_sha384_init(areq);
if (ret)
return ret;
return safexcel_ahash_finup(areq);
}
struct safexcel_alg_template safexcel_alg_sha384 = {
.type = SAFEXCEL_ALG_TYPE_AHASH,
.engines = EIP97IES | EIP197B | EIP197D,
.alg.ahash = {
.init = safexcel_sha384_init,
.update = safexcel_ahash_update,
.final = safexcel_ahash_final,
.finup = safexcel_ahash_finup,
.digest = safexcel_sha384_digest,
.export = safexcel_ahash_export,
.import = safexcel_ahash_import,
.halg = {
.digestsize = SHA384_DIGEST_SIZE,
.statesize = sizeof(struct safexcel_ahash_export_state),
.base = {
.cra_name = "sha384",
.cra_driver_name = "safexcel-sha384",
.cra_priority = 300,
.cra_flags = CRYPTO_ALG_ASYNC |
CRYPTO_ALG_KERN_DRIVER_ONLY,
.cra_blocksize = SHA384_BLOCK_SIZE,
.cra_ctxsize = sizeof(struct safexcel_ahash_ctx),
.cra_init = safexcel_ahash_cra_init,
.cra_exit = safexcel_ahash_cra_exit,
.cra_module = THIS_MODULE,
},
},
},
};
static int safexcel_hmac_sha512_setkey(struct crypto_ahash *tfm, const u8 *key,
unsigned int keylen)
{
return safexcel_hmac_alg_setkey(tfm, key, keylen, "safexcel-sha512",
SHA512_DIGEST_SIZE);
}
static int safexcel_hmac_sha512_init(struct ahash_request *areq)
{
struct safexcel_ahash_req *req = ahash_request_ctx(areq);
safexcel_sha512_init(areq);
req->digest = CONTEXT_CONTROL_DIGEST_HMAC;
return 0;
}
static int safexcel_hmac_sha512_digest(struct ahash_request *areq)
{
int ret = safexcel_hmac_sha512_init(areq);
if (ret)
return ret;
return safexcel_ahash_finup(areq);
}
struct safexcel_alg_template safexcel_alg_hmac_sha512 = {
.type = SAFEXCEL_ALG_TYPE_AHASH,
.engines = EIP97IES | EIP197B | EIP197D,
.alg.ahash = {
.init = safexcel_hmac_sha512_init,
.update = safexcel_ahash_update,
.final = safexcel_ahash_final,
.finup = safexcel_ahash_finup,
.digest = safexcel_hmac_sha512_digest,
.setkey = safexcel_hmac_sha512_setkey,
.export = safexcel_ahash_export,
.import = safexcel_ahash_import,
.halg = {
.digestsize = SHA512_DIGEST_SIZE,
.statesize = sizeof(struct safexcel_ahash_export_state),
.base = {
.cra_name = "hmac(sha512)",
.cra_driver_name = "safexcel-hmac-sha512",
.cra_priority = 300,
.cra_flags = CRYPTO_ALG_ASYNC |
CRYPTO_ALG_KERN_DRIVER_ONLY,
.cra_blocksize = SHA512_BLOCK_SIZE,
.cra_ctxsize = sizeof(struct safexcel_ahash_ctx),
.cra_init = safexcel_ahash_cra_init,
.cra_exit = safexcel_ahash_cra_exit,
.cra_module = THIS_MODULE,
},
},
},
};
static int safexcel_hmac_sha384_setkey(struct crypto_ahash *tfm, const u8 *key,
unsigned int keylen)
{
return safexcel_hmac_alg_setkey(tfm, key, keylen, "safexcel-sha384",
SHA512_DIGEST_SIZE);
}
static int safexcel_hmac_sha384_init(struct ahash_request *areq)
{
struct safexcel_ahash_req *req = ahash_request_ctx(areq);
safexcel_sha384_init(areq);
req->digest = CONTEXT_CONTROL_DIGEST_HMAC;
return 0;
}
static int safexcel_hmac_sha384_digest(struct ahash_request *areq)
{
int ret = safexcel_hmac_sha384_init(areq);
if (ret)
return ret;
return safexcel_ahash_finup(areq);
}
struct safexcel_alg_template safexcel_alg_hmac_sha384 = {
.type = SAFEXCEL_ALG_TYPE_AHASH,
.engines = EIP97IES | EIP197B | EIP197D,
.alg.ahash = {
.init = safexcel_hmac_sha384_init,
.update = safexcel_ahash_update,
.final = safexcel_ahash_final,
.finup = safexcel_ahash_finup,
.digest = safexcel_hmac_sha384_digest,
.setkey = safexcel_hmac_sha384_setkey,
.export = safexcel_ahash_export,
.import = safexcel_ahash_import,
.halg = {
.digestsize = SHA384_DIGEST_SIZE,
.statesize = sizeof(struct safexcel_ahash_export_state),
.base = {
.cra_name = "hmac(sha384)",
.cra_driver_name = "safexcel-hmac-sha384",
.cra_priority = 300,
.cra_flags = CRYPTO_ALG_ASYNC |
CRYPTO_ALG_KERN_DRIVER_ONLY,
.cra_blocksize = SHA384_BLOCK_SIZE,
.cra_ctxsize = sizeof(struct safexcel_ahash_ctx),
.cra_init = safexcel_ahash_cra_init,
.cra_exit = safexcel_ahash_cra_exit,
.cra_module = THIS_MODULE,
},
},
},
};
static int safexcel_md5_init(struct ahash_request *areq)
{
struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));
struct safexcel_ahash_req *req = ahash_request_ctx(areq);
memset(req, 0, sizeof(*req));
req->state[0] = MD5_H0;
req->state[1] = MD5_H1;
req->state[2] = MD5_H2;
req->state[3] = MD5_H3;
ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_MD5;
req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED;
req->state_sz = MD5_DIGEST_SIZE;
return 0;
}
static int safexcel_md5_digest(struct ahash_request *areq)
{
int ret = safexcel_md5_init(areq);
if (ret)
return ret;
return safexcel_ahash_finup(areq);
}
struct safexcel_alg_template safexcel_alg_md5 = {
.type = SAFEXCEL_ALG_TYPE_AHASH,
.engines = EIP97IES | EIP197B | EIP197D,
.alg.ahash = {
.init = safexcel_md5_init,
.update = safexcel_ahash_update,
.final = safexcel_ahash_final,
.finup = safexcel_ahash_finup,
.digest = safexcel_md5_digest,
.export = safexcel_ahash_export,
.import = safexcel_ahash_import,
.halg = {
.digestsize = MD5_DIGEST_SIZE,
.statesize = sizeof(struct safexcel_ahash_export_state),
.base = {
.cra_name = "md5",
.cra_driver_name = "safexcel-md5",
.cra_priority = 300,
.cra_flags = CRYPTO_ALG_ASYNC |
CRYPTO_ALG_KERN_DRIVER_ONLY,
.cra_blocksize = MD5_HMAC_BLOCK_SIZE,
.cra_ctxsize = sizeof(struct safexcel_ahash_ctx),
.cra_init = safexcel_ahash_cra_init,
.cra_exit = safexcel_ahash_cra_exit,
.cra_module = THIS_MODULE,
},
},
},
};
static int safexcel_hmac_md5_init(struct ahash_request *areq)
{
struct safexcel_ahash_req *req = ahash_request_ctx(areq);
safexcel_md5_init(areq);
req->digest = CONTEXT_CONTROL_DIGEST_HMAC;
return 0;
}
static int safexcel_hmac_md5_setkey(struct crypto_ahash *tfm, const u8 *key,
unsigned int keylen)
{
return safexcel_hmac_alg_setkey(tfm, key, keylen, "safexcel-md5",
MD5_DIGEST_SIZE);
}
static int safexcel_hmac_md5_digest(struct ahash_request *areq)
{
int ret = safexcel_hmac_md5_init(areq);
if (ret)
return ret;
return safexcel_ahash_finup(areq);
}
struct safexcel_alg_template safexcel_alg_hmac_md5 = {
.type = SAFEXCEL_ALG_TYPE_AHASH,
.engines = EIP97IES | EIP197B | EIP197D,
.alg.ahash = {
.init = safexcel_hmac_md5_init,
.update = safexcel_ahash_update,
.final = safexcel_ahash_final,
.finup = safexcel_ahash_finup,
.digest = safexcel_hmac_md5_digest,
.setkey = safexcel_hmac_md5_setkey,
.export = safexcel_ahash_export,
.import = safexcel_ahash_import,
.halg = {
.digestsize = MD5_DIGEST_SIZE,
.statesize = sizeof(struct safexcel_ahash_export_state),
.base = {
.cra_name = "hmac(md5)",
.cra_driver_name = "safexcel-hmac-md5",
.cra_priority = 300,
.cra_flags = CRYPTO_ALG_ASYNC |
CRYPTO_ALG_KERN_DRIVER_ONLY,
.cra_blocksize = MD5_HMAC_BLOCK_SIZE,
.cra_ctxsize = sizeof(struct safexcel_ahash_ctx),
.cra_init = safexcel_ahash_cra_init,
.cra_exit = safexcel_ahash_cra_exit,
.cra_module = THIS_MODULE,
},
},
},
};

65
drivers/crypto/inside-secure/safexcel_ring.c
View file

@ -1,11 +1,8 @@
// SPDX-License-Identifier: GPL-2.0
/*
* Copyright (C) 2017 Marvell
*
* Antoine Tenart <antoine.tenart@free-electrons.com>
*
* This file is licensed under the terms of the GNU General Public
* License version 2. This program is licensed "as is" without any
* warranty of any kind, whether express or implied.
*/
#include <linux/dma-mapping.h>
@ -14,8 +11,8 @@
#include "safexcel.h"
int safexcel_init_ring_descriptors(struct safexcel_crypto_priv *priv,
struct safexcel_ring *cdr,
struct safexcel_ring *rdr)
struct safexcel_desc_ring *cdr,
struct safexcel_desc_ring *rdr)
{
cdr->offset = sizeof(u32) * priv->config.cd_offset;
cdr->base = dmam_alloc_coherent(priv->dev,
@ -24,7 +21,7 @@ int safexcel_init_ring_descriptors(struct safexcel_crypto_priv *priv,
if (!cdr->base)
return -ENOMEM;
cdr->write = cdr->base;
cdr->base_end = cdr->base + cdr->offset * EIP197_DEFAULT_RING_SIZE;
cdr->base_end = cdr->base + cdr->offset * (EIP197_DEFAULT_RING_SIZE - 1);
cdr->read = cdr->base;
rdr->offset = sizeof(u32) * priv->config.rd_offset;
@ -34,7 +31,7 @@ int safexcel_init_ring_descriptors(struct safexcel_crypto_priv *priv,
if (!rdr->base)
return -ENOMEM;
rdr->write = rdr->base;
rdr->base_end = rdr->base + rdr->offset * EIP197_DEFAULT_RING_SIZE;
rdr->base_end = rdr->base + rdr->offset * (EIP197_DEFAULT_RING_SIZE - 1);
rdr->read = rdr->base;
return 0;
@ -46,49 +43,73 @@ inline int safexcel_select_ring(struct safexcel_crypto_priv *priv)
}
static void *safexcel_ring_next_wptr(struct safexcel_crypto_priv *priv,
struct safexcel_ring *ring)
struct safexcel_desc_ring *ring)
{
void *ptr = ring->write;
if (ring->nr == EIP197_DEFAULT_RING_SIZE - 1)
if ((ring->write == ring->read - ring->offset) ||
(ring->read == ring->base && ring->write == ring->base_end))
return ERR_PTR(-ENOMEM);
ring->write += ring->offset;
if (ring->write == ring->base_end)
ring->write = ring->base;
else
ring->write += ring->offset;
ring->nr++;
return ptr;
}
void *safexcel_ring_next_rptr(struct safexcel_crypto_priv *priv,
struct safexcel_ring *ring)
struct safexcel_desc_ring *ring)
{
void *ptr = ring->read;
if (!ring->nr)
if (ring->write == ring->read)
return ERR_PTR(-ENOENT);
ring->read += ring->offset;
if (ring->read == ring->base_end)
ring->read = ring->base;
else
ring->read += ring->offset;
ring->nr--;
return ptr;
}
void safexcel_ring_rollback_wptr(struct safexcel_crypto_priv *priv,
struct safexcel_ring *ring)
inline void *safexcel_ring_curr_rptr(struct safexcel_crypto_priv *priv,
int ring)
{
if (!ring->nr)
struct safexcel_desc_ring *rdr = &priv->ring[ring].rdr;
return rdr->read;
}
inline int safexcel_ring_first_rdr_index(struct safexcel_crypto_priv *priv,
int ring)
{
struct safexcel_desc_ring *rdr = &priv->ring[ring].rdr;
return (rdr->read - rdr->base) / rdr->offset;
}
inline int safexcel_ring_rdr_rdesc_index(struct safexcel_crypto_priv *priv,
int ring,
struct safexcel_result_desc *rdesc)
{
struct safexcel_desc_ring *rdr = &priv->ring[ring].rdr;
return ((void *)rdesc - rdr->base) / rdr->offset;
}
void safexcel_ring_rollback_wptr(struct safexcel_crypto_priv *priv,
struct safexcel_desc_ring *ring)
{
if (ring->write == ring->read)
return;
if (ring->write == ring->base)
ring->write = ring->base_end - ring->offset;
ring->write = ring->base_end;
else
ring->write -= ring->offset;
ring->nr--;
}
struct safexcel_command_desc *safexcel_add_cdesc(struct safexcel_crypto_priv *priv,

3
drivers/crypto/marvell/hash.c
View file

@ -1183,8 +1183,7 @@ static int mv_cesa_ahmac_setkey(const char *hash_alg_name,
u8 *opad;
int ret;
tfm = crypto_alloc_ahash(hash_alg_name, CRYPTO_ALG_TYPE_AHASH,
CRYPTO_ALG_TYPE_AHASH_MASK);
tfm = crypto_alloc_ahash(hash_alg_name, 0, 0);
if (IS_ERR(tfm))
return PTR_ERR(tfm);

3
drivers/crypto/n2_core.c
View file

@ -1487,8 +1487,7 @@ static int __n2_register_one_ahash(const struct n2_hash_tmpl *tmpl)
snprintf(base->cra_name, CRYPTO_MAX_ALG_NAME, "%s", tmpl->name);
snprintf(base->cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s-n2", tmpl->name);
base->cra_priority = N2_CRA_PRIORITY;
base->cra_flags = CRYPTO_ALG_TYPE_AHASH |
CRYPTO_ALG_KERN_DRIVER_ONLY |
base->cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY |
CRYPTO_ALG_NEED_FALLBACK;
base->cra_blocksize = tmpl->block_size;
base->cra_ctxsize = sizeof(struct n2_hash_ctx);

1
drivers/crypto/nx/nx-aes-xcbc.c
View file

@ -386,7 +386,6 @@ struct shash_alg nx_shash_aes_xcbc_alg = {
.cra_name = "xcbc(aes)",
.cra_driver_name = "xcbc-aes-nx",
.cra_priority = 300,
.cra_flags = CRYPTO_ALG_TYPE_SHASH,
.cra_blocksize = AES_BLOCK_SIZE,
.cra_module = THIS_MODULE,
.cra_ctxsize = sizeof(struct nx_crypto_ctx),

1
drivers/crypto/nx/nx-sha256.c
View file

@ -288,7 +288,6 @@ struct shash_alg nx_shash_sha256_alg = {
.cra_name = "sha256",
.cra_driver_name = "sha256-nx",
.cra_priority = 300,
.cra_flags = CRYPTO_ALG_TYPE_SHASH,
.cra_blocksize = SHA256_BLOCK_SIZE,
.cra_module = THIS_MODULE,
.cra_ctxsize = sizeof(struct nx_crypto_ctx),

1
drivers/crypto/nx/nx-sha512.c
View file

@ -294,7 +294,6 @@ struct shash_alg nx_shash_sha512_alg = {
.cra_name = "sha512",
.cra_driver_name = "sha512-nx",
.cra_priority = 300,
.cra_flags = CRYPTO_ALG_TYPE_SHASH,
.cra_blocksize = SHA512_BLOCK_SIZE,
.cra_module = THIS_MODULE,
.cra_ctxsize = sizeof(struct nx_crypto_ctx),

36
drivers/crypto/omap-sham.c
View file

@ -1464,8 +1464,7 @@ static struct ahash_alg algs_sha1_md5[] = {
.cra_name = "sha1",
.cra_driver_name = "omap-sha1",
.cra_priority = 400,
.cra_flags = CRYPTO_ALG_TYPE_AHASH |
CRYPTO_ALG_KERN_DRIVER_ONLY |
.cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY |
CRYPTO_ALG_ASYNC |
CRYPTO_ALG_NEED_FALLBACK,
.cra_blocksize = SHA1_BLOCK_SIZE,
@ -1487,8 +1486,7 @@ static struct ahash_alg algs_sha1_md5[] = {
.cra_name = "md5",
.cra_driver_name = "omap-md5",
.cra_priority = 400,
.cra_flags = CRYPTO_ALG_TYPE_AHASH |
CRYPTO_ALG_KERN_DRIVER_ONLY |
.cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY |
CRYPTO_ALG_ASYNC |
CRYPTO_ALG_NEED_FALLBACK,
.cra_blocksize = SHA1_BLOCK_SIZE,
@ -1511,8 +1509,7 @@ static struct ahash_alg algs_sha1_md5[] = {
.cra_name = "hmac(sha1)",
.cra_driver_name = "omap-hmac-sha1",
.cra_priority = 400,
.cra_flags = CRYPTO_ALG_TYPE_AHASH |
CRYPTO_ALG_KERN_DRIVER_ONLY |
.cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY |
CRYPTO_ALG_ASYNC |
CRYPTO_ALG_NEED_FALLBACK,
.cra_blocksize = SHA1_BLOCK_SIZE,
@ -1536,8 +1533,7 @@ static struct ahash_alg algs_sha1_md5[] = {
.cra_name = "hmac(md5)",
.cra_driver_name = "omap-hmac-md5",
.cra_priority = 400,
.cra_flags = CRYPTO_ALG_TYPE_AHASH |
CRYPTO_ALG_KERN_DRIVER_ONLY |
.cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY |
CRYPTO_ALG_ASYNC |
CRYPTO_ALG_NEED_FALLBACK,
.cra_blocksize = SHA1_BLOCK_SIZE,
@ -1564,8 +1560,7 @@ static struct ahash_alg algs_sha224_sha256[] = {
.cra_name = "sha224",
.cra_driver_name = "omap-sha224",
.cra_priority = 400,
.cra_flags = CRYPTO_ALG_TYPE_AHASH |
CRYPTO_ALG_ASYNC |
.cra_flags = CRYPTO_ALG_ASYNC |
CRYPTO_ALG_NEED_FALLBACK,
.cra_blocksize = SHA224_BLOCK_SIZE,
.cra_ctxsize = sizeof(struct omap_sham_ctx),
@ -1586,8 +1581,7 @@ static struct ahash_alg algs_sha224_sha256[] = {
.cra_name = "sha256",
.cra_driver_name = "omap-sha256",
.cra_priority = 400,
.cra_flags = CRYPTO_ALG_TYPE_AHASH |
CRYPTO_ALG_ASYNC |
.cra_flags = CRYPTO_ALG_ASYNC |
CRYPTO_ALG_NEED_FALLBACK,
.cra_blocksize = SHA256_BLOCK_SIZE,
.cra_ctxsize = sizeof(struct omap_sham_ctx),
@ -1609,8 +1603,7 @@ static struct ahash_alg algs_sha224_sha256[] = {
.cra_name = "hmac(sha224)",
.cra_driver_name = "omap-hmac-sha224",
.cra_priority = 400,
.cra_flags = CRYPTO_ALG_TYPE_AHASH |
CRYPTO_ALG_ASYNC |
.cra_flags = CRYPTO_ALG_ASYNC |
CRYPTO_ALG_NEED_FALLBACK,
.cra_blocksize = SHA224_BLOCK_SIZE,
.cra_ctxsize = sizeof(struct omap_sham_ctx) +
@ -1633,8 +1626,7 @@ static struct ahash_alg algs_sha224_sha256[] = {
.cra_name = "hmac(sha256)",
.cra_driver_name = "omap-hmac-sha256",
.cra_priority = 400,
.cra_flags = CRYPTO_ALG_TYPE_AHASH |
CRYPTO_ALG_ASYNC |
.cra_flags = CRYPTO_ALG_ASYNC |
CRYPTO_ALG_NEED_FALLBACK,
.cra_blocksize = SHA256_BLOCK_SIZE,
.cra_ctxsize = sizeof(struct omap_sham_ctx) +
@ -1659,8 +1651,7 @@ static struct ahash_alg algs_sha384_sha512[] = {
.cra_name = "sha384",
.cra_driver_name = "omap-sha384",
.cra_priority = 400,
.cra_flags = CRYPTO_ALG_TYPE_AHASH |
CRYPTO_ALG_ASYNC |
.cra_flags = CRYPTO_ALG_ASYNC |
CRYPTO_ALG_NEED_FALLBACK,
.cra_blocksize = SHA384_BLOCK_SIZE,
.cra_ctxsize = sizeof(struct omap_sham_ctx),
@ -1681,8 +1672,7 @@ static struct ahash_alg algs_sha384_sha512[] = {
.cra_name = "sha512",
.cra_driver_name = "omap-sha512",
.cra_priority = 400,
.cra_flags = CRYPTO_ALG_TYPE_AHASH |
CRYPTO_ALG_ASYNC |
.cra_flags = CRYPTO_ALG_ASYNC |
CRYPTO_ALG_NEED_FALLBACK,
.cra_blocksize = SHA512_BLOCK_SIZE,
.cra_ctxsize = sizeof(struct omap_sham_ctx),
@ -1704,8 +1694,7 @@ static struct ahash_alg algs_sha384_sha512[] = {
.cra_name = "hmac(sha384)",
.cra_driver_name = "omap-hmac-sha384",
.cra_priority = 400,
.cra_flags = CRYPTO_ALG_TYPE_AHASH |
CRYPTO_ALG_ASYNC |
.cra_flags = CRYPTO_ALG_ASYNC |
CRYPTO_ALG_NEED_FALLBACK,
.cra_blocksize = SHA384_BLOCK_SIZE,
.cra_ctxsize = sizeof(struct omap_sham_ctx) +
@ -1728,8 +1717,7 @@ static struct ahash_alg algs_sha384_sha512[] = {
.cra_name = "hmac(sha512)",
.cra_driver_name = "omap-hmac-sha512",
.cra_priority = 400,
.cra_flags = CRYPTO_ALG_TYPE_AHASH |
CRYPTO_ALG_ASYNC |
.cra_flags = CRYPTO_ALG_ASYNC |
CRYPTO_ALG_NEED_FALLBACK,
.cra_blocksize = SHA512_BLOCK_SIZE,
.cra_ctxsize = sizeof(struct omap_sham_ctx) +

8
drivers/crypto/padlock-sha.c
View file

@ -247,8 +247,7 @@ static struct shash_alg sha1_alg = {
.cra_name = "sha1",
.cra_driver_name = "sha1-padlock",
.cra_priority = PADLOCK_CRA_PRIORITY,
.cra_flags = CRYPTO_ALG_TYPE_SHASH |
CRYPTO_ALG_NEED_FALLBACK,
.cra_flags = CRYPTO_ALG_NEED_FALLBACK,
.cra_blocksize = SHA1_BLOCK_SIZE,
.cra_ctxsize = sizeof(struct padlock_sha_ctx),
.cra_module = THIS_MODULE,
@ -271,8 +270,7 @@ static struct shash_alg sha256_alg = {
.cra_name = "sha256",
.cra_driver_name = "sha256-padlock",
.cra_priority = PADLOCK_CRA_PRIORITY,
.cra_flags = CRYPTO_ALG_TYPE_SHASH |
CRYPTO_ALG_NEED_FALLBACK,
.cra_flags = CRYPTO_ALG_NEED_FALLBACK,
.cra_blocksize = SHA256_BLOCK_SIZE,
.cra_ctxsize = sizeof(struct padlock_sha_ctx),
.cra_module = THIS_MODULE,
@ -484,7 +482,6 @@ static struct shash_alg sha1_alg_nano = {
.cra_name = "sha1",
.cra_driver_name = "sha1-padlock-nano",
.cra_priority = PADLOCK_CRA_PRIORITY,
.cra_flags = CRYPTO_ALG_TYPE_SHASH,
.cra_blocksize = SHA1_BLOCK_SIZE,
.cra_module = THIS_MODULE,
}
@ -503,7 +500,6 @@ static struct shash_alg sha256_alg_nano = {
.cra_name = "sha256",
.cra_driver_name = "sha256-padlock-nano",
.cra_priority = PADLOCK_CRA_PRIORITY,
.cra_flags = CRYPTO_ALG_TYPE_SHASH,
.cra_blocksize = SHA256_BLOCK_SIZE,
.cra_module = THIS_MODULE,
}

2
drivers/crypto/qat/qat_common/adf_aer.c
View file

@ -163,7 +163,7 @@ static int adf_dev_aer_schedule_reset(struct adf_accel_dev *accel_dev,
return 0;
set_bit(ADF_STATUS_RESTARTING, &accel_dev->status);
reset_data = kzalloc(sizeof(*reset_data), GFP_ATOMIC);
reset_data = kzalloc(sizeof(*reset_data), GFP_KERNEL);
if (!reset_data)
return -ENOMEM;
reset_data->accel_dev = accel_dev;

3
drivers/crypto/qce/sha.c
View file

@ -378,8 +378,7 @@ static int qce_ahash_hmac_setkey(struct crypto_ahash *tfm, const u8 *key,
else
return -EINVAL;
ahash_tfm = crypto_alloc_ahash(alg_name, CRYPTO_ALG_TYPE_AHASH,
CRYPTO_ALG_TYPE_AHASH_MASK);
ahash_tfm = crypto_alloc_ahash(alg_name, 0, 0);
if (IS_ERR(ahash_tfm))
return PTR_ERR(ahash_tfm);

229
drivers/crypto/qcom-rng.c Normal file
View file

@ -0,0 +1,229 @@
// SPDX-License-Identifier: GPL-2.0
// Copyright (c) 2017-18 Linaro Limited
//
// Based on msm-rng.c and downstream driver
#include <crypto/internal/rng.h>
#include <linux/acpi.h>
#include <linux/clk.h>
#include <linux/crypto.h>
#include <linux/module.h>
#include <linux/of.h>
#include <linux/platform_device.h>
/* Device specific register offsets */
#define PRNG_DATA_OUT 0x0000
#define PRNG_STATUS 0x0004
#define PRNG_LFSR_CFG 0x0100
#define PRNG_CONFIG 0x0104
/* Device specific register masks and config values */
#define PRNG_LFSR_CFG_MASK 0x0000ffff
#define PRNG_LFSR_CFG_CLOCKS 0x0000dddd
#define PRNG_CONFIG_HW_ENABLE BIT(1)
#define PRNG_STATUS_DATA_AVAIL BIT(0)
#define WORD_SZ 4
struct qcom_rng {
struct mutex lock;
void __iomem *base;
struct clk *clk;
unsigned int skip_init;
};
struct qcom_rng_ctx {
struct qcom_rng *rng;
};
static struct qcom_rng *qcom_rng_dev;
static int qcom_rng_read(struct qcom_rng *rng, u8 *data, unsigned int max)
{
unsigned int currsize = 0;
u32 val;
/* read random data from hardware */
do {
val = readl_relaxed(rng->base + PRNG_STATUS);
if (!(val & PRNG_STATUS_DATA_AVAIL))
break;
val = readl_relaxed(rng->base + PRNG_DATA_OUT);
if (!val)
break;
if ((max - currsize) >= WORD_SZ) {
memcpy(data, &val, WORD_SZ);
data += WORD_SZ;
currsize += WORD_SZ;
} else {
/* copy only remaining bytes */
memcpy(data, &val, max - currsize);
break;
}
} while (currsize < max);
return currsize;
}
static int qcom_rng_generate(struct crypto_rng *tfm,
const u8 *src, unsigned int slen,
u8 *dstn, unsigned int dlen)
{
struct qcom_rng_ctx *ctx = crypto_rng_ctx(tfm);
struct qcom_rng *rng = ctx->rng;
int ret;
ret = clk_prepare_enable(rng->clk);
if (ret)
return ret;
mutex_lock(&rng->lock);
ret = qcom_rng_read(rng, dstn, dlen);
mutex_unlock(&rng->lock);
clk_disable_unprepare(rng->clk);
return 0;
}
static int qcom_rng_seed(struct crypto_rng *tfm, const u8 *seed,
unsigned int slen)
{
return 0;
}
static int qcom_rng_enable(struct qcom_rng *rng)
{
u32 val;
int ret;
ret = clk_prepare_enable(rng->clk);
if (ret)
return ret;
/* Enable PRNG only if it is not already enabled */
val = readl_relaxed(rng->base + PRNG_CONFIG);
if (val & PRNG_CONFIG_HW_ENABLE)
goto already_enabled;
val = readl_relaxed(rng->base + PRNG_LFSR_CFG);
val &= ~PRNG_LFSR_CFG_MASK;
val |= PRNG_LFSR_CFG_CLOCKS;
writel(val, rng->base + PRNG_LFSR_CFG);
val = readl_relaxed(rng->base + PRNG_CONFIG);
val |= PRNG_CONFIG_HW_ENABLE;
writel(val, rng->base + PRNG_CONFIG);
already_enabled:
clk_disable_unprepare(rng->clk);
return 0;
}
static int qcom_rng_init(struct crypto_tfm *tfm)
{
struct qcom_rng_ctx *ctx = crypto_tfm_ctx(tfm);
ctx->rng = qcom_rng_dev;
if (!ctx->rng->skip_init)
return qcom_rng_enable(ctx->rng);
return 0;
}
static struct rng_alg qcom_rng_alg = {
.generate = qcom_rng_generate,
.seed = qcom_rng_seed,
.seedsize = 0,
.base = {
.cra_name = "stdrng",
.cra_driver_name = "qcom-rng",
.cra_flags = CRYPTO_ALG_TYPE_RNG,
.cra_priority = 300,
.cra_ctxsize = sizeof(struct qcom_rng_ctx),
.cra_module = THIS_MODULE,
.cra_init = qcom_rng_init,
}
};
static int qcom_rng_probe(struct platform_device *pdev)
{
struct resource *res;
struct qcom_rng *rng;
int ret;
rng = devm_kzalloc(&pdev->dev, sizeof(*rng), GFP_KERNEL);
if (!rng)
return -ENOMEM;
platform_set_drvdata(pdev, rng);
mutex_init(&rng->lock);
res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
rng->base = devm_ioremap_resource(&pdev->dev, res);
if (IS_ERR(rng->base))
return PTR_ERR(rng->base);
/* ACPI systems have clk already on, so skip clk_get */
if (!has_acpi_companion(&pdev->dev)) {
rng->clk = devm_clk_get(&pdev->dev, "core");
if (IS_ERR(rng->clk))
return PTR_ERR(rng->clk);
}
rng->skip_init = (unsigned long)device_get_match_data(&pdev->dev);
qcom_rng_dev = rng;
ret = crypto_register_rng(&qcom_rng_alg);
if (ret) {
dev_err(&pdev->dev, "Register crypto rng failed: %d\n", ret);
qcom_rng_dev = NULL;
}
return ret;
}
static int qcom_rng_remove(struct platform_device *pdev)
{
crypto_unregister_rng(&qcom_rng_alg);
qcom_rng_dev = NULL;
return 0;
}
#if IS_ENABLED(CONFIG_ACPI)
static const struct acpi_device_id qcom_rng_acpi_match[] = {
{ .id = "QCOM8160", .driver_data = 1 },
{}
};
MODULE_DEVICE_TABLE(acpi, qcom_rng_acpi_match);
#endif
static const struct of_device_id qcom_rng_of_match[] = {
{ .compatible = "qcom,prng", .data = (void *)0},
{ .compatible = "qcom,prng-ee", .data = (void *)1},
{}
};
MODULE_DEVICE_TABLE(of, qcom_rng_of_match);
static struct platform_driver qcom_rng_driver = {
.probe = qcom_rng_probe,
.remove = qcom_rng_remove,
.driver = {
.name = KBUILD_MODNAME,
.of_match_table = of_match_ptr(qcom_rng_of_match),
.acpi_match_table = ACPI_PTR(qcom_rng_acpi_match),
}
};
module_platform_driver(qcom_rng_driver);
MODULE_ALIAS("platform:" KBUILD_MODNAME);
MODULE_DESCRIPTION("Qualcomm random number generator driver");
MODULE_LICENSE("GPL v2");

9
drivers/crypto/s5p-sss.c
View file

@ -1765,8 +1765,7 @@ static struct ahash_alg algs_sha1_md5_sha256[] = {
.cra_name = "sha1",
.cra_driver_name = "exynos-sha1",
.cra_priority = 100,
.cra_flags = CRYPTO_ALG_TYPE_AHASH |
CRYPTO_ALG_KERN_DRIVER_ONLY |
.cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY |
CRYPTO_ALG_ASYNC |
CRYPTO_ALG_NEED_FALLBACK,
.cra_blocksize = HASH_BLOCK_SIZE,
@ -1791,8 +1790,7 @@ static struct ahash_alg algs_sha1_md5_sha256[] = {
.cra_name = "md5",
.cra_driver_name = "exynos-md5",
.cra_priority = 100,
.cra_flags = CRYPTO_ALG_TYPE_AHASH |
CRYPTO_ALG_KERN_DRIVER_ONLY |
.cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY |
CRYPTO_ALG_ASYNC |
CRYPTO_ALG_NEED_FALLBACK,
.cra_blocksize = HASH_BLOCK_SIZE,
@ -1817,8 +1815,7 @@ static struct ahash_alg algs_sha1_md5_sha256[] = {
.cra_name = "sha256",
.cra_driver_name = "exynos-sha256",
.cra_priority = 100,
.cra_flags = CRYPTO_ALG_TYPE_AHASH |
CRYPTO_ALG_KERN_DRIVER_ONLY |
.cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY |
CRYPTO_ALG_ASYNC |
CRYPTO_ALG_NEED_FALLBACK,
.cra_blocksize = HASH_BLOCK_SIZE,

10
drivers/crypto/sahara.c
View file

@ -1253,8 +1253,7 @@ static struct ahash_alg sha_v3_algs[] = {
.cra_name = "sha1",
.cra_driver_name = "sahara-sha1",
.cra_priority = 300,
.cra_flags = CRYPTO_ALG_TYPE_AHASH |
CRYPTO_ALG_ASYNC |
.cra_flags = CRYPTO_ALG_ASYNC |
CRYPTO_ALG_NEED_FALLBACK,
.cra_blocksize = SHA1_BLOCK_SIZE,
.cra_ctxsize = sizeof(struct sahara_ctx),
@ -1280,8 +1279,7 @@ static struct ahash_alg sha_v4_algs[] = {
.cra_name = "sha256",
.cra_driver_name = "sahara-sha256",
.cra_priority = 300,
.cra_flags = CRYPTO_ALG_TYPE_AHASH |
CRYPTO_ALG_ASYNC |
.cra_flags = CRYPTO_ALG_ASYNC |
CRYPTO_ALG_NEED_FALLBACK,
.cra_blocksize = SHA256_BLOCK_SIZE,
.cra_ctxsize = sizeof(struct sahara_ctx),
@ -1351,7 +1349,7 @@ err_sha_v4_algs:
err_sha_v3_algs:
for (j = 0; j < k; j++)
crypto_unregister_ahash(&sha_v4_algs[j]);
crypto_unregister_ahash(&sha_v3_algs[j]);
err_aes_algs:
for (j = 0; j < i; j++)
@ -1367,7 +1365,7 @@ static void sahara_unregister_algs(struct sahara_dev *dev)
for (i = 0; i < ARRAY_SIZE(aes_algs); i++)
crypto_unregister_alg(&aes_algs[i]);
for (i = 0; i < ARRAY_SIZE(sha_v4_algs); i++)
for (i = 0; i < ARRAY_SIZE(sha_v3_algs); i++)
crypto_unregister_ahash(&sha_v3_algs[i]);
if (dev->version > SAHARA_VERSION_3)

62
drivers/crypto/stm32/stm32-cryp.c
View file

@ -11,6 +11,7 @@
#include <linux/module.h>
#include <linux/of_device.h>
#include <linux/platform_device.h>
#include <linux/pm_runtime.h>
#include <linux/reset.h>
#include <crypto/aes.h>
@ -105,6 +106,7 @@
#define GCM_CTR_INIT 2
#define _walked_in (cryp->in_walk.offset - cryp->in_sg->offset)
#define _walked_out (cryp->out_walk.offset - cryp->out_sg->offset)
#define CRYP_AUTOSUSPEND_DELAY 50
struct stm32_cryp_caps {
bool swap_final;
@ -519,6 +521,8 @@ static int stm32_cryp_hw_init(struct stm32_cryp *cryp)
int ret;
u32 cfg, hw_mode;
pm_runtime_get_sync(cryp->dev);
/* Disable interrupt */
stm32_cryp_write(cryp, CRYP_IMSCR, 0);
@ -638,6 +642,9 @@ static void stm32_cryp_finish_req(struct stm32_cryp *cryp, int err)
free_pages((unsigned long)buf_out, pages);
}
pm_runtime_mark_last_busy(cryp->dev);
pm_runtime_put_autosuspend(cryp->dev);
if (is_gcm(cryp) || is_ccm(cryp)) {
crypto_finalize_aead_request(cryp->engine, cryp->areq, err);
cryp->areq = NULL;
@ -1969,6 +1976,13 @@ static int stm32_cryp_probe(struct platform_device *pdev)
return ret;
}
pm_runtime_set_autosuspend_delay(dev, CRYP_AUTOSUSPEND_DELAY);
pm_runtime_use_autosuspend(dev);
pm_runtime_get_noresume(dev);
pm_runtime_set_active(dev);
pm_runtime_enable(dev);
rst = devm_reset_control_get(dev, NULL);
if (!IS_ERR(rst)) {
reset_control_assert(rst);
@ -2008,6 +2022,8 @@ static int stm32_cryp_probe(struct platform_device *pdev)
dev_info(dev, "Initialized\n");
pm_runtime_put_sync(dev);
return 0;
err_aead_algs:
@ -2020,6 +2036,11 @@ err_engine1:
list_del(&cryp->list);
spin_unlock(&cryp_list.lock);
pm_runtime_disable(dev);
pm_runtime_put_noidle(dev);
pm_runtime_disable(dev);
pm_runtime_put_noidle(dev);
clk_disable_unprepare(cryp->clk);
return ret;
@ -2028,10 +2049,15 @@ err_engine1:
static int stm32_cryp_remove(struct platform_device *pdev)
{
struct stm32_cryp *cryp = platform_get_drvdata(pdev);
int ret;
if (!cryp)
return -ENODEV;
ret = pm_runtime_get_sync(cryp->dev);
if (ret < 0)
return ret;
crypto_unregister_aeads(aead_algs, ARRAY_SIZE(aead_algs));
crypto_unregister_algs(crypto_algs, ARRAY_SIZE(crypto_algs));
@ -2041,16 +2067,52 @@ static int stm32_cryp_remove(struct platform_device *pdev)
list_del(&cryp->list);
spin_unlock(&cryp_list.lock);
pm_runtime_disable(cryp->dev);
pm_runtime_put_noidle(cryp->dev);
clk_disable_unprepare(cryp->clk);
return 0;
}
#ifdef CONFIG_PM
static int stm32_cryp_runtime_suspend(struct device *dev)
{
struct stm32_cryp *cryp = dev_get_drvdata(dev);
clk_disable_unprepare(cryp->clk);
return 0;
}
static int stm32_cryp_runtime_resume(struct device *dev)
{
struct stm32_cryp *cryp = dev_get_drvdata(dev);
int ret;
ret = clk_prepare_enable(cryp->clk);
if (ret) {
dev_err(cryp->dev, "Failed to prepare_enable clock\n");
return ret;
}
return 0;
}
#endif
static const struct dev_pm_ops stm32_cryp_pm_ops = {
SET_SYSTEM_SLEEP_PM_OPS(pm_runtime_force_suspend,
pm_runtime_force_resume)
SET_RUNTIME_PM_OPS(stm32_cryp_runtime_suspend,
stm32_cryp_runtime_resume, NULL)
};
static struct platform_driver stm32_cryp_driver = {
.probe = stm32_cryp_probe,
.remove = stm32_cryp_remove,
.driver = {
.name = DRIVER_NAME,
.pm = &stm32_cryp_pm_ops,
.of_match_table = stm32_dt_ids,
},
};

95
drivers/crypto/stm32/stm32-hash.c
View file

@ -31,6 +31,7 @@
#include <linux/module.h>
#include <linux/of_device.h>
#include <linux/platform_device.h>
#include <linux/pm_runtime.h>
#include <linux/reset.h>
#include <crypto/engine.h>
@ -121,6 +122,8 @@ enum stm32_hash_data_format {
#define HASH_QUEUE_LENGTH 16
#define HASH_DMA_THRESHOLD 50
#define HASH_AUTOSUSPEND_DELAY 50
struct stm32_hash_ctx {
struct crypto_engine_ctx enginectx;
struct stm32_hash_dev *hdev;
@ -814,12 +817,17 @@ static void stm32_hash_finish_req(struct ahash_request *req, int err)
rctx->flags |= HASH_FLAGS_ERRORS;
}
pm_runtime_mark_last_busy(hdev->dev);
pm_runtime_put_autosuspend(hdev->dev);
crypto_finalize_hash_request(hdev->engine, req, err);
}
static int stm32_hash_hw_init(struct stm32_hash_dev *hdev,
struct stm32_hash_request_ctx *rctx)
{
pm_runtime_get_sync(hdev->dev);
if (!(HASH_FLAGS_INIT & hdev->flags)) {
stm32_hash_write(hdev, HASH_CR, HASH_CR_INIT);
stm32_hash_write(hdev, HASH_STR, 0);
@ -967,6 +975,8 @@ static int stm32_hash_export(struct ahash_request *req, void *out)
u32 *preg;
unsigned int i;
pm_runtime_get_sync(hdev->dev);
while (!(stm32_hash_read(hdev, HASH_SR) & HASH_SR_DATA_INPUT_READY))
cpu_relax();
@ -982,6 +992,9 @@ static int stm32_hash_export(struct ahash_request *req, void *out)
for (i = 0; i < HASH_CSR_REGISTER_NUMBER; i++)
*preg++ = stm32_hash_read(hdev, HASH_CSR(i));
pm_runtime_mark_last_busy(hdev->dev);
pm_runtime_put_autosuspend(hdev->dev);
memcpy(out, rctx, sizeof(*rctx));
return 0;
@ -1000,6 +1013,8 @@ static int stm32_hash_import(struct ahash_request *req, const void *in)
preg = rctx->hw_context;
pm_runtime_get_sync(hdev->dev);
stm32_hash_write(hdev, HASH_IMR, *preg++);
stm32_hash_write(hdev, HASH_STR, *preg++);
stm32_hash_write(hdev, HASH_CR, *preg);
@ -1009,6 +1024,9 @@ static int stm32_hash_import(struct ahash_request *req, const void *in)
for (i = 0; i < HASH_CSR_REGISTER_NUMBER; i++)
stm32_hash_write(hdev, HASH_CSR(i), *preg++);
pm_runtime_mark_last_busy(hdev->dev);
pm_runtime_put_autosuspend(hdev->dev);
kfree(rctx->hw_context);
return 0;
@ -1132,8 +1150,7 @@ static struct ahash_alg algs_md5_sha1[] = {
.cra_name = "md5",
.cra_driver_name = "stm32-md5",
.cra_priority = 200,
.cra_flags = CRYPTO_ALG_TYPE_AHASH |
CRYPTO_ALG_ASYNC |
.cra_flags = CRYPTO_ALG_ASYNC |
CRYPTO_ALG_KERN_DRIVER_ONLY,
.cra_blocksize = MD5_HMAC_BLOCK_SIZE,
.cra_ctxsize = sizeof(struct stm32_hash_ctx),
@ -1159,8 +1176,7 @@ static struct ahash_alg algs_md5_sha1[] = {
.cra_name = "hmac(md5)",
.cra_driver_name = "stm32-hmac-md5",
.cra_priority = 200,
.cra_flags = CRYPTO_ALG_TYPE_AHASH |
CRYPTO_ALG_ASYNC |
.cra_flags = CRYPTO_ALG_ASYNC |
CRYPTO_ALG_KERN_DRIVER_ONLY,
.cra_blocksize = MD5_HMAC_BLOCK_SIZE,
.cra_ctxsize = sizeof(struct stm32_hash_ctx),
@ -1185,8 +1201,7 @@ static struct ahash_alg algs_md5_sha1[] = {
.cra_name = "sha1",
.cra_driver_name = "stm32-sha1",
.cra_priority = 200,
.cra_flags = CRYPTO_ALG_TYPE_AHASH |
CRYPTO_ALG_ASYNC |
.cra_flags = CRYPTO_ALG_ASYNC |
CRYPTO_ALG_KERN_DRIVER_ONLY,
.cra_blocksize = SHA1_BLOCK_SIZE,
.cra_ctxsize = sizeof(struct stm32_hash_ctx),
@ -1212,8 +1227,7 @@ static struct ahash_alg algs_md5_sha1[] = {
.cra_name = "hmac(sha1)",
.cra_driver_name = "stm32-hmac-sha1",
.cra_priority = 200,
.cra_flags = CRYPTO_ALG_TYPE_AHASH |
CRYPTO_ALG_ASYNC |
.cra_flags = CRYPTO_ALG_ASYNC |
CRYPTO_ALG_KERN_DRIVER_ONLY,
.cra_blocksize = SHA1_BLOCK_SIZE,
.cra_ctxsize = sizeof(struct stm32_hash_ctx),
@ -1241,8 +1255,7 @@ static struct ahash_alg algs_sha224_sha256[] = {
.cra_name = "sha224",
.cra_driver_name = "stm32-sha224",
.cra_priority = 200,
.cra_flags = CRYPTO_ALG_TYPE_AHASH |
CRYPTO_ALG_ASYNC |
.cra_flags = CRYPTO_ALG_ASYNC |
CRYPTO_ALG_KERN_DRIVER_ONLY,
.cra_blocksize = SHA224_BLOCK_SIZE,
.cra_ctxsize = sizeof(struct stm32_hash_ctx),
@ -1268,8 +1281,7 @@ static struct ahash_alg algs_sha224_sha256[] = {
.cra_name = "hmac(sha224)",
.cra_driver_name = "stm32-hmac-sha224",
.cra_priority = 200,
.cra_flags = CRYPTO_ALG_TYPE_AHASH |
CRYPTO_ALG_ASYNC |
.cra_flags = CRYPTO_ALG_ASYNC |
CRYPTO_ALG_KERN_DRIVER_ONLY,
.cra_blocksize = SHA224_BLOCK_SIZE,
.cra_ctxsize = sizeof(struct stm32_hash_ctx),
@ -1294,8 +1306,7 @@ static struct ahash_alg algs_sha224_sha256[] = {
.cra_name = "sha256",
.cra_driver_name = "stm32-sha256",
.cra_priority = 200,
.cra_flags = CRYPTO_ALG_TYPE_AHASH |
CRYPTO_ALG_ASYNC |
.cra_flags = CRYPTO_ALG_ASYNC |
CRYPTO_ALG_KERN_DRIVER_ONLY,
.cra_blocksize = SHA256_BLOCK_SIZE,
.cra_ctxsize = sizeof(struct stm32_hash_ctx),
@ -1321,8 +1332,7 @@ static struct ahash_alg algs_sha224_sha256[] = {
.cra_name = "hmac(sha256)",
.cra_driver_name = "stm32-hmac-sha256",
.cra_priority = 200,
.cra_flags = CRYPTO_ALG_TYPE_AHASH |
CRYPTO_ALG_ASYNC |
.cra_flags = CRYPTO_ALG_ASYNC |
CRYPTO_ALG_KERN_DRIVER_ONLY,
.cra_blocksize = SHA256_BLOCK_SIZE,
.cra_ctxsize = sizeof(struct stm32_hash_ctx),
@ -1482,6 +1492,13 @@ static int stm32_hash_probe(struct platform_device *pdev)
return ret;
}
pm_runtime_set_autosuspend_delay(dev, HASH_AUTOSUSPEND_DELAY);
pm_runtime_use_autosuspend(dev);
pm_runtime_get_noresume(dev);
pm_runtime_set_active(dev);
pm_runtime_enable(dev);
hdev->rst = devm_reset_control_get(&pdev->dev, NULL);
if (!IS_ERR(hdev->rst)) {
reset_control_assert(hdev->rst);
@ -1522,6 +1539,8 @@ static int stm32_hash_probe(struct platform_device *pdev)
dev_info(dev, "Init HASH done HW ver %x DMA mode %u\n",
stm32_hash_read(hdev, HASH_VER), hdev->dma_mode);
pm_runtime_put_sync(dev);
return 0;
err_algs:
@ -1535,6 +1554,9 @@ err_engine:
if (hdev->dma_lch)
dma_release_channel(hdev->dma_lch);
pm_runtime_disable(dev);
pm_runtime_put_noidle(dev);
clk_disable_unprepare(hdev->clk);
return ret;
@ -1543,11 +1565,16 @@ err_engine:
static int stm32_hash_remove(struct platform_device *pdev)
{
static struct stm32_hash_dev *hdev;
int ret;
hdev = platform_get_drvdata(pdev);
if (!hdev)
return -ENODEV;
ret = pm_runtime_get_sync(hdev->dev);
if (ret < 0)
return ret;
stm32_hash_unregister_algs(hdev);
crypto_engine_exit(hdev->engine);
@ -1559,16 +1586,52 @@ static int stm32_hash_remove(struct platform_device *pdev)
if (hdev->dma_lch)
dma_release_channel(hdev->dma_lch);
pm_runtime_disable(hdev->dev);
pm_runtime_put_noidle(hdev->dev);
clk_disable_unprepare(hdev->clk);
return 0;
}
#ifdef CONFIG_PM
static int stm32_hash_runtime_suspend(struct device *dev)
{
struct stm32_hash_dev *hdev = dev_get_drvdata(dev);
clk_disable_unprepare(hdev->clk);
return 0;
}
static int stm32_hash_runtime_resume(struct device *dev)
{
struct stm32_hash_dev *hdev = dev_get_drvdata(dev);
int ret;
ret = clk_prepare_enable(hdev->clk);
if (ret) {
dev_err(hdev->dev, "Failed to prepare_enable clock\n");
return ret;
}
return 0;
}
#endif
static const struct dev_pm_ops stm32_hash_pm_ops = {
SET_SYSTEM_SLEEP_PM_OPS(pm_runtime_force_suspend,
pm_runtime_force_resume)
SET_RUNTIME_PM_OPS(stm32_hash_runtime_suspend,
stm32_hash_runtime_resume, NULL)
};
static struct platform_driver stm32_hash_driver = {
.probe = stm32_hash_probe,
.remove = stm32_hash_remove,
.driver = {
.name = "stm32-hash",
.pm = &stm32_hash_pm_ops,
.of_match_table = stm32_hash_of_match,
}
};

71
drivers/crypto/stm32/stm32_crc32.c
View file

@ -6,8 +6,10 @@
#include <linux/bitrev.h>
#include <linux/clk.h>
#include <linux/crc32poly.h>
#include <linux/module.h>
#include <linux/platform_device.h>
#include <linux/pm_runtime.h>
#include <crypto/internal/hash.h>
@ -28,9 +30,7 @@
#define CRC_CR_REVERSE (BIT(7) | BIT(6) | BIT(5))
#define CRC_INIT_DEFAULT 0xFFFFFFFF
/* Polynomial reversed */
#define POLY_CRC32 0xEDB88320
#define POLY_CRC32C 0x82F63B78
#define CRC_AUTOSUSPEND_DELAY 50
struct stm32_crc {
struct list_head list;
@ -66,7 +66,7 @@ static int stm32_crc32_cra_init(struct crypto_tfm *tfm)
struct stm32_crc_ctx *mctx = crypto_tfm_ctx(tfm);
mctx->key = CRC_INIT_DEFAULT;
mctx->poly = POLY_CRC32;
mctx->poly = CRC32_POLY_LE;
return 0;
}
@ -75,7 +75,7 @@ static int stm32_crc32c_cra_init(struct crypto_tfm *tfm)
struct stm32_crc_ctx *mctx = crypto_tfm_ctx(tfm);
mctx->key = CRC_INIT_DEFAULT;
mctx->poly = POLY_CRC32C;
mctx->poly = CRC32C_POLY_LE;
return 0;
}
@ -106,6 +106,8 @@ static int stm32_crc_init(struct shash_desc *desc)
}
spin_unlock_bh(&crc_list.lock);
pm_runtime_get_sync(ctx->crc->dev);
/* Reset, set key, poly and configure in bit reverse mode */
writel_relaxed(bitrev32(mctx->key), ctx->crc->regs + CRC_INIT);
writel_relaxed(bitrev32(mctx->poly), ctx->crc->regs + CRC_POL);
@ -115,6 +117,9 @@ static int stm32_crc_init(struct shash_desc *desc)
ctx->partial = readl_relaxed(ctx->crc->regs + CRC_DR);
ctx->crc->nb_pending_bytes = 0;
pm_runtime_mark_last_busy(ctx->crc->dev);
pm_runtime_put_autosuspend(ctx->crc->dev);
return 0;
}
@ -126,6 +131,8 @@ static int stm32_crc_update(struct shash_desc *desc, const u8 *d8,
u32 *d32;
unsigned int i;
pm_runtime_get_sync(crc->dev);
if (unlikely(crc->nb_pending_bytes)) {
while (crc->nb_pending_bytes != sizeof(u32) && length) {
/* Fill in pending data */
@ -149,6 +156,9 @@ static int stm32_crc_update(struct shash_desc *desc, const u8 *d8,
/* Store partial result */
ctx->partial = readl_relaxed(crc->regs + CRC_DR);
pm_runtime_mark_last_busy(crc->dev);
pm_runtime_put_autosuspend(crc->dev);
/* Check for pending data (non 32 bits) */
length &= 3;
if (likely(!length))
@ -174,7 +184,7 @@ static int stm32_crc_final(struct shash_desc *desc, u8 *out)
struct stm32_crc_ctx *mctx = crypto_shash_ctx(desc->tfm);
/* Send computed CRC */
put_unaligned_le32(mctx->poly == POLY_CRC32C ?
put_unaligned_le32(mctx->poly == CRC32C_POLY_LE ?
~ctx->partial : ctx->partial, out);
return 0;
@ -272,6 +282,13 @@ static int stm32_crc_probe(struct platform_device *pdev)
return ret;
}
pm_runtime_set_autosuspend_delay(dev, CRC_AUTOSUSPEND_DELAY);
pm_runtime_use_autosuspend(dev);
pm_runtime_get_noresume(dev);
pm_runtime_set_active(dev);
pm_runtime_enable(dev);
platform_set_drvdata(pdev, crc);
spin_lock(&crc_list.lock);
@ -287,12 +304,18 @@ static int stm32_crc_probe(struct platform_device *pdev)
dev_info(dev, "Initialized\n");
pm_runtime_put_sync(dev);
return 0;
}
static int stm32_crc_remove(struct platform_device *pdev)
{
struct stm32_crc *crc = platform_get_drvdata(pdev);
int ret = pm_runtime_get_sync(crc->dev);
if (ret < 0)
return ret;
spin_lock(&crc_list.lock);
list_del(&crc->list);
@ -300,11 +323,46 @@ static int stm32_crc_remove(struct platform_device *pdev)
crypto_unregister_shashes(algs, ARRAY_SIZE(algs));
pm_runtime_disable(crc->dev);
pm_runtime_put_noidle(crc->dev);
clk_disable_unprepare(crc->clk);
return 0;
}
#ifdef CONFIG_PM
static int stm32_crc_runtime_suspend(struct device *dev)
{
struct stm32_crc *crc = dev_get_drvdata(dev);
clk_disable_unprepare(crc->clk);
return 0;
}
static int stm32_crc_runtime_resume(struct device *dev)
{
struct stm32_crc *crc = dev_get_drvdata(dev);
int ret;
ret = clk_prepare_enable(crc->clk);
if (ret) {
dev_err(crc->dev, "Failed to prepare_enable clock\n");
return ret;
}
return 0;
}
#endif
static const struct dev_pm_ops stm32_crc_pm_ops = {
SET_SYSTEM_SLEEP_PM_OPS(pm_runtime_force_suspend,
pm_runtime_force_resume)
SET_RUNTIME_PM_OPS(stm32_crc_runtime_suspend,
stm32_crc_runtime_resume, NULL)
};
static const struct of_device_id stm32_dt_ids[] = {
{ .compatible = "st,stm32f7-crc", },
{},
@ -316,6 +374,7 @@ static struct platform_driver stm32_crc_driver = {
.remove = stm32_crc_remove,
.driver = {
.name = DRIVER_NAME,
.pm = &stm32_crc_pm_ops,
.of_match_table = stm32_dt_ids,
},
};

20
drivers/crypto/sunxi-ss/sun4i-ss-core.c
View file

@ -45,11 +45,9 @@ static struct sun4i_ss_alg_template ss_algs[] = {
.cra_driver_name = "md5-sun4i-ss",
.cra_priority = 300,
.cra_alignmask = 3,
.cra_flags = CRYPTO_ALG_TYPE_AHASH,
.cra_blocksize = MD5_HMAC_BLOCK_SIZE,
.cra_ctxsize = sizeof(struct sun4i_req_ctx),
.cra_module = THIS_MODULE,
.cra_type = &crypto_ahash_type,
.cra_init = sun4i_hash_crainit
}
}
@ -73,11 +71,9 @@ static struct sun4i_ss_alg_template ss_algs[] = {
.cra_driver_name = "sha1-sun4i-ss",
.cra_priority = 300,
.cra_alignmask = 3,
.cra_flags = CRYPTO_ALG_TYPE_AHASH,
.cra_blocksize = SHA1_BLOCK_SIZE,
.cra_ctxsize = sizeof(struct sun4i_req_ctx),
.cra_module = THIS_MODULE,
.cra_type = &crypto_ahash_type,
.cra_init = sun4i_hash_crainit
}
}
@ -96,8 +92,7 @@ static struct sun4i_ss_alg_template ss_algs[] = {
.cra_driver_name = "cbc-aes-sun4i-ss",
.cra_priority = 300,
.cra_blocksize = AES_BLOCK_SIZE,
.cra_flags = CRYPTO_ALG_TYPE_SKCIPHER |
CRYPTO_ALG_KERN_DRIVER_ONLY,
.cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY,
.cra_ctxsize = sizeof(struct sun4i_tfm_ctx),
.cra_module = THIS_MODULE,
.cra_alignmask = 3,
@ -118,8 +113,7 @@ static struct sun4i_ss_alg_template ss_algs[] = {
.cra_driver_name = "ecb-aes-sun4i-ss",
.cra_priority = 300,
.cra_blocksize = AES_BLOCK_SIZE,
.cra_flags = CRYPTO_ALG_TYPE_SKCIPHER |
CRYPTO_ALG_KERN_DRIVER_ONLY,
.cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY,
.cra_ctxsize = sizeof(struct sun4i_tfm_ctx),
.cra_module = THIS_MODULE,
.cra_alignmask = 3,
@ -140,8 +134,7 @@ static struct sun4i_ss_alg_template ss_algs[] = {
.cra_driver_name = "cbc-des-sun4i-ss",
.cra_priority = 300,
.cra_blocksize = DES_BLOCK_SIZE,
.cra_flags = CRYPTO_ALG_TYPE_SKCIPHER |
CRYPTO_ALG_KERN_DRIVER_ONLY,
.cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY,
.cra_ctxsize = sizeof(struct sun4i_req_ctx),
.cra_module = THIS_MODULE,
.cra_alignmask = 3,
@ -161,8 +154,7 @@ static struct sun4i_ss_alg_template ss_algs[] = {
.cra_driver_name = "ecb-des-sun4i-ss",
.cra_priority = 300,
.cra_blocksize = DES_BLOCK_SIZE,
.cra_flags = CRYPTO_ALG_TYPE_SKCIPHER |
CRYPTO_ALG_KERN_DRIVER_ONLY,
.cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY,
.cra_ctxsize = sizeof(struct sun4i_req_ctx),
.cra_module = THIS_MODULE,
.cra_alignmask = 3,
@ -183,8 +175,7 @@ static struct sun4i_ss_alg_template ss_algs[] = {
.cra_driver_name = "cbc-des3-sun4i-ss",
.cra_priority = 300,
.cra_blocksize = DES3_EDE_BLOCK_SIZE,
.cra_flags = CRYPTO_ALG_TYPE_SKCIPHER |
CRYPTO_ALG_KERN_DRIVER_ONLY,
.cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY,
.cra_ctxsize = sizeof(struct sun4i_req_ctx),
.cra_module = THIS_MODULE,
.cra_alignmask = 3,
@ -205,7 +196,6 @@ static struct sun4i_ss_alg_template ss_algs[] = {
.cra_driver_name = "ecb-des3-sun4i-ss",
.cra_priority = 300,
.cra_blocksize = DES3_EDE_BLOCK_SIZE,
.cra_flags = CRYPTO_ALG_TYPE_SKCIPHER,
.cra_ctxsize = sizeof(struct sun4i_req_ctx),
.cra_module = THIS_MODULE,
.cra_alignmask = 3,

37
drivers/crypto/talitos.c
View file

@ -2822,8 +2822,7 @@ static struct talitos_alg_template driver_algs[] = {
.cra_name = "md5",
.cra_driver_name = "md5-talitos",
.cra_blocksize = MD5_HMAC_BLOCK_SIZE,
.cra_flags = CRYPTO_ALG_TYPE_AHASH |
CRYPTO_ALG_ASYNC,
.cra_flags = CRYPTO_ALG_ASYNC,
}
},
.desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
@ -2838,8 +2837,7 @@ static struct talitos_alg_template driver_algs[] = {
.cra_name = "sha1",
.cra_driver_name = "sha1-talitos",
.cra_blocksize = SHA1_BLOCK_SIZE,
.cra_flags = CRYPTO_ALG_TYPE_AHASH |
CRYPTO_ALG_ASYNC,
.cra_flags = CRYPTO_ALG_ASYNC,
}
},
.desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
@ -2854,8 +2852,7 @@ static struct talitos_alg_template driver_algs[] = {
.cra_name = "sha224",
.cra_driver_name = "sha224-talitos",
.cra_blocksize = SHA224_BLOCK_SIZE,
.cra_flags = CRYPTO_ALG_TYPE_AHASH |
CRYPTO_ALG_ASYNC,
.cra_flags = CRYPTO_ALG_ASYNC,
}
},
.desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
@ -2870,8 +2867,7 @@ static struct talitos_alg_template driver_algs[] = {
.cra_name = "sha256",
.cra_driver_name = "sha256-talitos",
.cra_blocksize = SHA256_BLOCK_SIZE,
.cra_flags = CRYPTO_ALG_TYPE_AHASH |
CRYPTO_ALG_ASYNC,
.cra_flags = CRYPTO_ALG_ASYNC,
}
},
.desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
@ -2886,8 +2882,7 @@ static struct talitos_alg_template driver_algs[] = {
.cra_name = "sha384",
.cra_driver_name = "sha384-talitos",
.cra_blocksize = SHA384_BLOCK_SIZE,
.cra_flags = CRYPTO_ALG_TYPE_AHASH |
CRYPTO_ALG_ASYNC,
.cra_flags = CRYPTO_ALG_ASYNC,
}
},
.desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
@ -2902,8 +2897,7 @@ static struct talitos_alg_template driver_algs[] = {
.cra_name = "sha512",
.cra_driver_name = "sha512-talitos",
.cra_blocksize = SHA512_BLOCK_SIZE,
.cra_flags = CRYPTO_ALG_TYPE_AHASH |
CRYPTO_ALG_ASYNC,
.cra_flags = CRYPTO_ALG_ASYNC,
}
},
.desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
@ -2918,8 +2912,7 @@ static struct talitos_alg_template driver_algs[] = {
.cra_name = "hmac(md5)",
.cra_driver_name = "hmac-md5-talitos",
.cra_blocksize = MD5_HMAC_BLOCK_SIZE,
.cra_flags = CRYPTO_ALG_TYPE_AHASH |
CRYPTO_ALG_ASYNC,
.cra_flags = CRYPTO_ALG_ASYNC,
}
},
.desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
@ -2934,8 +2927,7 @@ static struct talitos_alg_template driver_algs[] = {
.cra_name = "hmac(sha1)",
.cra_driver_name = "hmac-sha1-talitos",
.cra_blocksize = SHA1_BLOCK_SIZE,
.cra_flags = CRYPTO_ALG_TYPE_AHASH |
CRYPTO_ALG_ASYNC,
.cra_flags = CRYPTO_ALG_ASYNC,
}
},
.desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
@ -2950,8 +2942,7 @@ static struct talitos_alg_template driver_algs[] = {
.cra_name = "hmac(sha224)",
.cra_driver_name = "hmac-sha224-talitos",
.cra_blocksize = SHA224_BLOCK_SIZE,
.cra_flags = CRYPTO_ALG_TYPE_AHASH |
CRYPTO_ALG_ASYNC,
.cra_flags = CRYPTO_ALG_ASYNC,
}
},
.desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
@ -2966,8 +2957,7 @@ static struct talitos_alg_template driver_algs[] = {
.cra_name = "hmac(sha256)",
.cra_driver_name = "hmac-sha256-talitos",
.cra_blocksize = SHA256_BLOCK_SIZE,
.cra_flags = CRYPTO_ALG_TYPE_AHASH |
CRYPTO_ALG_ASYNC,
.cra_flags = CRYPTO_ALG_ASYNC,
}
},
.desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
@ -2982,8 +2972,7 @@ static struct talitos_alg_template driver_algs[] = {
.cra_name = "hmac(sha384)",
.cra_driver_name = "hmac-sha384-talitos",
.cra_blocksize = SHA384_BLOCK_SIZE,
.cra_flags = CRYPTO_ALG_TYPE_AHASH |
CRYPTO_ALG_ASYNC,
.cra_flags = CRYPTO_ALG_ASYNC,
}
},
.desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
@ -2998,8 +2987,7 @@ static struct talitos_alg_template driver_algs[] = {
.cra_name = "hmac(sha512)",
.cra_driver_name = "hmac-sha512-talitos",
.cra_blocksize = SHA512_BLOCK_SIZE,
.cra_flags = CRYPTO_ALG_TYPE_AHASH |
CRYPTO_ALG_ASYNC,
.cra_flags = CRYPTO_ALG_ASYNC,
}
},
.desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
@ -3186,7 +3174,6 @@ static struct talitos_crypto_alg *talitos_alg_alloc(struct device *dev,
alg = &t_alg->algt.alg.hash.halg.base;
alg->cra_init = talitos_cra_init_ahash;
alg->cra_exit = talitos_cra_exit;
alg->cra_type = &crypto_ahash_type;
t_alg->algt.alg.hash.init = ahash_init;
t_alg->algt.alg.hash.update = ahash_update;
t_alg->algt.alg.hash.final = ahash_final;

15
drivers/crypto/ux500/hash/hash_core.c
View file

@ -1524,8 +1524,7 @@ static struct hash_algo_template hash_algs[] = {
.halg.base = {
.cra_name = "sha1",
.cra_driver_name = "sha1-ux500",
.cra_flags = (CRYPTO_ALG_TYPE_AHASH |
CRYPTO_ALG_ASYNC),
.cra_flags = CRYPTO_ALG_ASYNC,
.cra_blocksize = SHA1_BLOCK_SIZE,
.cra_ctxsize = sizeof(struct hash_ctx),
.cra_init = hash_cra_init,
@ -1548,11 +1547,9 @@ static struct hash_algo_template hash_algs[] = {
.halg.base = {
.cra_name = "sha256",
.cra_driver_name = "sha256-ux500",
.cra_flags = (CRYPTO_ALG_TYPE_AHASH |
CRYPTO_ALG_ASYNC),
.cra_flags = CRYPTO_ALG_ASYNC,
.cra_blocksize = SHA256_BLOCK_SIZE,
.cra_ctxsize = sizeof(struct hash_ctx),
.cra_type = &crypto_ahash_type,
.cra_init = hash_cra_init,
.cra_module = THIS_MODULE,
}
@ -1574,11 +1571,9 @@ static struct hash_algo_template hash_algs[] = {
.halg.base = {
.cra_name = "hmac(sha1)",
.cra_driver_name = "hmac-sha1-ux500",
.cra_flags = (CRYPTO_ALG_TYPE_AHASH |
CRYPTO_ALG_ASYNC),
.cra_flags = CRYPTO_ALG_ASYNC,
.cra_blocksize = SHA1_BLOCK_SIZE,
.cra_ctxsize = sizeof(struct hash_ctx),
.cra_type = &crypto_ahash_type,
.cra_init = hash_cra_init,
.cra_module = THIS_MODULE,
}
@ -1600,11 +1595,9 @@ static struct hash_algo_template hash_algs[] = {
.halg.base = {
.cra_name = "hmac(sha256)",
.cra_driver_name = "hmac-sha256-ux500",
.cra_flags = (CRYPTO_ALG_TYPE_AHASH |
CRYPTO_ALG_ASYNC),
.cra_flags = CRYPTO_ALG_ASYNC,
.cra_blocksize = SHA256_BLOCK_SIZE,
.cra_ctxsize = sizeof(struct hash_ctx),
.cra_type = &crypto_ahash_type,
.cra_init = hash_cra_init,
.cra_module = THIS_MODULE,
}

116
drivers/crypto/virtio/virtio_crypto_algs.c
View file

@ -49,12 +49,18 @@ struct virtio_crypto_sym_request {
bool encrypt;
};
struct virtio_crypto_algo {
uint32_t algonum;
uint32_t service;
unsigned int active_devs;
struct crypto_alg algo;
};
/*
* The algs_lock protects the below global virtio_crypto_active_devs
* and crypto algorithms registion.
*/
static DEFINE_MUTEX(algs_lock);
static unsigned int virtio_crypto_active_devs;
static void virtio_crypto_ablkcipher_finalize_req(
struct virtio_crypto_sym_request *vc_sym_req,
struct ablkcipher_request *req,
@ -312,15 +318,21 @@ static int virtio_crypto_ablkcipher_setkey(struct crypto_ablkcipher *tfm,
unsigned int keylen)
{
struct virtio_crypto_ablkcipher_ctx *ctx = crypto_ablkcipher_ctx(tfm);
uint32_t alg;
int ret;
ret = virtio_crypto_alg_validate_key(keylen, &alg);
if (ret)
return ret;
if (!ctx->vcrypto) {
/* New key */
int node = virtio_crypto_get_current_node();
struct virtio_crypto *vcrypto =
virtcrypto_get_dev_node(node);
virtcrypto_get_dev_node(node,
VIRTIO_CRYPTO_SERVICE_CIPHER, alg);
if (!vcrypto) {
pr_err("virtio_crypto: Could not find a virtio device in the system\n");
pr_err("virtio_crypto: Could not find a virtio device in the system or unsupported algo\n");
return -ENODEV;
}
@ -371,12 +383,12 @@ __virtio_crypto_ablkcipher_do_req(struct virtio_crypto_sym_request *vc_sym_req,
/* Why 3? outhdr + iv + inhdr */
sg_total = src_nents + dst_nents + 3;
sgs = kcalloc_node(sg_total, sizeof(*sgs), GFP_ATOMIC,
sgs = kcalloc_node(sg_total, sizeof(*sgs), GFP_KERNEL,
dev_to_node(&vcrypto->vdev->dev));
if (!sgs)
return -ENOMEM;
req_data = kzalloc_node(sizeof(*req_data), GFP_ATOMIC,
req_data = kzalloc_node(sizeof(*req_data), GFP_KERNEL,
dev_to_node(&vcrypto->vdev->dev));
if (!req_data) {
kfree(sgs);
@ -571,57 +583,85 @@ static void virtio_crypto_ablkcipher_finalize_req(
virtcrypto_clear_request(&vc_sym_req->base);
}
static struct crypto_alg virtio_crypto_algs[] = { {
.cra_name = "cbc(aes)",
.cra_driver_name = "virtio_crypto_aes_cbc",
.cra_priority = 150,
.cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
.cra_blocksize = AES_BLOCK_SIZE,
.cra_ctxsize = sizeof(struct virtio_crypto_ablkcipher_ctx),
.cra_alignmask = 0,
.cra_module = THIS_MODULE,
.cra_type = &crypto_ablkcipher_type,
.cra_init = virtio_crypto_ablkcipher_init,
.cra_exit = virtio_crypto_ablkcipher_exit,
.cra_u = {
.ablkcipher = {
.setkey = virtio_crypto_ablkcipher_setkey,
.decrypt = virtio_crypto_ablkcipher_decrypt,
.encrypt = virtio_crypto_ablkcipher_encrypt,
.min_keysize = AES_MIN_KEY_SIZE,
.max_keysize = AES_MAX_KEY_SIZE,
.ivsize = AES_BLOCK_SIZE,
static struct virtio_crypto_algo virtio_crypto_algs[] = { {
.algonum = VIRTIO_CRYPTO_CIPHER_AES_CBC,
.service = VIRTIO_CRYPTO_SERVICE_CIPHER,
.algo = {
.cra_name = "cbc(aes)",
.cra_driver_name = "virtio_crypto_aes_cbc",
.cra_priority = 150,
.cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
.cra_blocksize = AES_BLOCK_SIZE,
.cra_ctxsize = sizeof(struct virtio_crypto_ablkcipher_ctx),
.cra_alignmask = 0,
.cra_module = THIS_MODULE,
.cra_type = &crypto_ablkcipher_type,
.cra_init = virtio_crypto_ablkcipher_init,
.cra_exit = virtio_crypto_ablkcipher_exit,
.cra_u = {
.ablkcipher = {
.setkey = virtio_crypto_ablkcipher_setkey,
.decrypt = virtio_crypto_ablkcipher_decrypt,
.encrypt = virtio_crypto_ablkcipher_encrypt,
.min_keysize = AES_MIN_KEY_SIZE,
.max_keysize = AES_MAX_KEY_SIZE,
.ivsize = AES_BLOCK_SIZE,
},
},
},
} };
int virtio_crypto_algs_register(void)
int virtio_crypto_algs_register(struct virtio_crypto *vcrypto)
{
int ret = 0;
int i = 0;
mutex_lock(&algs_lock);
if (++virtio_crypto_active_devs != 1)
goto unlock;
ret = crypto_register_algs(virtio_crypto_algs,
ARRAY_SIZE(virtio_crypto_algs));
if (ret)
virtio_crypto_active_devs--;
for (i = 0; i < ARRAY_SIZE(virtio_crypto_algs); i++) {
uint32_t service = virtio_crypto_algs[i].service;
uint32_t algonum = virtio_crypto_algs[i].algonum;
if (!virtcrypto_algo_is_supported(vcrypto, service, algonum))
continue;
if (virtio_crypto_algs[i].active_devs == 0) {
ret = crypto_register_alg(&virtio_crypto_algs[i].algo);
if (ret)
goto unlock;
}
virtio_crypto_algs[i].active_devs++;
dev_info(&vcrypto->vdev->dev, "Registered algo %s\n",
virtio_crypto_algs[i].algo.cra_name);
}
unlock:
mutex_unlock(&algs_lock);
return ret;
}
void virtio_crypto_algs_unregister(void)
void virtio_crypto_algs_unregister(struct virtio_crypto *vcrypto)
{
int i = 0;
mutex_lock(&algs_lock);
if (--virtio_crypto_active_devs != 0)
goto unlock;
crypto_unregister_algs(virtio_crypto_algs,
ARRAY_SIZE(virtio_crypto_algs));
for (i = 0; i < ARRAY_SIZE(virtio_crypto_algs); i++) {
uint32_t service = virtio_crypto_algs[i].service;
uint32_t algonum = virtio_crypto_algs[i].algonum;
if (virtio_crypto_algs[i].active_devs == 0 ||
!virtcrypto_algo_is_supported(vcrypto, service, algonum))
continue;
if (virtio_crypto_algs[i].active_devs == 1)
crypto_unregister_alg(&virtio_crypto_algs[i].algo);
virtio_crypto_algs[i].active_devs--;
}
unlock:
mutex_unlock(&algs_lock);
}

25
drivers/crypto/virtio/virtio_crypto_common.h
View file

@ -55,6 +55,20 @@ struct virtio_crypto {
/* Number of queue currently used by the driver */
u32 curr_queue;
/*
* Specifies the services mask which the device support,
* see VIRTIO_CRYPTO_SERVICE_*
*/
u32 crypto_services;
/* Detailed algorithms mask */
u32 cipher_algo_l;
u32 cipher_algo_h;
u32 hash_algo;
u32 mac_algo_l;
u32 mac_algo_h;
u32 aead_algo;
/* Maximum length of cipher key */
u32 max_cipher_key_len;
/* Maximum length of authenticated key */
@ -102,7 +116,12 @@ int virtcrypto_dev_in_use(struct virtio_crypto *vcrypto_dev);
int virtcrypto_dev_get(struct virtio_crypto *vcrypto_dev);
void virtcrypto_dev_put(struct virtio_crypto *vcrypto_dev);
int virtcrypto_dev_started(struct virtio_crypto *vcrypto_dev);
struct virtio_crypto *virtcrypto_get_dev_node(int node);
bool virtcrypto_algo_is_supported(struct virtio_crypto *vcrypto_dev,
uint32_t service,
uint32_t algo);
struct virtio_crypto *virtcrypto_get_dev_node(int node,
uint32_t service,
uint32_t algo);
int virtcrypto_dev_start(struct virtio_crypto *vcrypto);
void virtcrypto_dev_stop(struct virtio_crypto *vcrypto);
int virtio_crypto_ablkcipher_crypt_req(
@ -122,7 +141,7 @@ static inline int virtio_crypto_get_current_node(void)
return node;
}
int virtio_crypto_algs_register(void);
void virtio_crypto_algs_unregister(void);
int virtio_crypto_algs_register(struct virtio_crypto *vcrypto);
void virtio_crypto_algs_unregister(struct virtio_crypto *vcrypto);
#endif /* _VIRTIO_CRYPTO_COMMON_H */

29
drivers/crypto/virtio/virtio_crypto_core.c
View file

@ -303,6 +303,13 @@ static int virtcrypto_probe(struct virtio_device *vdev)
u32 max_data_queues = 0, max_cipher_key_len = 0;
u32 max_auth_key_len = 0;
u64 max_size = 0;
u32 cipher_algo_l = 0;
u32 cipher_algo_h = 0;
u32 hash_algo = 0;
u32 mac_algo_l = 0;
u32 mac_algo_h = 0;
u32 aead_algo = 0;
u32 crypto_services = 0;
if (!virtio_has_feature(vdev, VIRTIO_F_VERSION_1))
return -ENODEV;
@ -339,6 +346,20 @@ static int virtcrypto_probe(struct virtio_device *vdev)
max_auth_key_len, &max_auth_key_len);
virtio_cread(vdev, struct virtio_crypto_config,
max_size, &max_size);
virtio_cread(vdev, struct virtio_crypto_config,
crypto_services, &crypto_services);
virtio_cread(vdev, struct virtio_crypto_config,
cipher_algo_l, &cipher_algo_l);
virtio_cread(vdev, struct virtio_crypto_config,
cipher_algo_h, &cipher_algo_h);
virtio_cread(vdev, struct virtio_crypto_config,
hash_algo, &hash_algo);
virtio_cread(vdev, struct virtio_crypto_config,
mac_algo_l, &mac_algo_l);
virtio_cread(vdev, struct virtio_crypto_config,
mac_algo_h, &mac_algo_h);
virtio_cread(vdev, struct virtio_crypto_config,
aead_algo, &aead_algo);
/* Add virtio crypto device to global table */
err = virtcrypto_devmgr_add_dev(vcrypto);
@ -358,6 +379,14 @@ static int virtcrypto_probe(struct virtio_device *vdev)
vcrypto->max_cipher_key_len = max_cipher_key_len;
vcrypto->max_auth_key_len = max_auth_key_len;
vcrypto->max_size = max_size;
vcrypto->crypto_services = crypto_services;
vcrypto->cipher_algo_l = cipher_algo_l;
vcrypto->cipher_algo_h = cipher_algo_h;
vcrypto->mac_algo_l = mac_algo_l;
vcrypto->mac_algo_h = mac_algo_h;
vcrypto->hash_algo = hash_algo;
vcrypto->aead_algo = aead_algo;
dev_info(&vdev->dev,
"max_queues: %u, max_cipher_key_len: %u, max_auth_key_len: %u, max_size 0x%llx\n",

81
drivers/crypto/virtio/virtio_crypto_mgr.c
View file

@ -181,14 +181,20 @@ int virtcrypto_dev_started(struct virtio_crypto *vcrypto_dev)
/*
* virtcrypto_get_dev_node() - Get vcrypto_dev on the node.
* @node: Node id the driver works.
* @service: Crypto service that needs to be supported by the
* dev
* @algo: The algorithm number that needs to be supported by the
* dev
*
* Function returns the virtio crypto device used fewest on the node.
* Function returns the virtio crypto device used fewest on the node,
* and supports the given crypto service and algorithm.
*
* To be used by virtio crypto device specific drivers.
*
* Return: pointer to vcrypto_dev or NULL if not found.
*/
struct virtio_crypto *virtcrypto_get_dev_node(int node)
struct virtio_crypto *virtcrypto_get_dev_node(int node, uint32_t service,
uint32_t algo)
{
struct virtio_crypto *vcrypto_dev = NULL, *tmp_dev;
unsigned long best = ~0;
@ -199,7 +205,8 @@ struct virtio_crypto *virtcrypto_get_dev_node(int node)
if ((node == dev_to_node(&tmp_dev->vdev->dev) ||
dev_to_node(&tmp_dev->vdev->dev) < 0) &&
virtcrypto_dev_started(tmp_dev)) {
virtcrypto_dev_started(tmp_dev) &&
virtcrypto_algo_is_supported(tmp_dev, service, algo)) {
ctr = atomic_read(&tmp_dev->ref_count);
if (best > ctr) {
vcrypto_dev = tmp_dev;
@ -214,7 +221,9 @@ struct virtio_crypto *virtcrypto_get_dev_node(int node)
/* Get any started device */
list_for_each_entry(tmp_dev,
virtcrypto_devmgr_get_head(), list) {
if (virtcrypto_dev_started(tmp_dev)) {
if (virtcrypto_dev_started(tmp_dev) &&
virtcrypto_algo_is_supported(tmp_dev,
service, algo)) {
vcrypto_dev = tmp_dev;
break;
}
@ -240,7 +249,7 @@ struct virtio_crypto *virtcrypto_get_dev_node(int node)
*/
int virtcrypto_dev_start(struct virtio_crypto *vcrypto)
{
if (virtio_crypto_algs_register()) {
if (virtio_crypto_algs_register(vcrypto)) {
pr_err("virtio_crypto: Failed to register crypto algs\n");
return -EFAULT;
}
@ -260,5 +269,65 @@ int virtcrypto_dev_start(struct virtio_crypto *vcrypto)
*/
void virtcrypto_dev_stop(struct virtio_crypto *vcrypto)
{
virtio_crypto_algs_unregister();
virtio_crypto_algs_unregister(vcrypto);
}
/*
* vcrypto_algo_is_supported()
* @vcrypto: Pointer to virtio crypto device.
* @service: The bit number for service validate.
* See VIRTIO_CRYPTO_SERVICE_*
* @algo : The bit number for the algorithm to validate.
*
*
* Validate if the virtio crypto device supports a service and
* algo.
*
* Return true if device supports a service and algo.
*/
bool virtcrypto_algo_is_supported(struct virtio_crypto *vcrypto,
uint32_t service,
uint32_t algo)
{
uint32_t service_mask = 1u << service;
uint32_t algo_mask = 0;
bool low = true;
if (algo > 31) {
algo -= 32;
low = false;
}
if (!(vcrypto->crypto_services & service_mask))
return false;
switch (service) {
case VIRTIO_CRYPTO_SERVICE_CIPHER:
if (low)
algo_mask = vcrypto->cipher_algo_l;
else
algo_mask = vcrypto->cipher_algo_h;
break;
case VIRTIO_CRYPTO_SERVICE_HASH:
algo_mask = vcrypto->hash_algo;
break;
case VIRTIO_CRYPTO_SERVICE_MAC:
if (low)
algo_mask = vcrypto->mac_algo_l;
else
algo_mask = vcrypto->mac_algo_h;
break;
case VIRTIO_CRYPTO_SERVICE_AEAD:
algo_mask = vcrypto->aead_algo;
break;
}
if (!(algo_mask & (1u << algo)))
return false;
return true;
}

2
drivers/crypto/vmx/ghash.c
View file

@ -215,7 +215,7 @@ struct shash_alg p8_ghash_alg = {
.cra_name = "ghash",
.cra_driver_name = "p8_ghash",
.cra_priority = 1000,
.cra_flags = CRYPTO_ALG_TYPE_SHASH | CRYPTO_ALG_NEED_FALLBACK,
.cra_flags = CRYPTO_ALG_NEED_FALLBACK,
.cra_blocksize = GHASH_BLOCK_SIZE,
.cra_ctxsize = sizeof(struct p8_ghash_ctx),
.cra_module = THIS_MODULE,