A-Team_Digital_Solutions/android_kernel_motorola_sm6375
Watch
2
0
Fork
You've already forked android_kernel_motorola_sm6375
 0
mirror of https://github.com/BobTheBlinker/android_kernel_motorola_sm6375.git synced 2026-10-28 10:41:38 -04:00
Code Issues Projects Releases Packages Wiki Activity

Merge branch 'linus' of git://git.kernel.org/pub/scm/linux/kernel/git/herbert/crypto-2.6

Browse source 
Pull crypto updates from Herbert Xu:
 "API:
   - Add the ability to abort a skcipher walk.

  Algorithms:
   - Fix XTS to actually do the stealing.
   - Add library helpers for AES and DES for single-block users.
   - Add library helpers for SHA256.
   - Add new DES key verification helper.
   - Add surrounding bits for ESSIV generator.
   - Add accelerations for aegis128.
   - Add test vectors for lzo-rle.

  Drivers:
   - Add i.MX8MQ support to caam.
   - Add gcm/ccm/cfb/ofb aes support in inside-secure.
   - Add ofb/cfb aes support in media-tek.
   - Add HiSilicon ZIP accelerator support.

  Others:
   - Fix potential race condition in padata.
   - Use unbound workqueues in padata"

* 'linus' of git://git.kernel.org/pub/scm/linux/kernel/git/herbert/crypto-2.6: (311 commits)
  crypto: caam - Cast to long first before pointer conversion
  crypto: ccree - enable CTS support in AES-XTS
  crypto: inside-secure - Probe transform record cache RAM sizes
  crypto: inside-secure - Base RD fetchcount on actual RD FIFO size
  crypto: inside-secure - Base CD fetchcount on actual CD FIFO size
  crypto: inside-secure - Enable extended algorithms on newer HW
  crypto: inside-secure: Corrected configuration of EIP96_TOKEN_CTRL
  crypto: inside-secure - Add EIP97/EIP197 and endianness detection
  padata: remove cpu_index from the parallel_queue
  padata: unbind parallel jobs from specific CPUs
  padata: use separate workqueues for parallel and serial work
  padata, pcrypt: take CPU hotplug lock internally in padata_alloc_possible
  crypto: pcrypt - remove padata cpumask notifier
  padata: make padata_do_parallel find alternate callback CPU
  workqueue: require CPU hotplug read exclusion for apply_workqueue_attrs
  workqueue: unconfine alloc/apply/free_workqueue_attrs()
  padata: allocate workqueue internally
  arm64: dts: imx8mq: Add CAAM node
  random: Use wait_event_freezable() in add_hwgenerator_randomness()
  crypto: ux500 - Fix COMPILE_TEST warnings
  ...
This commit is contained in:
Linus Torvalds 2019-09-18 12:11:14 -07:00
commit 8b53c76533
297 changed files with 15605 additions and 18397 deletions
Download patch file Download diff file Expand all files Collapse all files

56
drivers/crypto/Kconfig
View file

@ -27,7 +27,7 @@ config CRYPTO_DEV_PADLOCK_AES
tristate "PadLock driver for AES algorithm"
depends on CRYPTO_DEV_PADLOCK
select CRYPTO_BLKCIPHER
select CRYPTO_AES
select CRYPTO_LIB_AES
help
Use VIA PadLock for AES algorithm.
@ -170,7 +170,7 @@ config CRYPTO_DES_S390
depends on S390
select CRYPTO_ALGAPI
select CRYPTO_BLKCIPHER
select CRYPTO_DES
select CRYPTO_LIB_DES
help
This is the s390 hardware accelerated implementation of the
DES cipher algorithm (FIPS 46-2), and Triple DES EDE (FIPS 46-3).
@ -209,12 +209,12 @@ config S390_PRNG
It is available as of z9.
config CRYPTO_GHASH_S390
tristate "GHASH digest algorithm"
tristate "GHASH hash function"
depends on S390
select CRYPTO_HASH
help
This is the s390 hardware accelerated implementation of the
GHASH message digest algorithm for GCM (Galois/Counter Mode).
This is the s390 hardware accelerated implementation of GHASH,
the hash function used in GCM (Galois/Counter mode).
It is available as of z196.
@ -234,8 +234,8 @@ config CRYPTO_CRC32_S390
config CRYPTO_DEV_MARVELL_CESA
tristate "Marvell's Cryptographic Engine driver"
depends on PLAT_ORION || ARCH_MVEBU
select CRYPTO_AES
select CRYPTO_DES
select CRYPTO_LIB_AES
select CRYPTO_LIB_DES
select CRYPTO_BLKCIPHER
select CRYPTO_HASH
select SRAM
@ -247,7 +247,7 @@ config CRYPTO_DEV_MARVELL_CESA
config CRYPTO_DEV_NIAGARA2
tristate "Niagara2 Stream Processing Unit driver"
select CRYPTO_DES
select CRYPTO_LIB_DES
select CRYPTO_BLKCIPHER
select CRYPTO_HASH
select CRYPTO_MD5
@ -264,7 +264,7 @@ config CRYPTO_DEV_NIAGARA2
config CRYPTO_DEV_HIFN_795X
tristate "Driver HIFN 795x crypto accelerator chips"
select CRYPTO_DES
select CRYPTO_LIB_DES
select CRYPTO_BLKCIPHER
select HW_RANDOM if CRYPTO_DEV_HIFN_795X_RNG
depends on PCI
@ -320,7 +320,7 @@ config CRYPTO_DEV_TALITOS2
config CRYPTO_DEV_IXP4XX
tristate "Driver for IXP4xx crypto hardware acceleration"
depends on ARCH_IXP4XX && IXP4XX_QMGR && IXP4XX_NPE
select CRYPTO_DES
select CRYPTO_LIB_DES
select CRYPTO_AEAD
select CRYPTO_AUTHENC
select CRYPTO_BLKCIPHER
@ -332,7 +332,7 @@ config CRYPTO_DEV_PPC4XX
depends on PPC && 4xx
select CRYPTO_HASH
select CRYPTO_AEAD
select CRYPTO_AES
select CRYPTO_LIB_AES
select CRYPTO_CCM
select CRYPTO_CTR
select CRYPTO_GCM
@ -386,7 +386,7 @@ config CRYPTO_DEV_OMAP_AES
config CRYPTO_DEV_OMAP_DES
tristate "Support for OMAP DES/3DES hw engine"
depends on ARCH_OMAP2PLUS
select CRYPTO_DES
select CRYPTO_LIB_DES
select CRYPTO_BLKCIPHER
select CRYPTO_ENGINE
help
@ -404,7 +404,7 @@ config CRYPTO_DEV_PICOXCELL
select CRYPTO_AES
select CRYPTO_AUTHENC
select CRYPTO_BLKCIPHER
select CRYPTO_DES
select CRYPTO_LIB_DES
select CRYPTO_CBC
select CRYPTO_ECB
select CRYPTO_SEQIV
@ -413,7 +413,7 @@ config CRYPTO_DEV_PICOXCELL
Picochip picoXcell SoC devices. Select this for IPSEC ESP offload
and for 3gpp Layer 2 ciphering support.
Saying m here will build a module named pipcoxcell_crypto.
Saying m here will build a module named picoxcell_crypto.
config CRYPTO_DEV_SAHARA
tristate "Support for SAHARA crypto accelerator"
@ -517,7 +517,7 @@ config CRYPTO_DEV_ATMEL_AES
config CRYPTO_DEV_ATMEL_TDES
tristate "Support for Atmel DES/TDES hw accelerator"
depends on ARCH_AT91 || COMPILE_TEST
select CRYPTO_DES
select CRYPTO_LIB_DES
select CRYPTO_BLKCIPHER
help
Some Atmel processors have DES/TDES hw accelerator.
@ -615,7 +615,7 @@ config CRYPTO_DEV_QCE
depends on ARCH_QCOM || COMPILE_TEST
depends on HAS_IOMEM
select CRYPTO_AES
select CRYPTO_DES
select CRYPTO_LIB_DES
select CRYPTO_ECB
select CRYPTO_CBC
select CRYPTO_XTS
@ -663,7 +663,7 @@ config CRYPTO_DEV_SUN4I_SS
select CRYPTO_MD5
select CRYPTO_SHA1
select CRYPTO_AES
select CRYPTO_DES
select CRYPTO_LIB_DES
select CRYPTO_BLKCIPHER
help
Some Allwinner SoC have a crypto accelerator named
@ -686,7 +686,7 @@ config CRYPTO_DEV_ROCKCHIP
tristate "Rockchip's Cryptographic Engine driver"
depends on OF && ARCH_ROCKCHIP
select CRYPTO_AES
select CRYPTO_DES
select CRYPTO_LIB_DES
select CRYPTO_MD5
select CRYPTO_SHA1
select CRYPTO_SHA256
@ -723,7 +723,7 @@ config CRYPTO_DEV_BCM_SPU
depends on MAILBOX
default m
select CRYPTO_AUTHENC
select CRYPTO_DES
select CRYPTO_LIB_DES
select CRYPTO_MD5
select CRYPTO_SHA1
select CRYPTO_SHA256
@ -737,12 +737,11 @@ source "drivers/crypto/stm32/Kconfig"
config CRYPTO_DEV_SAFEXCEL
tristate "Inside Secure's SafeXcel cryptographic engine driver"
depends on OF
depends on (ARM64 && ARCH_MVEBU) || (COMPILE_TEST && 64BIT)
select CRYPTO_AES
depends on OF || PCI || COMPILE_TEST
select CRYPTO_LIB_AES
select CRYPTO_AUTHENC
select CRYPTO_BLKCIPHER
select CRYPTO_DES
select CRYPTO_LIB_DES
select CRYPTO_HASH
select CRYPTO_HMAC
select CRYPTO_MD5
@ -750,10 +749,11 @@ config CRYPTO_DEV_SAFEXCEL
select CRYPTO_SHA256
select CRYPTO_SHA512
help
This driver interfaces with the SafeXcel EIP-197 cryptographic engine
designed by Inside Secure. Select this if you want to use CBC/ECB
chain mode, AES cipher mode and SHA1/SHA224/SHA256/SHA512 hash
algorithms.
This driver interfaces with the SafeXcel EIP-97 and EIP-197 cryptographic
engines designed by Inside Secure. It currently accelerates DES, 3DES and
AES block ciphers in ECB and CBC mode, as well as SHA1, SHA224, SHA256,
SHA384 and SHA512 hash algorithms for both basic hash and HMAC.
Additionally, it accelerates combined AES-CBC/HMAC-SHA AEAD operations.
config CRYPTO_DEV_ARTPEC6
tristate "Support for Axis ARTPEC-6/7 hardware crypto acceleration."
@ -780,7 +780,7 @@ config CRYPTO_DEV_CCREE
default n
select CRYPTO_HASH
select CRYPTO_BLKCIPHER
select CRYPTO_DES
select CRYPTO_LIB_DES
select CRYPTO_AEAD
select CRYPTO_AUTHENC
select CRYPTO_SHA1

24
drivers/crypto/amcc/crypto4xx_alg.c
View file

@ -527,28 +527,20 @@ static int crypto4xx_aes_gcm_validate_keylen(unsigned int keylen)
static int crypto4xx_compute_gcm_hash_key_sw(__le32 *hash_start, const u8 *key,
unsigned int keylen)
{
struct crypto_cipher *aes_tfm = NULL;
struct crypto_aes_ctx ctx;
uint8_t src[16] = { 0 };
int rc = 0;
int rc;
aes_tfm = crypto_alloc_cipher("aes", 0, CRYPTO_ALG_NEED_FALLBACK);
if (IS_ERR(aes_tfm)) {
rc = PTR_ERR(aes_tfm);
pr_warn("could not load aes cipher driver: %d\n", rc);
rc = aes_expandkey(&ctx, key, keylen);
if (rc) {
pr_err("aes_expandkey() failed: %d\n", rc);
return rc;
}
rc = crypto_cipher_setkey(aes_tfm, key, keylen);
if (rc) {
pr_err("setkey() failed: %d\n", rc);
goto out;
}
crypto_cipher_encrypt_one(aes_tfm, src, src);
aes_encrypt(&ctx, src, src);
crypto4xx_memcpy_to_le32(hash_start, src, 16);
out:
crypto_free_cipher(aes_tfm);
return rc;
memzero_explicit(&ctx, sizeof(ctx));
return 0;
}
int crypto4xx_setkey_aes_gcm(struct crypto_aead *cipher,

1
drivers/crypto/atmel-aes.c
View file

@ -2673,7 +2673,6 @@ static int atmel_aes_probe(struct platform_device *pdev)
/* Get the IRQ */
aes_dd->irq = platform_get_irq(pdev, 0);
if (aes_dd->irq < 0) {
dev_err(dev, "no IRQ resource info\n");
err = aes_dd->irq;
goto res_err;
}

12
drivers/crypto/atmel-i2c.c
View file

@ -21,6 +21,18 @@
#include <linux/workqueue.h>
#include "atmel-i2c.h"
static const struct {
u8 value;
const char *error_text;
} error_list[] = {
{ 0x01, "CheckMac or Verify miscompare" },
{ 0x03, "Parse Error" },
{ 0x05, "ECC Fault" },
{ 0x0F, "Execution Error" },
{ 0xEE, "Watchdog about to expire" },
{ 0xFF, "CRC or other communication error" },
};
/**
* atmel_i2c_checksum() - Generate 16-bit CRC as required by ATMEL ECC.
* CRC16 verification of the count, opcode, param1, param2 and data bytes.

12
drivers/crypto/atmel-i2c.h
View file

@ -62,18 +62,6 @@ struct atmel_i2c_cmd {
#define STATUS_NOERR 0x00
#define STATUS_WAKE_SUCCESSFUL 0x11
static const struct {
u8 value;
const char *error_text;
} error_list[] = {
{ 0x01, "CheckMac or Verify miscompare" },
{ 0x03, "Parse Error" },
{ 0x05, "ECC Fault" },
{ 0x0F, "Execution Error" },
{ 0xEE, "Watchdog about to expire" },
{ 0xFF, "CRC or other communication error" },
};
/* Definitions for eeprom organization */
#define CONFIG_ZONE 0

1
drivers/crypto/atmel-sha.c
View file

@ -2779,7 +2779,6 @@ static int atmel_sha_probe(struct platform_device *pdev)
/* Get the IRQ */
sha_dd->irq = platform_get_irq(pdev, 0);
if (sha_dd->irq < 0) {
dev_err(dev, "no IRQ resource info\n");
err = sha_dd->irq;
goto res_err;
}

3
drivers/crypto/atmel-sha204a.c
View file

@ -109,7 +109,7 @@ static int atmel_sha204a_probe(struct i2c_client *client,
i2c_priv->hwrng.read = atmel_sha204a_rng_read;
i2c_priv->hwrng.quality = 1024;
ret = hwrng_register(&i2c_priv->hwrng);
ret = devm_hwrng_register(&client->dev, &i2c_priv->hwrng);
if (ret)
dev_warn(&client->dev, "failed to register RNG (%d)\n", ret);
@ -127,7 +127,6 @@ static int atmel_sha204a_remove(struct i2c_client *client)
if (i2c_priv->hwrng.priv)
kfree((void *)i2c_priv->hwrng.priv);
hwrng_unregister(&i2c_priv->hwrng);
return 0;
}

29
drivers/crypto/atmel-tdes.c
View file

@ -33,7 +33,7 @@
#include <linux/cryptohash.h>
#include <crypto/scatterwalk.h>
#include <crypto/algapi.h>
#include <crypto/des.h>
#include <crypto/internal/des.h>
#include <crypto/hash.h>
#include <crypto/internal/hash.h>
#include <linux/platform_data/crypto-atmel.h>
@ -773,22 +773,12 @@ static void atmel_tdes_dma_cleanup(struct atmel_tdes_dev *dd)
static int atmel_des_setkey(struct crypto_ablkcipher *tfm, const u8 *key,
unsigned int keylen)
{
u32 tmp[DES_EXPKEY_WORDS];
int err;
struct crypto_tfm *ctfm = crypto_ablkcipher_tfm(tfm);
struct atmel_tdes_ctx *ctx = crypto_ablkcipher_ctx(tfm);
int err;
if (keylen != DES_KEY_SIZE) {
crypto_ablkcipher_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN);
return -EINVAL;
}
err = des_ekey(tmp, key);
if (err == 0 && (ctfm->crt_flags & CRYPTO_TFM_REQ_FORBID_WEAK_KEYS)) {
ctfm->crt_flags |= CRYPTO_TFM_RES_WEAK_KEY;
return -EINVAL;
}
err = verify_ablkcipher_des_key(tfm, key);
if (err)
return err;
memcpy(ctx->key, key, keylen);
ctx->keylen = keylen;
@ -800,15 +790,11 @@ static int atmel_tdes_setkey(struct crypto_ablkcipher *tfm, const u8 *key,
unsigned int keylen)
{
struct atmel_tdes_ctx *ctx = crypto_ablkcipher_ctx(tfm);
u32 flags;
int err;
flags = crypto_ablkcipher_get_flags(tfm);
err = __des3_verify_key(&flags, key);
if (unlikely(err)) {
crypto_ablkcipher_set_flags(tfm, flags);
err = verify_ablkcipher_des3_key(tfm, key);
if (err)
return err;
}
memcpy(ctx->key, key, keylen);
ctx->keylen = keylen;
@ -1281,7 +1267,6 @@ static int atmel_tdes_probe(struct platform_device *pdev)
/* Get the IRQ */
tdes_dd->irq = platform_get_irq(pdev, 0);
if (tdes_dd->irq < 0) {
dev_err(dev, "no IRQ resource info\n");
err = tdes_dd->irq;
goto res_err;
}

4
drivers/crypto/axis/artpec6_crypto.c
View file

@ -2854,7 +2854,6 @@ static int artpec6_crypto_probe(struct platform_device *pdev)
struct artpec6_crypto *ac;
struct device *dev = &pdev->dev;
void __iomem *base;
struct resource *res;
int irq;
int err;
@ -2867,8 +2866,7 @@ static int artpec6_crypto_probe(struct platform_device *pdev)
variant = (enum artpec6_crypto_variant)match->data;
res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
base = devm_ioremap_resource(&pdev->dev, res);
base = devm_platform_ioremap_resource(pdev, 0);
if (IS_ERR(base))
return PTR_ERR(base);

92
drivers/crypto/bcm/cipher.c
View file

@ -24,7 +24,7 @@
#include <crypto/aead.h>
#include <crypto/internal/aead.h>
#include <crypto/aes.h>
#include <crypto/des.h>
#include <crypto/internal/des.h>
#include <crypto/hmac.h>
#include <crypto/sha.h>
#include <crypto/md5.h>
@ -1802,24 +1802,13 @@ static int des_setkey(struct crypto_ablkcipher *cipher, const u8 *key,
unsigned int keylen)
{
struct iproc_ctx_s *ctx = crypto_ablkcipher_ctx(cipher);
u32 tmp[DES_EXPKEY_WORDS];
int err;
if (keylen == DES_KEY_SIZE) {
if (des_ekey(tmp, key) == 0) {
if (crypto_ablkcipher_get_flags(cipher) &
CRYPTO_TFM_REQ_FORBID_WEAK_KEYS) {
u32 flags = CRYPTO_TFM_RES_WEAK_KEY;
err = verify_ablkcipher_des_key(cipher, key);
if (err)
return err;
crypto_ablkcipher_set_flags(cipher, flags);
return -EINVAL;
}
}
ctx->cipher_type = CIPHER_TYPE_DES;
} else {
crypto_ablkcipher_set_flags(cipher, CRYPTO_TFM_RES_BAD_KEY_LEN);
return -EINVAL;
}
ctx->cipher_type = CIPHER_TYPE_DES;
return 0;
}
@ -1827,23 +1816,13 @@ static int threedes_setkey(struct crypto_ablkcipher *cipher, const u8 *key,
unsigned int keylen)
{
struct iproc_ctx_s *ctx = crypto_ablkcipher_ctx(cipher);
int err;
if (keylen == (DES_KEY_SIZE * 3)) {
u32 flags;
int ret;
err = verify_ablkcipher_des3_key(cipher, key);
if (err)
return err;
flags = crypto_ablkcipher_get_flags(cipher);
ret = __des3_verify_key(&flags, key);
if (unlikely(ret)) {
crypto_ablkcipher_set_flags(cipher, flags);
return ret;
}
ctx->cipher_type = CIPHER_TYPE_3DES;
} else {
crypto_ablkcipher_set_flags(cipher, CRYPTO_TFM_RES_BAD_KEY_LEN);
return -EINVAL;
}
ctx->cipher_type = CIPHER_TYPE_3DES;
return 0;
}
@ -2629,6 +2608,19 @@ static int aead_need_fallback(struct aead_request *req)
return 1;
}
/*
* RFC4106 and RFC4543 cannot handle the case where AAD is other than
* 16 or 20 bytes long. So use fallback in this case.
*/
if (ctx->cipher.mode == CIPHER_MODE_GCM &&
ctx->cipher.alg == CIPHER_ALG_AES &&
rctx->iv_ctr_len == GCM_RFC4106_IV_SIZE &&
req->assoclen != 16 && req->assoclen != 20) {
flow_log("RFC4106/RFC4543 needs fallback for assoclen"
" other than 16 or 20 bytes\n");
return 1;
}
payload_len = req->cryptlen;
if (spu->spu_type == SPU_TYPE_SPUM)
payload_len += req->assoclen;
@ -2855,40 +2847,16 @@ static int aead_authenc_setkey(struct crypto_aead *cipher,
switch (ctx->alg->cipher_info.alg) {
case CIPHER_ALG_DES:
if (ctx->enckeylen == DES_KEY_SIZE) {
u32 tmp[DES_EXPKEY_WORDS];
u32 flags = CRYPTO_TFM_RES_WEAK_KEY;
if (verify_aead_des_key(cipher, keys.enckey, keys.enckeylen))
return -EINVAL;
if (des_ekey(tmp, keys.enckey) == 0) {
if (crypto_aead_get_flags(cipher) &
CRYPTO_TFM_REQ_FORBID_WEAK_KEYS) {
crypto_aead_set_flags(cipher, flags);
return -EINVAL;
}
}
ctx->cipher_type = CIPHER_TYPE_DES;
} else {
goto badkey;
}
ctx->cipher_type = CIPHER_TYPE_DES;
break;
case CIPHER_ALG_3DES:
if (ctx->enckeylen == (DES_KEY_SIZE * 3)) {
u32 flags;
flags = crypto_aead_get_flags(cipher);
ret = __des3_verify_key(&flags, keys.enckey);
if (unlikely(ret)) {
crypto_aead_set_flags(cipher, flags);
return ret;
}
ctx->cipher_type = CIPHER_TYPE_3DES;
} else {
crypto_aead_set_flags(cipher,
CRYPTO_TFM_RES_BAD_KEY_LEN);
if (verify_aead_des3_key(cipher, keys.enckey, keys.enckeylen))
return -EINVAL;
}
ctx->cipher_type = CIPHER_TYPE_3DES;
break;
case CIPHER_ALG_AES:
switch (ctx->enckeylen) {

4
drivers/crypto/caam/Kconfig
View file

@ -98,7 +98,7 @@ config CRYPTO_DEV_FSL_CAAM_CRYPTO_API
select CRYPTO_AEAD
select CRYPTO_AUTHENC
select CRYPTO_BLKCIPHER
select CRYPTO_DES
select CRYPTO_LIB_DES
help
Selecting this will offload crypto for users of the
scatterlist crypto API (such as the linux native IPSec
@ -111,6 +111,7 @@ config CRYPTO_DEV_FSL_CAAM_CRYPTO_API_QI
select CRYPTO_DEV_FSL_CAAM_CRYPTO_API_DESC
select CRYPTO_AUTHENC
select CRYPTO_BLKCIPHER
select CRYPTO_DES
help
Selecting this will use CAAM Queue Interface (QI) for sending
& receiving crypto jobs to/from CAAM. This gives better performance
@ -161,6 +162,7 @@ config CRYPTO_DEV_FSL_DPAA2_CAAM
select CRYPTO_AUTHENC
select CRYPTO_AEAD
select CRYPTO_HASH
select CRYPTO_DES
help
CAAM driver for QorIQ Data Path Acceleration Architecture 2.
It handles DPSECI DPAA2 objects that sit on the Management Complex

1
drivers/crypto/caam/Makefile
View file

@ -30,3 +30,4 @@ endif
obj-$(CONFIG_CRYPTO_DEV_FSL_DPAA2_CAAM) += dpaa2_caam.o
dpaa2_caam-y := caamalg_qi2.o dpseci.o
dpaa2_caam-$(CONFIG_DEBUG_FS) += dpseci-debugfs.o

276
drivers/crypto/caam/caamalg.c
View file

@ -74,7 +74,7 @@
#define CHACHAPOLY_DESC_JOB_IO_LEN (AEAD_DESC_JOB_IO_LEN + CAAM_CMD_SZ * 6)
#define DESC_MAX_USED_BYTES (CAAM_DESC_BYTES_MAX - DESC_JOB_IO_LEN)
#define DESC_MAX_USED_BYTES (CAAM_DESC_BYTES_MAX - DESC_JOB_IO_LEN_MIN)
#define DESC_MAX_USED_LEN (DESC_MAX_USED_BYTES / CAAM_CMD_SZ)
struct caam_alg_entry {
@ -205,6 +205,18 @@ static int aead_set_sh_desc(struct crypto_aead *aead)
ctx->cdata.keylen - CTR_RFC3686_NONCE_SIZE);
}
/*
* In case |user key| > |derived key|, using DKP<imm,imm>
* would result in invalid opcodes (last bytes of user key) in
* the resulting descriptor. Use DKP<ptr,imm> instead => both
* virtual and dma key addresses are needed.
*/
ctx->adata.key_virt = ctx->key;
ctx->adata.key_dma = ctx->key_dma;
ctx->cdata.key_virt = ctx->key + ctx->adata.keylen_pad;
ctx->cdata.key_dma = ctx->key_dma + ctx->adata.keylen_pad;
data_len[0] = ctx->adata.keylen_pad;
data_len[1] = ctx->cdata.keylen;
@ -221,16 +233,6 @@ static int aead_set_sh_desc(struct crypto_aead *aead)
ARRAY_SIZE(data_len)) < 0)
return -EINVAL;
if (inl_mask & 1)
ctx->adata.key_virt = ctx->key;
else
ctx->adata.key_dma = ctx->key_dma;
if (inl_mask & 2)
ctx->cdata.key_virt = ctx->key + ctx->adata.keylen_pad;
else
ctx->cdata.key_dma = ctx->key_dma + ctx->adata.keylen_pad;
ctx->adata.key_inline = !!(inl_mask & 1);
ctx->cdata.key_inline = !!(inl_mask & 2);
@ -253,16 +255,6 @@ skip_enc:
ARRAY_SIZE(data_len)) < 0)
return -EINVAL;
if (inl_mask & 1)
ctx->adata.key_virt = ctx->key;
else
ctx->adata.key_dma = ctx->key_dma;
if (inl_mask & 2)
ctx->cdata.key_virt = ctx->key + ctx->adata.keylen_pad;
else
ctx->cdata.key_dma = ctx->key_dma + ctx->adata.keylen_pad;
ctx->adata.key_inline = !!(inl_mask & 1);
ctx->cdata.key_inline = !!(inl_mask & 2);
@ -287,16 +279,6 @@ skip_enc:
ARRAY_SIZE(data_len)) < 0)
return -EINVAL;
if (inl_mask & 1)
ctx->adata.key_virt = ctx->key;
else
ctx->adata.key_dma = ctx->key_dma;
if (inl_mask & 2)
ctx->cdata.key_virt = ctx->key + ctx->adata.keylen_pad;
else
ctx->cdata.key_dma = ctx->key_dma + ctx->adata.keylen_pad;
ctx->adata.key_inline = !!(inl_mask & 1);
ctx->cdata.key_inline = !!(inl_mask & 2);
@ -376,6 +358,11 @@ static int gcm_set_sh_desc(struct crypto_aead *aead)
static int gcm_setauthsize(struct crypto_aead *authenc, unsigned int authsize)
{
struct caam_ctx *ctx = crypto_aead_ctx(authenc);
int err;
err = crypto_gcm_check_authsize(authsize);
if (err)
return err;
ctx->authsize = authsize;
gcm_set_sh_desc(authenc);
@ -439,6 +426,11 @@ static int rfc4106_setauthsize(struct crypto_aead *authenc,
unsigned int authsize)
{
struct caam_ctx *ctx = crypto_aead_ctx(authenc);
int err;
err = crypto_rfc4106_check_authsize(authsize);
if (err)
return err;
ctx->authsize = authsize;
rfc4106_set_sh_desc(authenc);
@ -503,6 +495,9 @@ static int rfc4543_setauthsize(struct crypto_aead *authenc,
{
struct caam_ctx *ctx = crypto_aead_ctx(authenc);
if (authsize != 16)
return -EINVAL;
ctx->authsize = authsize;
rfc4543_set_sh_desc(authenc);
@ -633,33 +628,17 @@ static int des3_aead_setkey(struct crypto_aead *aead, const u8 *key,
unsigned int keylen)
{
struct crypto_authenc_keys keys;
u32 flags;
int err;
err = crypto_authenc_extractkeys(&keys, key, keylen);
if (unlikely(err))
goto badkey;
return err;
err = -EINVAL;
if (keys.enckeylen != DES3_EDE_KEY_SIZE)
goto badkey;
err = verify_aead_des3_key(aead, keys.enckey, keys.enckeylen) ?:
aead_setkey(aead, key, keylen);
flags = crypto_aead_get_flags(aead);
err = __des3_verify_key(&flags, keys.enckey);
if (unlikely(err)) {
crypto_aead_set_flags(aead, flags);
goto out;
}
err = aead_setkey(aead, key, keylen);
out:
memzero_explicit(&keys, sizeof(keys));
return err;
badkey:
crypto_aead_set_flags(aead, CRYPTO_TFM_RES_BAD_KEY_LEN);
goto out;
}
static int gcm_setkey(struct crypto_aead *aead,
@ -667,6 +646,13 @@ static int gcm_setkey(struct crypto_aead *aead,
{
struct caam_ctx *ctx = crypto_aead_ctx(aead);
struct device *jrdev = ctx->jrdev;
int err;
err = aes_check_keylen(keylen);
if (err) {
crypto_aead_set_flags(aead, CRYPTO_TFM_RES_BAD_KEY_LEN);
return err;
}
print_hex_dump_debug("key in @"__stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
@ -683,9 +669,13 @@ static int rfc4106_setkey(struct crypto_aead *aead,
{
struct caam_ctx *ctx = crypto_aead_ctx(aead);
struct device *jrdev = ctx->jrdev;
int err;
if (keylen < 4)
return -EINVAL;
err = aes_check_keylen(keylen - 4);
if (err) {
crypto_aead_set_flags(aead, CRYPTO_TFM_RES_BAD_KEY_LEN);
return err;
}
print_hex_dump_debug("key in @"__stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
@ -707,9 +697,13 @@ static int rfc4543_setkey(struct crypto_aead *aead,
{
struct caam_ctx *ctx = crypto_aead_ctx(aead);
struct device *jrdev = ctx->jrdev;
int err;
if (keylen < 4)
return -EINVAL;
err = aes_check_keylen(keylen - 4);
if (err) {
crypto_aead_set_flags(aead, CRYPTO_TFM_RES_BAD_KEY_LEN);
return err;
}
print_hex_dump_debug("key in @"__stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
@ -727,7 +721,7 @@ static int rfc4543_setkey(struct crypto_aead *aead,
}
static int skcipher_setkey(struct crypto_skcipher *skcipher, const u8 *key,
unsigned int keylen)
unsigned int keylen, const u32 ctx1_iv_off)
{
struct caam_ctx *ctx = crypto_skcipher_ctx(skcipher);
struct caam_skcipher_alg *alg =
@ -736,30 +730,10 @@ static int skcipher_setkey(struct crypto_skcipher *skcipher, const u8 *key,
struct device *jrdev = ctx->jrdev;
unsigned int ivsize = crypto_skcipher_ivsize(skcipher);
u32 *desc;
u32 ctx1_iv_off = 0;
const bool ctr_mode = ((ctx->cdata.algtype & OP_ALG_AAI_MASK) ==
OP_ALG_AAI_CTR_MOD128);
const bool is_rfc3686 = alg->caam.rfc3686;
print_hex_dump_debug("key in @"__stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
/*
* AES-CTR needs to load IV in CONTEXT1 reg
* at an offset of 128bits (16bytes)
* CONTEXT1[255:128] = IV
*/
if (ctr_mode)
ctx1_iv_off = 16;
/*
* RFC3686 specific:
* | CONTEXT1[255:128] = {NONCE, IV, COUNTER}
* | *key = {KEY, NONCE}
*/
if (is_rfc3686) {
ctx1_iv_off = 16 + CTR_RFC3686_NONCE_SIZE;
keylen -= CTR_RFC3686_NONCE_SIZE;
}
ctx->cdata.keylen = keylen;
ctx->cdata.key_virt = key;
@ -782,25 +756,86 @@ static int skcipher_setkey(struct crypto_skcipher *skcipher, const u8 *key,
return 0;
}
static int aes_skcipher_setkey(struct crypto_skcipher *skcipher,
const u8 *key, unsigned int keylen)
{
int err;
err = aes_check_keylen(keylen);
if (err) {
crypto_skcipher_set_flags(skcipher,
CRYPTO_TFM_RES_BAD_KEY_LEN);
return err;
}
return skcipher_setkey(skcipher, key, keylen, 0);
}
static int rfc3686_skcipher_setkey(struct crypto_skcipher *skcipher,
const u8 *key, unsigned int keylen)
{
u32 ctx1_iv_off;
int err;
/*
* RFC3686 specific:
* | CONTEXT1[255:128] = {NONCE, IV, COUNTER}
* | *key = {KEY, NONCE}
*/
ctx1_iv_off = 16 + CTR_RFC3686_NONCE_SIZE;
keylen -= CTR_RFC3686_NONCE_SIZE;
err = aes_check_keylen(keylen);
if (err) {
crypto_skcipher_set_flags(skcipher,
CRYPTO_TFM_RES_BAD_KEY_LEN);
return err;
}
return skcipher_setkey(skcipher, key, keylen, ctx1_iv_off);
}
static int ctr_skcipher_setkey(struct crypto_skcipher *skcipher,
const u8 *key, unsigned int keylen)
{
u32 ctx1_iv_off;
int err;
/*
* AES-CTR needs to load IV in CONTEXT1 reg
* at an offset of 128bits (16bytes)
* CONTEXT1[255:128] = IV
*/
ctx1_iv_off = 16;
err = aes_check_keylen(keylen);
if (err) {
crypto_skcipher_set_flags(skcipher,
CRYPTO_TFM_RES_BAD_KEY_LEN);
return err;
}
return skcipher_setkey(skcipher, key, keylen, ctx1_iv_off);
}
static int arc4_skcipher_setkey(struct crypto_skcipher *skcipher,
const u8 *key, unsigned int keylen)
{
return skcipher_setkey(skcipher, key, keylen, 0);
}
static int des_skcipher_setkey(struct crypto_skcipher *skcipher,
const u8 *key, unsigned int keylen)
{
u32 tmp[DES3_EDE_EXPKEY_WORDS];
struct crypto_tfm *tfm = crypto_skcipher_tfm(skcipher);
return verify_skcipher_des_key(skcipher, key) ?:
skcipher_setkey(skcipher, key, keylen, 0);
}
if (keylen == DES3_EDE_KEY_SIZE &&
__des3_ede_setkey(tmp, &tfm->crt_flags, key, DES3_EDE_KEY_SIZE)) {
return -EINVAL;
}
if (!des_ekey(tmp, key) && (crypto_skcipher_get_flags(skcipher) &
CRYPTO_TFM_REQ_FORBID_WEAK_KEYS)) {
crypto_skcipher_set_flags(skcipher,
CRYPTO_TFM_RES_WEAK_KEY);
return -EINVAL;
}
return skcipher_setkey(skcipher, key, keylen);
static int des3_skcipher_setkey(struct crypto_skcipher *skcipher,
const u8 *key, unsigned int keylen)
{
return verify_skcipher_des3_key(skcipher, key) ?:
skcipher_setkey(skcipher, key, keylen, 0);
}
static int xts_skcipher_setkey(struct crypto_skcipher *skcipher, const u8 *key,
@ -930,19 +965,20 @@ static void aead_encrypt_done(struct device *jrdev, u32 *desc, u32 err,
{
struct aead_request *req = context;
struct aead_edesc *edesc;
int ecode = 0;
dev_dbg(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err);
edesc = container_of(desc, struct aead_edesc, hw_desc[0]);
if (err)
caam_jr_strstatus(jrdev, err);
ecode = caam_jr_strstatus(jrdev, err);
aead_unmap(jrdev, edesc, req);
kfree(edesc);
aead_request_complete(req, err);
aead_request_complete(req, ecode);
}
static void aead_decrypt_done(struct device *jrdev, u32 *desc, u32 err,
@ -950,25 +986,20 @@ static void aead_decrypt_done(struct device *jrdev, u32 *desc, u32 err,
{
struct aead_request *req = context;
struct aead_edesc *edesc;
int ecode = 0;
dev_dbg(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err);
edesc = container_of(desc, struct aead_edesc, hw_desc[0]);
if (err)
caam_jr_strstatus(jrdev, err);
ecode = caam_jr_strstatus(jrdev, err);
aead_unmap(jrdev, edesc, req);
/*
* verify hw auth check passed else return -EBADMSG
*/
if ((err & JRSTA_CCBERR_ERRID_MASK) == JRSTA_CCBERR_ERRID_ICVCHK)
err = -EBADMSG;
kfree(edesc);
aead_request_complete(req, err);
aead_request_complete(req, ecode);
}
static void skcipher_encrypt_done(struct device *jrdev, u32 *desc, u32 err,
@ -978,13 +1009,14 @@ static void skcipher_encrypt_done(struct device *jrdev, u32 *desc, u32 err,
struct skcipher_edesc *edesc;
struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req);
int ivsize = crypto_skcipher_ivsize(skcipher);
int ecode = 0;
dev_dbg(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err);
edesc = container_of(desc, struct skcipher_edesc, hw_desc[0]);
if (err)
caam_jr_strstatus(jrdev, err);
ecode = caam_jr_strstatus(jrdev, err);
skcipher_unmap(jrdev, edesc, req);
@ -993,10 +1025,9 @@ static void skcipher_encrypt_done(struct device *jrdev, u32 *desc, u32 err,
* ciphertext block (CBC mode) or last counter (CTR mode).
* This is used e.g. by the CTS mode.
*/
if (ivsize) {
if (ivsize && !ecode) {
memcpy(req->iv, (u8 *)edesc->sec4_sg + edesc->sec4_sg_bytes,
ivsize);
print_hex_dump_debug("dstiv @"__stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, req->iv,
edesc->src_nents > 1 ? 100 : ivsize, 1);
@ -1008,7 +1039,7 @@ static void skcipher_encrypt_done(struct device *jrdev, u32 *desc, u32 err,
kfree(edesc);
skcipher_request_complete(req, err);
skcipher_request_complete(req, ecode);
}
static void skcipher_decrypt_done(struct device *jrdev, u32 *desc, u32 err,
@ -1018,12 +1049,13 @@ static void skcipher_decrypt_done(struct device *jrdev, u32 *desc, u32 err,
struct skcipher_edesc *edesc;
struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req);
int ivsize = crypto_skcipher_ivsize(skcipher);
int ecode = 0;
dev_dbg(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err);
edesc = container_of(desc, struct skcipher_edesc, hw_desc[0]);
if (err)
caam_jr_strstatus(jrdev, err);
ecode = caam_jr_strstatus(jrdev, err);
skcipher_unmap(jrdev, edesc, req);
@ -1032,7 +1064,7 @@ static void skcipher_decrypt_done(struct device *jrdev, u32 *desc, u32 err,
* ciphertext block (CBC mode) or last counter (CTR mode).
* This is used e.g. by the CTS mode.
*/
if (ivsize) {
if (ivsize && !ecode) {
memcpy(req->iv, (u8 *)edesc->sec4_sg + edesc->sec4_sg_bytes,
ivsize);
@ -1047,7 +1079,7 @@ static void skcipher_decrypt_done(struct device *jrdev, u32 *desc, u32 err,
kfree(edesc);
skcipher_request_complete(req, err);
skcipher_request_complete(req, ecode);
}
/*
@ -1525,10 +1557,7 @@ static int chachapoly_decrypt(struct aead_request *req)
static int ipsec_gcm_encrypt(struct aead_request *req)
{
if (req->assoclen < 8)
return -EINVAL;
return gcm_encrypt(req);
return crypto_ipsec_check_assoclen(req->assoclen) ? : gcm_encrypt(req);
}
static int aead_encrypt(struct aead_request *req)
@ -1602,10 +1631,7 @@ static int gcm_decrypt(struct aead_request *req)
static int ipsec_gcm_decrypt(struct aead_request *req)
{
if (req->assoclen < 8)
return -EINVAL;
return gcm_decrypt(req);
return crypto_ipsec_check_assoclen(req->assoclen) ? : gcm_decrypt(req);
}
static int aead_decrypt(struct aead_request *req)
@ -1817,6 +1843,9 @@ static int skcipher_encrypt(struct skcipher_request *req)
u32 *desc;
int ret = 0;
if (!req->cryptlen)
return 0;
/* allocate extended descriptor */
edesc = skcipher_edesc_alloc(req, DESC_JOB_IO_LEN * CAAM_CMD_SZ);
if (IS_ERR(edesc))
@ -1851,6 +1880,9 @@ static int skcipher_decrypt(struct skcipher_request *req)
u32 *desc;
int ret = 0;
if (!req->cryptlen)
return 0;
/* allocate extended descriptor */
edesc = skcipher_edesc_alloc(req, DESC_JOB_IO_LEN * CAAM_CMD_SZ);
if (IS_ERR(edesc))
@ -1883,7 +1915,7 @@ static struct caam_skcipher_alg driver_algs[] = {
.cra_driver_name = "cbc-aes-caam",
.cra_blocksize = AES_BLOCK_SIZE,
},
.setkey = skcipher_setkey,
.setkey = aes_skcipher_setkey,
.encrypt = skcipher_encrypt,
.decrypt = skcipher_decrypt,
.min_keysize = AES_MIN_KEY_SIZE,
@ -1899,7 +1931,7 @@ static struct caam_skcipher_alg driver_algs[] = {
.cra_driver_name = "cbc-3des-caam",
.cra_blocksize = DES3_EDE_BLOCK_SIZE,
},
.setkey = des_skcipher_setkey,
.setkey = des3_skcipher_setkey,
.encrypt = skcipher_encrypt,
.decrypt = skcipher_decrypt,
.min_keysize = DES3_EDE_KEY_SIZE,
@ -1931,7 +1963,7 @@ static struct caam_skcipher_alg driver_algs[] = {
.cra_driver_name = "ctr-aes-caam",
.cra_blocksize = 1,
},
.setkey = skcipher_setkey,
.setkey = ctr_skcipher_setkey,
.encrypt = skcipher_encrypt,
.decrypt = skcipher_decrypt,
.min_keysize = AES_MIN_KEY_SIZE,
@ -1949,7 +1981,7 @@ static struct caam_skcipher_alg driver_algs[] = {
.cra_driver_name = "rfc3686-ctr-aes-caam",
.cra_blocksize = 1,
},
.setkey = skcipher_setkey,
.setkey = rfc3686_skcipher_setkey,
.encrypt = skcipher_encrypt,
.decrypt = skcipher_decrypt,
.min_keysize = AES_MIN_KEY_SIZE +
@ -2003,7 +2035,7 @@ static struct caam_skcipher_alg driver_algs[] = {
.cra_driver_name = "ecb-aes-caam",
.cra_blocksize = AES_BLOCK_SIZE,
},
.setkey = skcipher_setkey,
.setkey = aes_skcipher_setkey,
.encrypt = skcipher_encrypt,
.decrypt = skcipher_decrypt,
.min_keysize = AES_MIN_KEY_SIZE,
@ -2018,7 +2050,7 @@ static struct caam_skcipher_alg driver_algs[] = {
.cra_driver_name = "ecb-des3-caam",
.cra_blocksize = DES3_EDE_BLOCK_SIZE,
},
.setkey = des_skcipher_setkey,
.setkey = des3_skcipher_setkey,
.encrypt = skcipher_encrypt,
.decrypt = skcipher_decrypt,
.min_keysize = DES3_EDE_KEY_SIZE,
@ -2033,7 +2065,7 @@ static struct caam_skcipher_alg driver_algs[] = {
.cra_driver_name = "ecb-arc4-caam",
.cra_blocksize = ARC4_BLOCK_SIZE,
},
.setkey = skcipher_setkey,
.setkey = arc4_skcipher_setkey,
.encrypt = skcipher_encrypt,
.decrypt = skcipher_decrypt,
.min_keysize = ARC4_MIN_KEY_SIZE,

56
drivers/crypto/caam/caamalg_desc.c
View file

@ -503,6 +503,7 @@ void cnstr_shdsc_aead_givencap(u32 * const desc, struct alginfo *cdata,
const bool is_qi, int era)
{
u32 geniv, moveiv;
u32 *wait_cmd;
/* Note: Context registers are saved. */
init_sh_desc_key_aead(desc, cdata, adata, is_rfc3686, nonce, era);
@ -598,6 +599,14 @@ copy_iv:
/* Will read cryptlen */
append_math_add(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
/*
* Wait for IV transfer (ofifo -> class2) to finish before starting
* ciphertext transfer (ofifo -> external memory).
*/
wait_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | JUMP_COND_NIFP);
set_jump_tgt_here(desc, wait_cmd);
append_seq_fifo_load(desc, 0, FIFOLD_CLASS_BOTH | KEY_VLF |
FIFOLD_TYPE_MSG1OUT2 | FIFOLD_TYPE_LASTBOTH);
append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | KEY_VLF);
@ -843,13 +852,16 @@ EXPORT_SYMBOL(cnstr_shdsc_gcm_decap);
* @ivsize: initialization vector size
* @icvsize: integrity check value (ICV) size (truncated or full)
* @is_qi: true when called from caam/qi
*
* Input sequence: AAD | PTXT
* Output sequence: AAD | CTXT | ICV
* AAD length (assoclen), which includes the IV length, is available in Math3.
*/
void cnstr_shdsc_rfc4106_encap(u32 * const desc, struct alginfo *cdata,
unsigned int ivsize, unsigned int icvsize,
const bool is_qi)
{
u32 *key_jump_cmd;
u32 *key_jump_cmd, *zero_cryptlen_jump_cmd, *skip_instructions;
init_sh_desc(desc, HDR_SHARE_SERIAL);
/* Skip key loading if it is loaded due to sharing */
@ -892,24 +904,26 @@ void cnstr_shdsc_rfc4106_encap(u32 * const desc, struct alginfo *cdata,
append_math_sub_imm_u32(desc, VARSEQINLEN, REG3, IMM, ivsize);
append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ);
/* Read assoc data */
/* Skip AAD */
append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF);
/* Read cryptlen and set this value into VARSEQOUTLEN */
append_math_sub(desc, VARSEQOUTLEN, SEQINLEN, REG3, CAAM_CMD_SZ);
/* If cryptlen is ZERO jump to AAD command */
zero_cryptlen_jump_cmd = append_jump(desc, JUMP_TEST_ALL |
JUMP_COND_MATH_Z);
/* Read AAD data */
append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
FIFOLD_TYPE_AAD | FIFOLD_TYPE_FLUSH1);
/* Workaround for erratum A-005473 (simultaneous SEQ FIFO skips) */
append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA);
/* Skip IV */
append_seq_fifo_load(desc, ivsize, FIFOLD_CLASS_SKIP);
/* Will read cryptlen bytes */
append_math_sub(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
/* Workaround for erratum A-005473 (simultaneous SEQ FIFO skips) */
append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLD_TYPE_MSG);
/* Skip assoc data */
append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF);
/* cryptlen = seqoutlen - assoclen */
append_math_sub(desc, VARSEQOUTLEN, VARSEQINLEN, REG0, CAAM_CMD_SZ);
append_math_add(desc, VARSEQINLEN, VARSEQOUTLEN, REG0, CAAM_CMD_SZ);
/* Write encrypted data */
append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | FIFOLDST_VLF);
@ -918,6 +932,18 @@ void cnstr_shdsc_rfc4106_encap(u32 * const desc, struct alginfo *cdata,
append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
FIFOLD_TYPE_MSG | FIFOLD_TYPE_LAST1);
/* Jump instructions to avoid double reading of AAD */
skip_instructions = append_jump(desc, JUMP_TEST_ALL);
/* There is no input data, cryptlen = 0 */
set_jump_tgt_here(desc, zero_cryptlen_jump_cmd);
/* Read AAD */
append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
FIFOLD_TYPE_AAD | FIFOLD_TYPE_LAST1);
set_jump_tgt_here(desc, skip_instructions);
/* Write ICV */
append_seq_store(desc, icvsize, LDST_CLASS_1_CCB |
LDST_SRCDST_BYTE_CONTEXT);

4
drivers/crypto/caam/caamalg_desc.h
View file

@ -12,7 +12,7 @@
#define DESC_AEAD_BASE (4 * CAAM_CMD_SZ)
#define DESC_AEAD_ENC_LEN (DESC_AEAD_BASE + 11 * CAAM_CMD_SZ)
#define DESC_AEAD_DEC_LEN (DESC_AEAD_BASE + 15 * CAAM_CMD_SZ)
#define DESC_AEAD_GIVENC_LEN (DESC_AEAD_ENC_LEN + 7 * CAAM_CMD_SZ)
#define DESC_AEAD_GIVENC_LEN (DESC_AEAD_ENC_LEN + 8 * CAAM_CMD_SZ)
#define DESC_QI_AEAD_ENC_LEN (DESC_AEAD_ENC_LEN + 3 * CAAM_CMD_SZ)
#define DESC_QI_AEAD_DEC_LEN (DESC_AEAD_DEC_LEN + 3 * CAAM_CMD_SZ)
#define DESC_QI_AEAD_GIVENC_LEN (DESC_AEAD_GIVENC_LEN + 3 * CAAM_CMD_SZ)
@ -31,7 +31,7 @@
#define DESC_QI_GCM_DEC_LEN (DESC_GCM_DEC_LEN + 3 * CAAM_CMD_SZ)
#define DESC_RFC4106_BASE (3 * CAAM_CMD_SZ)
#define DESC_RFC4106_ENC_LEN (DESC_RFC4106_BASE + 13 * CAAM_CMD_SZ)
#define DESC_RFC4106_ENC_LEN (DESC_RFC4106_BASE + 16 * CAAM_CMD_SZ)
#define DESC_RFC4106_DEC_LEN (DESC_RFC4106_BASE + 13 * CAAM_CMD_SZ)
#define DESC_QI_RFC4106_ENC_LEN (DESC_RFC4106_ENC_LEN + 5 * CAAM_CMD_SZ)
#define DESC_QI_RFC4106_DEC_LEN (DESC_RFC4106_DEC_LEN + 5 * CAAM_CMD_SZ)

257
drivers/crypto/caam/caamalg_qi.c
View file

@ -105,6 +105,18 @@ static int aead_set_sh_desc(struct crypto_aead *aead)
ctx->cdata.keylen - CTR_RFC3686_NONCE_SIZE);
}
/*
* In case |user key| > |derived key|, using DKP<imm,imm> would result
* in invalid opcodes (last bytes of user key) in the resulting
* descriptor. Use DKP<ptr,imm> instead => both virtual and dma key
* addresses are needed.
*/
ctx->adata.key_virt = ctx->key;
ctx->adata.key_dma = ctx->key_dma;
ctx->cdata.key_virt = ctx->key + ctx->adata.keylen_pad;
ctx->cdata.key_dma = ctx->key_dma + ctx->adata.keylen_pad;
data_len[0] = ctx->adata.keylen_pad;
data_len[1] = ctx->cdata.keylen;
@ -118,16 +130,6 @@ static int aead_set_sh_desc(struct crypto_aead *aead)
ARRAY_SIZE(data_len)) < 0)
return -EINVAL;
if (inl_mask & 1)
ctx->adata.key_virt = ctx->key;
else
ctx->adata.key_dma = ctx->key_dma;
if (inl_mask & 2)
ctx->cdata.key_virt = ctx->key + ctx->adata.keylen_pad;
else
ctx->cdata.key_dma = ctx->key_dma + ctx->adata.keylen_pad;
ctx->adata.key_inline = !!(inl_mask & 1);
ctx->cdata.key_inline = !!(inl_mask & 2);
@ -143,16 +145,6 @@ skip_enc:
ARRAY_SIZE(data_len)) < 0)
return -EINVAL;
if (inl_mask & 1)
ctx->adata.key_virt = ctx->key;
else
ctx->adata.key_dma = ctx->key_dma;
if (inl_mask & 2)
ctx->cdata.key_virt = ctx->key + ctx->adata.keylen_pad;
else
ctx->cdata.key_dma = ctx->key_dma + ctx->adata.keylen_pad;
ctx->adata.key_inline = !!(inl_mask & 1);
ctx->cdata.key_inline = !!(inl_mask & 2);
@ -171,16 +163,6 @@ skip_enc:
ARRAY_SIZE(data_len)) < 0)
return -EINVAL;
if (inl_mask & 1)
ctx->adata.key_virt = ctx->key;
else
ctx->adata.key_dma = ctx->key_dma;
if (inl_mask & 2)
ctx->cdata.key_virt = ctx->key + ctx->adata.keylen_pad;
else
ctx->cdata.key_dma = ctx->key_dma + ctx->adata.keylen_pad;
ctx->adata.key_inline = !!(inl_mask & 1);
ctx->cdata.key_inline = !!(inl_mask & 2);
@ -252,11 +234,10 @@ static int aead_setkey(struct crypto_aead *aead, const u8 *key,
dma_sync_single_for_device(jrdev->parent, ctx->key_dma,
ctx->adata.keylen_pad + keys.enckeylen,
ctx->dir);
#ifdef DEBUG
print_hex_dump(KERN_ERR, "ctx.key@" __stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, ctx->key,
ctx->adata.keylen_pad + keys.enckeylen, 1);
#endif
print_hex_dump_debug("ctx.key@" __stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, ctx->key,
ctx->adata.keylen_pad + keys.enckeylen, 1);
skip_split_key:
ctx->cdata.keylen = keys.enckeylen;
@ -296,33 +277,17 @@ static int des3_aead_setkey(struct crypto_aead *aead, const u8 *key,
unsigned int keylen)
{
struct crypto_authenc_keys keys;
u32 flags;
int err;
err = crypto_authenc_extractkeys(&keys, key, keylen);
if (unlikely(err))
goto badkey;
return err;
err = -EINVAL;
if (keys.enckeylen != DES3_EDE_KEY_SIZE)
goto badkey;
err = verify_aead_des3_key(aead, keys.enckey, keys.enckeylen) ?:
aead_setkey(aead, key, keylen);
flags = crypto_aead_get_flags(aead);
err = __des3_verify_key(&flags, keys.enckey);
if (unlikely(err)) {
crypto_aead_set_flags(aead, flags);
goto out;
}
err = aead_setkey(aead, key, keylen);
out:
memzero_explicit(&keys, sizeof(keys));
return err;
badkey:
crypto_aead_set_flags(aead, CRYPTO_TFM_RES_BAD_KEY_LEN);
goto out;
}
static int gcm_set_sh_desc(struct crypto_aead *aead)
@ -371,6 +336,11 @@ static int gcm_set_sh_desc(struct crypto_aead *aead)
static int gcm_setauthsize(struct crypto_aead *authenc, unsigned int authsize)
{
struct caam_ctx *ctx = crypto_aead_ctx(authenc);
int err;
err = crypto_gcm_check_authsize(authsize);
if (err)
return err;
ctx->authsize = authsize;
gcm_set_sh_desc(authenc);
@ -385,6 +355,12 @@ static int gcm_setkey(struct crypto_aead *aead,
struct device *jrdev = ctx->jrdev;
int ret;
ret = aes_check_keylen(keylen);
if (ret) {
crypto_aead_set_flags(aead, CRYPTO_TFM_RES_BAD_KEY_LEN);
return ret;
}
print_hex_dump_debug("key in @" __stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
@ -466,6 +442,11 @@ static int rfc4106_setauthsize(struct crypto_aead *authenc,
unsigned int authsize)
{
struct caam_ctx *ctx = crypto_aead_ctx(authenc);
int err;
err = crypto_rfc4106_check_authsize(authsize);
if (err)
return err;
ctx->authsize = authsize;
rfc4106_set_sh_desc(authenc);
@ -480,8 +461,11 @@ static int rfc4106_setkey(struct crypto_aead *aead,
struct device *jrdev = ctx->jrdev;
int ret;
if (keylen < 4)
return -EINVAL;
ret = aes_check_keylen(keylen - 4);
if (ret) {
crypto_aead_set_flags(aead, CRYPTO_TFM_RES_BAD_KEY_LEN);
return ret;
}
print_hex_dump_debug("key in @" __stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
@ -569,6 +553,9 @@ static int rfc4543_setauthsize(struct crypto_aead *authenc,
{
struct caam_ctx *ctx = crypto_aead_ctx(authenc);
if (authsize != 16)
return -EINVAL;
ctx->authsize = authsize;
rfc4543_set_sh_desc(authenc);
@ -582,8 +569,11 @@ static int rfc4543_setkey(struct crypto_aead *aead,
struct device *jrdev = ctx->jrdev;
int ret;
if (keylen < 4)
return -EINVAL;
ret = aes_check_keylen(keylen - 4);
if (ret) {
crypto_aead_set_flags(aead, CRYPTO_TFM_RES_BAD_KEY_LEN);
return ret;
}
print_hex_dump_debug("key in @" __stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
@ -624,7 +614,7 @@ static int rfc4543_setkey(struct crypto_aead *aead,
}
static int skcipher_setkey(struct crypto_skcipher *skcipher, const u8 *key,
unsigned int keylen)
unsigned int keylen, const u32 ctx1_iv_off)
{
struct caam_ctx *ctx = crypto_skcipher_ctx(skcipher);
struct caam_skcipher_alg *alg =
@ -632,33 +622,12 @@ static int skcipher_setkey(struct crypto_skcipher *skcipher, const u8 *key,
skcipher);
struct device *jrdev = ctx->jrdev;
unsigned int ivsize = crypto_skcipher_ivsize(skcipher);
u32 ctx1_iv_off = 0;
const bool ctr_mode = ((ctx->cdata.algtype & OP_ALG_AAI_MASK) ==
OP_ALG_AAI_CTR_MOD128);
const bool is_rfc3686 = alg->caam.rfc3686;
int ret = 0;
print_hex_dump_debug("key in @" __stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
/*
* AES-CTR needs to load IV in CONTEXT1 reg
* at an offset of 128bits (16bytes)
* CONTEXT1[255:128] = IV
*/
if (ctr_mode)
ctx1_iv_off = 16;
/*
* RFC3686 specific:
* | CONTEXT1[255:128] = {NONCE, IV, COUNTER}
* | *key = {KEY, NONCE}
*/
if (is_rfc3686) {
ctx1_iv_off = 16 + CTR_RFC3686_NONCE_SIZE;
keylen -= CTR_RFC3686_NONCE_SIZE;
}
ctx->cdata.keylen = keylen;
ctx->cdata.key_virt = key;
ctx->cdata.key_inline = true;
@ -694,11 +663,80 @@ badkey:
return -EINVAL;
}
static int aes_skcipher_setkey(struct crypto_skcipher *skcipher,
const u8 *key, unsigned int keylen)
{
int err;
err = aes_check_keylen(keylen);
if (err) {
crypto_skcipher_set_flags(skcipher,
CRYPTO_TFM_RES_BAD_KEY_LEN);
return err;
}
return skcipher_setkey(skcipher, key, keylen, 0);
}
static int rfc3686_skcipher_setkey(struct crypto_skcipher *skcipher,
const u8 *key, unsigned int keylen)
{
u32 ctx1_iv_off;
int err;
/*
* RFC3686 specific:
* | CONTEXT1[255:128] = {NONCE, IV, COUNTER}
* | *key = {KEY, NONCE}
*/
ctx1_iv_off = 16 + CTR_RFC3686_NONCE_SIZE;
keylen -= CTR_RFC3686_NONCE_SIZE;
err = aes_check_keylen(keylen);
if (err) {
crypto_skcipher_set_flags(skcipher,
CRYPTO_TFM_RES_BAD_KEY_LEN);
return err;
}
return skcipher_setkey(skcipher, key, keylen, ctx1_iv_off);
}
static int ctr_skcipher_setkey(struct crypto_skcipher *skcipher,
const u8 *key, unsigned int keylen)
{
u32 ctx1_iv_off;
int err;
/*
* AES-CTR needs to load IV in CONTEXT1 reg
* at an offset of 128bits (16bytes)
* CONTEXT1[255:128] = IV
*/
ctx1_iv_off = 16;
err = aes_check_keylen(keylen);
if (err) {
crypto_skcipher_set_flags(skcipher,
CRYPTO_TFM_RES_BAD_KEY_LEN);
return err;
}
return skcipher_setkey(skcipher, key, keylen, ctx1_iv_off);
}
static int des3_skcipher_setkey(struct crypto_skcipher *skcipher,
const u8 *key, unsigned int keylen)
{
return unlikely(des3_verify_key(skcipher, key)) ?:
skcipher_setkey(skcipher, key, keylen);
return verify_skcipher_des3_key(skcipher, key) ?:
skcipher_setkey(skcipher, key, keylen, 0);
}
static int des_skcipher_setkey(struct crypto_skcipher *skcipher,
const u8 *key, unsigned int keylen)
{
return verify_skcipher_des_key(skcipher, key) ?:
skcipher_setkey(skcipher, key, keylen, 0);
}
static int xts_skcipher_setkey(struct crypto_skcipher *skcipher, const u8 *key,
@ -884,20 +922,8 @@ static void aead_done(struct caam_drv_req *drv_req, u32 status)
qidev = caam_ctx->qidev;
if (unlikely(status)) {
u32 ssrc = status & JRSTA_SSRC_MASK;
u8 err_id = status & JRSTA_CCBERR_ERRID_MASK;
caam_jr_strstatus(qidev, status);
/*
* verify hw auth check passed else return -EBADMSG
*/
if (ssrc == JRSTA_SSRC_CCB_ERROR &&
err_id == JRSTA_CCBERR_ERRID_ICVCHK)
ecode = -EBADMSG;
else
ecode = -EIO;
}
if (unlikely(status))
ecode = caam_jr_strstatus(qidev, status);
edesc = container_of(drv_req, typeof(*edesc), drv_req);
aead_unmap(qidev, edesc, aead_req);
@ -1168,18 +1194,14 @@ static int aead_decrypt(struct aead_request *req)
static int ipsec_gcm_encrypt(struct aead_request *req)
{
if (req->assoclen < 8)
return -EINVAL;
return aead_crypt(req, true);
return crypto_ipsec_check_assoclen(req->assoclen) ? : aead_crypt(req,
true);
}
static int ipsec_gcm_decrypt(struct aead_request *req)
{
if (req->assoclen < 8)
return -EINVAL;
return aead_crypt(req, false);
return crypto_ipsec_check_assoclen(req->assoclen) ? : aead_crypt(req,
false);
}
static void skcipher_done(struct caam_drv_req *drv_req, u32 status)
@ -1190,13 +1212,14 @@ static void skcipher_done(struct caam_drv_req *drv_req, u32 status)
struct caam_ctx *caam_ctx = crypto_skcipher_ctx(skcipher);
struct device *qidev = caam_ctx->qidev;
int ivsize = crypto_skcipher_ivsize(skcipher);
int ecode = 0;
dev_dbg(qidev, "%s %d: status 0x%x\n", __func__, __LINE__, status);
edesc = container_of(drv_req, typeof(*edesc), drv_req);
if (status)
caam_jr_strstatus(qidev, status);
ecode = caam_jr_strstatus(qidev, status);
print_hex_dump_debug("dstiv @" __stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, req->iv,
@ -1212,10 +1235,12 @@ static void skcipher_done(struct caam_drv_req *drv_req, u32 status)
* ciphertext block (CBC mode) or last counter (CTR mode).
* This is used e.g. by the CTS mode.
*/
memcpy(req->iv, (u8 *)&edesc->sgt[0] + edesc->qm_sg_bytes, ivsize);
if (!ecode)
memcpy(req->iv, (u8 *)&edesc->sgt[0] + edesc->qm_sg_bytes,
ivsize);
qi_cache_free(edesc);
skcipher_request_complete(req, status);
skcipher_request_complete(req, ecode);
}
static struct skcipher_edesc *skcipher_edesc_alloc(struct skcipher_request *req,
@ -1377,6 +1402,9 @@ static inline int skcipher_crypt(struct skcipher_request *req, bool encrypt)
struct caam_ctx *ctx = crypto_skcipher_ctx(skcipher);
int ret;
if (!req->cryptlen)
return 0;
if (unlikely(caam_congested))
return -EAGAIN;
@ -1414,7 +1442,7 @@ static struct caam_skcipher_alg driver_algs[] = {
.cra_driver_name = "cbc-aes-caam-qi",
.cra_blocksize = AES_BLOCK_SIZE,
},
.setkey = skcipher_setkey,
.setkey = aes_skcipher_setkey,
.encrypt = skcipher_encrypt,
.decrypt = skcipher_decrypt,
.min_keysize = AES_MIN_KEY_SIZE,
@ -1446,7 +1474,7 @@ static struct caam_skcipher_alg driver_algs[] = {
.cra_driver_name = "cbc-des-caam-qi",
.cra_blocksize = DES_BLOCK_SIZE,
},
.setkey = skcipher_setkey,
.setkey = des_skcipher_setkey,
.encrypt = skcipher_encrypt,
.decrypt = skcipher_decrypt,
.min_keysize = DES_KEY_SIZE,
@ -1462,7 +1490,7 @@ static struct caam_skcipher_alg driver_algs[] = {
.cra_driver_name = "ctr-aes-caam-qi",
.cra_blocksize = 1,
},
.setkey = skcipher_setkey,
.setkey = ctr_skcipher_setkey,
.encrypt = skcipher_encrypt,
.decrypt = skcipher_decrypt,
.min_keysize = AES_MIN_KEY_SIZE,
@ -1480,7 +1508,7 @@ static struct caam_skcipher_alg driver_algs[] = {
.cra_driver_name = "rfc3686-ctr-aes-caam-qi",
.cra_blocksize = 1,
},
.setkey = skcipher_setkey,
.setkey = rfc3686_skcipher_setkey,
.encrypt = skcipher_encrypt,
.decrypt = skcipher_decrypt,
.min_keysize = AES_MIN_KEY_SIZE +
@ -2523,10 +2551,9 @@ int caam_qi_algapi_init(struct device *ctrldev)
unsigned int md_limit = SHA512_DIGEST_SIZE;
bool registered = false;
if (caam_dpaa2) {
dev_info(ctrldev, "caam/qi frontend driver not suitable for DPAA 2.x, aborting...\n");
return -ENODEV;
}
/* Make sure this runs only on (DPAA 1.x) QI */
if (!priv->qi_present || caam_dpaa2)
return 0;
/*
* Register crypto algorithms the device supports.

327
drivers/crypto/caam/caamalg_qi2.c
View file

@ -15,6 +15,7 @@
#include "key_gen.h"
#include "caamalg_desc.h"
#include "caamhash_desc.h"
#include "dpseci-debugfs.h"
#include <linux/fsl/mc.h>
#include <soc/fsl/dpaa2-io.h>
#include <soc/fsl/dpaa2-fd.h>
@ -198,6 +199,18 @@ static int aead_set_sh_desc(struct crypto_aead *aead)
ctx->cdata.keylen - CTR_RFC3686_NONCE_SIZE);
}
/*
* In case |user key| > |derived key|, using DKP<imm,imm> would result
* in invalid opcodes (last bytes of user key) in the resulting
* descriptor. Use DKP<ptr,imm> instead => both virtual and dma key
* addresses are needed.
*/
ctx->adata.key_virt = ctx->key;
ctx->adata.key_dma = ctx->key_dma;
ctx->cdata.key_virt = ctx->key + ctx->adata.keylen_pad;
ctx->cdata.key_dma = ctx->key_dma + ctx->adata.keylen_pad;
data_len[0] = ctx->adata.keylen_pad;
data_len[1] = ctx->cdata.keylen;
@ -209,16 +222,6 @@ static int aead_set_sh_desc(struct crypto_aead *aead)
ARRAY_SIZE(data_len)) < 0)
return -EINVAL;
if (inl_mask & 1)
ctx->adata.key_virt = ctx->key;
else
ctx->adata.key_dma = ctx->key_dma;
if (inl_mask & 2)
ctx->cdata.key_virt = ctx->key + ctx->adata.keylen_pad;
else
ctx->cdata.key_dma = ctx->key_dma + ctx->adata.keylen_pad;
ctx->adata.key_inline = !!(inl_mask & 1);
ctx->cdata.key_inline = !!(inl_mask & 2);
@ -247,16 +250,6 @@ static int aead_set_sh_desc(struct crypto_aead *aead)
ARRAY_SIZE(data_len)) < 0)
return -EINVAL;
if (inl_mask & 1)
ctx->adata.key_virt = ctx->key;
else
ctx->adata.key_dma = ctx->key_dma;
if (inl_mask & 2)
ctx->cdata.key_virt = ctx->key + ctx->adata.keylen_pad;
else
ctx->cdata.key_dma = ctx->key_dma + ctx->adata.keylen_pad;
ctx->adata.key_inline = !!(inl_mask & 1);
ctx->cdata.key_inline = !!(inl_mask & 2);
@ -329,7 +322,6 @@ static int des3_aead_setkey(struct crypto_aead *aead, const u8 *key,
unsigned int keylen)
{
struct crypto_authenc_keys keys;
u32 flags;
int err;
err = crypto_authenc_extractkeys(&keys, key, keylen);
@ -340,14 +332,8 @@ static int des3_aead_setkey(struct crypto_aead *aead, const u8 *key,
if (keys.enckeylen != DES3_EDE_KEY_SIZE)
goto badkey;
flags = crypto_aead_get_flags(aead);
err = __des3_verify_key(&flags, keys.enckey);
if (unlikely(err)) {
crypto_aead_set_flags(aead, flags);
goto out;
}
err = aead_setkey(aead, key, keylen);
err = crypto_des3_ede_verify_key(crypto_aead_tfm(aead), keys.enckey) ?:
aead_setkey(aead, key, keylen);
out:
memzero_explicit(&keys, sizeof(keys));
@ -719,6 +705,11 @@ static int gcm_set_sh_desc(struct crypto_aead *aead)
static int gcm_setauthsize(struct crypto_aead *authenc, unsigned int authsize)
{
struct caam_ctx *ctx = crypto_aead_ctx(authenc);
int err;
err = crypto_gcm_check_authsize(authsize);
if (err)
return err;
ctx->authsize = authsize;
gcm_set_sh_desc(authenc);
@ -731,7 +722,13 @@ static int gcm_setkey(struct crypto_aead *aead,
{
struct caam_ctx *ctx = crypto_aead_ctx(aead);
struct device *dev = ctx->dev;
int ret;
ret = aes_check_keylen(keylen);
if (ret) {
crypto_aead_set_flags(aead, CRYPTO_TFM_RES_BAD_KEY_LEN);
return ret;
}
print_hex_dump_debug("key in @" __stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
@ -805,6 +802,11 @@ static int rfc4106_setauthsize(struct crypto_aead *authenc,
unsigned int authsize)
{
struct caam_ctx *ctx = crypto_aead_ctx(authenc);
int err;
err = crypto_rfc4106_check_authsize(authsize);
if (err)
return err;
ctx->authsize = authsize;
rfc4106_set_sh_desc(authenc);
@ -817,9 +819,13 @@ static int rfc4106_setkey(struct crypto_aead *aead,
{
struct caam_ctx *ctx = crypto_aead_ctx(aead);
struct device *dev = ctx->dev;
int ret;
if (keylen < 4)
return -EINVAL;
ret = aes_check_keylen(keylen - 4);
if (ret) {
crypto_aead_set_flags(aead, CRYPTO_TFM_RES_BAD_KEY_LEN);
return ret;
}
print_hex_dump_debug("key in @" __stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
@ -900,6 +906,9 @@ static int rfc4543_setauthsize(struct crypto_aead *authenc,
{
struct caam_ctx *ctx = crypto_aead_ctx(authenc);
if (authsize != 16)
return -EINVAL;
ctx->authsize = authsize;
rfc4543_set_sh_desc(authenc);
@ -911,9 +920,13 @@ static int rfc4543_setkey(struct crypto_aead *aead,
{
struct caam_ctx *ctx = crypto_aead_ctx(aead);
struct device *dev = ctx->dev;
int ret;
if (keylen < 4)
return -EINVAL;
ret = aes_check_keylen(keylen - 4);
if (ret) {
crypto_aead_set_flags(aead, CRYPTO_TFM_RES_BAD_KEY_LEN);
return ret;
}
print_hex_dump_debug("key in @" __stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
@ -931,7 +944,7 @@ static int rfc4543_setkey(struct crypto_aead *aead,
}
static int skcipher_setkey(struct crypto_skcipher *skcipher, const u8 *key,
unsigned int keylen)
unsigned int keylen, const u32 ctx1_iv_off)
{
struct caam_ctx *ctx = crypto_skcipher_ctx(skcipher);
struct caam_skcipher_alg *alg =
@ -941,34 +954,11 @@ static int skcipher_setkey(struct crypto_skcipher *skcipher, const u8 *key,
struct caam_flc *flc;
unsigned int ivsize = crypto_skcipher_ivsize(skcipher);
u32 *desc;
u32 ctx1_iv_off = 0;
const bool ctr_mode = ((ctx->cdata.algtype & OP_ALG_AAI_MASK) ==
OP_ALG_AAI_CTR_MOD128) &&
((ctx->cdata.algtype & OP_ALG_ALGSEL_MASK) !=
OP_ALG_ALGSEL_CHACHA20);
const bool is_rfc3686 = alg->caam.rfc3686;
print_hex_dump_debug("key in @" __stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
/*
* AES-CTR needs to load IV in CONTEXT1 reg
* at an offset of 128bits (16bytes)
* CONTEXT1[255:128] = IV
*/
if (ctr_mode)
ctx1_iv_off = 16;
/*
* RFC3686 specific:
* | CONTEXT1[255:128] = {NONCE, IV, COUNTER}
* | *key = {KEY, NONCE}
*/
if (is_rfc3686) {
ctx1_iv_off = 16 + CTR_RFC3686_NONCE_SIZE;
keylen -= CTR_RFC3686_NONCE_SIZE;
}
ctx->cdata.keylen = keylen;
ctx->cdata.key_virt = key;
ctx->cdata.key_inline = true;
@ -996,11 +986,92 @@ static int skcipher_setkey(struct crypto_skcipher *skcipher, const u8 *key,
return 0;
}
static int des3_skcipher_setkey(struct crypto_skcipher *skcipher,
const u8 *key, unsigned int keylen)
static int aes_skcipher_setkey(struct crypto_skcipher *skcipher,
const u8 *key, unsigned int keylen)
{
return unlikely(des3_verify_key(skcipher, key)) ?:
skcipher_setkey(skcipher, key, keylen);
int err;
err = aes_check_keylen(keylen);
if (err) {
crypto_skcipher_set_flags(skcipher,
CRYPTO_TFM_RES_BAD_KEY_LEN);
return err;
}
return skcipher_setkey(skcipher, key, keylen, 0);
}
static int rfc3686_skcipher_setkey(struct crypto_skcipher *skcipher,
const u8 *key, unsigned int keylen)
{
u32 ctx1_iv_off;
int err;
/*
* RFC3686 specific:
* | CONTEXT1[255:128] = {NONCE, IV, COUNTER}
* | *key = {KEY, NONCE}
*/
ctx1_iv_off = 16 + CTR_RFC3686_NONCE_SIZE;
keylen -= CTR_RFC3686_NONCE_SIZE;
err = aes_check_keylen(keylen);
if (err) {
crypto_skcipher_set_flags(skcipher,
CRYPTO_TFM_RES_BAD_KEY_LEN);
return err;
}
return skcipher_setkey(skcipher, key, keylen, ctx1_iv_off);
}
static int ctr_skcipher_setkey(struct crypto_skcipher *skcipher,
const u8 *key, unsigned int keylen)
{
u32 ctx1_iv_off;
int err;
/*
* AES-CTR needs to load IV in CONTEXT1 reg
* at an offset of 128bits (16bytes)
* CONTEXT1[255:128] = IV
*/
ctx1_iv_off = 16;
err = aes_check_keylen(keylen);
if (err) {
crypto_skcipher_set_flags(skcipher,
CRYPTO_TFM_RES_BAD_KEY_LEN);
return err;
}
return skcipher_setkey(skcipher, key, keylen, ctx1_iv_off);
}
static int chacha20_skcipher_setkey(struct crypto_skcipher *skcipher,
const u8 *key, unsigned int keylen)
{
if (keylen != CHACHA_KEY_SIZE) {
crypto_skcipher_set_flags(skcipher,
CRYPTO_TFM_RES_BAD_KEY_LEN);
return -EINVAL;
}
return skcipher_setkey(skcipher, key, keylen, 0);
}
static int des_skcipher_setkey(struct crypto_skcipher *skcipher,
const u8 *key, unsigned int keylen)
{
return verify_skcipher_des_key(skcipher, key) ?:
skcipher_setkey(skcipher, key, keylen, 0);
}
static int des3_skcipher_setkey(struct crypto_skcipher *skcipher,
const u8 *key, unsigned int keylen)
{
return verify_skcipher_des3_key(skcipher, key) ?:
skcipher_setkey(skcipher, key, keylen, 0);
}
static int xts_skcipher_setkey(struct crypto_skcipher *skcipher, const u8 *key,
@ -1227,10 +1298,8 @@ static void aead_encrypt_done(void *cbk_ctx, u32 status)
dev_dbg(ctx->dev, "%s %d: err 0x%x\n", __func__, __LINE__, status);
if (unlikely(status)) {
caam_qi2_strstatus(ctx->dev, status);
ecode = -EIO;
}
if (unlikely(status))
ecode = caam_qi2_strstatus(ctx->dev, status);
aead_unmap(ctx->dev, edesc, req);
qi_cache_free(edesc);
@ -1250,17 +1319,8 @@ static void aead_decrypt_done(void *cbk_ctx, u32 status)
dev_dbg(ctx->dev, "%s %d: err 0x%x\n", __func__, __LINE__, status);
if (unlikely(status)) {
caam_qi2_strstatus(ctx->dev, status);
/*
* verify hw auth check passed else return -EBADMSG
*/
if ((status & JRSTA_CCBERR_ERRID_MASK) ==
JRSTA_CCBERR_ERRID_ICVCHK)
ecode = -EBADMSG;
else
ecode = -EIO;
}
if (unlikely(status))
ecode = caam_qi2_strstatus(ctx->dev, status);
aead_unmap(ctx->dev, edesc, req);
qi_cache_free(edesc);
@ -1325,18 +1385,12 @@ static int aead_decrypt(struct aead_request *req)
static int ipsec_gcm_encrypt(struct aead_request *req)
{
if (req->assoclen < 8)
return -EINVAL;
return aead_encrypt(req);
return crypto_ipsec_check_assoclen(req->assoclen) ? : aead_encrypt(req);
}
static int ipsec_gcm_decrypt(struct aead_request *req)
{
if (req->assoclen < 8)
return -EINVAL;
return aead_decrypt(req);
return crypto_ipsec_check_assoclen(req->assoclen) ? : aead_decrypt(req);
}
static void skcipher_encrypt_done(void *cbk_ctx, u32 status)
@ -1352,10 +1406,8 @@ static void skcipher_encrypt_done(void *cbk_ctx, u32 status)
dev_dbg(ctx->dev, "%s %d: err 0x%x\n", __func__, __LINE__, status);
if (unlikely(status)) {
caam_qi2_strstatus(ctx->dev, status);
ecode = -EIO;
}
if (unlikely(status))
ecode = caam_qi2_strstatus(ctx->dev, status);
print_hex_dump_debug("dstiv @" __stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, req->iv,
@ -1371,7 +1423,9 @@ static void skcipher_encrypt_done(void *cbk_ctx, u32 status)
* ciphertext block (CBC mode) or last counter (CTR mode).
* This is used e.g. by the CTS mode.
*/
memcpy(req->iv, (u8 *)&edesc->sgt[0] + edesc->qm_sg_bytes, ivsize);
if (!ecode)
memcpy(req->iv, (u8 *)&edesc->sgt[0] + edesc->qm_sg_bytes,
ivsize);
qi_cache_free(edesc);
skcipher_request_complete(req, ecode);
@ -1390,10 +1444,8 @@ static void skcipher_decrypt_done(void *cbk_ctx, u32 status)
dev_dbg(ctx->dev, "%s %d: err 0x%x\n", __func__, __LINE__, status);
if (unlikely(status)) {
caam_qi2_strstatus(ctx->dev, status);
ecode = -EIO;
}
if (unlikely(status))
ecode = caam_qi2_strstatus(ctx->dev, status);
print_hex_dump_debug("dstiv @" __stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, req->iv,
@ -1409,7 +1461,9 @@ static void skcipher_decrypt_done(void *cbk_ctx, u32 status)
* ciphertext block (CBC mode) or last counter (CTR mode).
* This is used e.g. by the CTS mode.
*/
memcpy(req->iv, (u8 *)&edesc->sgt[0] + edesc->qm_sg_bytes, ivsize);
if (!ecode)
memcpy(req->iv, (u8 *)&edesc->sgt[0] + edesc->qm_sg_bytes,
ivsize);
qi_cache_free(edesc);
skcipher_request_complete(req, ecode);
@ -1423,6 +1477,9 @@ static int skcipher_encrypt(struct skcipher_request *req)
struct caam_request *caam_req = skcipher_request_ctx(req);
int ret;
if (!req->cryptlen)
return 0;
/* allocate extended descriptor */
edesc = skcipher_edesc_alloc(req);
if (IS_ERR(edesc))
@ -1451,6 +1508,8 @@ static int skcipher_decrypt(struct skcipher_request *req)
struct caam_request *caam_req = skcipher_request_ctx(req);
int ret;
if (!req->cryptlen)
return 0;
/* allocate extended descriptor */
edesc = skcipher_edesc_alloc(req);
if (IS_ERR(edesc))
@ -1545,7 +1604,7 @@ static struct caam_skcipher_alg driver_algs[] = {
.cra_driver_name = "cbc-aes-caam-qi2",
.cra_blocksize = AES_BLOCK_SIZE,
},
.setkey = skcipher_setkey,
.setkey = aes_skcipher_setkey,
.encrypt = skcipher_encrypt,
.decrypt = skcipher_decrypt,
.min_keysize = AES_MIN_KEY_SIZE,
@ -1577,7 +1636,7 @@ static struct caam_skcipher_alg driver_algs[] = {
.cra_driver_name = "cbc-des-caam-qi2",
.cra_blocksize = DES_BLOCK_SIZE,
},
.setkey = skcipher_setkey,
.setkey = des_skcipher_setkey,
.encrypt = skcipher_encrypt,
.decrypt = skcipher_decrypt,
.min_keysize = DES_KEY_SIZE,
@ -1593,7 +1652,7 @@ static struct caam_skcipher_alg driver_algs[] = {
.cra_driver_name = "ctr-aes-caam-qi2",
.cra_blocksize = 1,
},
.setkey = skcipher_setkey,
.setkey = ctr_skcipher_setkey,
.encrypt = skcipher_encrypt,
.decrypt = skcipher_decrypt,
.min_keysize = AES_MIN_KEY_SIZE,
@ -1611,7 +1670,7 @@ static struct caam_skcipher_alg driver_algs[] = {
.cra_driver_name = "rfc3686-ctr-aes-caam-qi2",
.cra_blocksize = 1,
},
.setkey = skcipher_setkey,
.setkey = rfc3686_skcipher_setkey,
.encrypt = skcipher_encrypt,
.decrypt = skcipher_decrypt,
.min_keysize = AES_MIN_KEY_SIZE +
@ -1650,7 +1709,7 @@ static struct caam_skcipher_alg driver_algs[] = {
.cra_driver_name = "chacha20-caam-qi2",
.cra_blocksize = 1,
},
.setkey = skcipher_setkey,
.setkey = chacha20_skcipher_setkey,
.encrypt = skcipher_encrypt,
.decrypt = skcipher_decrypt,
.min_keysize = CHACHA_KEY_SIZE,
@ -2918,6 +2977,7 @@ enum hash_optype {
/**
* caam_hash_ctx - ahash per-session context
* @flc: Flow Contexts array
* @key: authentication key
* @flc_dma: I/O virtual addresses of the Flow Contexts
* @dev: dpseci device
* @ctx_len: size of Context Register
@ -2925,6 +2985,7 @@ enum hash_optype {
*/
struct caam_hash_ctx {
struct caam_flc flc[HASH_NUM_OP];
u8 key[CAAM_MAX_HASH_BLOCK_SIZE] ____cacheline_aligned;
dma_addr_t flc_dma[HASH_NUM_OP];
struct device *dev;
int ctx_len;
@ -3094,10 +3155,7 @@ static void split_key_sh_done(void *cbk_ctx, u32 err)
dev_dbg(res->dev, "%s %d: err 0x%x\n", __func__, __LINE__, err);
if (err)
caam_qi2_strstatus(res->dev, err);
res->err = err;
res->err = err ? caam_qi2_strstatus(res->dev, err) : 0;
complete(&res->completion);
}
@ -3228,6 +3286,19 @@ static int ahash_setkey(struct crypto_ahash *ahash, const u8 *key,
ctx->adata.key_virt = key;
ctx->adata.key_inline = true;
/*
* In case |user key| > |derived key|, using DKP<imm,imm> would result
* in invalid opcodes (last bytes of user key) in the resulting
* descriptor. Use DKP<ptr,imm> instead => both virtual and dma key
* addresses are needed.
*/
if (keylen > ctx->adata.keylen_pad) {
memcpy(ctx->key, key, keylen);
dma_sync_single_for_device(ctx->dev, ctx->adata.key_dma,
ctx->adata.keylen_pad,
DMA_TO_DEVICE);
}
ret = ahash_set_sh_desc(ahash);
kfree(hashed_key);
return ret;
@ -3282,10 +3353,8 @@ static void ahash_done(void *cbk_ctx, u32 status)
dev_dbg(ctx->dev, "%s %d: err 0x%x\n", __func__, __LINE__, status);
if (unlikely(status)) {
caam_qi2_strstatus(ctx->dev, status);
ecode = -EIO;
}
if (unlikely(status))
ecode = caam_qi2_strstatus(ctx->dev, status);
ahash_unmap_ctx(ctx->dev, edesc, req, DMA_FROM_DEVICE);
memcpy(req->result, state->caam_ctx, digestsize);
@ -3310,10 +3379,8 @@ static void ahash_done_bi(void *cbk_ctx, u32 status)
dev_dbg(ctx->dev, "%s %d: err 0x%x\n", __func__, __LINE__, status);
if (unlikely(status)) {
caam_qi2_strstatus(ctx->dev, status);
ecode = -EIO;
}
if (unlikely(status))
ecode = caam_qi2_strstatus(ctx->dev, status);
ahash_unmap_ctx(ctx->dev, edesc, req, DMA_BIDIRECTIONAL);
switch_buf(state);
@ -3343,10 +3410,8 @@ static void ahash_done_ctx_src(void *cbk_ctx, u32 status)
dev_dbg(ctx->dev, "%s %d: err 0x%x\n", __func__, __LINE__, status);
if (unlikely(status)) {
caam_qi2_strstatus(ctx->dev, status);
ecode = -EIO;
}
if (unlikely(status))
ecode = caam_qi2_strstatus(ctx->dev, status);
ahash_unmap_ctx(ctx->dev, edesc, req, DMA_BIDIRECTIONAL);
memcpy(req->result, state->caam_ctx, digestsize);
@ -3371,10 +3436,8 @@ static void ahash_done_ctx_dst(void *cbk_ctx, u32 status)
dev_dbg(ctx->dev, "%s %d: err 0x%x\n", __func__, __LINE__, status);
if (unlikely(status)) {
caam_qi2_strstatus(ctx->dev, status);
ecode = -EIO;
}
if (unlikely(status))
ecode = caam_qi2_strstatus(ctx->dev, status);
ahash_unmap_ctx(ctx->dev, edesc, req, DMA_FROM_DEVICE);
switch_buf(state);
@ -4466,11 +4529,27 @@ static int caam_hash_cra_init(struct crypto_tfm *tfm)
ctx->dev = caam_hash->dev;
if (alg->setkey) {
ctx->adata.key_dma = dma_map_single_attrs(ctx->dev, ctx->key,
ARRAY_SIZE(ctx->key),
DMA_TO_DEVICE,
DMA_ATTR_SKIP_CPU_SYNC);
if (dma_mapping_error(ctx->dev, ctx->adata.key_dma)) {
dev_err(ctx->dev, "unable to map key\n");
return -ENOMEM;
}
}
dma_addr = dma_map_single_attrs(ctx->dev, ctx->flc, sizeof(ctx->flc),
DMA_BIDIRECTIONAL,
DMA_ATTR_SKIP_CPU_SYNC);
if (dma_mapping_error(ctx->dev, dma_addr)) {
dev_err(ctx->dev, "unable to map shared descriptors\n");
if (ctx->adata.key_dma)
dma_unmap_single_attrs(ctx->dev, ctx->adata.key_dma,
ARRAY_SIZE(ctx->key),
DMA_TO_DEVICE,
DMA_ATTR_SKIP_CPU_SYNC);
return -ENOMEM;
}
@ -4496,6 +4575,10 @@ static void caam_hash_cra_exit(struct crypto_tfm *tfm)
dma_unmap_single_attrs(ctx->dev, ctx->flc_dma[0], sizeof(ctx->flc),
DMA_BIDIRECTIONAL, DMA_ATTR_SKIP_CPU_SYNC);
if (ctx->adata.key_dma)
dma_unmap_single_attrs(ctx->dev, ctx->adata.key_dma,
ARRAY_SIZE(ctx->key), DMA_TO_DEVICE,
DMA_ATTR_SKIP_CPU_SYNC);
}
static struct caam_hash_alg *caam_hash_alloc(struct device *dev,
@ -4700,7 +4783,7 @@ static void dpaa2_caam_process_fd(struct dpaa2_caam_priv *priv,
fd_err = dpaa2_fd_get_ctrl(fd) & FD_CTRL_ERR_MASK;
if (unlikely(fd_err))
dev_err(priv->dev, "FD error: %08x\n", fd_err);
dev_err_ratelimited(priv->dev, "FD error: %08x\n", fd_err);
/*
* FD[ADDR] is guaranteed to be valid, irrespective of errors reported
@ -5098,6 +5181,8 @@ static int dpaa2_caam_probe(struct fsl_mc_device *dpseci_dev)
goto err_bind;
}
dpaa2_dpseci_debugfs_init(priv);
/* register crypto algorithms the device supports */
for (i = 0; i < ARRAY_SIZE(driver_algs); i++) {
struct caam_skcipher_alg *t_alg = driver_algs + i;
@ -5265,6 +5350,8 @@ static int __cold dpaa2_caam_remove(struct fsl_mc_device *ls_dev)
dev = &ls_dev->dev;
priv = dev_get_drvdata(dev);
dpaa2_dpseci_debugfs_exit(priv);
for (i = 0; i < ARRAY_SIZE(driver_aeads); i++) {
struct caam_aead_alg *t_alg = driver_aeads + i;

31
drivers/crypto/caam/caamalg_qi2.h
View file

@ -10,12 +10,13 @@
#include <soc/fsl/dpaa2-io.h>
#include <soc/fsl/dpaa2-fd.h>
#include <linux/threads.h>
#include <linux/netdevice.h>
#include "dpseci.h"
#include "desc_constr.h"
#define DPAA2_CAAM_STORE_SIZE 16
/* NAPI weight *must* be a multiple of the store size. */
#define DPAA2_CAAM_NAPI_WEIGHT 64
#define DPAA2_CAAM_NAPI_WEIGHT 512
/* The congestion entrance threshold was chosen so that on LS2088
* we support the maximum throughput for the available memory
@ -64,6 +65,7 @@ struct dpaa2_caam_priv {
struct iommu_domain *domain;
struct dpaa2_caam_priv_per_cpu __percpu *ppriv;
struct dentry *dfs_root;
};
/**
@ -90,33 +92,6 @@ struct dpaa2_caam_priv_per_cpu {
struct dpaa2_io *dpio;
};
/*
* The CAAM QI hardware constructs a job descriptor which points
* to shared descriptor (as pointed by context_a of FQ to CAAM).
* When the job descriptor is executed by deco, the whole job
* descriptor together with shared descriptor gets loaded in
* deco buffer which is 64 words long (each 32-bit).
*
* The job descriptor constructed by QI hardware has layout:
*
* HEADER (1 word)
* Shdesc ptr (1 or 2 words)
* SEQ_OUT_PTR (1 word)
* Out ptr (1 or 2 words)
* Out length (1 word)
* SEQ_IN_PTR (1 word)
* In ptr (1 or 2 words)
* In length (1 word)
*
* The shdesc ptr is used to fetch shared descriptor contents
* into deco buffer.
*
* Apart from shdesc contents, the total number of words that
* get loaded in deco buffer are '8' or '11'. The remaining words
* in deco buffer can be used for storing shared descriptor.
*/
#define MAX_SDLEN ((CAAM_DESC_BYTES_MAX - DESC_JOB_IO_LEN) / CAAM_CMD_SZ)
/* Length of a single buffer in the QI driver memory cache */
#define CAAM_QI_MEMCACHE_SIZE 512

116
drivers/crypto/caam/caamhash.c
View file

@ -95,8 +95,8 @@ struct caam_hash_ctx {
dma_addr_t sh_desc_update_first_dma;
dma_addr_t sh_desc_fin_dma;
dma_addr_t sh_desc_digest_dma;
dma_addr_t key_dma;
enum dma_data_direction dir;
enum dma_data_direction key_dir;
struct device *jrdev;
int ctx_len;
struct alginfo adata;
@ -282,13 +282,10 @@ static int axcbc_set_sh_desc(struct crypto_ahash *ahash)
struct device *jrdev = ctx->jrdev;
u32 *desc;
/* key is loaded from memory for UPDATE and FINALIZE states */
ctx->adata.key_dma = ctx->key_dma;
/* shared descriptor for ahash_update */
desc = ctx->sh_desc_update;
cnstr_shdsc_sk_hash(desc, &ctx->adata, OP_ALG_AS_UPDATE,
ctx->ctx_len, ctx->ctx_len, 0);
ctx->ctx_len, ctx->ctx_len);
dma_sync_single_for_device(jrdev, ctx->sh_desc_update_dma,
desc_bytes(desc), ctx->dir);
print_hex_dump_debug("axcbc update shdesc@" __stringify(__LINE__)" : ",
@ -298,7 +295,7 @@ static int axcbc_set_sh_desc(struct crypto_ahash *ahash)
/* shared descriptor for ahash_{final,finup} */
desc = ctx->sh_desc_fin;
cnstr_shdsc_sk_hash(desc, &ctx->adata, OP_ALG_AS_FINALIZE,
digestsize, ctx->ctx_len, 0);
digestsize, ctx->ctx_len);
dma_sync_single_for_device(jrdev, ctx->sh_desc_fin_dma,
desc_bytes(desc), ctx->dir);
print_hex_dump_debug("axcbc finup shdesc@" __stringify(__LINE__)" : ",
@ -311,7 +308,7 @@ static int axcbc_set_sh_desc(struct crypto_ahash *ahash)
/* shared descriptor for first invocation of ahash_update */
desc = ctx->sh_desc_update_first;
cnstr_shdsc_sk_hash(desc, &ctx->adata, OP_ALG_AS_INIT, ctx->ctx_len,
ctx->ctx_len, ctx->key_dma);
ctx->ctx_len);
dma_sync_single_for_device(jrdev, ctx->sh_desc_update_first_dma,
desc_bytes(desc), ctx->dir);
print_hex_dump_debug("axcbc update first shdesc@" __stringify(__LINE__)
@ -321,7 +318,7 @@ static int axcbc_set_sh_desc(struct crypto_ahash *ahash)
/* shared descriptor for ahash_digest */
desc = ctx->sh_desc_digest;
cnstr_shdsc_sk_hash(desc, &ctx->adata, OP_ALG_AS_INITFINAL,
digestsize, ctx->ctx_len, 0);
digestsize, ctx->ctx_len);
dma_sync_single_for_device(jrdev, ctx->sh_desc_digest_dma,
desc_bytes(desc), ctx->dir);
print_hex_dump_debug("axcbc digest shdesc@" __stringify(__LINE__)" : ",
@ -340,7 +337,7 @@ static int acmac_set_sh_desc(struct crypto_ahash *ahash)
/* shared descriptor for ahash_update */
desc = ctx->sh_desc_update;
cnstr_shdsc_sk_hash(desc, &ctx->adata, OP_ALG_AS_UPDATE,
ctx->ctx_len, ctx->ctx_len, 0);
ctx->ctx_len, ctx->ctx_len);
dma_sync_single_for_device(jrdev, ctx->sh_desc_update_dma,
desc_bytes(desc), ctx->dir);
print_hex_dump_debug("acmac update shdesc@" __stringify(__LINE__)" : ",
@ -350,7 +347,7 @@ static int acmac_set_sh_desc(struct crypto_ahash *ahash)
/* shared descriptor for ahash_{final,finup} */
desc = ctx->sh_desc_fin;
cnstr_shdsc_sk_hash(desc, &ctx->adata, OP_ALG_AS_FINALIZE,
digestsize, ctx->ctx_len, 0);
digestsize, ctx->ctx_len);
dma_sync_single_for_device(jrdev, ctx->sh_desc_fin_dma,
desc_bytes(desc), ctx->dir);
print_hex_dump_debug("acmac finup shdesc@" __stringify(__LINE__)" : ",
@ -360,7 +357,7 @@ static int acmac_set_sh_desc(struct crypto_ahash *ahash)
/* shared descriptor for first invocation of ahash_update */
desc = ctx->sh_desc_update_first;
cnstr_shdsc_sk_hash(desc, &ctx->adata, OP_ALG_AS_INIT, ctx->ctx_len,
ctx->ctx_len, 0);
ctx->ctx_len);
dma_sync_single_for_device(jrdev, ctx->sh_desc_update_first_dma,
desc_bytes(desc), ctx->dir);
print_hex_dump_debug("acmac update first shdesc@" __stringify(__LINE__)
@ -370,7 +367,7 @@ static int acmac_set_sh_desc(struct crypto_ahash *ahash)
/* shared descriptor for ahash_digest */
desc = ctx->sh_desc_digest;
cnstr_shdsc_sk_hash(desc, &ctx->adata, OP_ALG_AS_INITFINAL,
digestsize, ctx->ctx_len, 0);
digestsize, ctx->ctx_len);
dma_sync_single_for_device(jrdev, ctx->sh_desc_digest_dma,
desc_bytes(desc), ctx->dir);
print_hex_dump_debug("acmac digest shdesc@" __stringify(__LINE__)" : ",
@ -480,6 +477,18 @@ static int ahash_setkey(struct crypto_ahash *ahash,
goto bad_free_key;
memcpy(ctx->key, key, keylen);
/*
* In case |user key| > |derived key|, using DKP<imm,imm>
* would result in invalid opcodes (last bytes of user key) in
* the resulting descriptor. Use DKP<ptr,imm> instead => both
* virtual and dma key addresses are needed.
*/
if (keylen > ctx->adata.keylen_pad)
dma_sync_single_for_device(ctx->jrdev,
ctx->adata.key_dma,
ctx->adata.keylen_pad,
DMA_TO_DEVICE);
} else {
ret = gen_split_key(ctx->jrdev, ctx->key, &ctx->adata, key,
keylen, CAAM_MAX_HASH_KEY_SIZE);
@ -501,8 +510,14 @@ static int axcbc_setkey(struct crypto_ahash *ahash, const u8 *key,
struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
struct device *jrdev = ctx->jrdev;
if (keylen != AES_KEYSIZE_128) {
crypto_ahash_set_flags(ahash, CRYPTO_TFM_RES_BAD_KEY_LEN);
return -EINVAL;
}
memcpy(ctx->key, key, keylen);
dma_sync_single_for_device(jrdev, ctx->key_dma, keylen, DMA_TO_DEVICE);
dma_sync_single_for_device(jrdev, ctx->adata.key_dma, keylen,
DMA_TO_DEVICE);
ctx->adata.keylen = keylen;
print_hex_dump_debug("axcbc ctx.key@" __stringify(__LINE__)" : ",
@ -515,6 +530,13 @@ static int acmac_setkey(struct crypto_ahash *ahash, const u8 *key,
unsigned int keylen)
{
struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
int err;
err = aes_check_keylen(keylen);
if (err) {
crypto_ahash_set_flags(ahash, CRYPTO_TFM_RES_BAD_KEY_LEN);
return err;
}
/* key is immediate data for all cmac shared descriptors */
ctx->adata.key_virt = key;
@ -538,7 +560,7 @@ struct ahash_edesc {
dma_addr_t sec4_sg_dma;
int src_nents;
int sec4_sg_bytes;
u32 hw_desc[DESC_JOB_IO_LEN / sizeof(u32)] ____cacheline_aligned;
u32 hw_desc[DESC_JOB_IO_LEN_MAX / sizeof(u32)] ____cacheline_aligned;
struct sec4_sg_entry sec4_sg[0];
};
@ -584,12 +606,13 @@ static void ahash_done(struct device *jrdev, u32 *desc, u32 err,
int digestsize = crypto_ahash_digestsize(ahash);
struct caam_hash_state *state = ahash_request_ctx(req);
struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
int ecode = 0;
dev_dbg(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err);
edesc = container_of(desc, struct ahash_edesc, hw_desc[0]);
if (err)
caam_jr_strstatus(jrdev, err);
ecode = caam_jr_strstatus(jrdev, err);
ahash_unmap_ctx(jrdev, edesc, req, digestsize, DMA_FROM_DEVICE);
memcpy(req->result, state->caam_ctx, digestsize);
@ -599,7 +622,7 @@ static void ahash_done(struct device *jrdev, u32 *desc, u32 err,
DUMP_PREFIX_ADDRESS, 16, 4, state->caam_ctx,
ctx->ctx_len, 1);
req->base.complete(&req->base, err);
req->base.complete(&req->base, ecode);
}
static void ahash_done_bi(struct device *jrdev, u32 *desc, u32 err,
@ -611,12 +634,13 @@ static void ahash_done_bi(struct device *jrdev, u32 *desc, u32 err,
struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
struct caam_hash_state *state = ahash_request_ctx(req);
int digestsize = crypto_ahash_digestsize(ahash);
int ecode = 0;
dev_dbg(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err);
edesc = container_of(desc, struct ahash_edesc, hw_desc[0]);
if (err)
caam_jr_strstatus(jrdev, err);
ecode = caam_jr_strstatus(jrdev, err);
ahash_unmap_ctx(jrdev, edesc, req, ctx->ctx_len, DMA_BIDIRECTIONAL);
switch_buf(state);
@ -630,7 +654,7 @@ static void ahash_done_bi(struct device *jrdev, u32 *desc, u32 err,
DUMP_PREFIX_ADDRESS, 16, 4, req->result,
digestsize, 1);
req->base.complete(&req->base, err);
req->base.complete(&req->base, ecode);
}
static void ahash_done_ctx_src(struct device *jrdev, u32 *desc, u32 err,
@ -642,12 +666,13 @@ static void ahash_done_ctx_src(struct device *jrdev, u32 *desc, u32 err,
int digestsize = crypto_ahash_digestsize(ahash);
struct caam_hash_state *state = ahash_request_ctx(req);
struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
int ecode = 0;
dev_dbg(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err);
edesc = container_of(desc, struct ahash_edesc, hw_desc[0]);
if (err)
caam_jr_strstatus(jrdev, err);
ecode = caam_jr_strstatus(jrdev, err);
ahash_unmap_ctx(jrdev, edesc, req, digestsize, DMA_BIDIRECTIONAL);
memcpy(req->result, state->caam_ctx, digestsize);
@ -657,7 +682,7 @@ static void ahash_done_ctx_src(struct device *jrdev, u32 *desc, u32 err,
DUMP_PREFIX_ADDRESS, 16, 4, state->caam_ctx,
ctx->ctx_len, 1);
req->base.complete(&req->base, err);
req->base.complete(&req->base, ecode);
}
static void ahash_done_ctx_dst(struct device *jrdev, u32 *desc, u32 err,
@ -669,12 +694,13 @@ static void ahash_done_ctx_dst(struct device *jrdev, u32 *desc, u32 err,
struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
struct caam_hash_state *state = ahash_request_ctx(req);
int digestsize = crypto_ahash_digestsize(ahash);
int ecode = 0;
dev_dbg(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err);
edesc = container_of(desc, struct ahash_edesc, hw_desc[0]);
if (err)
caam_jr_strstatus(jrdev, err);
ecode = caam_jr_strstatus(jrdev, err);
ahash_unmap_ctx(jrdev, edesc, req, ctx->ctx_len, DMA_FROM_DEVICE);
switch_buf(state);
@ -688,7 +714,7 @@ static void ahash_done_ctx_dst(struct device *jrdev, u32 *desc, u32 err,
DUMP_PREFIX_ADDRESS, 16, 4, req->result,
digestsize, 1);
req->base.complete(&req->base, err);
req->base.complete(&req->base, ecode);
}
/*
@ -1812,40 +1838,50 @@ static int caam_hash_cra_init(struct crypto_tfm *tfm)
if (is_xcbc_aes(caam_hash->alg_type)) {
ctx->dir = DMA_TO_DEVICE;
ctx->key_dir = DMA_BIDIRECTIONAL;
ctx->adata.algtype = OP_TYPE_CLASS1_ALG | caam_hash->alg_type;
ctx->ctx_len = 48;
ctx->key_dma = dma_map_single_attrs(ctx->jrdev, ctx->key,
ARRAY_SIZE(ctx->key),
DMA_BIDIRECTIONAL,
DMA_ATTR_SKIP_CPU_SYNC);
if (dma_mapping_error(ctx->jrdev, ctx->key_dma)) {
dev_err(ctx->jrdev, "unable to map key\n");
caam_jr_free(ctx->jrdev);
return -ENOMEM;
}
} else if (is_cmac_aes(caam_hash->alg_type)) {
ctx->dir = DMA_TO_DEVICE;
ctx->key_dir = DMA_NONE;
ctx->adata.algtype = OP_TYPE_CLASS1_ALG | caam_hash->alg_type;
ctx->ctx_len = 32;
} else {
ctx->dir = priv->era >= 6 ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE;
if (priv->era >= 6) {
ctx->dir = DMA_BIDIRECTIONAL;
ctx->key_dir = alg->setkey ? DMA_TO_DEVICE : DMA_NONE;
} else {
ctx->dir = DMA_TO_DEVICE;
ctx->key_dir = DMA_NONE;
}
ctx->adata.algtype = OP_TYPE_CLASS2_ALG | caam_hash->alg_type;
ctx->ctx_len = runninglen[(ctx->adata.algtype &
OP_ALG_ALGSEL_SUBMASK) >>
OP_ALG_ALGSEL_SHIFT];
}
if (ctx->key_dir != DMA_NONE) {
ctx->adata.key_dma = dma_map_single_attrs(ctx->jrdev, ctx->key,
ARRAY_SIZE(ctx->key),
ctx->key_dir,
DMA_ATTR_SKIP_CPU_SYNC);
if (dma_mapping_error(ctx->jrdev, ctx->adata.key_dma)) {
dev_err(ctx->jrdev, "unable to map key\n");
caam_jr_free(ctx->jrdev);
return -ENOMEM;
}
}
dma_addr = dma_map_single_attrs(ctx->jrdev, ctx->sh_desc_update,
offsetof(struct caam_hash_ctx, key),
ctx->dir, DMA_ATTR_SKIP_CPU_SYNC);
if (dma_mapping_error(ctx->jrdev, dma_addr)) {
dev_err(ctx->jrdev, "unable to map shared descriptors\n");
if (is_xcbc_aes(caam_hash->alg_type))
dma_unmap_single_attrs(ctx->jrdev, ctx->key_dma,
if (ctx->key_dir != DMA_NONE)
dma_unmap_single_attrs(ctx->jrdev, ctx->adata.key_dma,
ARRAY_SIZE(ctx->key),
DMA_BIDIRECTIONAL,
ctx->key_dir,
DMA_ATTR_SKIP_CPU_SYNC);
caam_jr_free(ctx->jrdev);
@ -1878,9 +1914,9 @@ static void caam_hash_cra_exit(struct crypto_tfm *tfm)
dma_unmap_single_attrs(ctx->jrdev, ctx->sh_desc_update_dma,
offsetof(struct caam_hash_ctx, key),
ctx->dir, DMA_ATTR_SKIP_CPU_SYNC);
if (is_xcbc_aes(ctx->adata.algtype))
dma_unmap_single_attrs(ctx->jrdev, ctx->key_dma,
ARRAY_SIZE(ctx->key), DMA_BIDIRECTIONAL,
if (ctx->key_dir != DMA_NONE)
dma_unmap_single_attrs(ctx->jrdev, ctx->adata.key_dma,
ARRAY_SIZE(ctx->key), ctx->key_dir,
DMA_ATTR_SKIP_CPU_SYNC);
caam_jr_free(ctx->jrdev);
}
@ -1971,7 +2007,7 @@ int caam_algapi_hash_init(struct device *ctrldev)
* is not present.
*/
if (!md_inst)
return -ENODEV;
return 0;
/* Limit digest size based on LP256 */
if (md_vid == CHA_VER_VID_MD_LP256)

5
drivers/crypto/caam/caamhash_desc.c
View file

@ -83,10 +83,9 @@ EXPORT_SYMBOL(cnstr_shdsc_ahash);
* @state: algorithm state OP_ALG_AS_{INIT, FINALIZE, INITFINALIZE, UPDATE}
* @digestsize: algorithm's digest size
* @ctx_len: size of Context Register
* @key_dma: I/O Virtual Address of the key
*/
void cnstr_shdsc_sk_hash(u32 * const desc, struct alginfo *adata, u32 state,
int digestsize, int ctx_len, dma_addr_t key_dma)
int digestsize, int ctx_len)
{
u32 *skip_key_load;
@ -136,7 +135,7 @@ void cnstr_shdsc_sk_hash(u32 * const desc, struct alginfo *adata, u32 state,
LDST_SRCDST_BYTE_CONTEXT);
if (is_xcbc_aes(adata->algtype) && state == OP_ALG_AS_INIT)
/* Save K1 */
append_fifo_store(desc, key_dma, adata->keylen,
append_fifo_store(desc, adata->key_dma, adata->keylen,
LDST_CLASS_1_CCB | FIFOST_TYPE_KEY_KEK);
}
EXPORT_SYMBOL(cnstr_shdsc_sk_hash);

2
drivers/crypto/caam/caamhash_desc.h
View file

@ -25,5 +25,5 @@ void cnstr_shdsc_ahash(u32 * const desc, struct alginfo *adata, u32 state,
int digestsize, int ctx_len, bool import_ctx, int era);
void cnstr_shdsc_sk_hash(u32 * const desc, struct alginfo *adata, u32 state,
int digestsize, int ctx_len, dma_addr_t key_dma);
int digestsize, int ctx_len);
#endif /* _CAAMHASH_DESC_H_ */

99
drivers/crypto/caam/caampkc.c
View file

@ -17,18 +17,29 @@
#include "sg_sw_sec4.h"
#include "caampkc.h"
#define DESC_RSA_PUB_LEN (2 * CAAM_CMD_SZ + sizeof(struct rsa_pub_pdb))
#define DESC_RSA_PUB_LEN (2 * CAAM_CMD_SZ + SIZEOF_RSA_PUB_PDB)
#define DESC_RSA_PRIV_F1_LEN (2 * CAAM_CMD_SZ + \
sizeof(struct rsa_priv_f1_pdb))
SIZEOF_RSA_PRIV_F1_PDB)
#define DESC_RSA_PRIV_F2_LEN (2 * CAAM_CMD_SZ + \
sizeof(struct rsa_priv_f2_pdb))
SIZEOF_RSA_PRIV_F2_PDB)
#define DESC_RSA_PRIV_F3_LEN (2 * CAAM_CMD_SZ + \
sizeof(struct rsa_priv_f3_pdb))
SIZEOF_RSA_PRIV_F3_PDB)
#define CAAM_RSA_MAX_INPUT_SIZE 512 /* for a 4096-bit modulus */
/* buffer filled with zeros, used for padding */
static u8 *zero_buffer;
/*
* variable used to avoid double free of resources in case
* algorithm registration was unsuccessful
*/
static bool init_done;
struct caam_akcipher_alg {
struct akcipher_alg akcipher;
bool registered;
};
static void rsa_io_unmap(struct device *dev, struct rsa_edesc *edesc,
struct akcipher_request *req)
{
@ -107,9 +118,10 @@ static void rsa_pub_done(struct device *dev, u32 *desc, u32 err, void *context)
{
struct akcipher_request *req = context;
struct rsa_edesc *edesc;
int ecode = 0;
if (err)
caam_jr_strstatus(dev, err);
ecode = caam_jr_strstatus(dev, err);
edesc = container_of(desc, struct rsa_edesc, hw_desc[0]);
@ -117,7 +129,7 @@ static void rsa_pub_done(struct device *dev, u32 *desc, u32 err, void *context)
rsa_io_unmap(dev, edesc, req);
kfree(edesc);
akcipher_request_complete(req, err);
akcipher_request_complete(req, ecode);
}
static void rsa_priv_f1_done(struct device *dev, u32 *desc, u32 err,
@ -125,9 +137,10 @@ static void rsa_priv_f1_done(struct device *dev, u32 *desc, u32 err,
{
struct akcipher_request *req = context;
struct rsa_edesc *edesc;
int ecode = 0;
if (err)
caam_jr_strstatus(dev, err);
ecode = caam_jr_strstatus(dev, err);
edesc = container_of(desc, struct rsa_edesc, hw_desc[0]);
@ -135,7 +148,7 @@ static void rsa_priv_f1_done(struct device *dev, u32 *desc, u32 err,
rsa_io_unmap(dev, edesc, req);
kfree(edesc);
akcipher_request_complete(req, err);
akcipher_request_complete(req, ecode);
}
static void rsa_priv_f2_done(struct device *dev, u32 *desc, u32 err,
@ -143,9 +156,10 @@ static void rsa_priv_f2_done(struct device *dev, u32 *desc, u32 err,
{
struct akcipher_request *req = context;
struct rsa_edesc *edesc;
int ecode = 0;
if (err)
caam_jr_strstatus(dev, err);
ecode = caam_jr_strstatus(dev, err);
edesc = container_of(desc, struct rsa_edesc, hw_desc[0]);
@ -153,7 +167,7 @@ static void rsa_priv_f2_done(struct device *dev, u32 *desc, u32 err,
rsa_io_unmap(dev, edesc, req);
kfree(edesc);
akcipher_request_complete(req, err);
akcipher_request_complete(req, ecode);
}
static void rsa_priv_f3_done(struct device *dev, u32 *desc, u32 err,
@ -161,9 +175,10 @@ static void rsa_priv_f3_done(struct device *dev, u32 *desc, u32 err,
{
struct akcipher_request *req = context;
struct rsa_edesc *edesc;
int ecode = 0;
if (err)
caam_jr_strstatus(dev, err);
ecode = caam_jr_strstatus(dev, err);
edesc = container_of(desc, struct rsa_edesc, hw_desc[0]);
@ -171,7 +186,7 @@ static void rsa_priv_f3_done(struct device *dev, u32 *desc, u32 err,
rsa_io_unmap(dev, edesc, req);
kfree(edesc);
akcipher_request_complete(req, err);
akcipher_request_complete(req, ecode);
}
/**
@ -867,7 +882,7 @@ static int caam_rsa_set_pub_key(struct crypto_akcipher *tfm, const void *key,
return ret;
/* Copy key in DMA zone */
rsa_key->e = kzalloc(raw_key.e_sz, GFP_DMA | GFP_KERNEL);
rsa_key->e = kmemdup(raw_key.e, raw_key.e_sz, GFP_DMA | GFP_KERNEL);
if (!rsa_key->e)
goto err;
@ -889,8 +904,6 @@ static int caam_rsa_set_pub_key(struct crypto_akcipher *tfm, const void *key,
rsa_key->e_sz = raw_key.e_sz;
rsa_key->n_sz = raw_key.n_sz;
memcpy(rsa_key->e, raw_key.e, raw_key.e_sz);
return 0;
err:
caam_rsa_free_key(rsa_key);
@ -971,11 +984,11 @@ static int caam_rsa_set_priv_key(struct crypto_akcipher *tfm, const void *key,
return ret;
/* Copy key in DMA zone */
rsa_key->d = kzalloc(raw_key.d_sz, GFP_DMA | GFP_KERNEL);
rsa_key->d = kmemdup(raw_key.d, raw_key.d_sz, GFP_DMA | GFP_KERNEL);
if (!rsa_key->d)
goto err;
rsa_key->e = kzalloc(raw_key.e_sz, GFP_DMA | GFP_KERNEL);
rsa_key->e = kmemdup(raw_key.e, raw_key.e_sz, GFP_DMA | GFP_KERNEL);
if (!rsa_key->e)
goto err;
@ -998,9 +1011,6 @@ static int caam_rsa_set_priv_key(struct crypto_akcipher *tfm, const void *key,
rsa_key->e_sz = raw_key.e_sz;
rsa_key->n_sz = raw_key.n_sz;
memcpy(rsa_key->d, raw_key.d, raw_key.d_sz);
memcpy(rsa_key->e, raw_key.e, raw_key.e_sz);
caam_rsa_set_priv_key_form(ctx, &raw_key);
return 0;
@ -1053,22 +1063,24 @@ static void caam_rsa_exit_tfm(struct crypto_akcipher *tfm)
caam_jr_free(ctx->dev);
}
static struct akcipher_alg caam_rsa = {
.encrypt = caam_rsa_enc,
.decrypt = caam_rsa_dec,
.set_pub_key = caam_rsa_set_pub_key,
.set_priv_key = caam_rsa_set_priv_key,
.max_size = caam_rsa_max_size,
.init = caam_rsa_init_tfm,
.exit = caam_rsa_exit_tfm,
.reqsize = sizeof(struct caam_rsa_req_ctx),
.base = {
.cra_name = "rsa",
.cra_driver_name = "rsa-caam",
.cra_priority = 3000,
.cra_module = THIS_MODULE,
.cra_ctxsize = sizeof(struct caam_rsa_ctx),
},
static struct caam_akcipher_alg caam_rsa = {
.akcipher = {
.encrypt = caam_rsa_enc,
.decrypt = caam_rsa_dec,
.set_pub_key = caam_rsa_set_pub_key,
.set_priv_key = caam_rsa_set_priv_key,
.max_size = caam_rsa_max_size,
.init = caam_rsa_init_tfm,
.exit = caam_rsa_exit_tfm,
.reqsize = sizeof(struct caam_rsa_req_ctx),
.base = {
.cra_name = "rsa",
.cra_driver_name = "rsa-caam",
.cra_priority = 3000,
.cra_module = THIS_MODULE,
.cra_ctxsize = sizeof(struct caam_rsa_ctx),
},
}
};
/* Public Key Cryptography module initialization handler */
@ -1077,6 +1089,7 @@ int caam_pkc_init(struct device *ctrldev)
struct caam_drv_private *priv = dev_get_drvdata(ctrldev);
u32 pk_inst;
int err;
init_done = false;
/* Determine public key hardware accelerator presence. */
if (priv->era < 10)
@ -1095,12 +1108,15 @@ int caam_pkc_init(struct device *ctrldev)
if (!zero_buffer)
return -ENOMEM;
err = crypto_register_akcipher(&caam_rsa);
err = crypto_register_akcipher(&caam_rsa.akcipher);
if (err) {
kfree(zero_buffer);
dev_warn(ctrldev, "%s alg registration failed\n",
caam_rsa.base.cra_driver_name);
caam_rsa.akcipher.base.cra_driver_name);
} else {
init_done = true;
caam_rsa.registered = true;
dev_info(ctrldev, "caam pkc algorithms registered in /proc/crypto\n");
}
@ -1109,6 +1125,11 @@ int caam_pkc_init(struct device *ctrldev)
void caam_pkc_exit(void)
{
if (!init_done)
return;
if (caam_rsa.registered)
crypto_unregister_akcipher(&caam_rsa.akcipher);
kfree(zero_buffer);
crypto_unregister_akcipher(&caam_rsa);
}

19
drivers/crypto/caam/caamrng.c
View file

@ -53,7 +53,7 @@
L1_CACHE_BYTES)
/* length of descriptors */
#define DESC_JOB_O_LEN (CAAM_CMD_SZ * 2 + CAAM_PTR_SZ * 2)
#define DESC_JOB_O_LEN (CAAM_CMD_SZ * 2 + CAAM_PTR_SZ_MAX * 2)
#define DESC_RNG_LEN (3 * CAAM_CMD_SZ)
/* Buffer, its dma address and lock */
@ -80,6 +80,12 @@ struct caam_rng_ctx {
static struct caam_rng_ctx *rng_ctx;
/*
* Variable used to avoid double free of resources in case
* algorithm registration was unsuccessful
*/
static bool init_done;
static inline void rng_unmap_buf(struct device *jrdev, struct buf_data *bd)
{
if (bd->addr)
@ -296,6 +302,9 @@ static struct hwrng caam_rng = {
void caam_rng_exit(void)
{
if (!init_done)
return;
caam_jr_free(rng_ctx->jrdev);
hwrng_unregister(&caam_rng);
kfree(rng_ctx);
@ -307,6 +316,7 @@ int caam_rng_init(struct device *ctrldev)
u32 rng_inst;
struct caam_drv_private *priv = dev_get_drvdata(ctrldev);
int err;
init_done = false;
/* Check for an instantiated RNG before registration */
if (priv->era < 10)
@ -333,7 +343,12 @@ int caam_rng_init(struct device *ctrldev)
goto free_rng_ctx;
dev_info(dev, "registering rng-caam\n");
return hwrng_register(&caam_rng);
err = hwrng_register(&caam_rng);
if (!err) {
init_done = true;
return err;
}
free_rng_ctx:
kfree(rng_ctx);

2
drivers/crypto/caam/compat.h
View file

@ -32,7 +32,7 @@
#include <crypto/null.h>
#include <crypto/aes.h>
#include <crypto/ctr.h>
#include <crypto/des.h>
#include <crypto/internal/des.h>
#include <crypto/gcm.h>
#include <crypto/sha.h>
#include <crypto/md5.h>

257
drivers/crypto/caam/ctrl.c
View file

@ -25,16 +25,6 @@ EXPORT_SYMBOL(caam_dpaa2);
#include "qi.h"
#endif
/*
* i.MX targets tend to have clock control subsystems that can
* enable/disable clocking to our device.
*/
static inline struct clk *caam_drv_identify_clk(struct device *dev,
char *clk_name)
{
return caam_imx ? devm_clk_get(dev, clk_name) : NULL;
}
/*
* Descriptor to instantiate RNG State Handle 0 in normal mode and
* load the JDKEK, TDKEK and TDSK registers
@ -107,7 +97,12 @@ static inline int run_descriptor_deco0(struct device *ctrldev, u32 *desc,
int i;
if (ctrlpriv->virt_en == 1) {
if (ctrlpriv->virt_en == 1 ||
/*
* Apparently on i.MX8MQ it doesn't matter if virt_en == 1
* and the following steps should be performed regardless
*/
of_machine_is_compatible("fsl,imx8mq")) {
clrsetbits_32(&ctrl->deco_rsr, 0, DECORSR_JR0);
while (!(rd_reg32(&ctrl->deco_rsr) & DECORSR_VALID) &&
@ -342,13 +337,6 @@ static int caam_remove(struct platform_device *pdev)
/* Unmap controller region */
iounmap(ctrl);
/* shut clocks off before finalizing shutdown */
clk_disable_unprepare(ctrlpriv->caam_ipg);
if (ctrlpriv->caam_mem)
clk_disable_unprepare(ctrlpriv->caam_mem);
clk_disable_unprepare(ctrlpriv->caam_aclk);
if (ctrlpriv->caam_emi_slow)
clk_disable_unprepare(ctrlpriv->caam_emi_slow);
return 0;
}
@ -497,20 +485,99 @@ static const struct of_device_id caam_match[] = {
};
MODULE_DEVICE_TABLE(of, caam_match);
struct caam_imx_data {
const struct clk_bulk_data *clks;
int num_clks;
};
static const struct clk_bulk_data caam_imx6_clks[] = {
{ .id = "ipg" },
{ .id = "mem" },
{ .id = "aclk" },
{ .id = "emi_slow" },
};
static const struct caam_imx_data caam_imx6_data = {
.clks = caam_imx6_clks,
.num_clks = ARRAY_SIZE(caam_imx6_clks),
};
static const struct clk_bulk_data caam_imx7_clks[] = {
{ .id = "ipg" },
{ .id = "aclk" },
};
static const struct caam_imx_data caam_imx7_data = {
.clks = caam_imx7_clks,
.num_clks = ARRAY_SIZE(caam_imx7_clks),
};
static const struct clk_bulk_data caam_imx6ul_clks[] = {
{ .id = "ipg" },
{ .id = "mem" },
{ .id = "aclk" },
};
static const struct caam_imx_data caam_imx6ul_data = {
.clks = caam_imx6ul_clks,
.num_clks = ARRAY_SIZE(caam_imx6ul_clks),
};
static const struct soc_device_attribute caam_imx_soc_table[] = {
{ .soc_id = "i.MX6UL", .data = &caam_imx6ul_data },
{ .soc_id = "i.MX6*", .data = &caam_imx6_data },
{ .soc_id = "i.MX7*", .data = &caam_imx7_data },
{ .soc_id = "i.MX8MQ", .data = &caam_imx7_data },
{ .family = "Freescale i.MX" },
{ /* sentinel */ }
};
static void disable_clocks(void *data)
{
struct caam_drv_private *ctrlpriv = data;
clk_bulk_disable_unprepare(ctrlpriv->num_clks, ctrlpriv->clks);
}
static int init_clocks(struct device *dev, const struct caam_imx_data *data)
{
struct caam_drv_private *ctrlpriv = dev_get_drvdata(dev);
int ret;
ctrlpriv->num_clks = data->num_clks;
ctrlpriv->clks = devm_kmemdup(dev, data->clks,
data->num_clks * sizeof(data->clks[0]),
GFP_KERNEL);
if (!ctrlpriv->clks)
return -ENOMEM;
ret = devm_clk_bulk_get(dev, ctrlpriv->num_clks, ctrlpriv->clks);
if (ret) {
dev_err(dev,
"Failed to request all necessary clocks\n");
return ret;
}
ret = clk_bulk_prepare_enable(ctrlpriv->num_clks, ctrlpriv->clks);
if (ret) {
dev_err(dev,
"Failed to prepare/enable all necessary clocks\n");
return ret;
}
return devm_add_action_or_reset(dev, disable_clocks, ctrlpriv);
}
/* Probe routine for CAAM top (controller) level */
static int caam_probe(struct platform_device *pdev)
{
int ret, ring, gen_sk, ent_delay = RTSDCTL_ENT_DLY_MIN;
u64 caam_id;
static const struct soc_device_attribute imx_soc[] = {
{.family = "Freescale i.MX"},
{},
};
const struct soc_device_attribute *imx_soc_match;
struct device *dev;
struct device_node *nprop, *np;
struct caam_ctrl __iomem *ctrl;
struct caam_drv_private *ctrlpriv;
struct clk *clk;
#ifdef CONFIG_DEBUG_FS
struct caam_perfmon *perfmon;
#endif
@ -527,103 +594,68 @@ static int caam_probe(struct platform_device *pdev)
dev_set_drvdata(dev, ctrlpriv);
nprop = pdev->dev.of_node;
caam_imx = (bool)soc_device_match(imx_soc);
imx_soc_match = soc_device_match(caam_imx_soc_table);
caam_imx = (bool)imx_soc_match;
/* Enable clocking */
clk = caam_drv_identify_clk(&pdev->dev, "ipg");
if (IS_ERR(clk)) {
ret = PTR_ERR(clk);
dev_err(&pdev->dev,
"can't identify CAAM ipg clk: %d\n", ret);
return ret;
}
ctrlpriv->caam_ipg = clk;
if (imx_soc_match) {
if (!imx_soc_match->data) {
dev_err(dev, "No clock data provided for i.MX SoC");
return -EINVAL;
}
if (!of_machine_is_compatible("fsl,imx7d") &&
!of_machine_is_compatible("fsl,imx7s") &&
!of_machine_is_compatible("fsl,imx7ulp")) {
clk = caam_drv_identify_clk(&pdev->dev, "mem");
if (IS_ERR(clk)) {
ret = PTR_ERR(clk);
dev_err(&pdev->dev,
"can't identify CAAM mem clk: %d\n", ret);
ret = init_clocks(dev, imx_soc_match->data);
if (ret)
return ret;
}
ctrlpriv->caam_mem = clk;
}
clk = caam_drv_identify_clk(&pdev->dev, "aclk");
if (IS_ERR(clk)) {
ret = PTR_ERR(clk);
dev_err(&pdev->dev,
"can't identify CAAM aclk clk: %d\n", ret);
return ret;
}
ctrlpriv->caam_aclk = clk;
if (!of_machine_is_compatible("fsl,imx6ul") &&
!of_machine_is_compatible("fsl,imx7d") &&
!of_machine_is_compatible("fsl,imx7s") &&
!of_machine_is_compatible("fsl,imx7ulp")) {
clk = caam_drv_identify_clk(&pdev->dev, "emi_slow");
if (IS_ERR(clk)) {
ret = PTR_ERR(clk);
dev_err(&pdev->dev,
"can't identify CAAM emi_slow clk: %d\n", ret);
return ret;
}
ctrlpriv->caam_emi_slow = clk;
}
ret = clk_prepare_enable(ctrlpriv->caam_ipg);
if (ret < 0) {
dev_err(&pdev->dev, "can't enable CAAM ipg clock: %d\n", ret);
return ret;
}
if (ctrlpriv->caam_mem) {
ret = clk_prepare_enable(ctrlpriv->caam_mem);
if (ret < 0) {
dev_err(&pdev->dev, "can't enable CAAM secure mem clock: %d\n",
ret);
goto disable_caam_ipg;
}
}
ret = clk_prepare_enable(ctrlpriv->caam_aclk);
if (ret < 0) {
dev_err(&pdev->dev, "can't enable CAAM aclk clock: %d\n", ret);
goto disable_caam_mem;
}
if (ctrlpriv->caam_emi_slow) {
ret = clk_prepare_enable(ctrlpriv->caam_emi_slow);
if (ret < 0) {
dev_err(&pdev->dev, "can't enable CAAM emi slow clock: %d\n",
ret);
goto disable_caam_aclk;
}
}
/* Get configuration properties from device tree */
/* First, get register page */
ctrl = of_iomap(nprop, 0);
if (ctrl == NULL) {
if (!ctrl) {
dev_err(dev, "caam: of_iomap() failed\n");
ret = -ENOMEM;
goto disable_caam_emi_slow;
return -ENOMEM;
}
caam_little_end = !(bool)(rd_reg32(&ctrl->perfmon.status) &
(CSTA_PLEND | CSTA_ALT_PLEND));
/* Finding the page size for using the CTPR_MS register */
comp_params = rd_reg32(&ctrl->perfmon.comp_parms_ms);
pg_size = (comp_params & CTPR_MS_PG_SZ_MASK) >> CTPR_MS_PG_SZ_SHIFT;
if (comp_params & CTPR_MS_PS && rd_reg32(&ctrl->mcr) & MCFGR_LONG_PTR)
caam_ptr_sz = sizeof(u64);
else
caam_ptr_sz = sizeof(u32);
caam_dpaa2 = !!(comp_params & CTPR_MS_DPAA2);
ctrlpriv->qi_present = !!(comp_params & CTPR_MS_QI_MASK);
#ifdef CONFIG_CAAM_QI
/* If (DPAA 1.x) QI present, check whether dependencies are available */
if (ctrlpriv->qi_present && !caam_dpaa2) {
ret = qman_is_probed();
if (!ret) {
ret = -EPROBE_DEFER;
goto iounmap_ctrl;
} else if (ret < 0) {
dev_err(dev, "failing probe due to qman probe error\n");
ret = -ENODEV;
goto iounmap_ctrl;
}
ret = qman_portals_probed();
if (!ret) {
ret = -EPROBE_DEFER;
goto iounmap_ctrl;
} else if (ret < 0) {
dev_err(dev, "failing probe due to qman portals probe error\n");
ret = -ENODEV;
goto iounmap_ctrl;
}
}
#endif
/* Allocating the BLOCK_OFFSET based on the supported page size on
* the platform
*/
pg_size = (comp_params & CTPR_MS_PG_SZ_MASK) >> CTPR_MS_PG_SZ_SHIFT;
if (pg_size == 0)
BLOCK_OFFSET = PG_SIZE_4K;
else
@ -648,7 +680,6 @@ static int caam_probe(struct platform_device *pdev)
* In case of SoCs with Management Complex, MC f/w performs
* the configuration.
*/
caam_dpaa2 = !!(comp_params & CTPR_MS_DPAA2);
np = of_find_compatible_node(NULL, NULL, "fsl,qoriq-mc");
ctrlpriv->mc_en = !!np;
of_node_put(np);
@ -688,16 +719,7 @@ static int caam_probe(struct platform_device *pdev)
JRSTART_JR1_START | JRSTART_JR2_START |
JRSTART_JR3_START);
if (sizeof(dma_addr_t) == sizeof(u64)) {
if (caam_dpaa2)
ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(49));
else if (of_device_is_compatible(nprop, "fsl,sec-v5.0"))
ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(40));
else
ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(36));
} else {
ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(32));
}
ret = dma_set_mask_and_coherent(dev, caam_get_dma_mask(dev));
if (ret) {
dev_err(dev, "dma_set_mask_and_coherent failed (%d)\n", ret);
goto iounmap_ctrl;
@ -719,7 +741,6 @@ static int caam_probe(struct platform_device *pdev)
#endif
/* Check to see if (DPAA 1.x) QI present. If so, enable */
ctrlpriv->qi_present = !!(comp_params & CTPR_MS_QI_MASK);
if (ctrlpriv->qi_present && !caam_dpaa2) {
ctrlpriv->qi = (struct caam_queue_if __iomem __force *)
((__force uint8_t *)ctrl +
@ -908,16 +929,6 @@ shutdown_qi:
#endif
iounmap_ctrl:
iounmap(ctrl);
disable_caam_emi_slow:
if (ctrlpriv->caam_emi_slow)
clk_disable_unprepare(ctrlpriv->caam_emi_slow);
disable_caam_aclk:
clk_disable_unprepare(ctrlpriv->caam_aclk);
disable_caam_mem:
if (ctrlpriv->caam_mem)
clk_disable_unprepare(ctrlpriv->caam_mem);
disable_caam_ipg:
clk_disable_unprepare(ctrlpriv->caam_ipg);
return ret;
}

81
drivers/crypto/caam/desc_constr.h
View file

@ -14,9 +14,41 @@
#define IMMEDIATE (1 << 23)
#define CAAM_CMD_SZ sizeof(u32)
#define CAAM_PTR_SZ sizeof(dma_addr_t)
#define CAAM_PTR_SZ caam_ptr_sz
#define CAAM_PTR_SZ_MAX sizeof(dma_addr_t)
#define CAAM_PTR_SZ_MIN sizeof(u32)
#define CAAM_DESC_BYTES_MAX (CAAM_CMD_SZ * MAX_CAAM_DESCSIZE)
#define DESC_JOB_IO_LEN (CAAM_CMD_SZ * 5 + CAAM_PTR_SZ * 3)
#define __DESC_JOB_IO_LEN(n) (CAAM_CMD_SZ * 5 + (n) * 3)
#define DESC_JOB_IO_LEN __DESC_JOB_IO_LEN(CAAM_PTR_SZ)
#define DESC_JOB_IO_LEN_MAX __DESC_JOB_IO_LEN(CAAM_PTR_SZ_MAX)
#define DESC_JOB_IO_LEN_MIN __DESC_JOB_IO_LEN(CAAM_PTR_SZ_MIN)
/*
* The CAAM QI hardware constructs a job descriptor which points
* to shared descriptor (as pointed by context_a of FQ to CAAM).
* When the job descriptor is executed by deco, the whole job
* descriptor together with shared descriptor gets loaded in
* deco buffer which is 64 words long (each 32-bit).
*
* The job descriptor constructed by QI hardware has layout:
*
* HEADER (1 word)
* Shdesc ptr (1 or 2 words)
* SEQ_OUT_PTR (1 word)
* Out ptr (1 or 2 words)
* Out length (1 word)
* SEQ_IN_PTR (1 word)
* In ptr (1 or 2 words)
* In length (1 word)
*
* The shdesc ptr is used to fetch shared descriptor contents
* into deco buffer.
*
* Apart from shdesc contents, the total number of words that
* get loaded in deco buffer are '8' or '11'. The remaining words
* in deco buffer can be used for storing shared descriptor.
*/
#define MAX_SDLEN ((CAAM_DESC_BYTES_MAX - DESC_JOB_IO_LEN_MIN) / CAAM_CMD_SZ)
#ifdef DEBUG
#define PRINT_POS do { printk(KERN_DEBUG "%02d: %s\n", desc_len(desc),\
@ -37,6 +69,7 @@
(LDOFF_ENABLE_AUTO_NFIFO << LDST_OFFSET_SHIFT))
extern bool caam_little_end;
extern size_t caam_ptr_sz;
/*
* HW fetches 4 S/G table entries at a time, irrespective of how many entries
@ -103,9 +136,15 @@ static inline void init_job_desc_pdb(u32 * const desc, u32 options,
static inline void append_ptr(u32 * const desc, dma_addr_t ptr)
{
dma_addr_t *offset = (dma_addr_t *)desc_end(desc);
if (caam_ptr_sz == sizeof(dma_addr_t)) {
dma_addr_t *offset = (dma_addr_t *)desc_end(desc);
*offset = cpu_to_caam_dma(ptr);
*offset = cpu_to_caam_dma(ptr);
} else {
u32 *offset = (u32 *)desc_end(desc);
*offset = cpu_to_caam_dma(ptr);
}
(*desc) = cpu_to_caam32(caam32_to_cpu(*desc) +
CAAM_PTR_SZ / CAAM_CMD_SZ);
@ -457,8 +496,8 @@ do { \
* functions where it is used.
* @keylen: length of the provided algorithm key, in bytes
* @keylen_pad: padded length of the provided algorithm key, in bytes
* @key: address where algorithm key resides; virtual address if key_inline
* is true, dma (bus) address if key_inline is false.
* @key_dma: dma (bus) address where algorithm key resides
* @key_virt: virtual address where algorithm key resides
* @key_inline: true - key can be inlined in the descriptor; false - key is
* referenced by the descriptor
*/
@ -466,10 +505,8 @@ struct alginfo {
u32 algtype;
unsigned int keylen;
unsigned int keylen_pad;
union {
dma_addr_t key_dma;
const void *key_virt;
};
dma_addr_t key_dma;
const void *key_virt;
bool key_inline;
};
@ -535,14 +572,26 @@ static inline void append_proto_dkp(u32 * const desc, struct alginfo *adata)
if (adata->key_inline) {
int words;
append_operation(desc, OP_TYPE_UNI_PROTOCOL | protid |
OP_PCL_DKP_SRC_IMM | OP_PCL_DKP_DST_IMM |
adata->keylen);
append_data(desc, adata->key_virt, adata->keylen);
if (adata->keylen > adata->keylen_pad) {
append_operation(desc, OP_TYPE_UNI_PROTOCOL | protid |
OP_PCL_DKP_SRC_PTR |
OP_PCL_DKP_DST_IMM | adata->keylen);
append_ptr(desc, adata->key_dma);
words = (ALIGN(adata->keylen_pad, CAAM_CMD_SZ) -
CAAM_PTR_SZ) / CAAM_CMD_SZ;
} else {
append_operation(desc, OP_TYPE_UNI_PROTOCOL | protid |
OP_PCL_DKP_SRC_IMM |
OP_PCL_DKP_DST_IMM | adata->keylen);
append_data(desc, adata->key_virt, adata->keylen);
words = (ALIGN(adata->keylen_pad, CAAM_CMD_SZ) -
ALIGN(adata->keylen, CAAM_CMD_SZ)) /
CAAM_CMD_SZ;
}
/* Reserve space in descriptor buffer for the derived key */
words = (ALIGN(adata->keylen_pad, CAAM_CMD_SZ) -
ALIGN(adata->keylen, CAAM_CMD_SZ)) / CAAM_CMD_SZ;
if (words)
(*desc) = cpu_to_caam32(caam32_to_cpu(*desc) + words);
} else {

79
drivers/crypto/caam/dpseci-debugfs.c Normal file
View file

@ -0,0 +1,79 @@
// SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause)
/* Copyright 2019 NXP */
#include <linux/module.h>
#include <linux/device.h>
#include <linux/debugfs.h>
#include "dpseci-debugfs.h"
static int dpseci_dbg_fqs_show(struct seq_file *file, void *offset)
{
struct dpaa2_caam_priv *priv = (struct dpaa2_caam_priv *)file->private;
u32 fqid, fcnt, bcnt;
int i, err;
seq_printf(file, "FQ stats for %s:\n", dev_name(priv->dev));
seq_printf(file, "%s%16s%16s\n",
"Rx-VFQID",
"Pending frames",
"Pending bytes");
for (i = 0; i < priv->num_pairs; i++) {
fqid = priv->rx_queue_attr[i].fqid;
err = dpaa2_io_query_fq_count(NULL, fqid, &fcnt, &bcnt);
if (err)
continue;
seq_printf(file, "%5d%16u%16u\n", fqid, fcnt, bcnt);
}
seq_printf(file, "%s%16s%16s\n",
"Tx-VFQID",
"Pending frames",
"Pending bytes");
for (i = 0; i < priv->num_pairs; i++) {
fqid = priv->tx_queue_attr[i].fqid;
err = dpaa2_io_query_fq_count(NULL, fqid, &fcnt, &bcnt);
if (err)
continue;
seq_printf(file, "%5d%16u%16u\n", fqid, fcnt, bcnt);
}
return 0;
}
static int dpseci_dbg_fqs_open(struct inode *inode, struct file *file)
{
int err;
struct dpaa2_caam_priv *priv;
priv = (struct dpaa2_caam_priv *)inode->i_private;
err = single_open(file, dpseci_dbg_fqs_show, priv);
if (err < 0)
dev_err(priv->dev, "single_open() failed\n");
return err;
}
static const struct file_operations dpseci_dbg_fq_ops = {
.open = dpseci_dbg_fqs_open,
.read = seq_read,
.llseek = seq_lseek,
.release = single_release,
};
void dpaa2_dpseci_debugfs_init(struct dpaa2_caam_priv *priv)
{
priv->dfs_root = debugfs_create_dir(dev_name(priv->dev), NULL);
debugfs_create_file("fq_stats", 0444, priv->dfs_root, priv,
&dpseci_dbg_fq_ops);
}
void dpaa2_dpseci_debugfs_exit(struct dpaa2_caam_priv *priv)
{
debugfs_remove_recursive(priv->dfs_root);
}

18
drivers/crypto/caam/dpseci-debugfs.h Normal file
View file

@ -0,0 +1,18 @@
/* SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause) */
/* Copyright 2019 NXP */
#ifndef DPSECI_DEBUGFS_H
#define DPSECI_DEBUGFS_H
#include <linux/dcache.h>
#include "caamalg_qi2.h"
#ifdef CONFIG_DEBUG_FS
void dpaa2_dpseci_debugfs_init(struct dpaa2_caam_priv *priv);
void dpaa2_dpseci_debugfs_exit(struct dpaa2_caam_priv *priv);
#else
static inline void dpaa2_dpseci_debugfs_init(struct dpaa2_caam_priv *priv) {}
static inline void dpaa2_dpseci_debugfs_exit(struct dpaa2_caam_priv *priv) {}
#endif /* CONFIG_DEBUG_FS */
#endif /* DPSECI_DEBUGFS_H */

64
drivers/crypto/caam/error.c
View file

@ -56,6 +56,9 @@ EXPORT_SYMBOL(caam_little_end);
bool caam_imx;
EXPORT_SYMBOL(caam_imx);
size_t caam_ptr_sz;
EXPORT_SYMBOL(caam_ptr_sz);
static const struct {
u8 value;
const char *error_text;
@ -118,6 +121,7 @@ static const struct {
u8 value;
const char *error_text;
} qi_error_list[] = {
{ 0x00, "No error" },
{ 0x1F, "Job terminated by FQ or ICID flush" },
{ 0x20, "FD format error"},
{ 0x21, "FD command format error"},
@ -210,8 +214,8 @@ static const char * const rng_err_id_list[] = {
"Secure key generation",
};
static void report_ccb_status(struct device *jrdev, const u32 status,
const char *error)
static int report_ccb_status(struct device *jrdev, const u32 status,
const char *error)
{
u8 cha_id = (status & JRSTA_CCBERR_CHAID_MASK) >>
JRSTA_CCBERR_CHAID_SHIFT;
@ -247,22 +251,27 @@ static void report_ccb_status(struct device *jrdev, const u32 status,
* CCB ICV check failures are part of normal operation life;
* we leave the upper layers to do what they want with them.
*/
if (err_id != JRSTA_CCBERR_ERRID_ICVCHK)
dev_err(jrdev, "%08x: %s: %s %d: %s%s: %s%s\n",
status, error, idx_str, idx,
cha_str, cha_err_code,
err_str, err_err_code);
if (err_id == JRSTA_CCBERR_ERRID_ICVCHK)
return -EBADMSG;
dev_err_ratelimited(jrdev, "%08x: %s: %s %d: %s%s: %s%s\n", status,
error, idx_str, idx, cha_str, cha_err_code,
err_str, err_err_code);
return -EINVAL;
}
static void report_jump_status(struct device *jrdev, const u32 status,
const char *error)
static int report_jump_status(struct device *jrdev, const u32 status,
const char *error)
{
dev_err(jrdev, "%08x: %s: %s() not implemented\n",
status, error, __func__);
return -EINVAL;
}
static void report_deco_status(struct device *jrdev, const u32 status,
const char *error)
static int report_deco_status(struct device *jrdev, const u32 status,
const char *error)
{
u8 err_id = status & JRSTA_DECOERR_ERROR_MASK;
u8 idx = (status & JRSTA_DECOERR_INDEX_MASK) >>
@ -288,10 +297,12 @@ static void report_deco_status(struct device *jrdev, const u32 status,
dev_err(jrdev, "%08x: %s: %s %d: %s%s\n",
status, error, idx_str, idx, err_str, err_err_code);
return -EINVAL;
}
static void report_qi_status(struct device *qidev, const u32 status,
const char *error)
static int report_qi_status(struct device *qidev, const u32 status,
const char *error)
{
u8 err_id = status & JRSTA_QIERR_ERROR_MASK;
const char *err_str = "unidentified error value 0x";
@ -309,27 +320,33 @@ static void report_qi_status(struct device *qidev, const u32 status,
dev_err(qidev, "%08x: %s: %s%s\n",
status, error, err_str, err_err_code);
return -EINVAL;
}
static void report_jr_status(struct device *jrdev, const u32 status,
const char *error)
static int report_jr_status(struct device *jrdev, const u32 status,
const char *error)
{
dev_err(jrdev, "%08x: %s: %s() not implemented\n",
status, error, __func__);
return -EINVAL;
}
static void report_cond_code_status(struct device *jrdev, const u32 status,
const char *error)
static int report_cond_code_status(struct device *jrdev, const u32 status,
const char *error)
{
dev_err(jrdev, "%08x: %s: %s() not implemented\n",
status, error, __func__);
return -EINVAL;
}
void caam_strstatus(struct device *jrdev, u32 status, bool qi_v2)
int caam_strstatus(struct device *jrdev, u32 status, bool qi_v2)
{
static const struct stat_src {
void (*report_ssed)(struct device *jrdev, const u32 status,
const char *error);
int (*report_ssed)(struct device *jrdev, const u32 status,
const char *error);
const char *error;
} status_src[16] = {
{ NULL, "No error" },
@ -357,11 +374,14 @@ void caam_strstatus(struct device *jrdev, u32 status, bool qi_v2)
* Otherwise print the error source name.
*/
if (status_src[ssrc].report_ssed)
status_src[ssrc].report_ssed(jrdev, status, error);
else if (error)
return status_src[ssrc].report_ssed(jrdev, status, error);
if (error)
dev_err(jrdev, "%d: %s\n", ssrc, error);
else
dev_err(jrdev, "%d: unknown error source\n", ssrc);
return -EINVAL;
}
EXPORT_SYMBOL(caam_strstatus);

2
drivers/crypto/caam/error.h
View file

@ -12,7 +12,7 @@
#define CAAM_ERROR_STR_MAX 302
void caam_strstatus(struct device *dev, u32 status, bool qi_v2);
int caam_strstatus(struct device *dev, u32 status, bool qi_v2);
#define caam_jr_strstatus(jrdev, status) caam_strstatus(jrdev, status, false)
#define caam_qi2_strstatus(qidev, status) caam_strstatus(qidev, status, true)

32
drivers/crypto/caam/intern.h
View file

@ -10,6 +10,8 @@
#ifndef INTERN_H
#define INTERN_H
#include "ctrl.h"
/* Currently comes from Kconfig param as a ^2 (driver-required) */
#define JOBR_DEPTH (1 << CONFIG_CRYPTO_DEV_FSL_CAAM_RINGSIZE)
@ -53,10 +55,11 @@ struct caam_drv_private_jr {
spinlock_t inplock ____cacheline_aligned; /* Input ring index lock */
u32 inpring_avail; /* Number of free entries in input ring */
int head; /* entinfo (s/w ring) head index */
dma_addr_t *inpring; /* Base of input ring, alloc DMA-safe */
void *inpring; /* Base of input ring, alloc
* DMA-safe */
int out_ring_read_index; /* Output index "tail" */
int tail; /* entinfo (s/w ring) tail index */
struct jr_outentry *outring; /* Base of output ring, DMA-safe */
void *outring; /* Base of output ring, DMA-safe */
};
/*
@ -92,11 +95,8 @@ struct caam_drv_private {
Handles of the RNG4 block are initialized
by this driver */
struct clk *caam_ipg;
struct clk *caam_mem;
struct clk *caam_aclk;
struct clk *caam_emi_slow;
struct clk_bulk_data *clks;
int num_clks;
/*
* debugfs entries for developer view into driver/device
* variables at runtime.
@ -215,4 +215,22 @@ DEFINE_SIMPLE_ATTRIBUTE(caam_fops_u32_ro, caam_debugfs_u32_get, NULL, "%llu\n");
DEFINE_SIMPLE_ATTRIBUTE(caam_fops_u64_ro, caam_debugfs_u64_get, NULL, "%llu\n");
#endif
static inline u64 caam_get_dma_mask(struct device *dev)
{
struct device_node *nprop = dev->of_node;
if (caam_ptr_sz != sizeof(u64))
return DMA_BIT_MASK(32);
if (caam_dpaa2)
return DMA_BIT_MASK(49);
if (of_device_is_compatible(nprop, "fsl,sec-v5.0-job-ring") ||
of_device_is_compatible(nprop, "fsl,sec-v5.0"))
return DMA_BIT_MASK(40);
return DMA_BIT_MASK(36);
}
#endif /* INTERN_H */

124
drivers/crypto/caam/jr.c
View file

@ -108,25 +108,12 @@ static int caam_reset_hw_jr(struct device *dev)
static int caam_jr_shutdown(struct device *dev)
{
struct caam_drv_private_jr *jrp = dev_get_drvdata(dev);
dma_addr_t inpbusaddr, outbusaddr;
int ret;
ret = caam_reset_hw_jr(dev);
tasklet_kill(&jrp->irqtask);
/* Release interrupt */
free_irq(jrp->irq, dev);
/* Free rings */
inpbusaddr = rd_reg64(&jrp->rregs->inpring_base);
outbusaddr = rd_reg64(&jrp->rregs->outring_base);
dma_free_coherent(dev, sizeof(dma_addr_t) * JOBR_DEPTH,
jrp->inpring, inpbusaddr);
dma_free_coherent(dev, sizeof(struct jr_outentry) * JOBR_DEPTH,
jrp->outring, outbusaddr);
kfree(jrp->entinfo);
return ret;
}
@ -159,7 +146,6 @@ static int caam_jr_remove(struct platform_device *pdev)
ret = caam_jr_shutdown(jrdev);
if (ret)
dev_err(jrdev, "Failed to shut down job ring\n");
irq_dispose_mapping(jrpriv->irq);
return ret;
}
@ -224,7 +210,7 @@ static void caam_jr_dequeue(unsigned long devarg)
for (i = 0; CIRC_CNT(head, tail + i, JOBR_DEPTH) >= 1; i++) {
sw_idx = (tail + i) & (JOBR_DEPTH - 1);
if (jrp->outring[hw_idx].desc ==
if (jr_outentry_desc(jrp->outring, hw_idx) ==
caam_dma_to_cpu(jrp->entinfo[sw_idx].desc_addr_dma))
break; /* found */
}
@ -233,7 +219,8 @@ static void caam_jr_dequeue(unsigned long devarg)
/* Unmap just-run descriptor so we can post-process */
dma_unmap_single(dev,
caam_dma_to_cpu(jrp->outring[hw_idx].desc),
caam_dma_to_cpu(jr_outentry_desc(jrp->outring,
hw_idx)),
jrp->entinfo[sw_idx].desc_size,
DMA_TO_DEVICE);
@ -244,7 +231,8 @@ static void caam_jr_dequeue(unsigned long devarg)
usercall = jrp->entinfo[sw_idx].callbk;
userarg = jrp->entinfo[sw_idx].cbkarg;
userdesc = jrp->entinfo[sw_idx].desc_addr_virt;
userstatus = caam32_to_cpu(jrp->outring[hw_idx].jrstatus);
userstatus = caam32_to_cpu(jr_outentry_jrstatus(jrp->outring,
hw_idx));
/*
* Make sure all information from the job has been obtained
@ -399,7 +387,7 @@ int caam_jr_enqueue(struct device *dev, u32 *desc,
head_entry->cbkarg = areq;
head_entry->desc_addr_dma = desc_dma;
jrp->inpring[head] = cpu_to_caam_dma(desc_dma);
jr_inpentry_set(jrp->inpring, head, cpu_to_caam_dma(desc_dma));
/*
* Guarantee that the descriptor's DMA address has been written to
@ -441,35 +429,26 @@ static int caam_jr_init(struct device *dev)
jrp = dev_get_drvdata(dev);
tasklet_init(&jrp->irqtask, caam_jr_dequeue, (unsigned long)dev);
/* Connect job ring interrupt handler. */
error = request_irq(jrp->irq, caam_jr_interrupt, IRQF_SHARED,
dev_name(dev), dev);
if (error) {
dev_err(dev, "can't connect JobR %d interrupt (%d)\n",
jrp->ridx, jrp->irq);
goto out_kill_deq;
}
error = caam_reset_hw_jr(dev);
if (error)
goto out_free_irq;
return error;
error = -ENOMEM;
jrp->inpring = dma_alloc_coherent(dev, sizeof(*jrp->inpring) *
JOBR_DEPTH, &inpbusaddr, GFP_KERNEL);
jrp->inpring = dmam_alloc_coherent(dev, SIZEOF_JR_INPENTRY *
JOBR_DEPTH, &inpbusaddr,
GFP_KERNEL);
if (!jrp->inpring)
goto out_free_irq;
return -ENOMEM;
jrp->outring = dma_alloc_coherent(dev, sizeof(*jrp->outring) *
JOBR_DEPTH, &outbusaddr, GFP_KERNEL);
jrp->outring = dmam_alloc_coherent(dev, SIZEOF_JR_OUTENTRY *
JOBR_DEPTH, &outbusaddr,
GFP_KERNEL);
if (!jrp->outring)
goto out_free_inpring;
return -ENOMEM;
jrp->entinfo = kcalloc(JOBR_DEPTH, sizeof(*jrp->entinfo), GFP_KERNEL);
jrp->entinfo = devm_kcalloc(dev, JOBR_DEPTH, sizeof(*jrp->entinfo),
GFP_KERNEL);
if (!jrp->entinfo)
goto out_free_outring;
return -ENOMEM;
for (i = 0; i < JOBR_DEPTH; i++)
jrp->entinfo[i].desc_addr_dma = !0;
@ -493,22 +472,24 @@ static int caam_jr_init(struct device *dev)
(JOBR_INTC_COUNT_THLD << JRCFG_ICDCT_SHIFT) |
(JOBR_INTC_TIME_THLD << JRCFG_ICTT_SHIFT));
return 0;
tasklet_init(&jrp->irqtask, caam_jr_dequeue, (unsigned long)dev);
/* Connect job ring interrupt handler. */
error = devm_request_irq(dev, jrp->irq, caam_jr_interrupt, IRQF_SHARED,
dev_name(dev), dev);
if (error) {
dev_err(dev, "can't connect JobR %d interrupt (%d)\n",
jrp->ridx, jrp->irq);
tasklet_kill(&jrp->irqtask);
}
out_free_outring:
dma_free_coherent(dev, sizeof(struct jr_outentry) * JOBR_DEPTH,
jrp->outring, outbusaddr);
out_free_inpring:
dma_free_coherent(dev, sizeof(dma_addr_t) * JOBR_DEPTH,
jrp->inpring, inpbusaddr);
dev_err(dev, "can't allocate job rings for %d\n", jrp->ridx);
out_free_irq:
free_irq(jrp->irq, dev);
out_kill_deq:
tasklet_kill(&jrp->irqtask);
return error;
}
static void caam_jr_irq_dispose_mapping(void *data)
{
irq_dispose_mapping((unsigned long)data);
}
/*
* Probe routine for each detected JobR subsystem.
@ -520,6 +501,7 @@ static int caam_jr_probe(struct platform_device *pdev)
struct caam_job_ring __iomem *ctrl;
struct caam_drv_private_jr *jrpriv;
static int total_jobrs;
struct resource *r;
int error;
jrdev = &pdev->dev;
@ -535,45 +517,43 @@ static int caam_jr_probe(struct platform_device *pdev)
nprop = pdev->dev.of_node;
/* Get configuration properties from device tree */
/* First, get register page */
ctrl = of_iomap(nprop, 0);
r = platform_get_resource(pdev, IORESOURCE_MEM, 0);
if (!r) {
dev_err(jrdev, "platform_get_resource() failed\n");
return -ENOMEM;
}
ctrl = devm_ioremap(jrdev, r->start, resource_size(r));
if (!ctrl) {
dev_err(jrdev, "of_iomap() failed\n");
dev_err(jrdev, "devm_ioremap() failed\n");
return -ENOMEM;
}
jrpriv->rregs = (struct caam_job_ring __iomem __force *)ctrl;
if (sizeof(dma_addr_t) == sizeof(u64)) {
if (caam_dpaa2)
error = dma_set_mask_and_coherent(jrdev,
DMA_BIT_MASK(49));
else if (of_device_is_compatible(nprop,
"fsl,sec-v5.0-job-ring"))
error = dma_set_mask_and_coherent(jrdev,
DMA_BIT_MASK(40));
else
error = dma_set_mask_and_coherent(jrdev,
DMA_BIT_MASK(36));
} else {
error = dma_set_mask_and_coherent(jrdev, DMA_BIT_MASK(32));
}
error = dma_set_mask_and_coherent(jrdev, caam_get_dma_mask(jrdev));
if (error) {
dev_err(jrdev, "dma_set_mask_and_coherent failed (%d)\n",
error);
iounmap(ctrl);
return error;
}
/* Identify the interrupt */
jrpriv->irq = irq_of_parse_and_map(nprop, 0);
if (!jrpriv->irq) {
dev_err(jrdev, "irq_of_parse_and_map failed\n");
return -EINVAL;
}
error = devm_add_action_or_reset(jrdev, caam_jr_irq_dispose_mapping,
(void *)(unsigned long)jrpriv->irq);
if (error)
return error;
/* Now do the platform independent part */
error = caam_jr_init(jrdev); /* now turn on hardware */
if (error) {
irq_dispose_mapping(jrpriv->irq);
iounmap(ctrl);
if (error)
return error;
}
jrpriv->dev = jrdev;
spin_lock(&driver_data.jr_alloc_lock);

14
drivers/crypto/caam/key_gen.c
View file

@ -15,13 +15,14 @@ void split_key_done(struct device *dev, u32 *desc, u32 err,
void *context)
{
struct split_key_result *res = context;
int ecode = 0;
dev_dbg(dev, "%s %d: err 0x%x\n", __func__, __LINE__, err);
if (err)
caam_jr_strstatus(dev, err);
ecode = caam_jr_strstatus(dev, err);
res->err = err;
res->err = ecode;
complete(&res->completion);
}
@ -47,18 +48,20 @@ int gen_split_key(struct device *jrdev, u8 *key_out,
u32 *desc;
struct split_key_result result;
dma_addr_t dma_addr;
unsigned int local_max;
int ret = -ENOMEM;
adata->keylen = split_key_len(adata->algtype & OP_ALG_ALGSEL_MASK);
adata->keylen_pad = split_key_pad_len(adata->algtype &
OP_ALG_ALGSEL_MASK);
local_max = max(keylen, adata->keylen_pad);
dev_dbg(jrdev, "split keylen %d split keylen padded %d\n",
adata->keylen, adata->keylen_pad);
print_hex_dump_debug("ctx.key@" __stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, key_in, keylen, 1);
if (adata->keylen_pad > max_keylen)
if (local_max > max_keylen)
return -EINVAL;
desc = kmalloc(CAAM_CMD_SZ * 6 + CAAM_PTR_SZ * 2, GFP_KERNEL | GFP_DMA);
@ -69,8 +72,7 @@ int gen_split_key(struct device *jrdev, u8 *key_out,
memcpy(key_out, key_in, keylen);
dma_addr = dma_map_single(jrdev, key_out, adata->keylen_pad,
DMA_BIDIRECTIONAL);
dma_addr = dma_map_single(jrdev, key_out, local_max, DMA_BIDIRECTIONAL);
if (dma_mapping_error(jrdev, dma_addr)) {
dev_err(jrdev, "unable to map key memory\n");
goto out_free;
@ -116,7 +118,7 @@ int gen_split_key(struct device *jrdev, u8 *key_out,
adata->keylen_pad, 1);
}
dma_unmap_single(jrdev, dma_addr, adata->keylen_pad, DMA_BIDIRECTIONAL);
dma_unmap_single(jrdev, dma_addr, local_max, DMA_BIDIRECTIONAL);
out_free:
kfree(desc);
return ret;

16
drivers/crypto/caam/pdb.h
View file

@ -512,7 +512,9 @@ struct rsa_pub_pdb {
dma_addr_t n_dma;
dma_addr_t e_dma;
u32 f_len;
} __packed;
};
#define SIZEOF_RSA_PUB_PDB (2 * sizeof(u32) + 4 * caam_ptr_sz)
/**
* RSA Decrypt PDB - Private Key Form #1
@ -528,7 +530,9 @@ struct rsa_priv_f1_pdb {
dma_addr_t f_dma;
dma_addr_t n_dma;
dma_addr_t d_dma;
} __packed;
};
#define SIZEOF_RSA_PRIV_F1_PDB (sizeof(u32) + 4 * caam_ptr_sz)
/**
* RSA Decrypt PDB - Private Key Form #2
@ -554,7 +558,9 @@ struct rsa_priv_f2_pdb {
dma_addr_t tmp1_dma;
dma_addr_t tmp2_dma;
u32 p_q_len;
} __packed;
};
#define SIZEOF_RSA_PRIV_F2_PDB (2 * sizeof(u32) + 7 * caam_ptr_sz)
/**
* RSA Decrypt PDB - Private Key Form #3
@ -586,6 +592,8 @@ struct rsa_priv_f3_pdb {
dma_addr_t tmp1_dma;
dma_addr_t tmp2_dma;
u32 p_q_len;
} __packed;
};
#define SIZEOF_RSA_PRIV_F3_PDB (2 * sizeof(u32) + 9 * caam_ptr_sz)
#endif

8
drivers/crypto/caam/pkc_desc.c
View file

@ -13,7 +13,7 @@
/* Descriptor for RSA Public operation */
void init_rsa_pub_desc(u32 *desc, struct rsa_pub_pdb *pdb)
{
init_job_desc_pdb(desc, 0, sizeof(*pdb));
init_job_desc_pdb(desc, 0, SIZEOF_RSA_PUB_PDB);
append_cmd(desc, pdb->sgf);
append_ptr(desc, pdb->f_dma);
append_ptr(desc, pdb->g_dma);
@ -26,7 +26,7 @@ void init_rsa_pub_desc(u32 *desc, struct rsa_pub_pdb *pdb)
/* Descriptor for RSA Private operation - Private Key Form #1 */
void init_rsa_priv_f1_desc(u32 *desc, struct rsa_priv_f1_pdb *pdb)
{
init_job_desc_pdb(desc, 0, sizeof(*pdb));
init_job_desc_pdb(desc, 0, SIZEOF_RSA_PRIV_F1_PDB);
append_cmd(desc, pdb->sgf);
append_ptr(desc, pdb->g_dma);
append_ptr(desc, pdb->f_dma);
@ -39,7 +39,7 @@ void init_rsa_priv_f1_desc(u32 *desc, struct rsa_priv_f1_pdb *pdb)
/* Descriptor for RSA Private operation - Private Key Form #2 */
void init_rsa_priv_f2_desc(u32 *desc, struct rsa_priv_f2_pdb *pdb)
{
init_job_desc_pdb(desc, 0, sizeof(*pdb));
init_job_desc_pdb(desc, 0, SIZEOF_RSA_PRIV_F2_PDB);
append_cmd(desc, pdb->sgf);
append_ptr(desc, pdb->g_dma);
append_ptr(desc, pdb->f_dma);
@ -56,7 +56,7 @@ void init_rsa_priv_f2_desc(u32 *desc, struct rsa_priv_f2_pdb *pdb)
/* Descriptor for RSA Private operation - Private Key Form #3 */
void init_rsa_priv_f3_desc(u32 *desc, struct rsa_priv_f3_pdb *pdb)
{
init_job_desc_pdb(desc, 0, sizeof(*pdb));
init_job_desc_pdb(desc, 0, SIZEOF_RSA_PRIV_F3_PDB);
append_cmd(desc, pdb->sgf);
append_ptr(desc, pdb->g_dma);
append_ptr(desc, pdb->f_dma);

10
drivers/crypto/caam/qi.c
View file

@ -163,7 +163,10 @@ static void caam_fq_ern_cb(struct qman_portal *qm, struct qman_fq *fq,
dma_unmap_single(drv_req->drv_ctx->qidev, qm_fd_addr(fd),
sizeof(drv_req->fd_sgt), DMA_BIDIRECTIONAL);
drv_req->cbk(drv_req, -EIO);
if (fd->status)
drv_req->cbk(drv_req, be32_to_cpu(fd->status));
else
drv_req->cbk(drv_req, JRSTA_SSRC_QI);
}
static struct qman_fq *create_caam_req_fq(struct device *qidev,
@ -574,8 +577,9 @@ static enum qman_cb_dqrr_result caam_rsp_fq_dqrr_cb(struct qman_portal *p,
if (ssrc != JRSTA_SSRC_CCB_ERROR ||
err_id != JRSTA_CCBERR_ERRID_ICVCHK)
dev_err(qidev, "Error: %#x in CAAM response FD\n",
status);
dev_err_ratelimited(qidev,
"Error: %#x in CAAM response FD\n",
status);
}
if (unlikely(qm_fd_get_format(fd) != qm_fd_compound)) {

26
drivers/crypto/caam/qi.h
View file

@ -14,32 +14,6 @@
#include "desc.h"
#include "desc_constr.h"
/*
* CAAM hardware constructs a job descriptor which points to a shared descriptor
* (as pointed by context_a of to-CAAM FQ).
* When the job descriptor is executed by DECO, the whole job descriptor
* together with shared descriptor gets loaded in DECO buffer, which is
* 64 words (each 32-bit) long.
*
* The job descriptor constructed by CAAM hardware has the following layout:
*
* HEADER (1 word)
* Shdesc ptr (1 or 2 words)
* SEQ_OUT_PTR (1 word)
* Out ptr (1 or 2 words)
* Out length (1 word)
* SEQ_IN_PTR (1 word)
* In ptr (1 or 2 words)
* In length (1 word)
*
* The shdesc ptr is used to fetch shared descriptor contents into DECO buffer.
*
* Apart from shdesc contents, the total number of words that get loaded in DECO
* buffer are '8' or '11'. The remaining words in DECO buffer can be used for
* storing shared descriptor.
*/
#define MAX_SDLEN ((CAAM_DESC_BYTES_MAX - DESC_JOB_IO_LEN) / CAAM_CMD_SZ)
/* Length of a single buffer in the QI driver memory cache */
#define CAAM_QI_MEMCACHE_SIZE 768

143
drivers/crypto/caam/regs.h
View file

@ -12,6 +12,7 @@
#include <linux/types.h>
#include <linux/bitops.h>
#include <linux/io.h>
#include <linux/io-64-nonatomic-hi-lo.h>
/*
* Architecture-specific register access methods
@ -70,6 +71,7 @@
extern bool caam_little_end;
extern bool caam_imx;
extern size_t caam_ptr_sz;
#define caam_to_cpu(len) \
static inline u##len caam##len ## _to_cpu(u##len val) \
@ -137,45 +139,37 @@ static inline void clrsetbits_32(void __iomem *reg, u32 clear, u32 set)
* base + 0x0000 : least-significant 32 bits
* base + 0x0004 : most-significant 32 bits
*/
#ifdef CONFIG_64BIT
static inline void wr_reg64(void __iomem *reg, u64 data)
{
if (caam_little_end)
iowrite64(data, reg);
else
iowrite64be(data, reg);
}
static inline u64 rd_reg64(void __iomem *reg)
{
if (caam_little_end)
return ioread64(reg);
else
return ioread64be(reg);
}
#else /* CONFIG_64BIT */
static inline void wr_reg64(void __iomem *reg, u64 data)
{
if (!caam_imx && caam_little_end) {
wr_reg32((u32 __iomem *)(reg) + 1, data >> 32);
wr_reg32((u32 __iomem *)(reg), data);
if (caam_little_end) {
if (caam_imx) {
iowrite32(data >> 32, (u32 __iomem *)(reg));
iowrite32(data, (u32 __iomem *)(reg) + 1);
} else {
iowrite64(data, reg);
}
} else {
wr_reg32((u32 __iomem *)(reg), data >> 32);
wr_reg32((u32 __iomem *)(reg) + 1, data);
iowrite64be(data, reg);
}
}
static inline u64 rd_reg64(void __iomem *reg)
{
if (!caam_imx && caam_little_end)
return ((u64)rd_reg32((u32 __iomem *)(reg) + 1) << 32 |
(u64)rd_reg32((u32 __iomem *)(reg)));
if (caam_little_end) {
if (caam_imx) {
u32 low, high;
return ((u64)rd_reg32((u32 __iomem *)(reg)) << 32 |
(u64)rd_reg32((u32 __iomem *)(reg) + 1));
high = ioread32(reg);
low = ioread32(reg + sizeof(u32));
return low + ((u64)high << 32);
} else {
return ioread64(reg);
}
} else {
return ioread64be(reg);
}
}
#endif /* CONFIG_64BIT */
static inline u64 cpu_to_caam_dma64(dma_addr_t value)
{
@ -195,22 +189,89 @@ static inline u64 caam_dma64_to_cpu(u64 value)
return caam64_to_cpu(value);
}
#ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT
#define cpu_to_caam_dma(value) cpu_to_caam_dma64(value)
#define caam_dma_to_cpu(value) caam_dma64_to_cpu(value)
#else
#define cpu_to_caam_dma(value) cpu_to_caam32(value)
#define caam_dma_to_cpu(value) caam32_to_cpu(value)
#endif /* CONFIG_ARCH_DMA_ADDR_T_64BIT */
static inline u64 cpu_to_caam_dma(u64 value)
{
if (IS_ENABLED(CONFIG_ARCH_DMA_ADDR_T_64BIT) &&
caam_ptr_sz == sizeof(u64))
return cpu_to_caam_dma64(value);
else
return cpu_to_caam32(value);
}
static inline u64 caam_dma_to_cpu(u64 value)
{
if (IS_ENABLED(CONFIG_ARCH_DMA_ADDR_T_64BIT) &&
caam_ptr_sz == sizeof(u64))
return caam_dma64_to_cpu(value);
else
return caam32_to_cpu(value);
}
/*
* jr_outentry
* Represents each entry in a JobR output ring
*/
struct jr_outentry {
dma_addr_t desc;/* Pointer to completed descriptor */
u32 jrstatus; /* Status for completed descriptor */
} __packed;
static inline void jr_outentry_get(void *outring, int hw_idx, dma_addr_t *desc,
u32 *jrstatus)
{
if (caam_ptr_sz == sizeof(u32)) {
struct {
u32 desc;
u32 jrstatus;
} __packed *outentry = outring;
*desc = outentry[hw_idx].desc;
*jrstatus = outentry[hw_idx].jrstatus;
} else {
struct {
dma_addr_t desc;/* Pointer to completed descriptor */
u32 jrstatus; /* Status for completed descriptor */
} __packed *outentry = outring;
*desc = outentry[hw_idx].desc;
*jrstatus = outentry[hw_idx].jrstatus;
}
}
#define SIZEOF_JR_OUTENTRY (caam_ptr_sz + sizeof(u32))
static inline dma_addr_t jr_outentry_desc(void *outring, int hw_idx)
{
dma_addr_t desc;
u32 unused;
jr_outentry_get(outring, hw_idx, &desc, &unused);
return desc;
}
static inline u32 jr_outentry_jrstatus(void *outring, int hw_idx)
{
dma_addr_t unused;
u32 jrstatus;
jr_outentry_get(outring, hw_idx, &unused, &jrstatus);
return jrstatus;
}
static inline void jr_inpentry_set(void *inpring, int hw_idx, dma_addr_t val)
{
if (caam_ptr_sz == sizeof(u32)) {
u32 *inpentry = inpring;
inpentry[hw_idx] = val;
} else {
dma_addr_t *inpentry = inpring;
inpentry[hw_idx] = val;
}
}
#define SIZEOF_JR_INPENTRY caam_ptr_sz
/* Version registers (Era 10+) e80-eff */
struct version_regs {
@ -338,6 +399,7 @@ struct caam_perfmon {
u32 cha_rev_ls; /* CRNR - CHA Rev No. Least significant half*/
#define CTPR_MS_QI_SHIFT 25
#define CTPR_MS_QI_MASK (0x1ull << CTPR_MS_QI_SHIFT)
#define CTPR_MS_PS BIT(17)
#define CTPR_MS_DPAA2 BIT(13)
#define CTPR_MS_VIRT_EN_INCL 0x00000001
#define CTPR_MS_VIRT_EN_POR 0x00000002
@ -641,6 +703,7 @@ struct caam_job_ring {
#define JRSTA_SSRC_CCB_ERROR 0x20000000
#define JRSTA_SSRC_JUMP_HALT_USER 0x30000000
#define JRSTA_SSRC_DECO 0x40000000
#define JRSTA_SSRC_QI 0x50000000
#define JRSTA_SSRC_JRERROR 0x60000000
#define JRSTA_SSRC_JUMP_HALT_CC 0x70000000

26
drivers/crypto/cavium/cpt/cptvf_algs.c
View file

@ -7,7 +7,7 @@
#include <crypto/aes.h>
#include <crypto/algapi.h>
#include <crypto/authenc.h>
#include <crypto/des.h>
#include <crypto/internal/des.h>
#include <crypto/xts.h>
#include <linux/crypto.h>
#include <linux/err.h>
@ -322,31 +322,15 @@ static int cvm_cfb_aes_setkey(struct crypto_ablkcipher *cipher, const u8 *key,
static int cvm_cbc_des3_setkey(struct crypto_ablkcipher *cipher, const u8 *key,
u32 keylen)
{
u32 flags = crypto_ablkcipher_get_flags(cipher);
int err;
err = __des3_verify_key(&flags, key);
if (unlikely(err)) {
crypto_ablkcipher_set_flags(cipher, flags);
return err;
}
return cvm_setkey(cipher, key, keylen, DES3_CBC);
return verify_ablkcipher_des3_key(cipher, key) ?:
cvm_setkey(cipher, key, keylen, DES3_CBC);
}
static int cvm_ecb_des3_setkey(struct crypto_ablkcipher *cipher, const u8 *key,
u32 keylen)
{
u32 flags = crypto_ablkcipher_get_flags(cipher);
int err;
err = __des3_verify_key(&flags, key);
if (unlikely(err)) {
crypto_ablkcipher_set_flags(cipher, flags);
return err;
}
return cvm_setkey(cipher, key, keylen, DES3_ECB);
return verify_ablkcipher_des3_key(cipher, key) ?:
cvm_setkey(cipher, key, keylen, DES3_ECB);
}
static int cvm_enc_dec_init(struct crypto_tfm *tfm)

2
drivers/crypto/cavium/nitrox/Kconfig
View file

@ -6,7 +6,7 @@ config CRYPTO_DEV_NITROX
tristate
select CRYPTO_BLKCIPHER
select CRYPTO_AES
select CRYPTO_DES
select CRYPTO_LIB_DES
select FW_LOADER
config CRYPTO_DEV_NITROX_CNN55XX

235
drivers/crypto/cavium/nitrox/nitrox_csr.h
View file

@ -40,9 +40,77 @@
#define EMU_FUSE_MAPX(_i) (0x1402708 + ((_i) * 0x40000))
/* UCD registers */
#define UCD_SE_EID_UCODE_BLOCK_NUMX(_i) (0x12C0000 + ((_i) * 0x1000))
#define UCD_AE_EID_UCODE_BLOCK_NUMX(_i) (0x12C0008 + ((_i) * 0x800))
#define UCD_UCODE_LOAD_BLOCK_NUM 0x12C0010
#define UCD_UCODE_LOAD_IDX_DATAX(_i) (0x12C0018 + ((_i) * 0x20))
#define UCD_SE_EID_UCODE_BLOCK_NUMX(_i) (0x12C0000 + ((_i) * 0x1000))
#define UCD_SE_CNTX(_i) (0x12C0040 + ((_i) * 0x1000))
#define UCD_AE_CNTX(_i) (0x12C0048 + ((_i) * 0x800))
/* AQM registers */
#define AQM_CTL 0x1300000
#define AQM_INT 0x1300008
#define AQM_DBELL_OVF_LO 0x1300010
#define AQM_DBELL_OVF_HI 0x1300018
#define AQM_DBELL_OVF_LO_W1S 0x1300020
#define AQM_DBELL_OVF_LO_ENA_W1C 0x1300028
#define AQM_DBELL_OVF_LO_ENA_W1S 0x1300030
#define AQM_DBELL_OVF_HI_W1S 0x1300038
#define AQM_DBELL_OVF_HI_ENA_W1C 0x1300040
#define AQM_DBELL_OVF_HI_ENA_W1S 0x1300048
#define AQM_DMA_RD_ERR_LO 0x1300050
#define AQM_DMA_RD_ERR_HI 0x1300058
#define AQM_DMA_RD_ERR_LO_W1S 0x1300060
#define AQM_DMA_RD_ERR_LO_ENA_W1C 0x1300068
#define AQM_DMA_RD_ERR_LO_ENA_W1S 0x1300070
#define AQM_DMA_RD_ERR_HI_W1S 0x1300078
#define AQM_DMA_RD_ERR_HI_ENA_W1C 0x1300080
#define AQM_DMA_RD_ERR_HI_ENA_W1S 0x1300088
#define AQM_EXEC_NA_LO 0x1300090
#define AQM_EXEC_NA_HI 0x1300098
#define AQM_EXEC_NA_LO_W1S 0x13000A0
#define AQM_EXEC_NA_LO_ENA_W1C 0x13000A8
#define AQM_EXEC_NA_LO_ENA_W1S 0x13000B0
#define AQM_EXEC_NA_HI_W1S 0x13000B8
#define AQM_EXEC_NA_HI_ENA_W1C 0x13000C0
#define AQM_EXEC_NA_HI_ENA_W1S 0x13000C8
#define AQM_EXEC_ERR_LO 0x13000D0
#define AQM_EXEC_ERR_HI 0x13000D8
#define AQM_EXEC_ERR_LO_W1S 0x13000E0
#define AQM_EXEC_ERR_LO_ENA_W1C 0x13000E8
#define AQM_EXEC_ERR_LO_ENA_W1S 0x13000F0
#define AQM_EXEC_ERR_HI_W1S 0x13000F8
#define AQM_EXEC_ERR_HI_ENA_W1C 0x1300100
#define AQM_EXEC_ERR_HI_ENA_W1S 0x1300108
#define AQM_ECC_INT 0x1300110
#define AQM_ECC_INT_W1S 0x1300118
#define AQM_ECC_INT_ENA_W1C 0x1300120
#define AQM_ECC_INT_ENA_W1S 0x1300128
#define AQM_ECC_CTL 0x1300130
#define AQM_BIST_STATUS 0x1300138
#define AQM_CMD_INF_THRX(x) (0x1300400 + ((x) * 0x8))
#define AQM_CMD_INFX(x) (0x1300800 + ((x) * 0x8))
#define AQM_GRP_EXECMSK_LOX(x) (0x1300C00 + ((x) * 0x10))
#define AQM_GRP_EXECMSK_HIX(x) (0x1300C08 + ((x) * 0x10))
#define AQM_ACTIVITY_STAT_LO 0x1300C80
#define AQM_ACTIVITY_STAT_HI 0x1300C88
#define AQM_Q_CMD_PROCX(x) (0x1301000 + ((x) * 0x8))
#define AQM_PERF_CTL_LO 0x1301400
#define AQM_PERF_CTL_HI 0x1301408
#define AQM_PERF_CNT 0x1301410
#define AQMQ_DRBLX(x) (0x20000 + ((x) * 0x40000))
#define AQMQ_QSZX(x) (0x20008 + ((x) * 0x40000))
#define AQMQ_BADRX(x) (0x20010 + ((x) * 0x40000))
#define AQMQ_NXT_CMDX(x) (0x20018 + ((x) * 0x40000))
#define AQMQ_CMD_CNTX(x) (0x20020 + ((x) * 0x40000))
#define AQMQ_CMP_THRX(x) (0x20028 + ((x) * 0x40000))
#define AQMQ_CMP_CNTX(x) (0x20030 + ((x) * 0x40000))
#define AQMQ_TIM_LDX(x) (0x20038 + ((x) * 0x40000))
#define AQMQ_TIMERX(x) (0x20040 + ((x) * 0x40000))
#define AQMQ_ENX(x) (0x20048 + ((x) * 0x40000))
#define AQMQ_ACTIVITY_STATX(x) (0x20050 + ((x) * 0x40000))
#define AQM_VF_CMP_STATX(x) (0x28000 + ((x) * 0x40000))
/* NPS core registers */
#define NPS_CORE_GBL_VFCFG 0x1000000
@ -134,6 +202,171 @@
/* PEM registers */
#define PEM0_INT 0x1080428
/**
* struct ucd_core_eid_ucode_block_num - Core Eid to Ucode Blk Mapping Registers
* @ucode_len: Ucode length identifier 32KB or 64KB
* @ucode_blk: Ucode Block Number
*/
union ucd_core_eid_ucode_block_num {
u64 value;
struct {
#if (defined(__BIG_ENDIAN_BITFIELD))
u64 raz_4_63 : 60;
u64 ucode_len : 1;
u64 ucode_blk : 3;
#else
u64 ucode_blk : 3;
u64 ucode_len : 1;
u64 raz_4_63 : 60;
#endif
};
};
/**
* struct aqm_grp_execmsk_lo - Available AE engines for the group
* @exec_0_to_39: AE engines 0 to 39 status
*/
union aqm_grp_execmsk_lo {
u64 value;
struct {
#if (defined(__BIG_ENDIAN_BITFIELD))
u64 raz_40_63 : 24;
u64 exec_0_to_39 : 40;
#else
u64 exec_0_to_39 : 40;
u64 raz_40_63 : 24;
#endif
};
};
/**
* struct aqm_grp_execmsk_hi - Available AE engines for the group
* @exec_40_to_79: AE engines 40 to 79 status
*/
union aqm_grp_execmsk_hi {
u64 value;
struct {
#if (defined(__BIG_ENDIAN_BITFIELD))
u64 raz_40_63 : 24;
u64 exec_40_to_79 : 40;
#else
u64 exec_40_to_79 : 40;
u64 raz_40_63 : 24;
#endif
};
};
/**
* struct aqmq_drbl - AQM Queue Doorbell Counter Registers
* @dbell_count: Doorbell Counter
*/
union aqmq_drbl {
u64 value;
struct {
#if (defined(__BIG_ENDIAN_BITFIELD))
u64 raz_32_63 : 32;
u64 dbell_count : 32;
#else
u64 dbell_count : 32;
u64 raz_32_63 : 32;
#endif
};
};
/**
* struct aqmq_qsz - AQM Queue Host Queue Size Registers
* @host_queue_size: Size, in numbers of 'aqmq_command_s' command
* of the Host Ring.
*/
union aqmq_qsz {
u64 value;
struct {
#if (defined(__BIG_ENDIAN_BITFIELD))
u64 raz_32_63 : 32;
u64 host_queue_size : 32;
#else
u64 host_queue_size : 32;
u64 raz_32_63 : 32;
#endif
};
};
/**
* struct aqmq_cmp_thr - AQM Queue Commands Completed Threshold Registers
* @commands_completed_threshold: Count of 'aqmq_command_s' commands executed
* by AE engines for which completion interrupt is asserted.
*/
union aqmq_cmp_thr {
u64 value;
struct {
#if (defined(__BIG_ENDIAN_BITFIELD))
u64 raz_32_63 : 32;
u64 commands_completed_threshold : 32;
#else
u64 commands_completed_threshold : 32;
u64 raz_32_63 : 32;
#endif
};
};
/**
* struct aqmq_cmp_cnt - AQM Queue Commands Completed Count Registers
* @resend: Bit to request completion interrupt Resend.
* @completion_status: Command completion status of the ring.
* @commands_completed_count: Count of 'aqmq_command_s' commands executed by
* AE engines.
*/
union aqmq_cmp_cnt {
u64 value;
struct {
#if (defined(__BIG_ENDIAN_BITFIELD))
u64 raz_34_63 : 30;
u64 resend : 1;
u64 completion_status : 1;
u64 commands_completed_count : 32;
#else
u64 commands_completed_count : 32;
u64 completion_status : 1;
u64 resend : 1;
u64 raz_34_63 : 30;
#endif
};
};
/**
* struct aqmq_en - AQM Queue Enable Registers
* @queue_status: 1 = AQMQ is enabled, 0 = AQMQ is disabled
*/
union aqmq_en {
u64 value;
struct {
#if (defined(__BIG_ENDIAN_BITFIELD))
u64 raz_1_63 : 63;
u64 queue_enable : 1;
#else
u64 queue_enable : 1;
u64 raz_1_63 : 63;
#endif
};
};
/**
* struct aqmq_activity_stat - AQM Queue Activity Status Registers
* @queue_active: 1 = AQMQ is active, 0 = AQMQ is quiescent
*/
union aqmq_activity_stat {
u64 value;
struct {
#if (defined(__BIG_ENDIAN_BITFIELD))
u64 raz_1_63 : 63;
u64 queue_active : 1;
#else
u64 queue_active : 1;
u64 raz_1_63 : 63;
#endif
};
};
/**
* struct emu_fuse_map - EMU Fuse Map Registers
* @ae_fuse: Fuse settings for AE 19..0

3
drivers/crypto/cavium/nitrox/nitrox_debugfs.c
View file

@ -9,7 +9,8 @@ static int firmware_show(struct seq_file *s, void *v)
{
struct nitrox_device *ndev = s->private;
seq_printf(s, "Version: %s\n", ndev->hw.fw_name);
seq_printf(s, "Version: %s\n", ndev->hw.fw_name[0]);
seq_printf(s, "Version: %s\n", ndev->hw.fw_name[1]);
return 0;
}

8
drivers/crypto/cavium/nitrox/nitrox_dev.h
View file

@ -10,6 +10,10 @@
#define VERSION_LEN 32
/* Maximum queues in PF mode */
#define MAX_PF_QUEUES 64
/* Maximum device queues */
#define MAX_DEV_QUEUES (MAX_PF_QUEUES)
/* Maximum UCD Blocks */
#define CNN55XX_MAX_UCD_BLOCKS 8
/**
* struct nitrox_cmdq - NITROX command queue
@ -74,7 +78,7 @@ struct nitrox_cmdq {
*/
struct nitrox_hw {
char partname[IFNAMSIZ * 2];
char fw_name[VERSION_LEN];
char fw_name[CNN55XX_MAX_UCD_BLOCKS][VERSION_LEN];
int freq;
u16 vendor_id;
@ -206,6 +210,7 @@ enum vf_mode {
* @mode: Device mode PF/VF
* @ctx_pool: DMA pool for crypto context
* @pkt_inq: Packet input rings
* @aqmq: AQM command queues
* @qvec: MSI-X queue vectors information
* @iov: SR-IOV informatin
* @num_vecs: number of MSI-X vectors
@ -232,6 +237,7 @@ struct nitrox_device {
struct dma_pool *ctx_pool;
struct nitrox_cmdq *pkt_inq;
struct nitrox_cmdq *aqmq[MAX_DEV_QUEUES] ____cacheline_aligned_in_smp;
struct nitrox_q_vector *qvec;
struct nitrox_iov iov;

158
drivers/crypto/cavium/nitrox/nitrox_hal.c
View file

@ -241,12 +241,12 @@ void nitrox_config_pkt_solicit_ports(struct nitrox_device *ndev)
}
/**
* enable_nps_interrupts - enable NPS interrutps
* enable_nps_core_interrupts - enable NPS core interrutps
* @ndev: NITROX device.
*
* This includes NPS core, packet in and slc interrupts.
* This includes NPS core interrupts.
*/
static void enable_nps_interrupts(struct nitrox_device *ndev)
static void enable_nps_core_interrupts(struct nitrox_device *ndev)
{
union nps_core_int_ena_w1s core_int;
@ -258,18 +258,9 @@ static void enable_nps_interrupts(struct nitrox_device *ndev)
core_int.s.npco_dma_malform = 1;
core_int.s.host_nps_wr_err = 1;
nitrox_write_csr(ndev, NPS_CORE_INT_ENA_W1S, core_int.value);
/* NPS packet in ring interrupts */
nitrox_write_csr(ndev, NPS_PKT_IN_RERR_LO_ENA_W1S, (~0ULL));
nitrox_write_csr(ndev, NPS_PKT_IN_RERR_HI_ENA_W1S, (~0ULL));
nitrox_write_csr(ndev, NPS_PKT_IN_ERR_TYPE_ENA_W1S, (~0ULL));
/* NPS packet slc port interrupts */
nitrox_write_csr(ndev, NPS_PKT_SLC_RERR_HI_ENA_W1S, (~0ULL));
nitrox_write_csr(ndev, NPS_PKT_SLC_RERR_LO_ENA_W1S, (~0ULL));
nitrox_write_csr(ndev, NPS_PKT_SLC_ERR_TYPE_ENA_W1S, (~0uLL));
}
void nitrox_config_nps_unit(struct nitrox_device *ndev)
void nitrox_config_nps_core_unit(struct nitrox_device *ndev)
{
union nps_core_gbl_vfcfg core_gbl_vfcfg;
@ -281,12 +272,149 @@ void nitrox_config_nps_unit(struct nitrox_device *ndev)
core_gbl_vfcfg.s.ilk_disable = 1;
core_gbl_vfcfg.s.cfg = __NDEV_MODE_PF;
nitrox_write_csr(ndev, NPS_CORE_GBL_VFCFG, core_gbl_vfcfg.value);
/* enable nps core interrupts */
enable_nps_core_interrupts(ndev);
}
/**
* enable_nps_pkt_interrupts - enable NPS packet interrutps
* @ndev: NITROX device.
*
* This includes NPS packet in and slc interrupts.
*/
static void enable_nps_pkt_interrupts(struct nitrox_device *ndev)
{
/* NPS packet in ring interrupts */
nitrox_write_csr(ndev, NPS_PKT_IN_RERR_LO_ENA_W1S, (~0ULL));
nitrox_write_csr(ndev, NPS_PKT_IN_RERR_HI_ENA_W1S, (~0ULL));
nitrox_write_csr(ndev, NPS_PKT_IN_ERR_TYPE_ENA_W1S, (~0ULL));
/* NPS packet slc port interrupts */
nitrox_write_csr(ndev, NPS_PKT_SLC_RERR_HI_ENA_W1S, (~0ULL));
nitrox_write_csr(ndev, NPS_PKT_SLC_RERR_LO_ENA_W1S, (~0ULL));
nitrox_write_csr(ndev, NPS_PKT_SLC_ERR_TYPE_ENA_W1S, (~0uLL));
}
void nitrox_config_nps_pkt_unit(struct nitrox_device *ndev)
{
/* config input and solicit ports */
nitrox_config_pkt_input_rings(ndev);
nitrox_config_pkt_solicit_ports(ndev);
/* enable interrupts */
enable_nps_interrupts(ndev);
/* enable nps packet interrupts */
enable_nps_pkt_interrupts(ndev);
}
static void reset_aqm_ring(struct nitrox_device *ndev, int ring)
{
union aqmq_en aqmq_en_reg;
union aqmq_activity_stat activity_stat;
union aqmq_cmp_cnt cmp_cnt;
int max_retries = MAX_CSR_RETRIES;
u64 offset;
/* step 1: disable the queue */
offset = AQMQ_ENX(ring);
aqmq_en_reg.value = 0;
aqmq_en_reg.queue_enable = 0;
nitrox_write_csr(ndev, offset, aqmq_en_reg.value);
/* step 2: wait for AQMQ_ACTIVITY_STATX[QUEUE_ACTIVE] to clear */
usleep_range(100, 150);
offset = AQMQ_ACTIVITY_STATX(ring);
do {
activity_stat.value = nitrox_read_csr(ndev, offset);
if (!activity_stat.queue_active)
break;
udelay(50);
} while (max_retries--);
/* step 3: clear commands completed count */
offset = AQMQ_CMP_CNTX(ring);
cmp_cnt.value = nitrox_read_csr(ndev, offset);
nitrox_write_csr(ndev, offset, cmp_cnt.value);
usleep_range(50, 100);
}
void enable_aqm_ring(struct nitrox_device *ndev, int ring)
{
union aqmq_en aqmq_en_reg;
u64 offset;
offset = AQMQ_ENX(ring);
aqmq_en_reg.value = 0;
aqmq_en_reg.queue_enable = 1;
nitrox_write_csr(ndev, offset, aqmq_en_reg.value);
usleep_range(50, 100);
}
void nitrox_config_aqm_rings(struct nitrox_device *ndev)
{
int ring;
for (ring = 0; ring < ndev->nr_queues; ring++) {
struct nitrox_cmdq *cmdq = ndev->aqmq[ring];
union aqmq_drbl drbl;
union aqmq_qsz qsize;
union aqmq_cmp_thr cmp_thr;
u64 offset;
/* steps 1 - 3 */
reset_aqm_ring(ndev, ring);
/* step 4: clear doorbell count of ring */
offset = AQMQ_DRBLX(ring);
drbl.value = 0;
drbl.dbell_count = 0xFFFFFFFF;
nitrox_write_csr(ndev, offset, drbl.value);
/* step 5: configure host ring details */
/* set host address for next command of ring */
offset = AQMQ_NXT_CMDX(ring);
nitrox_write_csr(ndev, offset, 0ULL);
/* set host address of ring base */
offset = AQMQ_BADRX(ring);
nitrox_write_csr(ndev, offset, cmdq->dma);
/* set ring size */
offset = AQMQ_QSZX(ring);
qsize.value = 0;
qsize.host_queue_size = ndev->qlen;
nitrox_write_csr(ndev, offset, qsize.value);
/* set command completion threshold */
offset = AQMQ_CMP_THRX(ring);
cmp_thr.value = 0;
cmp_thr.commands_completed_threshold = 1;
nitrox_write_csr(ndev, offset, cmp_thr.value);
/* step 6: enable the queue */
enable_aqm_ring(ndev, ring);
}
}
static void enable_aqm_interrupts(struct nitrox_device *ndev)
{
/* clear interrupt enable bits */
nitrox_write_csr(ndev, AQM_DBELL_OVF_LO_ENA_W1S, (~0ULL));
nitrox_write_csr(ndev, AQM_DBELL_OVF_HI_ENA_W1S, (~0ULL));
nitrox_write_csr(ndev, AQM_DMA_RD_ERR_LO_ENA_W1S, (~0ULL));
nitrox_write_csr(ndev, AQM_DMA_RD_ERR_HI_ENA_W1S, (~0ULL));
nitrox_write_csr(ndev, AQM_EXEC_NA_LO_ENA_W1S, (~0ULL));
nitrox_write_csr(ndev, AQM_EXEC_NA_HI_ENA_W1S, (~0ULL));
nitrox_write_csr(ndev, AQM_EXEC_ERR_LO_ENA_W1S, (~0ULL));
nitrox_write_csr(ndev, AQM_EXEC_ERR_HI_ENA_W1S, (~0ULL));
}
void nitrox_config_aqm_unit(struct nitrox_device *ndev)
{
/* config aqm command queues */
nitrox_config_aqm_rings(ndev);
/* enable aqm interrupts */
enable_aqm_interrupts(ndev);
}
void nitrox_config_pom_unit(struct nitrox_device *ndev)

6
drivers/crypto/cavium/nitrox/nitrox_hal.h
View file

@ -4,10 +4,13 @@
#include "nitrox_dev.h"
void nitrox_config_aqm_rings(struct nitrox_device *ndev);
void nitrox_config_aqm_unit(struct nitrox_device *ndev);
void nitrox_config_emu_unit(struct nitrox_device *ndev);
void nitrox_config_pkt_input_rings(struct nitrox_device *ndev);
void nitrox_config_pkt_solicit_ports(struct nitrox_device *ndev);
void nitrox_config_nps_unit(struct nitrox_device *ndev);
void nitrox_config_nps_core_unit(struct nitrox_device *ndev);
void nitrox_config_nps_pkt_unit(struct nitrox_device *ndev);
void nitrox_config_pom_unit(struct nitrox_device *ndev);
void nitrox_config_rand_unit(struct nitrox_device *ndev);
void nitrox_config_efl_unit(struct nitrox_device *ndev);
@ -15,6 +18,7 @@ void nitrox_config_bmi_unit(struct nitrox_device *ndev);
void nitrox_config_bmo_unit(struct nitrox_device *ndev);
void nitrox_config_lbc_unit(struct nitrox_device *ndev);
void invalidate_lbc(struct nitrox_device *ndev);
void enable_aqm_ring(struct nitrox_device *ndev, int qno);
void enable_pkt_input_ring(struct nitrox_device *ndev, int ring);
void enable_pkt_solicit_port(struct nitrox_device *ndev, int port);
void config_nps_core_vfcfg_mode(struct nitrox_device *ndev, enum vf_mode mode);

66
drivers/crypto/cavium/nitrox/nitrox_lib.c
View file

@ -19,6 +19,8 @@
/* packet inuput ring alignments */
#define PKTIN_Q_ALIGN_BYTES 16
/* AQM Queue input alignments */
#define AQM_Q_ALIGN_BYTES 32
static int nitrox_cmdq_init(struct nitrox_cmdq *cmdq, int align_bytes)
{
@ -57,11 +59,15 @@ static void nitrox_cmdq_reset(struct nitrox_cmdq *cmdq)
static void nitrox_cmdq_cleanup(struct nitrox_cmdq *cmdq)
{
struct nitrox_device *ndev = cmdq->ndev;
struct nitrox_device *ndev;
if (!cmdq)
return;
if (!cmdq->unalign_base)
return;
ndev = cmdq->ndev;
cancel_work_sync(&cmdq->backlog_qflush);
dma_free_coherent(DEV(ndev), cmdq->qsize,
@ -78,6 +84,57 @@ static void nitrox_cmdq_cleanup(struct nitrox_cmdq *cmdq)
cmdq->instr_size = 0;
}
static void nitrox_free_aqm_queues(struct nitrox_device *ndev)
{
int i;
for (i = 0; i < ndev->nr_queues; i++) {
nitrox_cmdq_cleanup(ndev->aqmq[i]);
kzfree(ndev->aqmq[i]);
ndev->aqmq[i] = NULL;
}
}
static int nitrox_alloc_aqm_queues(struct nitrox_device *ndev)
{
int i, err;
for (i = 0; i < ndev->nr_queues; i++) {
struct nitrox_cmdq *cmdq;
u64 offset;
cmdq = kzalloc_node(sizeof(*cmdq), GFP_KERNEL, ndev->node);
if (!cmdq) {
err = -ENOMEM;
goto aqmq_fail;
}
cmdq->ndev = ndev;
cmdq->qno = i;
cmdq->instr_size = sizeof(struct aqmq_command_s);
/* AQM Queue Doorbell Counter Register Address */
offset = AQMQ_DRBLX(i);
cmdq->dbell_csr_addr = NITROX_CSR_ADDR(ndev, offset);
/* AQM Queue Commands Completed Count Register Address */
offset = AQMQ_CMD_CNTX(i);
cmdq->compl_cnt_csr_addr = NITROX_CSR_ADDR(ndev, offset);
err = nitrox_cmdq_init(cmdq, AQM_Q_ALIGN_BYTES);
if (err) {
kzfree(cmdq);
goto aqmq_fail;
}
ndev->aqmq[i] = cmdq;
}
return 0;
aqmq_fail:
nitrox_free_aqm_queues(ndev);
return err;
}
static void nitrox_free_pktin_queues(struct nitrox_device *ndev)
{
int i;
@ -222,6 +279,12 @@ int nitrox_common_sw_init(struct nitrox_device *ndev)
if (err)
destroy_crypto_dma_pool(ndev);
err = nitrox_alloc_aqm_queues(ndev);
if (err) {
nitrox_free_pktin_queues(ndev);
destroy_crypto_dma_pool(ndev);
}
return err;
}
@ -231,6 +294,7 @@ int nitrox_common_sw_init(struct nitrox_device *ndev)
*/
void nitrox_common_sw_cleanup(struct nitrox_device *ndev)
{
nitrox_free_aqm_queues(ndev);
nitrox_free_pktin_queues(ndev);
destroy_crypto_dma_pool(ndev);
}

150
drivers/crypto/cavium/nitrox/nitrox_main.c
View file

@ -17,12 +17,17 @@
#define CNN55XX_DEV_ID 0x12
#define UCODE_HLEN 48
#define SE_GROUP 0
#define DEFAULT_SE_GROUP 0
#define DEFAULT_AE_GROUP 0
#define DRIVER_VERSION "1.1"
#define DRIVER_VERSION "1.2"
#define CNN55XX_UCD_BLOCK_SIZE 32768
#define CNN55XX_MAX_UCODE_SIZE (CNN55XX_UCD_BLOCK_SIZE * 2)
#define FW_DIR "cavium/"
/* SE microcode */
#define SE_FW FW_DIR "cnn55xx_se.fw"
/* AE microcode */
#define AE_FW FW_DIR "cnn55xx_ae.fw"
static const char nitrox_driver_name[] = "CNN55XX";
@ -72,10 +77,10 @@ struct ucode {
/**
* write_to_ucd_unit - Write Firmware to NITROX UCD unit
*/
static void write_to_ucd_unit(struct nitrox_device *ndev,
struct ucode *ucode)
static void write_to_ucd_unit(struct nitrox_device *ndev, u32 ucode_size,
u64 *ucode_data, int block_num)
{
u32 code_size = be32_to_cpu(ucode->code_size) * 2;
u32 code_size;
u64 offset, data;
int i = 0;
@ -96,11 +101,12 @@ static void write_to_ucd_unit(struct nitrox_device *ndev,
/* set the block number */
offset = UCD_UCODE_LOAD_BLOCK_NUM;
nitrox_write_csr(ndev, offset, 0);
nitrox_write_csr(ndev, offset, block_num);
code_size = ucode_size;
code_size = roundup(code_size, 8);
while (code_size) {
data = ucode->code[i];
data = ucode_data[i];
/* write 8 bytes at a time */
offset = UCD_UCODE_LOAD_IDX_DATAX(i);
nitrox_write_csr(ndev, offset, data);
@ -108,29 +114,23 @@ static void write_to_ucd_unit(struct nitrox_device *ndev,
i++;
}
/* put all SE cores in group 0 */
offset = POM_GRP_EXECMASKX(SE_GROUP);
nitrox_write_csr(ndev, offset, (~0ULL));
for (i = 0; i < ndev->hw.se_cores; i++) {
/*
* write block number and firware length
* bit:<2:0> block number
* bit:3 is set SE uses 32KB microcode
* bit:3 is clear SE uses 64KB microcode
*/
offset = UCD_SE_EID_UCODE_BLOCK_NUMX(i);
nitrox_write_csr(ndev, offset, 0x8);
}
usleep_range(300, 400);
}
static int nitrox_load_fw(struct nitrox_device *ndev, const char *fw_name)
static int nitrox_load_fw(struct nitrox_device *ndev)
{
const struct firmware *fw;
const char *fw_name;
struct ucode *ucode;
int ret;
u64 *ucode_data;
u64 offset;
union ucd_core_eid_ucode_block_num core_2_eid_val;
union aqm_grp_execmsk_lo aqm_grp_execmask_lo;
union aqm_grp_execmsk_hi aqm_grp_execmask_hi;
u32 ucode_size;
int ret, i = 0;
fw_name = SE_FW;
dev_info(DEV(ndev), "Loading firmware \"%s\"\n", fw_name);
ret = request_firmware(&fw, fw_name, DEV(ndev));
@ -140,13 +140,101 @@ static int nitrox_load_fw(struct nitrox_device *ndev, const char *fw_name)
}
ucode = (struct ucode *)fw->data;
/* copy the firmware version */
memcpy(ndev->hw.fw_name, ucode->version, (VERSION_LEN - 2));
ndev->hw.fw_name[VERSION_LEN - 1] = '\0';
write_to_ucd_unit(ndev, ucode);
ucode_size = be32_to_cpu(ucode->code_size) * 2;
if (!ucode_size || ucode_size > CNN55XX_MAX_UCODE_SIZE) {
dev_err(DEV(ndev), "Invalid ucode size: %u for firmware %s\n",
ucode_size, fw_name);
release_firmware(fw);
return -EINVAL;
}
ucode_data = ucode->code;
/* copy the firmware version */
memcpy(&ndev->hw.fw_name[0][0], ucode->version, (VERSION_LEN - 2));
ndev->hw.fw_name[0][VERSION_LEN - 1] = '\0';
/* Load SE Firmware on UCD Block 0 */
write_to_ucd_unit(ndev, ucode_size, ucode_data, 0);
release_firmware(fw);
/* put all SE cores in DEFAULT_SE_GROUP */
offset = POM_GRP_EXECMASKX(DEFAULT_SE_GROUP);
nitrox_write_csr(ndev, offset, (~0ULL));
/* write block number and firmware length
* bit:<2:0> block number
* bit:3 is set SE uses 32KB microcode
* bit:3 is clear SE uses 64KB microcode
*/
core_2_eid_val.value = 0ULL;
core_2_eid_val.ucode_blk = 0;
if (ucode_size <= CNN55XX_UCD_BLOCK_SIZE)
core_2_eid_val.ucode_len = 1;
else
core_2_eid_val.ucode_len = 0;
for (i = 0; i < ndev->hw.se_cores; i++) {
offset = UCD_SE_EID_UCODE_BLOCK_NUMX(i);
nitrox_write_csr(ndev, offset, core_2_eid_val.value);
}
fw_name = AE_FW;
dev_info(DEV(ndev), "Loading firmware \"%s\"\n", fw_name);
ret = request_firmware(&fw, fw_name, DEV(ndev));
if (ret < 0) {
dev_err(DEV(ndev), "failed to get firmware %s\n", fw_name);
return ret;
}
ucode = (struct ucode *)fw->data;
ucode_size = be32_to_cpu(ucode->code_size) * 2;
if (!ucode_size || ucode_size > CNN55XX_MAX_UCODE_SIZE) {
dev_err(DEV(ndev), "Invalid ucode size: %u for firmware %s\n",
ucode_size, fw_name);
release_firmware(fw);
return -EINVAL;
}
ucode_data = ucode->code;
/* copy the firmware version */
memcpy(&ndev->hw.fw_name[1][0], ucode->version, (VERSION_LEN - 2));
ndev->hw.fw_name[1][VERSION_LEN - 1] = '\0';
/* Load AE Firmware on UCD Block 2 */
write_to_ucd_unit(ndev, ucode_size, ucode_data, 2);
release_firmware(fw);
/* put all AE cores in DEFAULT_AE_GROUP */
offset = AQM_GRP_EXECMSK_LOX(DEFAULT_AE_GROUP);
aqm_grp_execmask_lo.exec_0_to_39 = 0xFFFFFFFFFFULL;
nitrox_write_csr(ndev, offset, aqm_grp_execmask_lo.value);
offset = AQM_GRP_EXECMSK_HIX(DEFAULT_AE_GROUP);
aqm_grp_execmask_hi.exec_40_to_79 = 0xFFFFFFFFFFULL;
nitrox_write_csr(ndev, offset, aqm_grp_execmask_hi.value);
/* write block number and firmware length
* bit:<2:0> block number
* bit:3 is set SE uses 32KB microcode
* bit:3 is clear SE uses 64KB microcode
*/
core_2_eid_val.value = 0ULL;
core_2_eid_val.ucode_blk = 0;
if (ucode_size <= CNN55XX_UCD_BLOCK_SIZE)
core_2_eid_val.ucode_len = 1;
else
core_2_eid_val.ucode_len = 0;
for (i = 0; i < ndev->hw.ae_cores; i++) {
offset = UCD_AE_EID_UCODE_BLOCK_NUMX(i);
nitrox_write_csr(ndev, offset, core_2_eid_val.value);
}
return 0;
}
@ -299,7 +387,9 @@ static int nitrox_pf_hw_init(struct nitrox_device *ndev)
/* get cores information */
nitrox_get_hwinfo(ndev);
nitrox_config_nps_unit(ndev);
nitrox_config_nps_core_unit(ndev);
nitrox_config_aqm_unit(ndev);
nitrox_config_nps_pkt_unit(ndev);
nitrox_config_pom_unit(ndev);
nitrox_config_efl_unit(ndev);
/* configure IO units */
@ -309,8 +399,8 @@ static int nitrox_pf_hw_init(struct nitrox_device *ndev)
nitrox_config_lbc_unit(ndev);
nitrox_config_rand_unit(ndev);
/* load firmware on SE cores */
err = nitrox_load_fw(ndev, SE_FW);
/* load firmware on cores */
err = nitrox_load_fw(ndev);
if (err)
return err;

30
drivers/crypto/cavium/nitrox/nitrox_req.h
View file

@ -399,6 +399,36 @@ struct nps_pkt_instr {
u64 fdata[2];
};
/**
* struct aqmq_command_s - The 32 byte command for AE processing.
* @opcode: Request opcode
* @param1: Request control parameter 1
* @param2: Request control parameter 2
* @dlen: Input length
* @dptr: Input pointer points to buffer in remote host
* @rptr: Result pointer points to buffer in remote host
* @grp: AQM Group (0..7)
* @cptr: Context pointer
*/
struct aqmq_command_s {
__be16 opcode;
__be16 param1;
__be16 param2;
__be16 dlen;
__be64 dptr;
__be64 rptr;
union {
__be64 word3;
#if defined(__BIG_ENDIAN_BITFIELD)
u64 grp : 3;
u64 cptr : 61;
#else
u64 cptr : 61;
u64 grp : 3;
#endif
};
};
/**
* struct ctx_hdr - Book keeping data about the crypto context
* @pool: Pool used to allocate crypto context

4
drivers/crypto/cavium/nitrox/nitrox_skcipher.c
View file

@ -7,7 +7,7 @@
#include <crypto/aes.h>
#include <crypto/skcipher.h>
#include <crypto/ctr.h>
#include <crypto/des.h>
#include <crypto/internal/des.h>
#include <crypto/xts.h>
#include "nitrox_dev.h"
@ -257,7 +257,7 @@ static int nitrox_aes_decrypt(struct skcipher_request *skreq)
static int nitrox_3des_setkey(struct crypto_skcipher *cipher,
const u8 *key, unsigned int keylen)
{
return unlikely(des3_verify_key(cipher, key)) ?:
return verify_skcipher_des3_key(cipher, key) ?:
nitrox_skcipher_setkey(cipher, 0, key, keylen);
}

3
drivers/crypto/cavium/nitrox/nitrox_sriov.c
View file

@ -109,6 +109,9 @@ static int nitrox_pf_reinit(struct nitrox_device *ndev)
return err;
}
/* configure the AQM queues */
nitrox_config_aqm_rings(ndev);
/* configure the packet queues */
nitrox_config_pkt_input_rings(ndev);
nitrox_config_pkt_solicit_ports(ndev);

3
drivers/crypto/cavium/zip/zip_main.c
View file

@ -593,6 +593,7 @@ static const struct file_operations zip_stats_fops = {
.owner = THIS_MODULE,
.open = zip_stats_open,
.read = seq_read,
.release = single_release,
};
static int zip_clear_open(struct inode *inode, struct file *file)
@ -604,6 +605,7 @@ static const struct file_operations zip_clear_fops = {
.owner = THIS_MODULE,
.open = zip_clear_open,
.read = seq_read,
.release = single_release,
};
static int zip_regs_open(struct inode *inode, struct file *file)
@ -615,6 +617,7 @@ static const struct file_operations zip_regs_fops = {
.owner = THIS_MODULE,
.open = zip_regs_open,
.read = seq_read,
.release = single_release,
};
/* Root directory for thunderx_zip debugfs entry */

9
drivers/crypto/ccp/Kconfig
View file

@ -30,6 +30,7 @@ config CRYPTO_DEV_CCP_CRYPTO
select CRYPTO_BLKCIPHER
select CRYPTO_AUTHENC
select CRYPTO_RSA
select CRYPTO_LIB_AES
help
Support for using the cryptographic API with the AMD Cryptographic
Coprocessor. This module supports offload of SHA and AES algorithms.
@ -45,3 +46,11 @@ config CRYPTO_DEV_SP_PSP
management commands in Secure Encrypted Virtualization (SEV) mode,
along with software-based Trusted Execution Environment (TEE) to
enable third-party trusted applications.
config CRYPTO_DEV_CCP_DEBUGFS
bool "Enable CCP Internals in DebugFS"
default n
depends on CRYPTO_DEV_SP_CCP
help
Expose CCP device information such as operation statistics, feature
information, and descriptor queue contents.

4
drivers/crypto/ccp/Makefile
View file

@ -5,8 +5,8 @@ ccp-$(CONFIG_CRYPTO_DEV_SP_CCP) += ccp-dev.o \
ccp-ops.o \
ccp-dev-v3.o \
ccp-dev-v5.o \
ccp-dmaengine.o \
ccp-debugfs.o
ccp-dmaengine.o
ccp-$(CONFIG_CRYPTO_DEV_CCP_DEBUGFS) += ccp-debugfs.o
ccp-$(CONFIG_PCI) += sp-pci.o
ccp-$(CONFIG_CRYPTO_DEV_SP_PSP) += psp-dev.o

25
drivers/crypto/ccp/ccp-crypto-aes-cmac.c
View file

@ -261,6 +261,7 @@ static int ccp_aes_cmac_setkey(struct crypto_ahash *tfm, const u8 *key,
ccp_crypto_ahash_alg(crypto_ahash_tfm(tfm));
u64 k0_hi, k0_lo, k1_hi, k1_lo, k2_hi, k2_lo;
u64 rb_hi = 0x00, rb_lo = 0x87;
struct crypto_aes_ctx aes;
__be64 *gk;
int ret;
@ -284,14 +285,14 @@ static int ccp_aes_cmac_setkey(struct crypto_ahash *tfm, const u8 *key,
ctx->u.aes.key_len = 0;
/* Set the key for the AES cipher used to generate the keys */
ret = crypto_cipher_setkey(ctx->u.aes.tfm_cipher, key, key_len);
ret = aes_expandkey(&aes, key, key_len);
if (ret)
return ret;
/* Encrypt a block of zeroes - use key area in context */
memset(ctx->u.aes.key, 0, sizeof(ctx->u.aes.key));
crypto_cipher_encrypt_one(ctx->u.aes.tfm_cipher, ctx->u.aes.key,
ctx->u.aes.key);
aes_encrypt(&aes, ctx->u.aes.key, ctx->u.aes.key);
memzero_explicit(&aes, sizeof(aes));
/* Generate K1 and K2 */
k0_hi = be64_to_cpu(*((__be64 *)ctx->u.aes.key));
@ -336,32 +337,15 @@ static int ccp_aes_cmac_cra_init(struct crypto_tfm *tfm)
{
struct ccp_ctx *ctx = crypto_tfm_ctx(tfm);
struct crypto_ahash *ahash = __crypto_ahash_cast(tfm);
struct crypto_cipher *cipher_tfm;
ctx->complete = ccp_aes_cmac_complete;
ctx->u.aes.key_len = 0;
crypto_ahash_set_reqsize(ahash, sizeof(struct ccp_aes_cmac_req_ctx));
cipher_tfm = crypto_alloc_cipher("aes", 0, CRYPTO_ALG_NEED_FALLBACK);
if (IS_ERR(cipher_tfm)) {
pr_warn("could not load aes cipher driver\n");
return PTR_ERR(cipher_tfm);
}
ctx->u.aes.tfm_cipher = cipher_tfm;
return 0;
}
static void ccp_aes_cmac_cra_exit(struct crypto_tfm *tfm)
{
struct ccp_ctx *ctx = crypto_tfm_ctx(tfm);
if (ctx->u.aes.tfm_cipher)
crypto_free_cipher(ctx->u.aes.tfm_cipher);
ctx->u.aes.tfm_cipher = NULL;
}
int ccp_register_aes_cmac_algs(struct list_head *head)
{
struct ccp_crypto_ahash_alg *ccp_alg;
@ -401,7 +385,6 @@ int ccp_register_aes_cmac_algs(struct list_head *head)
base->cra_ctxsize = sizeof(struct ccp_ctx);
base->cra_priority = CCP_CRA_PRIORITY;
base->cra_init = ccp_aes_cmac_cra_init;
base->cra_exit = ccp_aes_cmac_cra_exit;
base->cra_module = THIS_MODULE;
ret = crypto_register_ahash(alg);

3
drivers/crypto/ccp/ccp-crypto-aes-xts.c
View file

@ -116,9 +116,6 @@ static int ccp_aes_xts_crypt(struct ablkcipher_request *req,
if (!ctx->u.aes.key_len)
return -EINVAL;
if (req->nbytes & (AES_BLOCK_SIZE - 1))
return -EINVAL;
if (!req->info)
return -EINVAL;

7
drivers/crypto/ccp/ccp-crypto-des3.c
View file

@ -14,7 +14,7 @@
#include <linux/crypto.h>
#include <crypto/algapi.h>
#include <crypto/scatterwalk.h>
#include <crypto/des.h>
#include <crypto/internal/des.h>
#include "ccp-crypto.h"
@ -39,11 +39,10 @@ static int ccp_des3_setkey(struct crypto_ablkcipher *tfm, const u8 *key,
struct ccp_ctx *ctx = crypto_tfm_ctx(crypto_ablkcipher_tfm(tfm));
struct ccp_crypto_ablkcipher_alg *alg =
ccp_crypto_ablkcipher_alg(crypto_ablkcipher_tfm(tfm));
u32 *flags = &tfm->base.crt_flags;
int err;
err = __des3_verify_key(flags, key);
if (unlikely(err))
err = verify_ablkcipher_des3_key(tfm, key);
if (err)
return err;
/* It's not clear that there is any support for a keysize of 112.

4
drivers/crypto/ccp/ccp-crypto-main.c
View file

@ -405,8 +405,10 @@ static int ccp_crypto_init(void)
int ret;
ret = ccp_present();
if (ret)
if (ret) {
pr_err("Cannot load: there are no available CCPs\n");
return ret;
}
spin_lock_init(&req_queue_lock);
INIT_LIST_HEAD(&req_queue.cmds);

8
drivers/crypto/ccp/ccp-crypto.h
View file

@ -12,7 +12,6 @@
#include <linux/list.h>
#include <linux/wait.h>
#include <linux/pci.h>
#include <linux/ccp.h>
#include <crypto/algapi.h>
#include <crypto/aes.h>
@ -24,6 +23,10 @@
#include <crypto/akcipher.h>
#include <crypto/internal/rsa.h>
/* We want the module name in front of our messages */
#undef pr_fmt
#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
#define CCP_LOG_LEVEL KERN_INFO
#define CCP_CRA_PRIORITY 300
@ -87,9 +90,6 @@ struct ccp_aes_ctx {
/* Fallback cipher for XTS with unsupported unit sizes */
struct crypto_sync_skcipher *tfm_skcipher;
/* Cipher used to generate CMAC K1/K2 keys */
struct crypto_cipher *tfm_cipher;
enum ccp_engine engine;
enum ccp_aes_type type;
enum ccp_aes_mode mode;

3
drivers/crypto/ccp/ccp-dev-v3.c
View file

@ -10,7 +10,6 @@
#include <linux/module.h>
#include <linux/kernel.h>
#include <linux/pci.h>
#include <linux/kthread.h>
#include <linux/interrupt.h>
#include <linux/ccp.h>
@ -379,7 +378,7 @@ static int ccp_init(struct ccp_device *ccp)
/* Find available queues */
ccp->qim = 0;
qmr = ioread32(ccp->io_regs + Q_MASK_REG);
for (i = 0; i < MAX_HW_QUEUES; i++) {
for (i = 0; (i < MAX_HW_QUEUES) && (ccp->cmd_q_count < ccp->max_q_count); i++) {
if (!(qmr & (1 << i)))
continue;

26
drivers/crypto/ccp/ccp-dev-v5.c
View file

@ -2,16 +2,13 @@
/*
* AMD Cryptographic Coprocessor (CCP) driver
*
* Copyright (C) 2016,2017 Advanced Micro Devices, Inc.
* Copyright (C) 2016,2019 Advanced Micro Devices, Inc.
*
* Author: Gary R Hook <gary.hook@amd.com>
*/
#include <linux/module.h>
#include <linux/kernel.h>
#include <linux/pci.h>
#include <linux/kthread.h>
#include <linux/debugfs.h>
#include <linux/dma-mapping.h>
#include <linux/interrupt.h>
#include <linux/compiler.h>
@ -792,8 +789,7 @@ static int ccp5_init(struct ccp_device *ccp)
/* Find available queues */
qmr = ioread32(ccp->io_regs + Q_MASK_REG);
for (i = 0; i < MAX_HW_QUEUES; i++) {
for (i = 0; (i < MAX_HW_QUEUES) && (ccp->cmd_q_count < ccp->max_q_count); i++) {
if (!(qmr & (1 << i)))
continue;
@ -806,6 +802,7 @@ static int ccp5_init(struct ccp_device *ccp)
if (!dma_pool) {
dev_err(dev, "unable to allocate dma pool\n");
ret = -ENOMEM;
goto e_pool;
}
cmd_q = &ccp->cmd_q[ccp->cmd_q_count];
@ -819,9 +816,9 @@ static int ccp5_init(struct ccp_device *ccp)
/* Page alignment satisfies our needs for N <= 128 */
BUILD_BUG_ON(COMMANDS_PER_QUEUE > 128);
cmd_q->qsize = Q_SIZE(Q_DESC_SIZE);
cmd_q->qbase = dma_alloc_coherent(dev, cmd_q->qsize,
&cmd_q->qbase_dma,
GFP_KERNEL);
cmd_q->qbase = dmam_alloc_coherent(dev, cmd_q->qsize,
&cmd_q->qbase_dma,
GFP_KERNEL);
if (!cmd_q->qbase) {
dev_err(dev, "unable to allocate command queue\n");
ret = -ENOMEM;
@ -970,8 +967,10 @@ static int ccp5_init(struct ccp_device *ccp)
if (ret)
goto e_hwrng;
#ifdef CONFIG_CRYPTO_DEV_CCP_DEBUGFS
/* Set up debugfs entries */
ccp5_debugfs_setup(ccp);
#endif
return 0;
@ -995,7 +994,6 @@ e_pool:
static void ccp5_destroy(struct ccp_device *ccp)
{
struct device *dev = ccp->dev;
struct ccp_cmd_queue *cmd_q;
struct ccp_cmd *cmd;
unsigned int i;
@ -1009,11 +1007,13 @@ static void ccp5_destroy(struct ccp_device *ccp)
/* Remove this device from the list of available units first */
ccp_del_device(ccp);
#ifdef CONFIG_CRYPTO_DEV_CCP_DEBUGFS
/* We're in the process of tearing down the entire driver;
* when all the devices are gone clean up debugfs
*/
if (ccp_present())
ccp5_debugfs_destroy();
#endif
/* Disable and clear interrupts */
ccp5_disable_queue_interrupts(ccp);
@ -1036,12 +1036,6 @@ static void ccp5_destroy(struct ccp_device *ccp)
sp_free_ccp_irq(ccp->sp, ccp);
for (i = 0; i < ccp->cmd_q_count; i++) {
cmd_q = &ccp->cmd_q[i];
dma_free_coherent(dev, cmd_q->qsize, cmd_q->qbase,
cmd_q->qbase_dma);
}
/* Flush the cmd and backlog queue */
while (!list_empty(&ccp->cmd)) {
/* Invoke the callback directly with an error code */

29
drivers/crypto/ccp/ccp-dev.c
View file

@ -2,12 +2,13 @@
/*
* AMD Cryptographic Coprocessor (CCP) driver
*
* Copyright (C) 2013,2017 Advanced Micro Devices, Inc.
* Copyright (C) 2013,2019 Advanced Micro Devices, Inc.
*
* Author: Tom Lendacky <thomas.lendacky@amd.com>
* Author: Gary R Hook <gary.hook@amd.com>
*/
#include <linux/module.h>
#include <linux/kernel.h>
#include <linux/kthread.h>
#include <linux/sched.h>
@ -19,6 +20,7 @@
#include <linux/delay.h>
#include <linux/hw_random.h>
#include <linux/cpu.h>
#include <linux/atomic.h>
#ifdef CONFIG_X86
#include <asm/cpu_device_id.h>
#endif
@ -26,6 +28,19 @@
#include "ccp-dev.h"
#define MAX_CCPS 32
/* Limit CCP use to a specifed number of queues per device */
static unsigned int nqueues = 0;
module_param(nqueues, uint, 0444);
MODULE_PARM_DESC(nqueues, "Number of queues per CCP (minimum 1; default: all available)");
/* Limit the maximum number of configured CCPs */
static atomic_t dev_count = ATOMIC_INIT(0);
static unsigned int max_devs = MAX_CCPS;
module_param(max_devs, uint, 0444);
MODULE_PARM_DESC(max_devs, "Maximum number of CCPs to enable (default: all; 0 disables all CCPs)");
struct ccp_tasklet_data {
struct completion completion;
struct ccp_cmd *cmd;
@ -594,12 +609,24 @@ int ccp_dev_init(struct sp_device *sp)
struct ccp_device *ccp;
int ret;
/*
* Check how many we have so far, and stop after reaching
* that number
*/
if (atomic_inc_return(&dev_count) > max_devs)
return 0; /* don't fail the load */
ret = -ENOMEM;
ccp = ccp_alloc_struct(sp);
if (!ccp)
goto e_err;
sp->ccp_data = ccp;
if (!nqueues || (nqueues > MAX_HW_QUEUES))
ccp->max_q_count = MAX_HW_QUEUES;
else
ccp->max_q_count = nqueues;
ccp->vdata = (struct ccp_vdata *)sp->dev_vdata->ccp_vdata;
if (!ccp->vdata || !ccp->vdata->version) {
ret = -ENODEV;

3
drivers/crypto/ccp/ccp-dev.h
View file

@ -12,11 +12,11 @@
#define __CCP_DEV_H__
#include <linux/device.h>
#include <linux/pci.h>
#include <linux/spinlock.h>
#include <linux/mutex.h>
#include <linux/list.h>
#include <linux/wait.h>
#include <linux/dma-direction.h>
#include <linux/dmapool.h>
#include <linux/hw_random.h>
#include <linux/bitops.h>
@ -379,6 +379,7 @@ struct ccp_device {
*/
struct ccp_cmd_queue cmd_q[MAX_HW_QUEUES];
unsigned int cmd_q_count;
unsigned int max_q_count;
/* Support for the CCP True RNG
*/

13
drivers/crypto/ccp/ccp-dmaengine.c
View file

@ -2,13 +2,14 @@
/*
* AMD Cryptographic Coprocessor (CCP) driver
*
* Copyright (C) 2016,2017 Advanced Micro Devices, Inc.
* Copyright (C) 2016,2019 Advanced Micro Devices, Inc.
*
* Author: Gary R Hook <gary.hook@amd.com>
*/
#include <linux/module.h>
#include <linux/kernel.h>
#include <linux/dma-mapping.h>
#include <linux/dmaengine.h>
#include <linux/spinlock.h>
#include <linux/mutex.h>
@ -35,6 +36,10 @@ static unsigned int dma_chan_attr = CCP_DMA_DFLT;
module_param(dma_chan_attr, uint, 0444);
MODULE_PARM_DESC(dma_chan_attr, "Set DMA channel visibility: 0 (default) = device defaults, 1 = make private, 2 = make public");
static unsigned int dmaengine = 1;
module_param(dmaengine, uint, 0444);
MODULE_PARM_DESC(dmaengine, "Register services with the DMA subsystem (any non-zero value, default: 1)");
static unsigned int ccp_get_dma_chan_attr(struct ccp_device *ccp)
{
switch (dma_chan_attr) {
@ -637,6 +642,9 @@ int ccp_dmaengine_register(struct ccp_device *ccp)
unsigned int i;
int ret;
if (!dmaengine)
return 0;
ccp->ccp_dma_chan = devm_kcalloc(ccp->dev, ccp->cmd_q_count,
sizeof(*(ccp->ccp_dma_chan)),
GFP_KERNEL);
@ -740,6 +748,9 @@ void ccp_dmaengine_unregister(struct ccp_device *ccp)
{
struct dma_device *dma_dev = &ccp->dma_dev;
if (!dmaengine)
return;
dma_async_device_unregister(dma_dev);
kmem_cache_destroy(ccp->dma_desc_cache);

56
drivers/crypto/ccp/ccp-ops.c
View file

@ -10,7 +10,6 @@
#include <linux/module.h>
#include <linux/kernel.h>
#include <linux/pci.h>
#include <linux/interrupt.h>
#include <crypto/scatterwalk.h>
#include <crypto/des.h>
@ -150,14 +149,13 @@ static int ccp_init_dm_workarea(struct ccp_dm_workarea *wa,
if (len <= CCP_DMAPOOL_MAX_SIZE) {
wa->dma_pool = cmd_q->dma_pool;
wa->address = dma_pool_alloc(wa->dma_pool, GFP_KERNEL,
wa->address = dma_pool_zalloc(wa->dma_pool, GFP_KERNEL,
&wa->dma.address);
if (!wa->address)
return -ENOMEM;
wa->dma.length = CCP_DMAPOOL_MAX_SIZE;
memset(wa->address, 0, CCP_DMAPOOL_MAX_SIZE);
} else {
wa->address = kzalloc(len, GFP_KERNEL);
if (!wa->address)
@ -455,8 +453,8 @@ static int ccp_copy_from_sb(struct ccp_cmd_queue *cmd_q,
return ccp_copy_to_from_sb(cmd_q, wa, jobid, sb, byte_swap, true);
}
static int ccp_run_aes_cmac_cmd(struct ccp_cmd_queue *cmd_q,
struct ccp_cmd *cmd)
static noinline_for_stack int
ccp_run_aes_cmac_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
{
struct ccp_aes_engine *aes = &cmd->u.aes;
struct ccp_dm_workarea key, ctx;
@ -611,8 +609,8 @@ e_key:
return ret;
}
static int ccp_run_aes_gcm_cmd(struct ccp_cmd_queue *cmd_q,
struct ccp_cmd *cmd)
static noinline_for_stack int
ccp_run_aes_gcm_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
{
struct ccp_aes_engine *aes = &cmd->u.aes;
struct ccp_dm_workarea key, ctx, final_wa, tag;
@ -894,7 +892,8 @@ e_key:
return ret;
}
static int ccp_run_aes_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
static noinline_for_stack int
ccp_run_aes_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
{
struct ccp_aes_engine *aes = &cmd->u.aes;
struct ccp_dm_workarea key, ctx;
@ -904,12 +903,6 @@ static int ccp_run_aes_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
bool in_place = false;
int ret;
if (aes->mode == CCP_AES_MODE_CMAC)
return ccp_run_aes_cmac_cmd(cmd_q, cmd);
if (aes->mode == CCP_AES_MODE_GCM)
return ccp_run_aes_gcm_cmd(cmd_q, cmd);
if (!((aes->key_len == AES_KEYSIZE_128) ||
(aes->key_len == AES_KEYSIZE_192) ||
(aes->key_len == AES_KEYSIZE_256)))
@ -1076,8 +1069,8 @@ e_key:
return ret;
}
static int ccp_run_xts_aes_cmd(struct ccp_cmd_queue *cmd_q,
struct ccp_cmd *cmd)
static noinline_for_stack int
ccp_run_xts_aes_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
{
struct ccp_xts_aes_engine *xts = &cmd->u.xts;
struct ccp_dm_workarea key, ctx;
@ -1276,7 +1269,8 @@ e_key:
return ret;
}
static int ccp_run_des3_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
static noinline_for_stack int
ccp_run_des3_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
{
struct ccp_des3_engine *des3 = &cmd->u.des3;
@ -1472,7 +1466,8 @@ e_key:
return ret;
}
static int ccp_run_sha_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
static noinline_for_stack int
ccp_run_sha_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
{
struct ccp_sha_engine *sha = &cmd->u.sha;
struct ccp_dm_workarea ctx;
@ -1816,7 +1811,8 @@ e_ctx:
return ret;
}
static int ccp_run_rsa_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
static noinline_for_stack int
ccp_run_rsa_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
{
struct ccp_rsa_engine *rsa = &cmd->u.rsa;
struct ccp_dm_workarea exp, src, dst;
@ -1947,8 +1943,8 @@ e_sb:
return ret;
}
static int ccp_run_passthru_cmd(struct ccp_cmd_queue *cmd_q,
struct ccp_cmd *cmd)
static noinline_for_stack int
ccp_run_passthru_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
{
struct ccp_passthru_engine *pt = &cmd->u.passthru;
struct ccp_dm_workarea mask;
@ -2079,7 +2075,8 @@ e_mask:
return ret;
}
static int ccp_run_passthru_nomap_cmd(struct ccp_cmd_queue *cmd_q,
static noinline_for_stack int
ccp_run_passthru_nomap_cmd(struct ccp_cmd_queue *cmd_q,
struct ccp_cmd *cmd)
{
struct ccp_passthru_nomap_engine *pt = &cmd->u.passthru_nomap;
@ -2420,7 +2417,8 @@ e_src:
return ret;
}
static int ccp_run_ecc_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
static noinline_for_stack int
ccp_run_ecc_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
{
struct ccp_ecc_engine *ecc = &cmd->u.ecc;
@ -2457,7 +2455,17 @@ int ccp_run_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
switch (cmd->engine) {
case CCP_ENGINE_AES:
ret = ccp_run_aes_cmd(cmd_q, cmd);
switch (cmd->u.aes.mode) {
case CCP_AES_MODE_CMAC:
ret = ccp_run_aes_cmac_cmd(cmd_q, cmd);
break;
case CCP_AES_MODE_GCM:
ret = ccp_run_aes_gcm_cmd(cmd_q, cmd);
break;
default:
ret = ccp_run_aes_cmd(cmd_q, cmd);
break;
}
break;
case CCP_ENGINE_XTS_AES_128:
ret = ccp_run_xts_aes_cmd(cmd_q, cmd);

1
drivers/crypto/ccp/psp-dev.h
View file

@ -11,7 +11,6 @@
#define __PSP_DEV_H__
#include <linux/device.h>
#include <linux/pci.h>
#include <linux/spinlock.h>
#include <linux/mutex.h>
#include <linux/list.h>

1
drivers/crypto/ccp/sp-dev.h
View file

@ -13,7 +13,6 @@
#define __SP_DEV_H__
#include <linux/device.h>
#include <linux/pci.h>
#include <linux/spinlock.h>
#include <linux/mutex.h>
#include <linux/list.h>

4
drivers/crypto/ccp/sp-platform.c
View file

@ -125,7 +125,6 @@ static int sp_platform_probe(struct platform_device *pdev)
struct sp_platform *sp_platform;
struct device *dev = &pdev->dev;
enum dev_dma_attr attr;
struct resource *ior;
int ret;
ret = -ENOMEM;
@ -146,8 +145,7 @@ static int sp_platform_probe(struct platform_device *pdev)
goto e_err;
}
ior = platform_get_resource(pdev, IORESOURCE_MEM, 0);
sp->io_map = devm_ioremap_resource(dev, ior);
sp->io_map = devm_platform_ioremap_resource(pdev, 0);
if (IS_ERR(sp->io_map)) {
ret = PTR_ERR(sp->io_map);
goto e_err;

2
drivers/crypto/ccree/Makefile
View file

@ -2,7 +2,7 @@
# Copyright (C) 2012-2019 ARM Limited (or its affiliates).
obj-$(CONFIG_CRYPTO_DEV_CCREE) := ccree.o
ccree-y := cc_driver.o cc_buffer_mgr.o cc_request_mgr.o cc_cipher.o cc_hash.o cc_aead.o cc_ivgen.o cc_sram_mgr.o
ccree-y := cc_driver.o cc_buffer_mgr.o cc_request_mgr.o cc_cipher.o cc_hash.o cc_aead.o cc_sram_mgr.o
ccree-$(CONFIG_CRYPTO_FIPS) += cc_fips.o
ccree-$(CONFIG_DEBUG_FS) += cc_debugfs.o
ccree-$(CONFIG_PM) += cc_pm.o

129
drivers/crypto/ccree/cc_aead.c
View file

@ -6,7 +6,7 @@
#include <crypto/algapi.h>
#include <crypto/internal/aead.h>
#include <crypto/authenc.h>
#include <crypto/des.h>
#include <crypto/internal/des.h>
#include <linux/rtnetlink.h>
#include "cc_driver.h"
#include "cc_buffer_mgr.h"
@ -236,31 +236,17 @@ static void cc_aead_complete(struct device *dev, void *cc_req, int err)
/* In case of payload authentication failure, MUST NOT
* revealed the decrypted message --> zero its memory.
*/
cc_zero_sgl(areq->dst, areq_ctx->cryptlen);
sg_zero_buffer(areq->dst, sg_nents(areq->dst),
areq->cryptlen, 0);
err = -EBADMSG;
}
} else { /*ENCRYPT*/
if (areq_ctx->is_icv_fragmented) {
u32 skip = areq->cryptlen + areq_ctx->dst_offset;
/*ENCRYPT*/
} else if (areq_ctx->is_icv_fragmented) {
u32 skip = areq->cryptlen + areq_ctx->dst_offset;
cc_copy_sg_portion(dev, areq_ctx->mac_buf,
areq_ctx->dst_sgl, skip,
(skip + ctx->authsize),
CC_SG_FROM_BUF);
}
/* If an IV was generated, copy it back to the user provided
* buffer.
*/
if (areq_ctx->backup_giv) {
if (ctx->cipher_mode == DRV_CIPHER_CTR)
memcpy(areq_ctx->backup_giv, areq_ctx->ctr_iv +
CTR_RFC3686_NONCE_SIZE,
CTR_RFC3686_IV_SIZE);
else if (ctx->cipher_mode == DRV_CIPHER_CCM)
memcpy(areq_ctx->backup_giv, areq_ctx->ctr_iv +
CCM_BLOCK_IV_OFFSET, CCM_BLOCK_IV_SIZE);
}
cc_copy_sg_portion(dev, areq_ctx->mac_buf, areq_ctx->dst_sgl,
skip, (skip + ctx->authsize),
CC_SG_FROM_BUF);
}
done:
aead_request_complete(areq, err);
@ -663,33 +649,17 @@ static int cc_des3_aead_setkey(struct crypto_aead *aead, const u8 *key,
unsigned int keylen)
{
struct crypto_authenc_keys keys;
u32 flags;
int err;
err = crypto_authenc_extractkeys(&keys, key, keylen);
if (unlikely(err))
goto badkey;
return err;
err = -EINVAL;
if (keys.enckeylen != DES3_EDE_KEY_SIZE)
goto badkey;
err = verify_aead_des3_key(aead, keys.enckey, keys.enckeylen) ?:
cc_aead_setkey(aead, key, keylen);
flags = crypto_aead_get_flags(aead);
err = __des3_verify_key(&flags, keys.enckey);
if (unlikely(err)) {
crypto_aead_set_flags(aead, flags);
goto out;
}
err = cc_aead_setkey(aead, key, keylen);
out:
memzero_explicit(&keys, sizeof(keys));
return err;
badkey:
crypto_aead_set_flags(aead, CRYPTO_TFM_RES_BAD_KEY_LEN);
goto out;
}
static int cc_rfc4309_ccm_setkey(struct crypto_aead *tfm, const u8 *key,
@ -1975,9 +1945,8 @@ static int cc_proc_aead(struct aead_request *req,
*/
memcpy(areq_ctx->ctr_iv, ctx->ctr_nonce,
CTR_RFC3686_NONCE_SIZE);
if (!areq_ctx->backup_giv) /*User none-generated IV*/
memcpy(areq_ctx->ctr_iv + CTR_RFC3686_NONCE_SIZE,
req->iv, CTR_RFC3686_IV_SIZE);
memcpy(areq_ctx->ctr_iv + CTR_RFC3686_NONCE_SIZE, req->iv,
CTR_RFC3686_IV_SIZE);
/* Initialize counter portion of counter block */
*(__be32 *)(areq_ctx->ctr_iv + CTR_RFC3686_NONCE_SIZE +
CTR_RFC3686_IV_SIZE) = cpu_to_be32(1);
@ -2023,40 +1992,6 @@ static int cc_proc_aead(struct aead_request *req,
goto exit;
}
/* do we need to generate IV? */
if (areq_ctx->backup_giv) {
/* set the DMA mapped IV address*/
if (ctx->cipher_mode == DRV_CIPHER_CTR) {
cc_req.ivgen_dma_addr[0] =
areq_ctx->gen_ctx.iv_dma_addr +
CTR_RFC3686_NONCE_SIZE;
cc_req.ivgen_dma_addr_len = 1;
} else if (ctx->cipher_mode == DRV_CIPHER_CCM) {
/* In ccm, the IV needs to exist both inside B0 and
* inside the counter.It is also copied to iv_dma_addr
* for other reasons (like returning it to the user).
* So, using 3 (identical) IV outputs.
*/
cc_req.ivgen_dma_addr[0] =
areq_ctx->gen_ctx.iv_dma_addr +
CCM_BLOCK_IV_OFFSET;
cc_req.ivgen_dma_addr[1] =
sg_dma_address(&areq_ctx->ccm_adata_sg) +
CCM_B0_OFFSET + CCM_BLOCK_IV_OFFSET;
cc_req.ivgen_dma_addr[2] =
sg_dma_address(&areq_ctx->ccm_adata_sg) +
CCM_CTR_COUNT_0_OFFSET + CCM_BLOCK_IV_OFFSET;
cc_req.ivgen_dma_addr_len = 3;
} else {
cc_req.ivgen_dma_addr[0] =
areq_ctx->gen_ctx.iv_dma_addr;
cc_req.ivgen_dma_addr_len = 1;
}
/* set the IV size (8/16 B long)*/
cc_req.ivgen_size = crypto_aead_ivsize(tfm);
}
/* STAT_PHASE_2: Create sequence */
/* Load MLLI tables to SRAM if necessary */
@ -2107,7 +2042,6 @@ static int cc_aead_encrypt(struct aead_request *req)
/* No generated IV required */
areq_ctx->backup_iv = req->iv;
areq_ctx->assoclen = req->assoclen;
areq_ctx->backup_giv = NULL;
areq_ctx->is_gcm4543 = false;
areq_ctx->plaintext_authenticate_only = false;
@ -2139,7 +2073,6 @@ static int cc_rfc4309_ccm_encrypt(struct aead_request *req)
/* No generated IV required */
areq_ctx->backup_iv = req->iv;
areq_ctx->assoclen = req->assoclen;
areq_ctx->backup_giv = NULL;
areq_ctx->is_gcm4543 = true;
cc_proc_rfc4309_ccm(req);
@ -2161,7 +2094,6 @@ static int cc_aead_decrypt(struct aead_request *req)
/* No generated IV required */
areq_ctx->backup_iv = req->iv;
areq_ctx->assoclen = req->assoclen;
areq_ctx->backup_giv = NULL;
areq_ctx->is_gcm4543 = false;
areq_ctx->plaintext_authenticate_only = false;
@ -2191,7 +2123,6 @@ static int cc_rfc4309_ccm_decrypt(struct aead_request *req)
/* No generated IV required */
areq_ctx->backup_iv = req->iv;
areq_ctx->assoclen = req->assoclen;
areq_ctx->backup_giv = NULL;
areq_ctx->is_gcm4543 = true;
cc_proc_rfc4309_ccm(req);
@ -2311,8 +2242,6 @@ static int cc_rfc4106_gcm_encrypt(struct aead_request *req)
/* No generated IV required */
areq_ctx->backup_iv = req->iv;
areq_ctx->assoclen = req->assoclen;
areq_ctx->backup_giv = NULL;
areq_ctx->plaintext_authenticate_only = false;
cc_proc_rfc4_gcm(req);
@ -2328,9 +2257,16 @@ out:
static int cc_rfc4543_gcm_encrypt(struct aead_request *req)
{
/* Very similar to cc_aead_encrypt() above. */
struct crypto_aead *tfm = crypto_aead_reqtfm(req);
struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);
struct device *dev = drvdata_to_dev(ctx->drvdata);
struct aead_req_ctx *areq_ctx = aead_request_ctx(req);
int rc;
int rc = -EINVAL;
if (!valid_assoclen(req)) {
dev_err(dev, "invalid Assoclen:%u\n", req->assoclen);
goto out;
}
memset(areq_ctx, 0, sizeof(*areq_ctx));
@ -2340,7 +2276,6 @@ static int cc_rfc4543_gcm_encrypt(struct aead_request *req)
/* No generated IV required */
areq_ctx->backup_iv = req->iv;
areq_ctx->assoclen = req->assoclen;
areq_ctx->backup_giv = NULL;
cc_proc_rfc4_gcm(req);
areq_ctx->is_gcm4543 = true;
@ -2348,7 +2283,7 @@ static int cc_rfc4543_gcm_encrypt(struct aead_request *req)
rc = cc_proc_aead(req, DRV_CRYPTO_DIRECTION_ENCRYPT);
if (rc != -EINPROGRESS && rc != -EBUSY)
req->iv = areq_ctx->backup_iv;
out:
return rc;
}
@ -2372,8 +2307,6 @@ static int cc_rfc4106_gcm_decrypt(struct aead_request *req)
/* No generated IV required */
areq_ctx->backup_iv = req->iv;
areq_ctx->assoclen = req->assoclen;
areq_ctx->backup_giv = NULL;
areq_ctx->plaintext_authenticate_only = false;
cc_proc_rfc4_gcm(req);
@ -2389,9 +2322,16 @@ out:
static int cc_rfc4543_gcm_decrypt(struct aead_request *req)
{
/* Very similar to cc_aead_decrypt() above. */
struct crypto_aead *tfm = crypto_aead_reqtfm(req);
struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);
struct device *dev = drvdata_to_dev(ctx->drvdata);
struct aead_req_ctx *areq_ctx = aead_request_ctx(req);
int rc;
int rc = -EINVAL;
if (!valid_assoclen(req)) {
dev_err(dev, "invalid Assoclen:%u\n", req->assoclen);
goto out;
}
memset(areq_ctx, 0, sizeof(*areq_ctx));
@ -2401,7 +2341,6 @@ static int cc_rfc4543_gcm_decrypt(struct aead_request *req)
/* No generated IV required */
areq_ctx->backup_iv = req->iv;
areq_ctx->assoclen = req->assoclen;
areq_ctx->backup_giv = NULL;
cc_proc_rfc4_gcm(req);
areq_ctx->is_gcm4543 = true;
@ -2409,7 +2348,7 @@ static int cc_rfc4543_gcm_decrypt(struct aead_request *req)
rc = cc_proc_aead(req, DRV_CRYPTO_DIRECTION_DECRYPT);
if (rc != -EINPROGRESS && rc != -EBUSY)
req->iv = areq_ctx->backup_iv;
out:
return rc;
}

3
drivers/crypto/ccree/cc_aead.h
View file

@ -65,8 +65,7 @@ struct aead_req_ctx {
unsigned int hw_iv_size ____cacheline_aligned;
/* used to prevent cache coherence problem */
u8 backup_mac[MAX_MAC_SIZE];
u8 *backup_iv; /*store iv for generated IV flow*/
u8 *backup_giv; /*store iv for rfc3686(ctr) flow*/
u8 *backup_iv; /* store orig iv */
u32 assoclen; /* internal assoclen */
dma_addr_t mac_buf_dma_addr; /* internal ICV DMA buffer */
/* buffer for internal ccm configurations */

21
drivers/crypto/ccree/cc_buffer_mgr.c
View file

@ -99,27 +99,6 @@ static unsigned int cc_get_sgl_nents(struct device *dev,
return nents;
}
/**
* cc_zero_sgl() - Zero scatter scatter list data.
*
* @sgl:
*/
void cc_zero_sgl(struct scatterlist *sgl, u32 data_len)
{
struct scatterlist *current_sg = sgl;
int sg_index = 0;
while (sg_index <= data_len) {
if (!current_sg) {
/* reached the end of the sgl --> just return back */
return;
}
memset(sg_virt(current_sg), 0, current_sg->length);
sg_index += current_sg->length;
current_sg = sg_next(current_sg);
}
}
/**
* cc_copy_sg_portion() - Copy scatter list data,
* from to_skip to end, to dest and vice versa

2
drivers/crypto/ccree/cc_buffer_mgr.h
View file

@ -66,6 +66,4 @@ void cc_unmap_hash_request(struct device *dev, void *ctx,
void cc_copy_sg_portion(struct device *dev, u8 *dest, struct scatterlist *sg,
u32 to_skip, u32 end, enum cc_sg_cpy_direct direct);
void cc_zero_sgl(struct scatterlist *sgl, u32 data_len);
#endif /*__BUFFER_MGR_H__*/

31
drivers/crypto/ccree/cc_cipher.c
View file

@ -5,7 +5,7 @@
#include <linux/module.h>
#include <crypto/algapi.h>
#include <crypto/internal/skcipher.h>
#include <crypto/des.h>
#include <crypto/internal/des.h>
#include <crypto/xts.h>
#include <crypto/sm4.h>
#include <crypto/scatterwalk.h>
@ -116,10 +116,6 @@ static int validate_data_size(struct cc_cipher_ctx *ctx_p,
case S_DIN_to_AES:
switch (ctx_p->cipher_mode) {
case DRV_CIPHER_XTS:
if (size >= AES_BLOCK_SIZE &&
IS_ALIGNED(size, AES_BLOCK_SIZE))
return 0;
break;
case DRV_CIPHER_CBC_CTS:
if (size >= AES_BLOCK_SIZE)
return 0;
@ -411,16 +407,9 @@ static int cc_cipher_setkey(struct crypto_skcipher *sktfm, const u8 *key,
* HW does the expansion on its own.
*/
if (ctx_p->flow_mode == S_DIN_to_DES) {
u32 tmp[DES3_EDE_EXPKEY_WORDS];
if (keylen == DES3_EDE_KEY_SIZE &&
__des3_ede_setkey(tmp, &tfm->crt_flags, key,
DES3_EDE_KEY_SIZE)) {
dev_dbg(dev, "weak 3DES key");
return -EINVAL;
} else if (!des_ekey(tmp, key) &&
(crypto_tfm_get_flags(tfm) &
CRYPTO_TFM_REQ_FORBID_WEAK_KEYS)) {
tfm->crt_flags |= CRYPTO_TFM_RES_WEAK_KEY;
if ((keylen == DES3_EDE_KEY_SIZE &&
verify_skcipher_des3_key(sktfm, key)) ||
verify_skcipher_des_key(sktfm, key)) {
dev_dbg(dev, "weak DES key");
return -EINVAL;
}
@ -945,7 +934,7 @@ static const struct cc_alg_template skcipher_algs[] = {
{
.name = "xts(paes)",
.driver_name = "xts-paes-ccree",
.blocksize = AES_BLOCK_SIZE,
.blocksize = 1,
.template_skcipher = {
.setkey = cc_cipher_sethkey,
.encrypt = cc_cipher_encrypt,
@ -963,7 +952,7 @@ static const struct cc_alg_template skcipher_algs[] = {
{
.name = "xts512(paes)",
.driver_name = "xts-paes-du512-ccree",
.blocksize = AES_BLOCK_SIZE,
.blocksize = 1,
.template_skcipher = {
.setkey = cc_cipher_sethkey,
.encrypt = cc_cipher_encrypt,
@ -982,7 +971,7 @@ static const struct cc_alg_template skcipher_algs[] = {
{
.name = "xts4096(paes)",
.driver_name = "xts-paes-du4096-ccree",
.blocksize = AES_BLOCK_SIZE,
.blocksize = 1,
.template_skcipher = {
.setkey = cc_cipher_sethkey,
.encrypt = cc_cipher_encrypt,
@ -1203,7 +1192,7 @@ static const struct cc_alg_template skcipher_algs[] = {
{
.name = "xts(aes)",
.driver_name = "xts-aes-ccree",
.blocksize = AES_BLOCK_SIZE,
.blocksize = 1,
.template_skcipher = {
.setkey = cc_cipher_setkey,
.encrypt = cc_cipher_encrypt,
@ -1220,7 +1209,7 @@ static const struct cc_alg_template skcipher_algs[] = {
{
.name = "xts512(aes)",
.driver_name = "xts-aes-du512-ccree",
.blocksize = AES_BLOCK_SIZE,
.blocksize = 1,
.template_skcipher = {
.setkey = cc_cipher_setkey,
.encrypt = cc_cipher_encrypt,
@ -1238,7 +1227,7 @@ static const struct cc_alg_template skcipher_algs[] = {
{
.name = "xts4096(aes)",
.driver_name = "xts-aes-du4096-ccree",
.blocksize = AES_BLOCK_SIZE,
.blocksize = 1,
.template_skcipher = {
.setkey = cc_cipher_setkey,
.encrypt = cc_cipher_encrypt,

18
drivers/crypto/ccree/cc_driver.c
View file

@ -22,7 +22,6 @@
#include "cc_cipher.h"
#include "cc_aead.h"
#include "cc_hash.h"
#include "cc_ivgen.h"
#include "cc_sram_mgr.h"
#include "cc_pm.h"
#include "cc_fips.h"
@ -339,10 +338,8 @@ static int init_cc_resources(struct platform_device *plat_dev)
/* Then IRQ */
new_drvdata->irq = platform_get_irq(plat_dev, 0);
if (new_drvdata->irq < 0) {
dev_err(dev, "Failed getting IRQ resource\n");
if (new_drvdata->irq < 0)
return new_drvdata->irq;
}
init_completion(&new_drvdata->hw_queue_avail);
@ -421,7 +418,7 @@ static int init_cc_resources(struct platform_device *plat_dev)
}
break;
default:
dev_err(dev, "Unsupported engines configration.\n");
dev_err(dev, "Unsupported engines configuration.\n");
rc = -EINVAL;
goto post_clk_err;
}
@ -503,17 +500,11 @@ static int init_cc_resources(struct platform_device *plat_dev)
goto post_buf_mgr_err;
}
rc = cc_ivgen_init(new_drvdata);
if (rc) {
dev_err(dev, "cc_ivgen_init failed\n");
goto post_buf_mgr_err;
}
/* Allocate crypto algs */
rc = cc_cipher_alloc(new_drvdata);
if (rc) {
dev_err(dev, "cc_cipher_alloc failed\n");
goto post_ivgen_err;
goto post_buf_mgr_err;
}
/* hash must be allocated before aead since hash exports APIs */
@ -544,8 +535,6 @@ post_hash_err:
cc_hash_free(new_drvdata);
post_cipher_err:
cc_cipher_free(new_drvdata);
post_ivgen_err:
cc_ivgen_fini(new_drvdata);
post_buf_mgr_err:
cc_buffer_mgr_fini(new_drvdata);
post_req_mgr_err:
@ -577,7 +566,6 @@ static void cleanup_cc_resources(struct platform_device *plat_dev)
cc_aead_free(drvdata);
cc_hash_free(drvdata);
cc_cipher_free(drvdata);
cc_ivgen_fini(drvdata);
cc_pm_fini(drvdata);
cc_buffer_mgr_fini(drvdata);
cc_req_mgr_fini(drvdata);

10
drivers/crypto/ccree/cc_driver.h
View file

@ -126,15 +126,6 @@ struct cc_cpp_req {
struct cc_crypto_req {
void (*user_cb)(struct device *dev, void *req, int err);
void *user_arg;
dma_addr_t ivgen_dma_addr[CC_MAX_IVGEN_DMA_ADDRESSES];
/* For the first 'ivgen_dma_addr_len' addresses of this array,
* generated IV would be placed in it by send_request().
* Same generated IV for all addresses!
*/
/* Amount of 'ivgen_dma_addr' elements to be filled. */
unsigned int ivgen_dma_addr_len;
/* The generated IV size required, 8/16 B allowed. */
unsigned int ivgen_size;
struct completion seq_compl; /* request completion */
struct cc_cpp_req cpp;
};
@ -158,7 +149,6 @@ struct cc_drvdata {
void *aead_handle;
void *request_mgr_handle;
void *fips_handle;
void *ivgen_handle;
void *sram_mgr_handle;
void *debugfs;
struct clk *clk;

31
drivers/crypto/ccree/cc_fips.c
View file

@ -3,6 +3,7 @@
#include <linux/kernel.h>
#include <linux/fips.h>
#include <linux/notifier.h>
#include "cc_driver.h"
#include "cc_fips.h"
@ -11,6 +12,8 @@ static void fips_dsr(unsigned long devarg);
struct cc_fips_handle {
struct tasklet_struct tasklet;
struct notifier_block nb;
struct cc_drvdata *drvdata;
};
/* The function called once at driver entry point to check
@ -21,7 +24,13 @@ static bool cc_get_tee_fips_status(struct cc_drvdata *drvdata)
u32 reg;
reg = cc_ioread(drvdata, CC_REG(GPR_HOST));
return (reg == (CC_FIPS_SYNC_TEE_STATUS | CC_FIPS_SYNC_MODULE_OK));
/* Did the TEE report status? */
if (reg & CC_FIPS_SYNC_TEE_STATUS)
/* Yes. Is it OK? */
return (reg & CC_FIPS_SYNC_MODULE_OK);
/* No. It's either not in use or will be reported later */
return true;
}
/*
@ -40,6 +49,21 @@ void cc_set_ree_fips_status(struct cc_drvdata *drvdata, bool status)
cc_iowrite(drvdata, CC_REG(HOST_GPR0), val);
}
/* Push REE side FIPS test failure to TEE side */
static int cc_ree_fips_failure(struct notifier_block *nb, unsigned long unused1,
void *unused2)
{
struct cc_fips_handle *fips_h =
container_of(nb, struct cc_fips_handle, nb);
struct cc_drvdata *drvdata = fips_h->drvdata;
struct device *dev = drvdata_to_dev(drvdata);
cc_set_ree_fips_status(drvdata, false);
dev_info(dev, "Notifying TEE of FIPS test failure...\n");
return NOTIFY_OK;
}
void cc_fips_fini(struct cc_drvdata *drvdata)
{
struct cc_fips_handle *fips_h = drvdata->fips_handle;
@ -47,6 +71,8 @@ void cc_fips_fini(struct cc_drvdata *drvdata)
if (drvdata->hw_rev < CC_HW_REV_712 || !fips_h)
return;
atomic_notifier_chain_unregister(&fips_fail_notif_chain, &fips_h->nb);
/* Kill tasklet */
tasklet_kill(&fips_h->tasklet);
drvdata->fips_handle = NULL;
@ -118,6 +144,9 @@ int cc_fips_init(struct cc_drvdata *p_drvdata)
dev_dbg(dev, "Initializing fips tasklet\n");
tasklet_init(&fips_h->tasklet, fips_dsr, (unsigned long)p_drvdata);
fips_h->drvdata = p_drvdata;
fips_h->nb.notifier_call = cc_ree_fips_failure;
atomic_notifier_chain_register(&fips_fail_notif_chain, &fips_h->nb);
cc_tee_handle_fips_error(p_drvdata);

153
drivers/crypto/ccree/cc_hash.c
View file

@ -25,27 +25,27 @@ struct cc_hash_handle {
struct list_head hash_list;
};
static const u32 digest_len_init[] = {
static const u32 cc_digest_len_init[] = {
0x00000040, 0x00000000, 0x00000000, 0x00000000 };
static const u32 md5_init[] = {
static const u32 cc_md5_init[] = {
SHA1_H3, SHA1_H2, SHA1_H1, SHA1_H0 };
static const u32 sha1_init[] = {
static const u32 cc_sha1_init[] = {
SHA1_H4, SHA1_H3, SHA1_H2, SHA1_H1, SHA1_H0 };
static const u32 sha224_init[] = {
static const u32 cc_sha224_init[] = {
SHA224_H7, SHA224_H6, SHA224_H5, SHA224_H4,
SHA224_H3, SHA224_H2, SHA224_H1, SHA224_H0 };
static const u32 sha256_init[] = {
static const u32 cc_sha256_init[] = {
SHA256_H7, SHA256_H6, SHA256_H5, SHA256_H4,
SHA256_H3, SHA256_H2, SHA256_H1, SHA256_H0 };
static const u32 digest_len_sha512_init[] = {
static const u32 cc_digest_len_sha512_init[] = {
0x00000080, 0x00000000, 0x00000000, 0x00000000 };
static u64 sha384_init[] = {
static u64 cc_sha384_init[] = {
SHA384_H7, SHA384_H6, SHA384_H5, SHA384_H4,
SHA384_H3, SHA384_H2, SHA384_H1, SHA384_H0 };
static u64 sha512_init[] = {
static u64 cc_sha512_init[] = {
SHA512_H7, SHA512_H6, SHA512_H5, SHA512_H4,
SHA512_H3, SHA512_H2, SHA512_H1, SHA512_H0 };
static const u32 sm3_init[] = {
static const u32 cc_sm3_init[] = {
SM3_IVH, SM3_IVG, SM3_IVF, SM3_IVE,
SM3_IVD, SM3_IVC, SM3_IVB, SM3_IVA };
@ -144,10 +144,11 @@ static void cc_init_req(struct device *dev, struct ahash_req_ctx *state,
if (ctx->hash_mode == DRV_HASH_SHA512 ||
ctx->hash_mode == DRV_HASH_SHA384)
memcpy(state->digest_bytes_len,
digest_len_sha512_init,
cc_digest_len_sha512_init,
ctx->hash_len);
else
memcpy(state->digest_bytes_len, digest_len_init,
memcpy(state->digest_bytes_len,
cc_digest_len_init,
ctx->hash_len);
}
@ -1873,26 +1874,26 @@ int cc_init_hash_sram(struct cc_drvdata *drvdata)
int rc = 0;
/* Copy-to-sram digest-len */
cc_set_sram_desc(digest_len_init, sram_buff_ofs,
ARRAY_SIZE(digest_len_init), larval_seq,
cc_set_sram_desc(cc_digest_len_init, sram_buff_ofs,
ARRAY_SIZE(cc_digest_len_init), larval_seq,
&larval_seq_len);
rc = send_request_init(drvdata, larval_seq, larval_seq_len);
if (rc)
goto init_digest_const_err;
sram_buff_ofs += sizeof(digest_len_init);
sram_buff_ofs += sizeof(cc_digest_len_init);
larval_seq_len = 0;
if (large_sha_supported) {
/* Copy-to-sram digest-len for sha384/512 */
cc_set_sram_desc(digest_len_sha512_init, sram_buff_ofs,
ARRAY_SIZE(digest_len_sha512_init),
cc_set_sram_desc(cc_digest_len_sha512_init, sram_buff_ofs,
ARRAY_SIZE(cc_digest_len_sha512_init),
larval_seq, &larval_seq_len);
rc = send_request_init(drvdata, larval_seq, larval_seq_len);
if (rc)
goto init_digest_const_err;
sram_buff_ofs += sizeof(digest_len_sha512_init);
sram_buff_ofs += sizeof(cc_digest_len_sha512_init);
larval_seq_len = 0;
}
@ -1900,64 +1901,64 @@ int cc_init_hash_sram(struct cc_drvdata *drvdata)
hash_handle->larval_digest_sram_addr = sram_buff_ofs;
/* Copy-to-sram initial SHA* digests */
cc_set_sram_desc(md5_init, sram_buff_ofs, ARRAY_SIZE(md5_init),
cc_set_sram_desc(cc_md5_init, sram_buff_ofs, ARRAY_SIZE(cc_md5_init),
larval_seq, &larval_seq_len);
rc = send_request_init(drvdata, larval_seq, larval_seq_len);
if (rc)
goto init_digest_const_err;
sram_buff_ofs += sizeof(md5_init);
sram_buff_ofs += sizeof(cc_md5_init);
larval_seq_len = 0;
cc_set_sram_desc(sha1_init, sram_buff_ofs,
ARRAY_SIZE(sha1_init), larval_seq,
cc_set_sram_desc(cc_sha1_init, sram_buff_ofs,
ARRAY_SIZE(cc_sha1_init), larval_seq,
&larval_seq_len);
rc = send_request_init(drvdata, larval_seq, larval_seq_len);
if (rc)
goto init_digest_const_err;
sram_buff_ofs += sizeof(sha1_init);
sram_buff_ofs += sizeof(cc_sha1_init);
larval_seq_len = 0;
cc_set_sram_desc(sha224_init, sram_buff_ofs,
ARRAY_SIZE(sha224_init), larval_seq,
cc_set_sram_desc(cc_sha224_init, sram_buff_ofs,
ARRAY_SIZE(cc_sha224_init), larval_seq,
&larval_seq_len);
rc = send_request_init(drvdata, larval_seq, larval_seq_len);
if (rc)
goto init_digest_const_err;
sram_buff_ofs += sizeof(sha224_init);
sram_buff_ofs += sizeof(cc_sha224_init);
larval_seq_len = 0;
cc_set_sram_desc(sha256_init, sram_buff_ofs,
ARRAY_SIZE(sha256_init), larval_seq,
cc_set_sram_desc(cc_sha256_init, sram_buff_ofs,
ARRAY_SIZE(cc_sha256_init), larval_seq,
&larval_seq_len);
rc = send_request_init(drvdata, larval_seq, larval_seq_len);
if (rc)
goto init_digest_const_err;
sram_buff_ofs += sizeof(sha256_init);
sram_buff_ofs += sizeof(cc_sha256_init);
larval_seq_len = 0;
if (sm3_supported) {
cc_set_sram_desc(sm3_init, sram_buff_ofs,
ARRAY_SIZE(sm3_init), larval_seq,
cc_set_sram_desc(cc_sm3_init, sram_buff_ofs,
ARRAY_SIZE(cc_sm3_init), larval_seq,
&larval_seq_len);
rc = send_request_init(drvdata, larval_seq, larval_seq_len);
if (rc)
goto init_digest_const_err;
sram_buff_ofs += sizeof(sm3_init);
sram_buff_ofs += sizeof(cc_sm3_init);
larval_seq_len = 0;
}
if (large_sha_supported) {
cc_set_sram_desc((u32 *)sha384_init, sram_buff_ofs,
(ARRAY_SIZE(sha384_init) * 2), larval_seq,
cc_set_sram_desc((u32 *)cc_sha384_init, sram_buff_ofs,
(ARRAY_SIZE(cc_sha384_init) * 2), larval_seq,
&larval_seq_len);
rc = send_request_init(drvdata, larval_seq, larval_seq_len);
if (rc)
goto init_digest_const_err;
sram_buff_ofs += sizeof(sha384_init);
sram_buff_ofs += sizeof(cc_sha384_init);
larval_seq_len = 0;
cc_set_sram_desc((u32 *)sha512_init, sram_buff_ofs,
(ARRAY_SIZE(sha512_init) * 2), larval_seq,
cc_set_sram_desc((u32 *)cc_sha512_init, sram_buff_ofs,
(ARRAY_SIZE(cc_sha512_init) * 2), larval_seq,
&larval_seq_len);
rc = send_request_init(drvdata, larval_seq, larval_seq_len);
if (rc)
@ -1986,8 +1987,8 @@ static void __init cc_swap_dwords(u32 *buf, unsigned long size)
*/
void __init cc_hash_global_init(void)
{
cc_swap_dwords((u32 *)&sha384_init, (ARRAY_SIZE(sha384_init) * 2));
cc_swap_dwords((u32 *)&sha512_init, (ARRAY_SIZE(sha512_init) * 2));
cc_swap_dwords((u32 *)&cc_sha384_init, (ARRAY_SIZE(cc_sha384_init) * 2));
cc_swap_dwords((u32 *)&cc_sha512_init, (ARRAY_SIZE(cc_sha512_init) * 2));
}
int cc_hash_alloc(struct cc_drvdata *drvdata)
@ -2006,18 +2007,18 @@ int cc_hash_alloc(struct cc_drvdata *drvdata)
INIT_LIST_HEAD(&hash_handle->hash_list);
drvdata->hash_handle = hash_handle;
sram_size_to_alloc = sizeof(digest_len_init) +
sizeof(md5_init) +
sizeof(sha1_init) +
sizeof(sha224_init) +
sizeof(sha256_init);
sram_size_to_alloc = sizeof(cc_digest_len_init) +
sizeof(cc_md5_init) +
sizeof(cc_sha1_init) +
sizeof(cc_sha224_init) +
sizeof(cc_sha256_init);
if (drvdata->hw_rev >= CC_HW_REV_713)
sram_size_to_alloc += sizeof(sm3_init);
sram_size_to_alloc += sizeof(cc_sm3_init);
if (drvdata->hw_rev >= CC_HW_REV_712)
sram_size_to_alloc += sizeof(digest_len_sha512_init) +
sizeof(sha384_init) + sizeof(sha512_init);
sram_size_to_alloc += sizeof(cc_digest_len_sha512_init) +
sizeof(cc_sha384_init) + sizeof(cc_sha512_init);
sram_buff = cc_sram_alloc(drvdata, sram_size_to_alloc);
if (sram_buff == NULL_SRAM_ADDR) {
@ -2258,22 +2259,22 @@ static const void *cc_larval_digest(struct device *dev, u32 mode)
{
switch (mode) {
case DRV_HASH_MD5:
return md5_init;
return cc_md5_init;
case DRV_HASH_SHA1:
return sha1_init;
return cc_sha1_init;
case DRV_HASH_SHA224:
return sha224_init;
return cc_sha224_init;
case DRV_HASH_SHA256:
return sha256_init;
return cc_sha256_init;
case DRV_HASH_SHA384:
return sha384_init;
return cc_sha384_init;
case DRV_HASH_SHA512:
return sha512_init;
return cc_sha512_init;
case DRV_HASH_SM3:
return sm3_init;
return cc_sm3_init;
default:
dev_err(dev, "Invalid hash mode (%d)\n", mode);
return md5_init;
return cc_md5_init;
}
}
@ -2301,40 +2302,40 @@ cc_sram_addr_t cc_larval_digest_addr(void *drvdata, u32 mode)
return (hash_handle->larval_digest_sram_addr);
case DRV_HASH_SHA1:
return (hash_handle->larval_digest_sram_addr +
sizeof(md5_init));
sizeof(cc_md5_init));
case DRV_HASH_SHA224:
return (hash_handle->larval_digest_sram_addr +
sizeof(md5_init) +
sizeof(sha1_init));
sizeof(cc_md5_init) +
sizeof(cc_sha1_init));
case DRV_HASH_SHA256:
return (hash_handle->larval_digest_sram_addr +
sizeof(md5_init) +
sizeof(sha1_init) +
sizeof(sha224_init));
sizeof(cc_md5_init) +
sizeof(cc_sha1_init) +
sizeof(cc_sha224_init));
case DRV_HASH_SM3:
return (hash_handle->larval_digest_sram_addr +
sizeof(md5_init) +
sizeof(sha1_init) +
sizeof(sha224_init) +
sizeof(sha256_init));
sizeof(cc_md5_init) +
sizeof(cc_sha1_init) +
sizeof(cc_sha224_init) +
sizeof(cc_sha256_init));
case DRV_HASH_SHA384:
addr = (hash_handle->larval_digest_sram_addr +
sizeof(md5_init) +
sizeof(sha1_init) +
sizeof(sha224_init) +
sizeof(sha256_init));
sizeof(cc_md5_init) +
sizeof(cc_sha1_init) +
sizeof(cc_sha224_init) +
sizeof(cc_sha256_init));
if (sm3_supported)
addr += sizeof(sm3_init);
addr += sizeof(cc_sm3_init);
return addr;
case DRV_HASH_SHA512:
addr = (hash_handle->larval_digest_sram_addr +
sizeof(md5_init) +
sizeof(sha1_init) +
sizeof(sha224_init) +
sizeof(sha256_init) +
sizeof(sha384_init));
sizeof(cc_md5_init) +
sizeof(cc_sha1_init) +
sizeof(cc_sha224_init) +
sizeof(cc_sha256_init) +
sizeof(cc_sha384_init));
if (sm3_supported)
addr += sizeof(sm3_init);
addr += sizeof(cc_sm3_init);
return addr;
default:
dev_err(dev, "Invalid hash mode (%d)\n", mode);
@ -2360,7 +2361,7 @@ cc_digest_len_addr(void *drvdata, u32 mode)
#if (CC_DEV_SHA_MAX > 256)
case DRV_HASH_SHA384:
case DRV_HASH_SHA512:
return digest_len_addr + sizeof(digest_len_init);
return digest_len_addr + sizeof(cc_digest_len_init);
#endif
default:
return digest_len_addr; /*to avoid kernel crash*/

276
drivers/crypto/ccree/cc_ivgen.c
View file

@ -1,276 +0,0 @@
// SPDX-License-Identifier: GPL-2.0
/* Copyright (C) 2012-2019 ARM Limited (or its affiliates). */
#include <crypto/ctr.h>
#include "cc_driver.h"
#include "cc_ivgen.h"
#include "cc_request_mgr.h"
#include "cc_sram_mgr.h"
#include "cc_buffer_mgr.h"
/* The max. size of pool *MUST* be <= SRAM total size */
#define CC_IVPOOL_SIZE 1024
/* The first 32B fraction of pool are dedicated to the
* next encryption "key" & "IV" for pool regeneration
*/
#define CC_IVPOOL_META_SIZE (CC_AES_IV_SIZE + AES_KEYSIZE_128)
#define CC_IVPOOL_GEN_SEQ_LEN 4
/**
* struct cc_ivgen_ctx -IV pool generation context
* @pool: the start address of the iv-pool resides in internal RAM
* @ctr_key_dma: address of pool's encryption key material in internal RAM
* @ctr_iv_dma: address of pool's counter iv in internal RAM
* @next_iv_ofs: the offset to the next available IV in pool
* @pool_meta: virt. address of the initial enc. key/IV
* @pool_meta_dma: phys. address of the initial enc. key/IV
*/
struct cc_ivgen_ctx {
cc_sram_addr_t pool;
cc_sram_addr_t ctr_key;
cc_sram_addr_t ctr_iv;
u32 next_iv_ofs;
u8 *pool_meta;
dma_addr_t pool_meta_dma;
};
/*!
* Generates CC_IVPOOL_SIZE of random bytes by
* encrypting 0's using AES128-CTR.
*
* \param ivgen iv-pool context
* \param iv_seq IN/OUT array to the descriptors sequence
* \param iv_seq_len IN/OUT pointer to the sequence length
*/
static int cc_gen_iv_pool(struct cc_ivgen_ctx *ivgen_ctx,
struct cc_hw_desc iv_seq[], unsigned int *iv_seq_len)
{
unsigned int idx = *iv_seq_len;
if ((*iv_seq_len + CC_IVPOOL_GEN_SEQ_LEN) > CC_IVPOOL_SEQ_LEN) {
/* The sequence will be longer than allowed */
return -EINVAL;
}
/* Setup key */
hw_desc_init(&iv_seq[idx]);
set_din_sram(&iv_seq[idx], ivgen_ctx->ctr_key, AES_KEYSIZE_128);
set_setup_mode(&iv_seq[idx], SETUP_LOAD_KEY0);
set_cipher_config0(&iv_seq[idx], DESC_DIRECTION_ENCRYPT_ENCRYPT);
set_flow_mode(&iv_seq[idx], S_DIN_to_AES);
set_key_size_aes(&iv_seq[idx], CC_AES_128_BIT_KEY_SIZE);
set_cipher_mode(&iv_seq[idx], DRV_CIPHER_CTR);
idx++;
/* Setup cipher state */
hw_desc_init(&iv_seq[idx]);
set_din_sram(&iv_seq[idx], ivgen_ctx->ctr_iv, CC_AES_IV_SIZE);
set_cipher_config0(&iv_seq[idx], DESC_DIRECTION_ENCRYPT_ENCRYPT);
set_flow_mode(&iv_seq[idx], S_DIN_to_AES);
set_setup_mode(&iv_seq[idx], SETUP_LOAD_STATE1);
set_key_size_aes(&iv_seq[idx], CC_AES_128_BIT_KEY_SIZE);
set_cipher_mode(&iv_seq[idx], DRV_CIPHER_CTR);
idx++;
/* Perform dummy encrypt to skip first block */
hw_desc_init(&iv_seq[idx]);
set_din_const(&iv_seq[idx], 0, CC_AES_IV_SIZE);
set_dout_sram(&iv_seq[idx], ivgen_ctx->pool, CC_AES_IV_SIZE);
set_flow_mode(&iv_seq[idx], DIN_AES_DOUT);
idx++;
/* Generate IV pool */
hw_desc_init(&iv_seq[idx]);
set_din_const(&iv_seq[idx], 0, CC_IVPOOL_SIZE);
set_dout_sram(&iv_seq[idx], ivgen_ctx->pool, CC_IVPOOL_SIZE);
set_flow_mode(&iv_seq[idx], DIN_AES_DOUT);
idx++;
*iv_seq_len = idx; /* Update sequence length */
/* queue ordering assures pool readiness */
ivgen_ctx->next_iv_ofs = CC_IVPOOL_META_SIZE;
return 0;
}
/*!
* Generates the initial pool in SRAM.
* This function should be invoked when resuming driver.
*
* \param drvdata
*
* \return int Zero for success, negative value otherwise.
*/
int cc_init_iv_sram(struct cc_drvdata *drvdata)
{
struct cc_ivgen_ctx *ivgen_ctx = drvdata->ivgen_handle;
struct cc_hw_desc iv_seq[CC_IVPOOL_SEQ_LEN];
unsigned int iv_seq_len = 0;
int rc;
/* Generate initial enc. key/iv */
get_random_bytes(ivgen_ctx->pool_meta, CC_IVPOOL_META_SIZE);
/* The first 32B reserved for the enc. Key/IV */
ivgen_ctx->ctr_key = ivgen_ctx->pool;
ivgen_ctx->ctr_iv = ivgen_ctx->pool + AES_KEYSIZE_128;
/* Copy initial enc. key and IV to SRAM at a single descriptor */
hw_desc_init(&iv_seq[iv_seq_len]);
set_din_type(&iv_seq[iv_seq_len], DMA_DLLI, ivgen_ctx->pool_meta_dma,
CC_IVPOOL_META_SIZE, NS_BIT);
set_dout_sram(&iv_seq[iv_seq_len], ivgen_ctx->pool,
CC_IVPOOL_META_SIZE);
set_flow_mode(&iv_seq[iv_seq_len], BYPASS);
iv_seq_len++;
/* Generate initial pool */
rc = cc_gen_iv_pool(ivgen_ctx, iv_seq, &iv_seq_len);
if (rc)
return rc;
/* Fire-and-forget */
return send_request_init(drvdata, iv_seq, iv_seq_len);
}
/*!
* Free iv-pool and ivgen context.
*
* \param drvdata
*/
void cc_ivgen_fini(struct cc_drvdata *drvdata)
{
struct cc_ivgen_ctx *ivgen_ctx = drvdata->ivgen_handle;
struct device *device = &drvdata->plat_dev->dev;
if (!ivgen_ctx)
return;
if (ivgen_ctx->pool_meta) {
memset(ivgen_ctx->pool_meta, 0, CC_IVPOOL_META_SIZE);
dma_free_coherent(device, CC_IVPOOL_META_SIZE,
ivgen_ctx->pool_meta,
ivgen_ctx->pool_meta_dma);
}
ivgen_ctx->pool = NULL_SRAM_ADDR;
}
/*!
* Allocates iv-pool and maps resources.
* This function generates the first IV pool.
*
* \param drvdata Driver's private context
*
* \return int Zero for success, negative value otherwise.
*/
int cc_ivgen_init(struct cc_drvdata *drvdata)
{
struct cc_ivgen_ctx *ivgen_ctx;
struct device *device = &drvdata->plat_dev->dev;
int rc;
/* Allocate "this" context */
ivgen_ctx = devm_kzalloc(device, sizeof(*ivgen_ctx), GFP_KERNEL);
if (!ivgen_ctx)
return -ENOMEM;
drvdata->ivgen_handle = ivgen_ctx;
/* Allocate pool's header for initial enc. key/IV */
ivgen_ctx->pool_meta = dma_alloc_coherent(device, CC_IVPOOL_META_SIZE,
&ivgen_ctx->pool_meta_dma,
GFP_KERNEL);
if (!ivgen_ctx->pool_meta) {
dev_err(device, "Not enough memory to allocate DMA of pool_meta (%u B)\n",
CC_IVPOOL_META_SIZE);
rc = -ENOMEM;
goto out;
}
/* Allocate IV pool in SRAM */
ivgen_ctx->pool = cc_sram_alloc(drvdata, CC_IVPOOL_SIZE);
if (ivgen_ctx->pool == NULL_SRAM_ADDR) {
dev_err(device, "SRAM pool exhausted\n");
rc = -ENOMEM;
goto out;
}
return cc_init_iv_sram(drvdata);
out:
cc_ivgen_fini(drvdata);
return rc;
}
/*!
* Acquires 16 Bytes IV from the iv-pool
*
* \param drvdata Driver private context
* \param iv_out_dma Array of physical IV out addresses
* \param iv_out_dma_len Length of iv_out_dma array (additional elements
* of iv_out_dma array are ignore)
* \param iv_out_size May be 8 or 16 bytes long
* \param iv_seq IN/OUT array to the descriptors sequence
* \param iv_seq_len IN/OUT pointer to the sequence length
*
* \return int Zero for success, negative value otherwise.
*/
int cc_get_iv(struct cc_drvdata *drvdata, dma_addr_t iv_out_dma[],
unsigned int iv_out_dma_len, unsigned int iv_out_size,
struct cc_hw_desc iv_seq[], unsigned int *iv_seq_len)
{
struct cc_ivgen_ctx *ivgen_ctx = drvdata->ivgen_handle;
unsigned int idx = *iv_seq_len;
struct device *dev = drvdata_to_dev(drvdata);
unsigned int t;
if (iv_out_size != CC_AES_IV_SIZE &&
iv_out_size != CTR_RFC3686_IV_SIZE) {
return -EINVAL;
}
if ((iv_out_dma_len + 1) > CC_IVPOOL_SEQ_LEN) {
/* The sequence will be longer than allowed */
return -EINVAL;
}
/* check that number of generated IV is limited to max dma address
* iv buffer size
*/
if (iv_out_dma_len > CC_MAX_IVGEN_DMA_ADDRESSES) {
/* The sequence will be longer than allowed */
return -EINVAL;
}
for (t = 0; t < iv_out_dma_len; t++) {
/* Acquire IV from pool */
hw_desc_init(&iv_seq[idx]);
set_din_sram(&iv_seq[idx], (ivgen_ctx->pool +
ivgen_ctx->next_iv_ofs),
iv_out_size);
set_dout_dlli(&iv_seq[idx], iv_out_dma[t], iv_out_size,
NS_BIT, 0);
set_flow_mode(&iv_seq[idx], BYPASS);
idx++;
}
/* Bypass operation is proceeded by crypto sequence, hence must
* assure bypass-write-transaction by a memory barrier
*/
hw_desc_init(&iv_seq[idx]);
set_din_no_dma(&iv_seq[idx], 0, 0xfffff0);
set_dout_no_dma(&iv_seq[idx], 0, 0, 1);
idx++;
*iv_seq_len = idx; /* update seq length */
/* Update iv index */
ivgen_ctx->next_iv_ofs += iv_out_size;
if ((CC_IVPOOL_SIZE - ivgen_ctx->next_iv_ofs) < CC_AES_IV_SIZE) {
dev_dbg(dev, "Pool exhausted, regenerating iv-pool\n");
/* pool is drained -regenerate it! */
return cc_gen_iv_pool(ivgen_ctx, iv_seq, iv_seq_len);
}
return 0;
}

55
drivers/crypto/ccree/cc_ivgen.h
View file

@ -1,55 +0,0 @@
/* SPDX-License-Identifier: GPL-2.0 */
/* Copyright (C) 2012-2019 ARM Limited (or its affiliates). */
#ifndef __CC_IVGEN_H__
#define __CC_IVGEN_H__
#include "cc_hw_queue_defs.h"
#define CC_IVPOOL_SEQ_LEN 8
/*!
* Allocates iv-pool and maps resources.
* This function generates the first IV pool.
*
* \param drvdata Driver's private context
*
* \return int Zero for success, negative value otherwise.
*/
int cc_ivgen_init(struct cc_drvdata *drvdata);
/*!
* Free iv-pool and ivgen context.
*
* \param drvdata
*/
void cc_ivgen_fini(struct cc_drvdata *drvdata);
/*!
* Generates the initial pool in SRAM.
* This function should be invoked when resuming DX driver.
*
* \param drvdata
*
* \return int Zero for success, negative value otherwise.
*/
int cc_init_iv_sram(struct cc_drvdata *drvdata);
/*!
* Acquires 16 Bytes IV from the iv-pool
*
* \param drvdata Driver private context
* \param iv_out_dma Array of physical IV out addresses
* \param iv_out_dma_len Length of iv_out_dma array (additional elements of
* iv_out_dma array are ignore)
* \param iv_out_size May be 8 or 16 bytes long
* \param iv_seq IN/OUT array to the descriptors sequence
* \param iv_seq_len IN/OUT pointer to the sequence length
*
* \return int Zero for success, negative value otherwise.
*/
int cc_get_iv(struct cc_drvdata *drvdata, dma_addr_t iv_out_dma[],
unsigned int iv_out_dma_len, unsigned int iv_out_size,
struct cc_hw_desc iv_seq[], unsigned int *iv_seq_len);
#endif /*__CC_IVGEN_H__*/

2
drivers/crypto/ccree/cc_pm.c
View file

@ -8,7 +8,6 @@
#include "cc_buffer_mgr.h"
#include "cc_request_mgr.h"
#include "cc_sram_mgr.h"
#include "cc_ivgen.h"
#include "cc_hash.h"
#include "cc_pm.h"
#include "cc_fips.h"
@ -73,7 +72,6 @@ int cc_pm_resume(struct device *dev)
/* must be after the queue resuming as it uses the HW queue*/
cc_init_hash_sram(drvdata);
cc_init_iv_sram(drvdata);
return 0;
}

47
drivers/crypto/ccree/cc_request_mgr.c
View file

@ -6,7 +6,6 @@
#include "cc_driver.h"
#include "cc_buffer_mgr.h"
#include "cc_request_mgr.h"
#include "cc_ivgen.h"
#include "cc_pm.h"
#define CC_MAX_POLL_ITER 10
@ -281,36 +280,12 @@ static int cc_queues_status(struct cc_drvdata *drvdata,
static int cc_do_send_request(struct cc_drvdata *drvdata,
struct cc_crypto_req *cc_req,
struct cc_hw_desc *desc, unsigned int len,
bool add_comp, bool ivgen)
bool add_comp)
{
struct cc_req_mgr_handle *req_mgr_h = drvdata->request_mgr_handle;
unsigned int used_sw_slots;
unsigned int iv_seq_len = 0;
unsigned int total_seq_len = len; /*initial sequence length*/
struct cc_hw_desc iv_seq[CC_IVPOOL_SEQ_LEN];
struct device *dev = drvdata_to_dev(drvdata);
int rc;
if (ivgen) {
dev_dbg(dev, "Acquire IV from pool into %d DMA addresses %pad, %pad, %pad, IV-size=%u\n",
cc_req->ivgen_dma_addr_len,
&cc_req->ivgen_dma_addr[0],
&cc_req->ivgen_dma_addr[1],
&cc_req->ivgen_dma_addr[2],
cc_req->ivgen_size);
/* Acquire IV from pool */
rc = cc_get_iv(drvdata, cc_req->ivgen_dma_addr,
cc_req->ivgen_dma_addr_len,
cc_req->ivgen_size, iv_seq, &iv_seq_len);
if (rc) {
dev_err(dev, "Failed to generate IV (rc=%d)\n", rc);
return rc;
}
total_seq_len += iv_seq_len;
}
used_sw_slots = ((req_mgr_h->req_queue_head -
req_mgr_h->req_queue_tail) &
@ -334,8 +309,6 @@ static int cc_do_send_request(struct cc_drvdata *drvdata,
wmb();
/* STAT_PHASE_4: Push sequence */
if (ivgen)
enqueue_seq(drvdata, iv_seq, iv_seq_len);
enqueue_seq(drvdata, desc, len);
@ -380,8 +353,6 @@ static void cc_proc_backlog(struct cc_drvdata *drvdata)
struct cc_bl_item *bli;
struct cc_crypto_req *creq;
void *req;
bool ivgen;
unsigned int total_len;
struct device *dev = drvdata_to_dev(drvdata);
int rc;
@ -406,12 +377,9 @@ static void cc_proc_backlog(struct cc_drvdata *drvdata)
bli->notif = true;
}
ivgen = !!creq->ivgen_dma_addr_len;
total_len = bli->len + (ivgen ? CC_IVPOOL_SEQ_LEN : 0);
spin_lock(&mgr->hw_lock);
rc = cc_queues_status(drvdata, mgr, total_len);
rc = cc_queues_status(drvdata, mgr, bli->len);
if (rc) {
/*
* There is still not room in the FIFO for
@ -423,7 +391,7 @@ static void cc_proc_backlog(struct cc_drvdata *drvdata)
}
rc = cc_do_send_request(drvdata, &bli->creq, bli->desc,
bli->len, false, ivgen);
bli->len, false);
spin_unlock(&mgr->hw_lock);
@ -447,8 +415,6 @@ int cc_send_request(struct cc_drvdata *drvdata, struct cc_crypto_req *cc_req,
{
int rc;
struct cc_req_mgr_handle *mgr = drvdata->request_mgr_handle;
bool ivgen = !!cc_req->ivgen_dma_addr_len;
unsigned int total_len = len + (ivgen ? CC_IVPOOL_SEQ_LEN : 0);
struct device *dev = drvdata_to_dev(drvdata);
bool backlog_ok = req->flags & CRYPTO_TFM_REQ_MAY_BACKLOG;
gfp_t flags = cc_gfp_flags(req);
@ -461,7 +427,7 @@ int cc_send_request(struct cc_drvdata *drvdata, struct cc_crypto_req *cc_req,
}
spin_lock_bh(&mgr->hw_lock);
rc = cc_queues_status(drvdata, mgr, total_len);
rc = cc_queues_status(drvdata, mgr, len);
#ifdef CC_DEBUG_FORCE_BACKLOG
if (backlog_ok)
@ -486,8 +452,7 @@ int cc_send_request(struct cc_drvdata *drvdata, struct cc_crypto_req *cc_req,
}
if (!rc)
rc = cc_do_send_request(drvdata, cc_req, desc, len, false,
ivgen);
rc = cc_do_send_request(drvdata, cc_req, desc, len, false);
spin_unlock_bh(&mgr->hw_lock);
return rc;
@ -527,7 +492,7 @@ int cc_send_sync_request(struct cc_drvdata *drvdata,
reinit_completion(&drvdata->hw_queue_avail);
}
rc = cc_do_send_request(drvdata, cc_req, desc, len, true, false);
rc = cc_do_send_request(drvdata, cc_req, desc, len, true);
spin_unlock_bh(&mgr->hw_lock);
if (rc != -EINPROGRESS) {

1
drivers/crypto/chelsio/Kconfig
View file

@ -2,6 +2,7 @@
config CRYPTO_DEV_CHELSIO
tristate "Chelsio Crypto Co-processor Driver"
depends on CHELSIO_T4
select CRYPTO_LIB_AES
select CRYPTO_SHA1
select CRYPTO_SHA256
select CRYPTO_SHA512

46
drivers/crypto/chelsio/chcr_algo.c
View file

@ -1023,22 +1023,21 @@ static int chcr_update_tweak(struct ablkcipher_request *req, u8 *iv,
struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req);
struct ablk_ctx *ablkctx = ABLK_CTX(c_ctx(tfm));
struct chcr_blkcipher_req_ctx *reqctx = ablkcipher_request_ctx(req);
struct crypto_cipher *cipher;
struct crypto_aes_ctx aes;
int ret, i;
u8 *key;
unsigned int keylen;
int round = reqctx->last_req_len / AES_BLOCK_SIZE;
int round8 = round / 8;
cipher = ablkctx->aes_generic;
memcpy(iv, reqctx->iv, AES_BLOCK_SIZE);
keylen = ablkctx->enckey_len / 2;
key = ablkctx->key + keylen;
ret = crypto_cipher_setkey(cipher, key, keylen);
ret = aes_expandkey(&aes, key, keylen);
if (ret)
goto out;
crypto_cipher_encrypt_one(cipher, iv, iv);
return ret;
aes_encrypt(&aes, iv, iv);
for (i = 0; i < round8; i++)
gf128mul_x8_ble((le128 *)iv, (le128 *)iv);
@ -1046,9 +1045,10 @@ static int chcr_update_tweak(struct ablkcipher_request *req, u8 *iv,
gf128mul_x_ble((le128 *)iv, (le128 *)iv);
if (!isfinal)
crypto_cipher_decrypt_one(cipher, iv, iv);
out:
return ret;
aes_decrypt(&aes, iv, iv);
memzero_explicit(&aes, sizeof(aes));
return 0;
}
static int chcr_update_cipher_iv(struct ablkcipher_request *req,
@ -1411,16 +1411,6 @@ static int chcr_cra_init(struct crypto_tfm *tfm)
return PTR_ERR(ablkctx->sw_cipher);
}
if (get_cryptoalg_subtype(tfm) == CRYPTO_ALG_SUB_TYPE_XTS) {
/* To update tweak*/
ablkctx->aes_generic = crypto_alloc_cipher("aes-generic", 0, 0);
if (IS_ERR(ablkctx->aes_generic)) {
pr_err("failed to allocate aes cipher for tweak\n");
return PTR_ERR(ablkctx->aes_generic);
}
} else
ablkctx->aes_generic = NULL;
tfm->crt_ablkcipher.reqsize = sizeof(struct chcr_blkcipher_req_ctx);
return chcr_device_init(crypto_tfm_ctx(tfm));
}
@ -1451,8 +1441,6 @@ static void chcr_cra_exit(struct crypto_tfm *tfm)
struct ablk_ctx *ablkctx = ABLK_CTX(ctx);
crypto_free_sync_skcipher(ablkctx->sw_cipher);
if (ablkctx->aes_generic)
crypto_free_cipher(ablkctx->aes_generic);
}
static int get_alg_config(struct algo_param *params,
@ -3364,9 +3352,9 @@ static int chcr_gcm_setkey(struct crypto_aead *aead, const u8 *key,
{
struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(aead));
struct chcr_gcm_ctx *gctx = GCM_CTX(aeadctx);
struct crypto_cipher *cipher;
unsigned int ck_size;
int ret = 0, key_ctx_size = 0;
struct crypto_aes_ctx aes;
aeadctx->enckey_len = 0;
crypto_aead_clear_flags(aeadctx->sw_cipher, CRYPTO_TFM_REQ_MASK);
@ -3409,23 +3397,15 @@ static int chcr_gcm_setkey(struct crypto_aead *aead, const u8 *key,
/* Calculate the H = CIPH(K, 0 repeated 16 times).
* It will go in key context
*/
cipher = crypto_alloc_cipher("aes-generic", 0, 0);
if (IS_ERR(cipher)) {
aeadctx->enckey_len = 0;
ret = -ENOMEM;
goto out;
}
ret = crypto_cipher_setkey(cipher, key, keylen);
ret = aes_expandkey(&aes, key, keylen);
if (ret) {
aeadctx->enckey_len = 0;
goto out1;
goto out;
}
memset(gctx->ghash_h, 0, AEAD_H_SIZE);
crypto_cipher_encrypt_one(cipher, gctx->ghash_h, gctx->ghash_h);
aes_encrypt(&aes, gctx->ghash_h, gctx->ghash_h);
memzero_explicit(&aes, sizeof(aes));
out1:
crypto_free_cipher(cipher);
out:
return ret;
}

20
drivers/crypto/chelsio/chcr_algo.h
View file

@ -333,26 +333,26 @@ struct phys_sge_pairs {
};
static const u32 sha1_init[SHA1_DIGEST_SIZE / 4] = {
static const u32 chcr_sha1_init[SHA1_DIGEST_SIZE / 4] = {
SHA1_H0, SHA1_H1, SHA1_H2, SHA1_H3, SHA1_H4,
};
static const u32 sha224_init[SHA256_DIGEST_SIZE / 4] = {
static const u32 chcr_sha224_init[SHA256_DIGEST_SIZE / 4] = {
SHA224_H0, SHA224_H1, SHA224_H2, SHA224_H3,
SHA224_H4, SHA224_H5, SHA224_H6, SHA224_H7,
};
static const u32 sha256_init[SHA256_DIGEST_SIZE / 4] = {
static const u32 chcr_sha256_init[SHA256_DIGEST_SIZE / 4] = {
SHA256_H0, SHA256_H1, SHA256_H2, SHA256_H3,
SHA256_H4, SHA256_H5, SHA256_H6, SHA256_H7,
};
static const u64 sha384_init[SHA512_DIGEST_SIZE / 8] = {
static const u64 chcr_sha384_init[SHA512_DIGEST_SIZE / 8] = {
SHA384_H0, SHA384_H1, SHA384_H2, SHA384_H3,
SHA384_H4, SHA384_H5, SHA384_H6, SHA384_H7,
};
static const u64 sha512_init[SHA512_DIGEST_SIZE / 8] = {
static const u64 chcr_sha512_init[SHA512_DIGEST_SIZE / 8] = {
SHA512_H0, SHA512_H1, SHA512_H2, SHA512_H3,
SHA512_H4, SHA512_H5, SHA512_H6, SHA512_H7,
};
@ -362,21 +362,21 @@ static inline void copy_hash_init_values(char *key, int digestsize)
u8 i;
__be32 *dkey = (__be32 *)key;
u64 *ldkey = (u64 *)key;
__be64 *sha384 = (__be64 *)sha384_init;
__be64 *sha512 = (__be64 *)sha512_init;
__be64 *sha384 = (__be64 *)chcr_sha384_init;
__be64 *sha512 = (__be64 *)chcr_sha512_init;
switch (digestsize) {
case SHA1_DIGEST_SIZE:
for (i = 0; i < SHA1_INIT_STATE; i++)
dkey[i] = cpu_to_be32(sha1_init[i]);
dkey[i] = cpu_to_be32(chcr_sha1_init[i]);
break;
case SHA224_DIGEST_SIZE:
for (i = 0; i < SHA224_INIT_STATE; i++)
dkey[i] = cpu_to_be32(sha224_init[i]);
dkey[i] = cpu_to_be32(chcr_sha224_init[i]);
break;
case SHA256_DIGEST_SIZE:
for (i = 0; i < SHA256_INIT_STATE; i++)
dkey[i] = cpu_to_be32(sha256_init[i]);
dkey[i] = cpu_to_be32(chcr_sha256_init[i]);
break;
case SHA384_DIGEST_SIZE:
for (i = 0; i < SHA384_INIT_STATE; i++)

1
drivers/crypto/chelsio/chcr_crypto.h
View file

@ -172,7 +172,6 @@ static inline struct chcr_context *h_ctx(struct crypto_ahash *tfm)
struct ablk_ctx {
struct crypto_sync_skcipher *sw_cipher;
struct crypto_cipher *aes_generic;
__be32 key_ctx_hdr;
unsigned int enckey_len;
unsigned char ciph_mode;

19
drivers/crypto/chelsio/chcr_ipsec.c
View file

@ -132,11 +132,11 @@ static inline int chcr_ipsec_setauthsize(struct xfrm_state *x,
static inline int chcr_ipsec_setkey(struct xfrm_state *x,
struct ipsec_sa_entry *sa_entry)
{
struct crypto_cipher *cipher;
int keylen = (x->aead->alg_key_len + 7) / 8;
unsigned char *key = x->aead->alg_key;
int ck_size, key_ctx_size = 0;
unsigned char ghash_h[AEAD_H_SIZE];
struct crypto_aes_ctx aes;
int ret = 0;
if (keylen > 3) {
@ -170,26 +170,19 @@ static inline int chcr_ipsec_setkey(struct xfrm_state *x,
/* Calculate the H = CIPH(K, 0 repeated 16 times).
* It will go in key context
*/
cipher = crypto_alloc_cipher("aes-generic", 0, 0);
if (IS_ERR(cipher)) {
sa_entry->enckey_len = 0;
ret = -ENOMEM;
goto out;
}
ret = crypto_cipher_setkey(cipher, key, keylen);
ret = aes_expandkey(&aes, key, keylen);
if (ret) {
sa_entry->enckey_len = 0;
goto out1;
goto out;
}
memset(ghash_h, 0, AEAD_H_SIZE);
crypto_cipher_encrypt_one(cipher, ghash_h, ghash_h);
aes_encrypt(&aes, ghash_h, ghash_h);
memzero_explicit(&aes, sizeof(aes));
memcpy(sa_entry->key + (DIV_ROUND_UP(sa_entry->enckey_len, 16) *
16), ghash_h, AEAD_H_SIZE);
sa_entry->kctx_len = ((DIV_ROUND_UP(sa_entry->enckey_len, 16)) << 4) +
AEAD_H_SIZE;
out1:
crypto_free_cipher(cipher);
out:
return ret;
}

20
drivers/crypto/chelsio/chtls/chtls_hw.c
View file

@ -213,8 +213,8 @@ static int chtls_key_info(struct chtls_sock *csk,
unsigned char key[AES_KEYSIZE_128];
struct tls12_crypto_info_aes_gcm_128 *gcm_ctx;
unsigned char ghash_h[AEAD_H_SIZE];
struct crypto_cipher *cipher;
int ck_size, key_ctx_size;
struct crypto_aes_ctx aes;
int ret;
gcm_ctx = (struct tls12_crypto_info_aes_gcm_128 *)
@ -234,18 +234,13 @@ static int chtls_key_info(struct chtls_sock *csk,
/* Calculate the H = CIPH(K, 0 repeated 16 times).
* It will go in key context
*/
cipher = crypto_alloc_cipher("aes", 0, 0);
if (IS_ERR(cipher)) {
ret = -ENOMEM;
goto out;
}
ret = crypto_cipher_setkey(cipher, key, keylen);
ret = aes_expandkey(&aes, key, keylen);
if (ret)
goto out1;
return ret;
memset(ghash_h, 0, AEAD_H_SIZE);
crypto_cipher_encrypt_one(cipher, ghash_h, ghash_h);
aes_encrypt(&aes, ghash_h, ghash_h);
memzero_explicit(&aes, sizeof(aes));
csk->tlshws.keylen = key_ctx_size;
/* Copy the Key context */
@ -269,10 +264,7 @@ static int chtls_key_info(struct chtls_sock *csk,
/* erase key info from driver */
memset(gcm_ctx->key, 0, keylen);
out1:
crypto_free_cipher(cipher);
out:
return ret;
return 0;
}
static void chtls_set_scmd(struct chtls_sock *csk)

4
drivers/crypto/exynos-rng.c
View file

@ -268,7 +268,6 @@ static struct rng_alg exynos_rng_alg = {
static int exynos_rng_probe(struct platform_device *pdev)
{
struct exynos_rng_dev *rng;
struct resource *res;
int ret;
if (exynos_rng_dev)
@ -289,8 +288,7 @@ static int exynos_rng_probe(struct platform_device *pdev)
return PTR_ERR(rng->clk);
}
res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
rng->mem = devm_ioremap_resource(&pdev->dev, res);
rng->mem = devm_platform_ioremap_resource(pdev, 0);
if (IS_ERR(rng->mem))
return PTR_ERR(rng->mem);

32
drivers/crypto/hifn_795x.c
View file

@ -21,7 +21,7 @@
#include <linux/ktime.h>
#include <crypto/algapi.h>
#include <crypto/des.h>
#include <crypto/internal/des.h>
static char hifn_pll_ref[sizeof("extNNN")] = "ext";
module_param_string(hifn_pll_ref, hifn_pll_ref, sizeof(hifn_pll_ref), 0444);
@ -1939,25 +1939,13 @@ static void hifn_flush(struct hifn_device *dev)
static int hifn_setkey(struct crypto_ablkcipher *cipher, const u8 *key,
unsigned int len)
{
struct crypto_tfm *tfm = crypto_ablkcipher_tfm(cipher);
struct hifn_context *ctx = crypto_tfm_ctx(tfm);
struct hifn_context *ctx = crypto_ablkcipher_ctx(cipher);
struct hifn_device *dev = ctx->dev;
int err;
if (len > HIFN_MAX_CRYPT_KEY_LENGTH) {
crypto_ablkcipher_set_flags(cipher, CRYPTO_TFM_RES_BAD_KEY_LEN);
return -1;
}
if (len == HIFN_DES_KEY_LENGTH) {
u32 tmp[DES_EXPKEY_WORDS];
int ret = des_ekey(tmp, key);
if (unlikely(ret == 0) &&
(tfm->crt_flags & CRYPTO_TFM_REQ_FORBID_WEAK_KEYS)) {
tfm->crt_flags |= CRYPTO_TFM_RES_WEAK_KEY;
return -EINVAL;
}
}
err = verify_ablkcipher_des_key(cipher, key);
if (err)
return err;
dev->flags &= ~HIFN_FLAG_OLD_KEY;
@ -1972,15 +1960,11 @@ static int hifn_des3_setkey(struct crypto_ablkcipher *cipher, const u8 *key,
{
struct hifn_context *ctx = crypto_ablkcipher_ctx(cipher);
struct hifn_device *dev = ctx->dev;
u32 flags;
int err;
flags = crypto_ablkcipher_get_flags(cipher);
err = __des3_verify_key(&flags, key);
if (unlikely(err)) {
crypto_ablkcipher_set_flags(cipher, flags);
err = verify_ablkcipher_des3_key(cipher, key);
if (err)
return err;
}
dev->flags &= ~HIFN_FLAG_OLD_KEY;

25
drivers/crypto/hisilicon/Kconfig
View file

@ -4,6 +4,7 @@ config CRYPTO_DEV_HISI_SEC
tristate "Support for Hisilicon SEC crypto block cipher accelerator"
select CRYPTO_BLKCIPHER
select CRYPTO_ALGAPI
select CRYPTO_LIB_DES
select SG_SPLIT
depends on ARM64 || COMPILE_TEST
depends on HAS_IOMEM
@ -12,3 +13,27 @@ config CRYPTO_DEV_HISI_SEC
To compile this as a module, choose M here: the module
will be called hisi_sec.
config CRYPTO_DEV_HISI_QM
tristate
depends on ARM64 && PCI && PCI_MSI
help
HiSilicon accelerator engines use a common queue management
interface. Specific engine driver may use this module.
config CRYPTO_HISI_SGL
tristate
depends on ARM64
help
HiSilicon accelerator engines use a common hardware scatterlist
interface for data format. Specific engine driver may use this
module.
config CRYPTO_DEV_HISI_ZIP
tristate "Support for HiSilicon ZIP accelerator"
depends on ARM64 && PCI && PCI_MSI
select CRYPTO_DEV_HISI_QM
select CRYPTO_HISI_SGL
select SG_SPLIT
help
Support for HiSilicon ZIP Driver

3
drivers/crypto/hisilicon/Makefile
View file

@ -1,2 +1,5 @@
# SPDX-License-Identifier: GPL-2.0
obj-$(CONFIG_CRYPTO_DEV_HISI_SEC) += sec/
obj-$(CONFIG_CRYPTO_DEV_HISI_QM) += qm.o
obj-$(CONFIG_CRYPTO_HISI_SGL) += sgl.o
obj-$(CONFIG_CRYPTO_DEV_HISI_ZIP) += zip/

1913
drivers/crypto/hisilicon/qm.c Normal file
View file

File diff suppressed because it is too large Load diff

215
drivers/crypto/hisilicon/qm.h Normal file
View file

@ -0,0 +1,215 @@
/* SPDX-License-Identifier: GPL-2.0 */
/* Copyright (c) 2019 HiSilicon Limited. */
#ifndef HISI_ACC_QM_H
#define HISI_ACC_QM_H
#include <linux/bitfield.h>
#include <linux/iopoll.h>
#include <linux/module.h>
#include <linux/pci.h>
/* qm user domain */
#define QM_ARUSER_M_CFG_1 0x100088
#define AXUSER_SNOOP_ENABLE BIT(30)
#define AXUSER_CMD_TYPE GENMASK(14, 12)
#define AXUSER_CMD_SMMU_NORMAL 1
#define AXUSER_NS BIT(6)
#define AXUSER_NO BIT(5)
#define AXUSER_FP BIT(4)
#define AXUSER_SSV BIT(0)
#define AXUSER_BASE (AXUSER_SNOOP_ENABLE | \
FIELD_PREP(AXUSER_CMD_TYPE, \
AXUSER_CMD_SMMU_NORMAL) | \
AXUSER_NS | AXUSER_NO | AXUSER_FP)
#define QM_ARUSER_M_CFG_ENABLE 0x100090
#define ARUSER_M_CFG_ENABLE 0xfffffffe
#define QM_AWUSER_M_CFG_1 0x100098
#define QM_AWUSER_M_CFG_ENABLE 0x1000a0
#define AWUSER_M_CFG_ENABLE 0xfffffffe
#define QM_WUSER_M_CFG_ENABLE 0x1000a8
#define WUSER_M_CFG_ENABLE 0xffffffff
/* qm cache */
#define QM_CACHE_CTL 0x100050
#define SQC_CACHE_ENABLE BIT(0)
#define CQC_CACHE_ENABLE BIT(1)
#define SQC_CACHE_WB_ENABLE BIT(4)
#define SQC_CACHE_WB_THRD GENMASK(10, 5)
#define CQC_CACHE_WB_ENABLE BIT(11)
#define CQC_CACHE_WB_THRD GENMASK(17, 12)
#define QM_AXI_M_CFG 0x1000ac
#define AXI_M_CFG 0xffff
#define QM_AXI_M_CFG_ENABLE 0x1000b0
#define AXI_M_CFG_ENABLE 0xffffffff
#define QM_PEH_AXUSER_CFG 0x1000cc
#define QM_PEH_AXUSER_CFG_ENABLE 0x1000d0
#define PEH_AXUSER_CFG 0x401001
#define PEH_AXUSER_CFG_ENABLE 0xffffffff
#define QM_DFX_MB_CNT_VF 0x104010
#define QM_DFX_DB_CNT_VF 0x104020
#define QM_DFX_SQE_CNT_VF_SQN 0x104030
#define QM_DFX_CQE_CNT_VF_CQN 0x104040
#define QM_DFX_QN_SHIFT 16
#define CURRENT_FUN_MASK GENMASK(5, 0)
#define CURRENT_Q_MASK GENMASK(31, 16)
#define QM_AXI_RRESP BIT(0)
#define QM_AXI_BRESP BIT(1)
#define QM_ECC_MBIT BIT(2)
#define QM_ECC_1BIT BIT(3)
#define QM_ACC_GET_TASK_TIMEOUT BIT(4)
#define QM_ACC_DO_TASK_TIMEOUT BIT(5)
#define QM_ACC_WB_NOT_READY_TIMEOUT BIT(6)
#define QM_SQ_CQ_VF_INVALID BIT(7)
#define QM_CQ_VF_INVALID BIT(8)
#define QM_SQ_VF_INVALID BIT(9)
#define QM_DB_TIMEOUT BIT(10)
#define QM_OF_FIFO_OF BIT(11)
#define QM_DB_RANDOM_INVALID BIT(12)
#define QM_BASE_NFE (QM_AXI_RRESP | QM_AXI_BRESP | QM_ECC_MBIT | \
QM_ACC_GET_TASK_TIMEOUT | QM_DB_TIMEOUT | \
QM_OF_FIFO_OF)
#define QM_BASE_CE QM_ECC_1BIT
#define QM_Q_DEPTH 1024
enum qp_state {
QP_STOP,
};
enum qm_hw_ver {
QM_HW_UNKNOWN = -1,
QM_HW_V1 = 0x20,
QM_HW_V2 = 0x21,
};
enum qm_fun_type {
QM_HW_PF,
QM_HW_VF,
};
enum qm_debug_file {
CURRENT_Q,
CLEAR_ENABLE,
DEBUG_FILE_NUM,
};
struct debugfs_file {
enum qm_debug_file index;
struct mutex lock;
struct qm_debug *debug;
};
struct qm_debug {
u32 curr_qm_qp_num;
struct dentry *debug_root;
struct dentry *qm_d;
struct debugfs_file files[DEBUG_FILE_NUM];
};
struct qm_dma {
void *va;
dma_addr_t dma;
size_t size;
};
struct hisi_qm_status {
u32 eq_head;
bool eqc_phase;
u32 aeq_head;
bool aeqc_phase;
unsigned long flags;
};
struct hisi_qm {
enum qm_hw_ver ver;
enum qm_fun_type fun_type;
const char *dev_name;
struct pci_dev *pdev;
void __iomem *io_base;
u32 sqe_size;
u32 qp_base;
u32 qp_num;
u32 ctrl_qp_num;
struct qm_dma qdma;
struct qm_sqc *sqc;
struct qm_cqc *cqc;
struct qm_eqe *eqe;
struct qm_aeqe *aeqe;
dma_addr_t sqc_dma;
dma_addr_t cqc_dma;
dma_addr_t eqe_dma;
dma_addr_t aeqe_dma;
struct hisi_qm_status status;
rwlock_t qps_lock;
unsigned long *qp_bitmap;
struct hisi_qp **qp_array;
struct mutex mailbox_lock;
const struct hisi_qm_hw_ops *ops;
struct qm_debug debug;
u32 error_mask;
u32 msi_mask;
bool use_dma_api;
};
struct hisi_qp_status {
atomic_t used;
u16 sq_tail;
u16 cq_head;
bool cqc_phase;
unsigned long flags;
};
struct hisi_qp_ops {
int (*fill_sqe)(void *sqe, void *q_parm, void *d_parm);
};
struct hisi_qp {
u32 qp_id;
u8 alg_type;
u8 req_type;
struct qm_dma qdma;
void *sqe;
struct qm_cqe *cqe;
dma_addr_t sqe_dma;
dma_addr_t cqe_dma;
struct hisi_qp_status qp_status;
struct hisi_qp_ops *hw_ops;
void *qp_ctx;
void (*req_cb)(struct hisi_qp *qp, void *data);
struct work_struct work;
struct workqueue_struct *wq;
struct hisi_qm *qm;
};
int hisi_qm_init(struct hisi_qm *qm);
void hisi_qm_uninit(struct hisi_qm *qm);
int hisi_qm_start(struct hisi_qm *qm);
int hisi_qm_stop(struct hisi_qm *qm);
struct hisi_qp *hisi_qm_create_qp(struct hisi_qm *qm, u8 alg_type);
int hisi_qm_start_qp(struct hisi_qp *qp, unsigned long arg);
int hisi_qm_stop_qp(struct hisi_qp *qp);
void hisi_qm_release_qp(struct hisi_qp *qp);
int hisi_qp_send(struct hisi_qp *qp, const void *msg);
int hisi_qm_get_vft(struct hisi_qm *qm, u32 *base, u32 *number);
int hisi_qm_set_vft(struct hisi_qm *qm, u32 fun_num, u32 base, u32 number);
int hisi_qm_debug_init(struct hisi_qm *qm);
void hisi_qm_hw_error_init(struct hisi_qm *qm, u32 ce, u32 nfe, u32 fe,
u32 msi);
int hisi_qm_hw_error_handle(struct hisi_qm *qm);
enum qm_hw_ver hisi_qm_get_hw_version(struct pci_dev *pdev);
void hisi_qm_debug_regs_clear(struct hisi_qm *qm);
#endif

18
drivers/crypto/hisilicon/sec/sec_algs.c
View file

@ -9,7 +9,7 @@
#include <crypto/aes.h>
#include <crypto/algapi.h>
#include <crypto/des.h>
#include <crypto/internal/des.h>
#include <crypto/skcipher.h>
#include <crypto/xts.h>
#include <crypto/internal/skcipher.h>
@ -347,25 +347,21 @@ static int sec_alg_skcipher_setkey_aes_xts(struct crypto_skcipher *tfm,
static int sec_alg_skcipher_setkey_des_ecb(struct crypto_skcipher *tfm,
const u8 *key, unsigned int keylen)
{
if (keylen != DES_KEY_SIZE)
return -EINVAL;
return sec_alg_skcipher_setkey(tfm, key, keylen, SEC_C_DES_ECB_64);
return verify_skcipher_des_key(tfm, key) ?:
sec_alg_skcipher_setkey(tfm, key, keylen, SEC_C_DES_ECB_64);
}
static int sec_alg_skcipher_setkey_des_cbc(struct crypto_skcipher *tfm,
const u8 *key, unsigned int keylen)
{
if (keylen != DES_KEY_SIZE)
return -EINVAL;
return sec_alg_skcipher_setkey(tfm, key, keylen, SEC_C_DES_CBC_64);
return verify_skcipher_des_key(tfm, key) ?:
sec_alg_skcipher_setkey(tfm, key, keylen, SEC_C_DES_CBC_64);
}
static int sec_alg_skcipher_setkey_3des_ecb(struct crypto_skcipher *tfm,
const u8 *key, unsigned int keylen)
{
return unlikely(des3_verify_key(tfm, key)) ?:
return verify_skcipher_des3_key(tfm, key) ?:
sec_alg_skcipher_setkey(tfm, key, keylen,
SEC_C_3DES_ECB_192_3KEY);
}
@ -373,7 +369,7 @@ static int sec_alg_skcipher_setkey_3des_ecb(struct crypto_skcipher *tfm,
static int sec_alg_skcipher_setkey_3des_cbc(struct crypto_skcipher *tfm,
const u8 *key, unsigned int keylen)
{
return unlikely(des3_verify_key(tfm, key)) ?:
return verify_skcipher_des3_key(tfm, key) ?:
sec_alg_skcipher_setkey(tfm, key, keylen,
SEC_C_3DES_CBC_192_3KEY);
}

214
drivers/crypto/hisilicon/sgl.c Normal file
View file

@ -0,0 +1,214 @@
// SPDX-License-Identifier: GPL-2.0
/* Copyright (c) 2019 HiSilicon Limited. */
#include <linux/dma-mapping.h>
#include <linux/module.h>
#include "./sgl.h"
#define HISI_ACC_SGL_SGE_NR_MIN 1
#define HISI_ACC_SGL_SGE_NR_MAX 255
#define HISI_ACC_SGL_SGE_NR_DEF 10
#define HISI_ACC_SGL_NR_MAX 256
#define HISI_ACC_SGL_ALIGN_SIZE 64
static int acc_sgl_sge_set(const char *val, const struct kernel_param *kp)
{
int ret;
u32 n;
if (!val)
return -EINVAL;
ret = kstrtou32(val, 10, &n);
if (ret != 0 || n > HISI_ACC_SGL_SGE_NR_MAX || n == 0)
return -EINVAL;
return param_set_int(val, kp);
}
static const struct kernel_param_ops acc_sgl_sge_ops = {
.set = acc_sgl_sge_set,
.get = param_get_int,
};
static u32 acc_sgl_sge_nr = HISI_ACC_SGL_SGE_NR_DEF;
module_param_cb(acc_sgl_sge_nr, &acc_sgl_sge_ops, &acc_sgl_sge_nr, 0444);
MODULE_PARM_DESC(acc_sgl_sge_nr, "Number of sge in sgl(1-255)");
struct acc_hw_sge {
dma_addr_t buf;
void *page_ctrl;
__le32 len;
__le32 pad;
__le32 pad0;
__le32 pad1;
};
/* use default sgl head size 64B */
struct hisi_acc_hw_sgl {
dma_addr_t next_dma;
__le16 entry_sum_in_chain;
__le16 entry_sum_in_sgl;
__le16 entry_length_in_sgl;
__le16 pad0;
__le64 pad1[5];
struct hisi_acc_hw_sgl *next;
struct acc_hw_sge sge_entries[];
} __aligned(1);
/**
* hisi_acc_create_sgl_pool() - Create a hw sgl pool.
* @dev: The device which hw sgl pool belongs to.
* @pool: Pointer of pool.
* @count: Count of hisi_acc_hw_sgl in pool.
*
* This function creates a hw sgl pool, after this user can get hw sgl memory
* from it.
*/
int hisi_acc_create_sgl_pool(struct device *dev,
struct hisi_acc_sgl_pool *pool, u32 count)
{
u32 sgl_size;
u32 size;
if (!dev || !pool || !count)
return -EINVAL;
sgl_size = sizeof(struct acc_hw_sge) * acc_sgl_sge_nr +
sizeof(struct hisi_acc_hw_sgl);
size = sgl_size * count;
pool->sgl = dma_alloc_coherent(dev, size, &pool->sgl_dma, GFP_KERNEL);
if (!pool->sgl)
return -ENOMEM;
pool->size = size;
pool->count = count;
pool->sgl_size = sgl_size;
return 0;
}
EXPORT_SYMBOL_GPL(hisi_acc_create_sgl_pool);
/**
* hisi_acc_free_sgl_pool() - Free a hw sgl pool.
* @dev: The device which hw sgl pool belongs to.
* @pool: Pointer of pool.
*
* This function frees memory of a hw sgl pool.
*/
void hisi_acc_free_sgl_pool(struct device *dev, struct hisi_acc_sgl_pool *pool)
{
dma_free_coherent(dev, pool->size, pool->sgl, pool->sgl_dma);
memset(pool, 0, sizeof(struct hisi_acc_sgl_pool));
}
EXPORT_SYMBOL_GPL(hisi_acc_free_sgl_pool);
struct hisi_acc_hw_sgl *acc_get_sgl(struct hisi_acc_sgl_pool *pool, u32 index,
dma_addr_t *hw_sgl_dma)
{
if (!pool || !hw_sgl_dma || index >= pool->count || !pool->sgl)
return ERR_PTR(-EINVAL);
*hw_sgl_dma = pool->sgl_dma + pool->sgl_size * index;
return (void *)pool->sgl + pool->sgl_size * index;
}
void acc_put_sgl(struct hisi_acc_sgl_pool *pool, u32 index) {}
static void sg_map_to_hw_sg(struct scatterlist *sgl,
struct acc_hw_sge *hw_sge)
{
hw_sge->buf = sgl->dma_address;
hw_sge->len = sgl->dma_length;
}
static void inc_hw_sgl_sge(struct hisi_acc_hw_sgl *hw_sgl)
{
hw_sgl->entry_sum_in_sgl++;
}
static void update_hw_sgl_sum_sge(struct hisi_acc_hw_sgl *hw_sgl, u16 sum)
{
hw_sgl->entry_sum_in_chain = sum;
}
/**
* hisi_acc_sg_buf_map_to_hw_sgl - Map a scatterlist to a hw sgl.
* @dev: The device which hw sgl belongs to.
* @sgl: Scatterlist which will be mapped to hw sgl.
* @pool: Pool which hw sgl memory will be allocated in.
* @index: Index of hisi_acc_hw_sgl in pool.
* @hw_sgl_dma: The dma address of allocated hw sgl.
*
* This function builds hw sgl according input sgl, user can use hw_sgl_dma
* as src/dst in its BD. Only support single hw sgl currently.
*/
struct hisi_acc_hw_sgl *
hisi_acc_sg_buf_map_to_hw_sgl(struct device *dev,
struct scatterlist *sgl,
struct hisi_acc_sgl_pool *pool,
u32 index, dma_addr_t *hw_sgl_dma)
{
struct hisi_acc_hw_sgl *curr_hw_sgl;
dma_addr_t curr_sgl_dma = 0;
struct acc_hw_sge *curr_hw_sge;
struct scatterlist *sg;
int sg_n = sg_nents(sgl);
int i, ret;
if (!dev || !sgl || !pool || !hw_sgl_dma || sg_n > acc_sgl_sge_nr)
return ERR_PTR(-EINVAL);
ret = dma_map_sg(dev, sgl, sg_n, DMA_BIDIRECTIONAL);
if (!ret)
return ERR_PTR(-EINVAL);
curr_hw_sgl = acc_get_sgl(pool, index, &curr_sgl_dma);
if (!curr_hw_sgl) {
ret = -ENOMEM;
goto err_unmap_sg;
}
curr_hw_sgl->entry_length_in_sgl = acc_sgl_sge_nr;
curr_hw_sge = curr_hw_sgl->sge_entries;
for_each_sg(sgl, sg, sg_n, i) {
sg_map_to_hw_sg(sg, curr_hw_sge);
inc_hw_sgl_sge(curr_hw_sgl);
curr_hw_sge++;
}
update_hw_sgl_sum_sge(curr_hw_sgl, acc_sgl_sge_nr);
*hw_sgl_dma = curr_sgl_dma;
return curr_hw_sgl;
err_unmap_sg:
dma_unmap_sg(dev, sgl, sg_n, DMA_BIDIRECTIONAL);
return ERR_PTR(ret);
}
EXPORT_SYMBOL_GPL(hisi_acc_sg_buf_map_to_hw_sgl);
/**
* hisi_acc_sg_buf_unmap() - Unmap allocated hw sgl.
* @dev: The device which hw sgl belongs to.
* @sgl: Related scatterlist.
* @hw_sgl: Virtual address of hw sgl.
* @hw_sgl_dma: DMA address of hw sgl.
* @pool: Pool which hw sgl is allocated in.
*
* This function unmaps allocated hw sgl.
*/
void hisi_acc_sg_buf_unmap(struct device *dev, struct scatterlist *sgl,
struct hisi_acc_hw_sgl *hw_sgl)
{
dma_unmap_sg(dev, sgl, sg_nents(sgl), DMA_BIDIRECTIONAL);
hw_sgl->entry_sum_in_chain = 0;
hw_sgl->entry_sum_in_sgl = 0;
hw_sgl->entry_length_in_sgl = 0;
}
EXPORT_SYMBOL_GPL(hisi_acc_sg_buf_unmap);
MODULE_LICENSE("GPL v2");
MODULE_AUTHOR("Zhou Wang <[email protected]>");
MODULE_DESCRIPTION("HiSilicon Accelerator SGL support");

24
drivers/crypto/hisilicon/sgl.h Normal file
View file

@ -0,0 +1,24 @@
/* SPDX-License-Identifier: GPL-2.0 */
/* Copyright (c) 2019 HiSilicon Limited. */
#ifndef HISI_ACC_SGL_H
#define HISI_ACC_SGL_H
struct hisi_acc_sgl_pool {
struct hisi_acc_hw_sgl *sgl;
dma_addr_t sgl_dma;
size_t size;
u32 count;
size_t sgl_size;
};
struct hisi_acc_hw_sgl *
hisi_acc_sg_buf_map_to_hw_sgl(struct device *dev,
struct scatterlist *sgl,
struct hisi_acc_sgl_pool *pool,
u32 index, dma_addr_t *hw_sgl_dma);
void hisi_acc_sg_buf_unmap(struct device *dev, struct scatterlist *sgl,
struct hisi_acc_hw_sgl *hw_sgl);
int hisi_acc_create_sgl_pool(struct device *dev, struct hisi_acc_sgl_pool *pool,
u32 count);
void hisi_acc_free_sgl_pool(struct device *dev, struct hisi_acc_sgl_pool *pool);
#endif

2
drivers/crypto/hisilicon/zip/Makefile Normal file
View file

@ -0,0 +1,2 @@
obj-$(CONFIG_CRYPTO_DEV_HISI_ZIP) += hisi_zip.o
hisi_zip-objs = zip_main.o zip_crypto.o

71
drivers/crypto/hisilicon/zip/zip.h Normal file
View file

@ -0,0 +1,71 @@
/* SPDX-License-Identifier: GPL-2.0 */
/* Copyright (c) 2019 HiSilicon Limited. */
#ifndef HISI_ZIP_H
#define HISI_ZIP_H
#undef pr_fmt
#define pr_fmt(fmt) "hisi_zip: " fmt
#include <linux/list.h>
#include "../qm.h"
#include "../sgl.h"
/* hisi_zip_sqe dw3 */
#define HZIP_BD_STATUS_M GENMASK(7, 0)
/* hisi_zip_sqe dw9 */
#define HZIP_REQ_TYPE_M GENMASK(7, 0)
#define HZIP_ALG_TYPE_ZLIB 0x02
#define HZIP_ALG_TYPE_GZIP 0x03
#define HZIP_BUF_TYPE_M GENMASK(11, 8)
#define HZIP_PBUFFER 0x0
#define HZIP_SGL 0x1
enum hisi_zip_error_type {
/* negative compression */
HZIP_NC_ERR = 0x0d,
};
struct hisi_zip_ctrl;
struct hisi_zip {
struct hisi_qm qm;
struct list_head list;
struct hisi_zip_ctrl *ctrl;
};
struct hisi_zip_sqe {
u32 consumed;
u32 produced;
u32 comp_data_length;
u32 dw3;
u32 input_data_length;
u32 lba_l;
u32 lba_h;
u32 dw7;
u32 dw8;
u32 dw9;
u32 dw10;
u32 priv_info;
u32 dw12;
u32 tag;
u32 dest_avail_out;
u32 rsvd0;
u32 comp_head_addr_l;
u32 comp_head_addr_h;
u32 source_addr_l;
u32 source_addr_h;
u32 dest_addr_l;
u32 dest_addr_h;
u32 stream_ctx_addr_l;
u32 stream_ctx_addr_h;
u32 cipher_key1_addr_l;
u32 cipher_key1_addr_h;
u32 cipher_key2_addr_l;
u32 cipher_key2_addr_h;
u32 rsvd1[4];
};
struct hisi_zip *find_zip_device(int node);
int hisi_zip_register_to_crypto(void);
void hisi_zip_unregister_from_crypto(void);
#endif

653
drivers/crypto/hisilicon/zip/zip_crypto.c Normal file
View file

@ -0,0 +1,653 @@
// SPDX-License-Identifier: GPL-2.0
/* Copyright (c) 2019 HiSilicon Limited. */
#include <crypto/internal/acompress.h>
#include <linux/bitfield.h>
#include <linux/dma-mapping.h>
#include <linux/scatterlist.h>
#include "zip.h"
#define HZIP_ZLIB_HEAD_SIZE 2
#define HZIP_GZIP_HEAD_SIZE 10
#define GZIP_HEAD_FHCRC_BIT BIT(1)
#define GZIP_HEAD_FEXTRA_BIT BIT(2)
#define GZIP_HEAD_FNAME_BIT BIT(3)
#define GZIP_HEAD_FCOMMENT_BIT BIT(4)
#define GZIP_HEAD_FLG_SHIFT 3
#define GZIP_HEAD_FEXTRA_SHIFT 10
#define GZIP_HEAD_FEXTRA_XLEN 2
#define GZIP_HEAD_FHCRC_SIZE 2
#define HZIP_CTX_Q_NUM 2
#define HZIP_GZIP_HEAD_BUF 256
#define HZIP_ALG_PRIORITY 300
static const u8 zlib_head[HZIP_ZLIB_HEAD_SIZE] = {0x78, 0x9c};
static const u8 gzip_head[HZIP_GZIP_HEAD_SIZE] = {0x1f, 0x8b, 0x08, 0x0, 0x0,
0x0, 0x0, 0x0, 0x0, 0x03};
enum hisi_zip_alg_type {
HZIP_ALG_TYPE_COMP = 0,
HZIP_ALG_TYPE_DECOMP = 1,
};
#define COMP_NAME_TO_TYPE(alg_name) \
(!strcmp((alg_name), "zlib-deflate") ? HZIP_ALG_TYPE_ZLIB : \
!strcmp((alg_name), "gzip") ? HZIP_ALG_TYPE_GZIP : 0) \
#define TO_HEAD_SIZE(req_type) \
(((req_type) == HZIP_ALG_TYPE_ZLIB) ? sizeof(zlib_head) : \
((req_type) == HZIP_ALG_TYPE_GZIP) ? sizeof(gzip_head) : 0) \
#define TO_HEAD(req_type) \
(((req_type) == HZIP_ALG_TYPE_ZLIB) ? zlib_head : \
((req_type) == HZIP_ALG_TYPE_GZIP) ? gzip_head : 0) \
struct hisi_zip_req {
struct acomp_req *req;
struct scatterlist *src;
struct scatterlist *dst;
size_t slen;
size_t dlen;
struct hisi_acc_hw_sgl *hw_src;
struct hisi_acc_hw_sgl *hw_dst;
dma_addr_t dma_src;
dma_addr_t dma_dst;
int req_id;
};
struct hisi_zip_req_q {
struct hisi_zip_req *q;
unsigned long *req_bitmap;
rwlock_t req_lock;
u16 size;
};
struct hisi_zip_qp_ctx {
struct hisi_qp *qp;
struct hisi_zip_sqe zip_sqe;
struct hisi_zip_req_q req_q;
struct hisi_acc_sgl_pool sgl_pool;
struct hisi_zip *zip_dev;
struct hisi_zip_ctx *ctx;
};
struct hisi_zip_ctx {
#define QPC_COMP 0
#define QPC_DECOMP 1
struct hisi_zip_qp_ctx qp_ctx[HZIP_CTX_Q_NUM];
};
static void hisi_zip_config_buf_type(struct hisi_zip_sqe *sqe, u8 buf_type)
{
u32 val;
val = (sqe->dw9) & ~HZIP_BUF_TYPE_M;
val |= FIELD_PREP(HZIP_BUF_TYPE_M, buf_type);
sqe->dw9 = val;
}
static void hisi_zip_config_tag(struct hisi_zip_sqe *sqe, u32 tag)
{
sqe->tag = tag;
}
static void hisi_zip_fill_sqe(struct hisi_zip_sqe *sqe, u8 req_type,
dma_addr_t s_addr, dma_addr_t d_addr, u32 slen,
u32 dlen)
{
memset(sqe, 0, sizeof(struct hisi_zip_sqe));
sqe->input_data_length = slen;
sqe->dw9 = FIELD_PREP(HZIP_REQ_TYPE_M, req_type);
sqe->dest_avail_out = dlen;
sqe->source_addr_l = lower_32_bits(s_addr);
sqe->source_addr_h = upper_32_bits(s_addr);
sqe->dest_addr_l = lower_32_bits(d_addr);
sqe->dest_addr_h = upper_32_bits(d_addr);
}
static int hisi_zip_create_qp(struct hisi_qm *qm, struct hisi_zip_qp_ctx *ctx,
int alg_type, int req_type)
{
struct hisi_qp *qp;
int ret;
qp = hisi_qm_create_qp(qm, alg_type);
if (IS_ERR(qp))
return PTR_ERR(qp);
qp->req_type = req_type;
qp->qp_ctx = ctx;
ctx->qp = qp;
ret = hisi_qm_start_qp(qp, 0);
if (ret < 0)
goto err_release_qp;
return 0;
err_release_qp:
hisi_qm_release_qp(qp);
return ret;
}
static void hisi_zip_release_qp(struct hisi_zip_qp_ctx *ctx)
{
hisi_qm_stop_qp(ctx->qp);
hisi_qm_release_qp(ctx->qp);
}
static int hisi_zip_ctx_init(struct hisi_zip_ctx *hisi_zip_ctx, u8 req_type)
{
struct hisi_zip *hisi_zip;
struct hisi_qm *qm;
int ret, i, j;
/* find the proper zip device */
hisi_zip = find_zip_device(cpu_to_node(smp_processor_id()));
if (!hisi_zip) {
pr_err("Failed to find a proper ZIP device!\n");
return -ENODEV;
}
qm = &hisi_zip->qm;
for (i = 0; i < HZIP_CTX_Q_NUM; i++) {
/* alg_type = 0 for compress, 1 for decompress in hw sqe */
ret = hisi_zip_create_qp(qm, &hisi_zip_ctx->qp_ctx[i], i,
req_type);
if (ret)
goto err;
hisi_zip_ctx->qp_ctx[i].zip_dev = hisi_zip;
}
return 0;
err:
for (j = i - 1; j >= 0; j--)
hisi_zip_release_qp(&hisi_zip_ctx->qp_ctx[j]);
return ret;
}
static void hisi_zip_ctx_exit(struct hisi_zip_ctx *hisi_zip_ctx)
{
int i;
for (i = 1; i >= 0; i--)
hisi_zip_release_qp(&hisi_zip_ctx->qp_ctx[i]);
}
static u16 get_extra_field_size(const u8 *start)
{
return *((u16 *)start) + GZIP_HEAD_FEXTRA_XLEN;
}
static u32 get_name_field_size(const u8 *start)
{
return strlen(start) + 1;
}
static u32 get_comment_field_size(const u8 *start)
{
return strlen(start) + 1;
}
static u32 __get_gzip_head_size(const u8 *src)
{
u8 head_flg = *(src + GZIP_HEAD_FLG_SHIFT);
u32 size = GZIP_HEAD_FEXTRA_SHIFT;
if (head_flg & GZIP_HEAD_FEXTRA_BIT)
size += get_extra_field_size(src + size);
if (head_flg & GZIP_HEAD_FNAME_BIT)
size += get_name_field_size(src + size);
if (head_flg & GZIP_HEAD_FCOMMENT_BIT)
size += get_comment_field_size(src + size);
if (head_flg & GZIP_HEAD_FHCRC_BIT)
size += GZIP_HEAD_FHCRC_SIZE;
return size;
}
static int hisi_zip_create_req_q(struct hisi_zip_ctx *ctx)
{
struct hisi_zip_req_q *req_q;
int i, ret;
for (i = 0; i < HZIP_CTX_Q_NUM; i++) {
req_q = &ctx->qp_ctx[i].req_q;
req_q->size = QM_Q_DEPTH;
req_q->req_bitmap = kcalloc(BITS_TO_LONGS(req_q->size),
sizeof(long), GFP_KERNEL);
if (!req_q->req_bitmap) {
ret = -ENOMEM;
if (i == 0)
return ret;
goto err_free_loop0;
}
rwlock_init(&req_q->req_lock);
req_q->q = kcalloc(req_q->size, sizeof(struct hisi_zip_req),
GFP_KERNEL);
if (!req_q->q) {
ret = -ENOMEM;
if (i == 0)
goto err_free_bitmap;
else
goto err_free_loop1;
}
}
return 0;
err_free_loop1:
kfree(ctx->qp_ctx[QPC_DECOMP].req_q.req_bitmap);
err_free_loop0:
kfree(ctx->qp_ctx[QPC_COMP].req_q.q);
err_free_bitmap:
kfree(ctx->qp_ctx[QPC_COMP].req_q.req_bitmap);
return ret;
}
static void hisi_zip_release_req_q(struct hisi_zip_ctx *ctx)
{
int i;
for (i = 0; i < HZIP_CTX_Q_NUM; i++) {
kfree(ctx->qp_ctx[i].req_q.q);
kfree(ctx->qp_ctx[i].req_q.req_bitmap);
}
}
static int hisi_zip_create_sgl_pool(struct hisi_zip_ctx *ctx)
{
struct hisi_zip_qp_ctx *tmp;
int i, ret;
for (i = 0; i < HZIP_CTX_Q_NUM; i++) {
tmp = &ctx->qp_ctx[i];
ret = hisi_acc_create_sgl_pool(&tmp->qp->qm->pdev->dev,
&tmp->sgl_pool,
QM_Q_DEPTH << 1);
if (ret < 0) {
if (i == 1)
goto err_free_sgl_pool0;
return -ENOMEM;
}
}
return 0;
err_free_sgl_pool0:
hisi_acc_free_sgl_pool(&ctx->qp_ctx[QPC_COMP].qp->qm->pdev->dev,
&ctx->qp_ctx[QPC_COMP].sgl_pool);
return -ENOMEM;
}
static void hisi_zip_release_sgl_pool(struct hisi_zip_ctx *ctx)
{
int i;
for (i = 0; i < HZIP_CTX_Q_NUM; i++)
hisi_acc_free_sgl_pool(&ctx->qp_ctx[i].qp->qm->pdev->dev,
&ctx->qp_ctx[i].sgl_pool);
}
static void hisi_zip_remove_req(struct hisi_zip_qp_ctx *qp_ctx,
struct hisi_zip_req *req)
{
struct hisi_zip_req_q *req_q = &qp_ctx->req_q;
if (qp_ctx->qp->alg_type == HZIP_ALG_TYPE_COMP)
kfree(req->dst);
else
kfree(req->src);
write_lock(&req_q->req_lock);
clear_bit(req->req_id, req_q->req_bitmap);
memset(req, 0, sizeof(struct hisi_zip_req));
write_unlock(&req_q->req_lock);
}
static void hisi_zip_acomp_cb(struct hisi_qp *qp, void *data)
{
struct hisi_zip_sqe *sqe = data;
struct hisi_zip_qp_ctx *qp_ctx = qp->qp_ctx;
struct hisi_zip_req_q *req_q = &qp_ctx->req_q;
struct hisi_zip_req *req = req_q->q + sqe->tag;
struct acomp_req *acomp_req = req->req;
struct device *dev = &qp->qm->pdev->dev;
u32 status, dlen, head_size;
int err = 0;
status = sqe->dw3 & HZIP_BD_STATUS_M;
if (status != 0 && status != HZIP_NC_ERR) {
dev_err(dev, "%scompress fail in qp%u: %u, output: %u\n",
(qp->alg_type == 0) ? "" : "de", qp->qp_id, status,
sqe->produced);
err = -EIO;
}
dlen = sqe->produced;
hisi_acc_sg_buf_unmap(dev, req->src, req->hw_src);
hisi_acc_sg_buf_unmap(dev, req->dst, req->hw_dst);
head_size = (qp->alg_type == 0) ? TO_HEAD_SIZE(qp->req_type) : 0;
acomp_req->dlen = dlen + head_size;
if (acomp_req->base.complete)
acomp_request_complete(acomp_req, err);
hisi_zip_remove_req(qp_ctx, req);
}
static void hisi_zip_set_acomp_cb(struct hisi_zip_ctx *ctx,
void (*fn)(struct hisi_qp *, void *))
{
int i;
for (i = 0; i < HZIP_CTX_Q_NUM; i++)
ctx->qp_ctx[i].qp->req_cb = fn;
}
static int hisi_zip_acomp_init(struct crypto_acomp *tfm)
{
const char *alg_name = crypto_tfm_alg_name(&tfm->base);
struct hisi_zip_ctx *ctx = crypto_tfm_ctx(&tfm->base);
int ret;
ret = hisi_zip_ctx_init(ctx, COMP_NAME_TO_TYPE(alg_name));
if (ret)
return ret;
ret = hisi_zip_create_req_q(ctx);
if (ret)
goto err_ctx_exit;
ret = hisi_zip_create_sgl_pool(ctx);
if (ret)
goto err_release_req_q;
hisi_zip_set_acomp_cb(ctx, hisi_zip_acomp_cb);
return 0;
err_release_req_q:
hisi_zip_release_req_q(ctx);
err_ctx_exit:
hisi_zip_ctx_exit(ctx);
return ret;
}
static void hisi_zip_acomp_exit(struct crypto_acomp *tfm)
{
struct hisi_zip_ctx *ctx = crypto_tfm_ctx(&tfm->base);
hisi_zip_set_acomp_cb(ctx, NULL);
hisi_zip_release_sgl_pool(ctx);
hisi_zip_release_req_q(ctx);
hisi_zip_ctx_exit(ctx);
}
static int add_comp_head(struct scatterlist *dst, u8 req_type)
{
int head_size = TO_HEAD_SIZE(req_type);
const u8 *head = TO_HEAD(req_type);
int ret;
ret = sg_copy_from_buffer(dst, sg_nents(dst), head, head_size);
if (ret != head_size)
return -ENOMEM;
return head_size;
}
static size_t get_gzip_head_size(struct scatterlist *sgl)
{
char buf[HZIP_GZIP_HEAD_BUF];
sg_copy_to_buffer(sgl, sg_nents(sgl), buf, sizeof(buf));
return __get_gzip_head_size(buf);
}
static size_t get_comp_head_size(struct scatterlist *src, u8 req_type)
{
switch (req_type) {
case HZIP_ALG_TYPE_ZLIB:
return TO_HEAD_SIZE(HZIP_ALG_TYPE_ZLIB);
case HZIP_ALG_TYPE_GZIP:
return get_gzip_head_size(src);
default:
pr_err("request type does not support!\n");
return -EINVAL;
}
}
static int get_sg_skip_bytes(struct scatterlist *sgl, size_t bytes,
size_t remains, struct scatterlist **out)
{
#define SPLIT_NUM 2
size_t split_sizes[SPLIT_NUM];
int out_mapped_nents[SPLIT_NUM];
split_sizes[0] = bytes;
split_sizes[1] = remains;
return sg_split(sgl, 0, 0, SPLIT_NUM, split_sizes, out,
out_mapped_nents, GFP_KERNEL);
}
static struct hisi_zip_req *hisi_zip_create_req(struct acomp_req *req,
struct hisi_zip_qp_ctx *qp_ctx,
size_t head_size, bool is_comp)
{
struct hisi_zip_req_q *req_q = &qp_ctx->req_q;
struct hisi_zip_req *q = req_q->q;
struct hisi_zip_req *req_cache;
struct scatterlist *out[2];
struct scatterlist *sgl;
size_t len;
int ret, req_id;
/*
* remove/add zlib/gzip head, as hardware operations do not include
* comp head. so split req->src to get sgl without heads in acomp, or
* add comp head to req->dst ahead of that hardware output compressed
* data in sgl splited from req->dst without comp head.
*/
if (is_comp) {
sgl = req->dst;
len = req->dlen - head_size;
} else {
sgl = req->src;
len = req->slen - head_size;
}
ret = get_sg_skip_bytes(sgl, head_size, len, out);
if (ret)
return ERR_PTR(ret);
/* sgl for comp head is useless, so free it now */
kfree(out[0]);
write_lock(&req_q->req_lock);
req_id = find_first_zero_bit(req_q->req_bitmap, req_q->size);
if (req_id >= req_q->size) {
write_unlock(&req_q->req_lock);
dev_dbg(&qp_ctx->qp->qm->pdev->dev, "req cache is full!\n");
kfree(out[1]);
return ERR_PTR(-EBUSY);
}
set_bit(req_id, req_q->req_bitmap);
req_cache = q + req_id;
req_cache->req_id = req_id;
req_cache->req = req;
if (is_comp) {
req_cache->src = req->src;
req_cache->dst = out[1];
req_cache->slen = req->slen;
req_cache->dlen = req->dlen - head_size;
} else {
req_cache->src = out[1];
req_cache->dst = req->dst;
req_cache->slen = req->slen - head_size;
req_cache->dlen = req->dlen;
}
write_unlock(&req_q->req_lock);
return req_cache;
}
static int hisi_zip_do_work(struct hisi_zip_req *req,
struct hisi_zip_qp_ctx *qp_ctx)
{
struct hisi_zip_sqe *zip_sqe = &qp_ctx->zip_sqe;
struct hisi_qp *qp = qp_ctx->qp;
struct device *dev = &qp->qm->pdev->dev;
struct hisi_acc_sgl_pool *pool = &qp_ctx->sgl_pool;
dma_addr_t input;
dma_addr_t output;
int ret;
if (!req->src || !req->slen || !req->dst || !req->dlen)
return -EINVAL;
req->hw_src = hisi_acc_sg_buf_map_to_hw_sgl(dev, req->src, pool,
req->req_id << 1, &input);
if (IS_ERR(req->hw_src))
return PTR_ERR(req->hw_src);
req->dma_src = input;
req->hw_dst = hisi_acc_sg_buf_map_to_hw_sgl(dev, req->dst, pool,
(req->req_id << 1) + 1,
&output);
if (IS_ERR(req->hw_dst)) {
ret = PTR_ERR(req->hw_dst);
goto err_unmap_input;
}
req->dma_dst = output;
hisi_zip_fill_sqe(zip_sqe, qp->req_type, input, output, req->slen,
req->dlen);
hisi_zip_config_buf_type(zip_sqe, HZIP_SGL);
hisi_zip_config_tag(zip_sqe, req->req_id);
/* send command to start a task */
ret = hisi_qp_send(qp, zip_sqe);
if (ret < 0)
goto err_unmap_output;
return -EINPROGRESS;
err_unmap_output:
hisi_acc_sg_buf_unmap(dev, req->dst, req->hw_dst);
err_unmap_input:
hisi_acc_sg_buf_unmap(dev, req->src, req->hw_src);
return ret;
}
static int hisi_zip_acompress(struct acomp_req *acomp_req)
{
struct hisi_zip_ctx *ctx = crypto_tfm_ctx(acomp_req->base.tfm);
struct hisi_zip_qp_ctx *qp_ctx = &ctx->qp_ctx[QPC_COMP];
struct hisi_zip_req *req;
size_t head_size;
int ret;
/* let's output compression head now */
head_size = add_comp_head(acomp_req->dst, qp_ctx->qp->req_type);
if (head_size < 0)
return -ENOMEM;
req = hisi_zip_create_req(acomp_req, qp_ctx, head_size, true);
if (IS_ERR(req))
return PTR_ERR(req);
ret = hisi_zip_do_work(req, qp_ctx);
if (ret != -EINPROGRESS)
hisi_zip_remove_req(qp_ctx, req);
return ret;
}
static int hisi_zip_adecompress(struct acomp_req *acomp_req)
{
struct hisi_zip_ctx *ctx = crypto_tfm_ctx(acomp_req->base.tfm);
struct hisi_zip_qp_ctx *qp_ctx = &ctx->qp_ctx[QPC_DECOMP];
struct hisi_zip_req *req;
size_t head_size;
int ret;
head_size = get_comp_head_size(acomp_req->src, qp_ctx->qp->req_type);
req = hisi_zip_create_req(acomp_req, qp_ctx, head_size, false);
if (IS_ERR(req))
return PTR_ERR(req);
ret = hisi_zip_do_work(req, qp_ctx);
if (ret != -EINPROGRESS)
hisi_zip_remove_req(qp_ctx, req);
return ret;
}
static struct acomp_alg hisi_zip_acomp_zlib = {
.init = hisi_zip_acomp_init,
.exit = hisi_zip_acomp_exit,
.compress = hisi_zip_acompress,
.decompress = hisi_zip_adecompress,
.base = {
.cra_name = "zlib-deflate",
.cra_driver_name = "hisi-zlib-acomp",
.cra_module = THIS_MODULE,
.cra_priority = HZIP_ALG_PRIORITY,
.cra_ctxsize = sizeof(struct hisi_zip_ctx),
}
};
static struct acomp_alg hisi_zip_acomp_gzip = {
.init = hisi_zip_acomp_init,
.exit = hisi_zip_acomp_exit,
.compress = hisi_zip_acompress,
.decompress = hisi_zip_adecompress,
.base = {
.cra_name = "gzip",
.cra_driver_name = "hisi-gzip-acomp",
.cra_module = THIS_MODULE,
.cra_priority = HZIP_ALG_PRIORITY,
.cra_ctxsize = sizeof(struct hisi_zip_ctx),
}
};
int hisi_zip_register_to_crypto(void)
{
int ret = 0;
ret = crypto_register_acomp(&hisi_zip_acomp_zlib);
if (ret) {
pr_err("Zlib acomp algorithm registration failed\n");
return ret;
}
ret = crypto_register_acomp(&hisi_zip_acomp_gzip);
if (ret) {
pr_err("Gzip acomp algorithm registration failed\n");
crypto_unregister_acomp(&hisi_zip_acomp_zlib);
}
return ret;
}
void hisi_zip_unregister_from_crypto(void)
{
crypto_unregister_acomp(&hisi_zip_acomp_gzip);
crypto_unregister_acomp(&hisi_zip_acomp_zlib);
}

1013
drivers/crypto/hisilicon/zip/zip_main.c Normal file
View file

File diff suppressed because it is too large Load diff

Some files were not shown because too many files have changed in this diff Show more