diff --git a/drivers/mmc/host/Kconfig b/drivers/mmc/host/Kconfig index e65d936ed123..8f696b2bcba7 100644 --- a/drivers/mmc/host/Kconfig +++ b/drivers/mmc/host/Kconfig @@ -1041,3 +1041,20 @@ config SDC_QTI help This configuration flag allows adding QTI code in MMC upstream driver. + +config MMC_CQHCI_CRYPTO + bool "CQHCI Crypto Engine Support" + depends on MMC_CQHCI && BLK_INLINE_ENCRYPTION + help + Enable Crypto Engine Support in CQHCI. + Enabling this makes it possible for the kernel to use the crypto + capabilities of the CQHCI device (if present) to perform crypto + operations on data being transferred to/from the device. + +config MMC_CQHCI_CRYPTO_QTI + bool "Vendor specific CQHCI Crypto Engine Support" + depends on MMC_CQHCI_CRYPTO + help + Enable Vendor Crypto Engine Support in CQHCI + Enabling this allows kernel to use CQHCI crypto operations defined + and implemented by QTI. diff --git a/drivers/mmc/host/Makefile b/drivers/mmc/host/Makefile index c14b439da34b..d44941db28e2 100644 --- a/drivers/mmc/host/Makefile +++ b/drivers/mmc/host/Makefile @@ -99,6 +99,8 @@ obj-$(CONFIG_MMC_SDHCI_OMAP) += sdhci-omap.o obj-$(CONFIG_MMC_SDHCI_SPRD) += sdhci-sprd.o obj-$(CONFIG_MMC_CQHCI) += cqhci.o obj-$(CONFIG_MMC_HSQ) += mmc_hsq.o +obj-$(CONFIG_MMC_CQHCI_CRYPTO) += cqhci-crypto.o +obj-$(CONFIG_MMC_CQHCI_CRYPTO_QTI) += cqhci-crypto-qti.o ifeq ($(CONFIG_CB710_DEBUG),y) CFLAGS-cb710-mmc += -DDEBUG diff --git a/drivers/mmc/host/cqhci-crypto-qti.c b/drivers/mmc/host/cqhci-crypto-qti.c new file mode 100644 index 000000000000..4f1f3c08c7f3 --- /dev/null +++ b/drivers/mmc/host/cqhci-crypto-qti.c @@ -0,0 +1,293 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2020, Linux Foundation. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 and + * only version 2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + */ + +#include +#include "sdhci.h" +#include "sdhci-pltfm.h" +#include "cqhci-crypto-qti.h" +#include + +#define RAW_SECRET_SIZE 32 +#define MINIMUM_DUN_SIZE 512 +#define MAXIMUM_DUN_SIZE 65536 + +static struct cqhci_host_crypto_variant_ops cqhci_crypto_qti_variant_ops = { + .host_init_crypto = cqhci_crypto_qti_init_crypto, + .enable = cqhci_crypto_qti_enable, + .disable = cqhci_crypto_qti_disable, + .resume = cqhci_crypto_qti_resume, + .debug = cqhci_crypto_qti_debug, +}; + +static bool ice_cap_idx_valid(struct cqhci_host *host, + unsigned int cap_idx) +{ + return cap_idx < host->crypto_capabilities.num_crypto_cap; +} + +static uint8_t get_data_unit_size_mask(unsigned int data_unit_size) +{ + if (data_unit_size < MINIMUM_DUN_SIZE || + data_unit_size > MAXIMUM_DUN_SIZE || + !is_power_of_2(data_unit_size)) + return 0; + + return data_unit_size / MINIMUM_DUN_SIZE; +} + + +void cqhci_crypto_qti_enable(struct cqhci_host *host) +{ + int err = 0; + + if (!cqhci_host_is_crypto_supported(host)) + return; + + host->caps |= CQHCI_CAP_CRYPTO_SUPPORT; + + err = crypto_qti_enable(host->crypto_vops->priv); + if (err) { + pr_err("%s: Error enabling crypto, err %d\n", + __func__, err); + cqhci_crypto_qti_disable(host); + } +} + +void cqhci_crypto_qti_disable(struct cqhci_host *host) +{ + cqhci_crypto_disable_spec(host); + crypto_qti_disable(host->crypto_vops->priv); +} + +static int cqhci_crypto_qti_keyslot_program(struct keyslot_manager *ksm, + const struct blk_crypto_key *key, + unsigned int slot) +{ + struct cqhci_host *host = keyslot_manager_private(ksm); + int err = 0; + u8 data_unit_mask; + int crypto_alg_id; + + crypto_alg_id = cqhci_crypto_cap_find(host, key->crypto_mode, + key->data_unit_size); + + if (!cqhci_is_crypto_enabled(host) || + !cqhci_keyslot_valid(host, slot) || + !ice_cap_idx_valid(host, crypto_alg_id)) { + return -EINVAL; + } + + data_unit_mask = get_data_unit_size_mask(key->data_unit_size); + + if (!(data_unit_mask & + host->crypto_cap_array[crypto_alg_id].sdus_mask)) { + return -EINVAL; + } + + err = crypto_qti_keyslot_program(host->crypto_vops->priv, key, + slot, data_unit_mask, crypto_alg_id); + if (err) + pr_err("%s: failed with error %d\n", __func__, err); + + return err; +} + +static int cqhci_crypto_qti_keyslot_evict(struct keyslot_manager *ksm, + const struct blk_crypto_key *key, + unsigned int slot) +{ + int err = 0; + struct cqhci_host *host = keyslot_manager_private(ksm); + + if (!cqhci_is_crypto_enabled(host) || + !cqhci_keyslot_valid(host, slot)) + return -EINVAL; + + err = crypto_qti_keyslot_evict(host->crypto_vops->priv, slot); + if (err) + pr_err("%s: failed with error %d\n", __func__, err); + + return err; +} + +static int cqhci_crypto_qti_derive_raw_secret(struct keyslot_manager *ksm, + const u8 *wrapped_key, unsigned int wrapped_key_size, + u8 *secret, unsigned int secret_size) +{ + int err = 0; + struct cqhci_host *host = keyslot_manager_private(ksm); + + err = crypto_qti_derive_raw_secret(host->crypto_vops->priv, wrapped_key, wrapped_key_size, + secret, secret_size); + return err; +} + +static const struct keyslot_mgmt_ll_ops cqhci_crypto_qti_ksm_ops = { + .keyslot_program = cqhci_crypto_qti_keyslot_program, + .keyslot_evict = cqhci_crypto_qti_keyslot_evict, + .derive_raw_secret = cqhci_crypto_qti_derive_raw_secret +}; + +enum blk_crypto_mode_num cqhci_blk_crypto_qti_mode_num_for_alg_dusize( + enum cqhci_crypto_alg cqhci_crypto_alg, + enum cqhci_crypto_key_size key_size) +{ + /* + * Currently the only mode that eMMC and blk-crypto both support. + */ + if (cqhci_crypto_alg == CQHCI_CRYPTO_ALG_AES_XTS && + key_size == CQHCI_CRYPTO_KEY_SIZE_256) + return BLK_ENCRYPTION_MODE_AES_256_XTS; + + return BLK_ENCRYPTION_MODE_INVALID; +} + +int cqhci_host_init_crypto_qti_spec(struct cqhci_host *host, + const struct keyslot_mgmt_ll_ops *ksm_ops) +{ + int cap_idx = 0; + int err = 0; + unsigned int crypto_modes_supported[BLK_ENCRYPTION_MODE_MAX]; + enum blk_crypto_mode_num blk_mode_num; + + /* Default to disabling crypto */ + host->caps &= ~CQHCI_CAP_CRYPTO_SUPPORT; + + if (!(cqhci_readl(host, CQHCI_CAP) & CQHCI_CAP_CS)) { + pr_debug("%s no crypto capability\n", __func__); + err = -ENODEV; + goto out; + } + + /* + * Crypto Capabilities should never be 0, because the + * config_array_ptr > 04h. So we use a 0 value to indicate that + * crypto init failed, and can't be enabled. + */ + host->crypto_capabilities.reg_val = cqhci_readl(host, CQHCI_CCAP); + host->crypto_cfg_register = + (u32)host->crypto_capabilities.config_array_ptr * 0x100; + host->crypto_cap_array = + devm_kcalloc(mmc_dev(host->mmc), + host->crypto_capabilities.num_crypto_cap, + sizeof(host->crypto_cap_array[0]), GFP_KERNEL); + if (!host->crypto_cap_array) { + err = -ENOMEM; + pr_err("%s failed to allocate memory\n", __func__); + goto out; + } + + memset(crypto_modes_supported, 0, sizeof(crypto_modes_supported)); + + /* + * Store all the capabilities now so that we don't need to repeatedly + * access the device each time we want to know its capabilities + */ + for (cap_idx = 0; cap_idx < host->crypto_capabilities.num_crypto_cap; + cap_idx++) { + host->crypto_cap_array[cap_idx].reg_val = + cpu_to_le32(cqhci_readl(host, + CQHCI_CRYPTOCAP + + cap_idx * sizeof(__le32))); + blk_mode_num = cqhci_blk_crypto_qti_mode_num_for_alg_dusize( + host->crypto_cap_array[cap_idx].algorithm_id, + host->crypto_cap_array[cap_idx].key_size); + if (blk_mode_num == BLK_ENCRYPTION_MODE_INVALID) + continue; + crypto_modes_supported[blk_mode_num] |= + host->crypto_cap_array[cap_idx].sdus_mask * 512; + } + + host->ksm = keyslot_manager_create(host->mmc->parent, + cqhci_num_keyslots(host), ksm_ops, + BLK_CRYPTO_FEATURE_STANDARD_KEYS | + BLK_CRYPTO_FEATURE_WRAPPED_KEYS, + crypto_modes_supported, + host); + + keyslot_manager_set_max_dun_bytes(host->ksm, sizeof(u32)); + + if (!host->ksm) { + err = -ENOMEM; + goto out; + } + /* + * In case host controller supports cryptographic operations + * then, it uses 128bit task descriptor. Upper 64 bits of task + * descriptor would be used to pass crypto specific informaton. + */ + host->caps |= CQHCI_TASK_DESC_SZ_128; + + return 0; + +out: + /* Indicate that init failed by setting crypto_capabilities to 0 */ + host->crypto_capabilities.reg_val = 0; + return err; +} + +int cqhci_crypto_qti_init_crypto(struct cqhci_host *host, + const struct keyslot_mgmt_ll_ops *ksm_ops) +{ + int err = 0; + struct resource *cqhci_ice_memres = NULL; + + cqhci_ice_memres = platform_get_resource_byname(host->pdev, + IORESOURCE_MEM, + "cqhci_ice"); + if (!cqhci_ice_memres) { + pr_debug("%s ICE not supported\n", __func__); + host->icemmio = NULL; + return PTR_ERR(cqhci_ice_memres); + } + + host->icemmio = devm_ioremap(&host->pdev->dev, + cqhci_ice_memres->start, + resource_size(cqhci_ice_memres)); + if (!host->icemmio) { + pr_err("%s failed to remap ice regs\n", __func__); + return PTR_ERR(host->icemmio); + } + + err = cqhci_host_init_crypto_qti_spec(host, &cqhci_crypto_qti_ksm_ops); + if (err) { + pr_err("%s: Error initiating crypto capabilities, err %d\n", + __func__, err); + return err; + } + + err = crypto_qti_init_crypto(&host->pdev->dev, + host->icemmio, (void **)&host->crypto_vops->priv); + if (err) { + pr_err("%s: Error initiating crypto, err %d\n", + __func__, err); + } + return err; +} + +int cqhci_crypto_qti_debug(struct cqhci_host *host) +{ + return crypto_qti_debug(host->crypto_vops->priv); +} + +void cqhci_crypto_qti_set_vops(struct cqhci_host *host) +{ + return cqhci_crypto_set_vops(host, &cqhci_crypto_qti_variant_ops); +} + +int cqhci_crypto_qti_resume(struct cqhci_host *host) +{ + return crypto_qti_resume(host->crypto_vops->priv); +} diff --git a/drivers/mmc/host/cqhci-crypto-qti.h b/drivers/mmc/host/cqhci-crypto-qti.h new file mode 100644 index 000000000000..2788e96b8c16 --- /dev/null +++ b/drivers/mmc/host/cqhci-crypto-qti.h @@ -0,0 +1,26 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (c) 2020, The Linux Foundation. All rights reserved. + */ + +#ifndef _UFSHCD_CRYPTO_QTI_H +#define _UFSHCD_CRYPTO_QTI_H + +#include "cqhci-crypto.h" + +void cqhci_crypto_qti_enable(struct cqhci_host *host); + +void cqhci_crypto_qti_disable(struct cqhci_host *host); + +#ifdef CONFIG_BLK_INLINE_ENCRYPTION +int cqhci_crypto_qti_init_crypto(struct cqhci_host *host, + const struct keyslot_mgmt_ll_ops *ksm_ops); +#endif + +int cqhci_crypto_qti_debug(struct cqhci_host *host); + +void cqhci_crypto_qti_set_vops(struct cqhci_host *host); + +int cqhci_crypto_qti_resume(struct cqhci_host *host); + +#endif /* _UFSHCD_ICE_QTI_H */ diff --git a/drivers/mmc/host/cqhci-crypto.c b/drivers/mmc/host/cqhci-crypto.c new file mode 100644 index 000000000000..b20c0597ad8d --- /dev/null +++ b/drivers/mmc/host/cqhci-crypto.c @@ -0,0 +1,532 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright 2020 Google LLC + * + * Copyright (c) 2020 The Linux Foundation. All rights reserved. + * + * drivers/mmc/host/cqhci-crypto.c - Qualcomm Technologies, Inc. + * + * Original source is taken from: + * https://android.googlesource.com/kernel/common/+/4bac1109a10c55d49c0aa4f7ebdc4bc53cc368e8 + * The driver caters to crypto engine support for UFS controllers. + * The crypto engine programming sequence, HW functionality and register + * offset is almost same in UFS and eMMC controllers. + */ + +#include +#include "cqhci-crypto.h" +#include "../core/queue.h" + +static bool cqhci_cap_idx_valid(struct cqhci_host *host, unsigned int cap_idx) +{ + return cap_idx < host->crypto_capabilities.num_crypto_cap; +} + +static u8 get_data_unit_size_mask(unsigned int data_unit_size) +{ + if (data_unit_size < 512 || data_unit_size > 65536 || + !is_power_of_2(data_unit_size)) + return 0; + + return data_unit_size / 512; +} + +static size_t get_keysize_bytes(enum cqhci_crypto_key_size size) +{ + switch (size) { + case CQHCI_CRYPTO_KEY_SIZE_128: + return 16; + case CQHCI_CRYPTO_KEY_SIZE_192: + return 24; + case CQHCI_CRYPTO_KEY_SIZE_256: + return 32; + case CQHCI_CRYPTO_KEY_SIZE_512: + return 64; + default: + return 0; + } +} + +int cqhci_crypto_cap_find(void *host_p, enum blk_crypto_mode_num crypto_mode, + unsigned int data_unit_size) +{ + struct cqhci_host *host = host_p; + enum cqhci_crypto_alg cqhci_alg; + u8 data_unit_mask; + int cap_idx; + enum cqhci_crypto_key_size cqhci_key_size; + union cqhci_crypto_cap_entry *ccap_array = host->crypto_cap_array; + + if (!cqhci_host_is_crypto_supported(host)) + return -EINVAL; + + switch (crypto_mode) { + case BLK_ENCRYPTION_MODE_AES_256_XTS: + cqhci_alg = CQHCI_CRYPTO_ALG_AES_XTS; + cqhci_key_size = CQHCI_CRYPTO_KEY_SIZE_256; + break; + default: + return -EINVAL; + } + + data_unit_mask = get_data_unit_size_mask(data_unit_size); + + for (cap_idx = 0; cap_idx < host->crypto_capabilities.num_crypto_cap; + cap_idx++) { + if (ccap_array[cap_idx].algorithm_id == cqhci_alg && + (ccap_array[cap_idx].sdus_mask & data_unit_mask) && + ccap_array[cap_idx].key_size == cqhci_key_size) + return cap_idx; + } + + return -EINVAL; +} +EXPORT_SYMBOL(cqhci_crypto_cap_find); + +/** + * cqhci_crypto_cfg_entry_write_key - Write a key into a crypto_cfg_entry + * + * Writes the key with the appropriate format - for AES_XTS, + * the first half of the key is copied as is, the second half is + * copied with an offset halfway into the cfg->crypto_key array. + * For the other supported crypto algs, the key is just copied. + * + * @cfg: The crypto config to write to + * @key: The key to write + * @cap: The crypto capability (which specifies the crypto alg and key size) + * + * Returns 0 on success, or -EINVAL + */ +static int cqhci_crypto_cfg_entry_write_key(union cqhci_crypto_cfg_entry *cfg, + const u8 *key, + union cqhci_crypto_cap_entry cap) +{ + size_t key_size_bytes = get_keysize_bytes(cap.key_size); + + if (key_size_bytes == 0) + return -EINVAL; + + switch (cap.algorithm_id) { + case CQHCI_CRYPTO_ALG_AES_XTS: + key_size_bytes *= 2; + if (key_size_bytes > CQHCI_CRYPTO_KEY_MAX_SIZE) + return -EINVAL; + + memcpy(cfg->crypto_key, key, key_size_bytes/2); + memcpy(cfg->crypto_key + CQHCI_CRYPTO_KEY_MAX_SIZE/2, + key + key_size_bytes/2, key_size_bytes/2); + return 0; + case CQHCI_CRYPTO_ALG_BITLOCKER_AES_CBC: + fallthrough; + case CQHCI_CRYPTO_ALG_AES_ECB: + fallthrough; + case CQHCI_CRYPTO_ALG_ESSIV_AES_CBC: + memcpy(cfg->crypto_key, key, key_size_bytes); + return 0; + } + + return -EINVAL; +} + +static void cqhci_program_key(struct cqhci_host *host, + const union cqhci_crypto_cfg_entry *cfg, + int slot) +{ + int i; + u32 slot_offset = host->crypto_cfg_register + slot * sizeof(*cfg); + + if (host->crypto_vops && host->crypto_vops->program_key) + host->crypto_vops->program_key(host, cfg, slot); + + /* Clear the dword 16 */ + cqhci_writel(host, 0, slot_offset + 16 * sizeof(cfg->reg_val[0])); + /* Ensure that CFGE is cleared before programming the key */ + wmb(); + for (i = 0; i < 16; i++) { + cqhci_writel(host, le32_to_cpu(cfg->reg_val[i]), + slot_offset + i * sizeof(cfg->reg_val[0])); + /* Spec says each dword in key must be written sequentially */ + wmb(); + } + /* Write dword 17 */ + cqhci_writel(host, le32_to_cpu(cfg->reg_val[17]), + slot_offset + 17 * sizeof(cfg->reg_val[0])); + /* Dword 16 must be written last */ + wmb(); + /* Write dword 16 */ + cqhci_writel(host, le32_to_cpu(cfg->reg_val[16]), + slot_offset + 16 * sizeof(cfg->reg_val[0])); + /*Ensure that dword 16 is written */ + wmb(); +} + +static void cqhci_crypto_clear_keyslot(struct cqhci_host *host, int slot) +{ + union cqhci_crypto_cfg_entry cfg = { {0} }; + + cqhci_program_key(host, &cfg, slot); +} + +static void cqhci_crypto_clear_all_keyslots(struct cqhci_host *host) +{ + int slot; + + for (slot = 0; slot < cqhci_num_keyslots(host); slot++) + cqhci_crypto_clear_keyslot(host, slot); +} + +static int cqhci_crypto_keyslot_program(struct keyslot_manager *ksm, + const struct blk_crypto_key *key, + unsigned int slot) +{ + struct cqhci_host *host = keyslot_manager_private(ksm); + int err = 0; + u8 data_unit_mask; + union cqhci_crypto_cfg_entry cfg; + int cap_idx; + + cap_idx = cqhci_crypto_cap_find(host, key->crypto_mode, + key->data_unit_size); + + if (!cqhci_is_crypto_enabled(host) || + !cqhci_keyslot_valid(host, slot) || + !cqhci_cap_idx_valid(host, cap_idx)) + return -EINVAL; + + data_unit_mask = get_data_unit_size_mask(key->data_unit_size); + + if (!(data_unit_mask & host->crypto_cap_array[cap_idx].sdus_mask)) + return -EINVAL; + + memset(&cfg, 0, sizeof(cfg)); + cfg.data_unit_size = data_unit_mask; + cfg.crypto_cap_idx = cap_idx; + cfg.config_enable |= CQHCI_CRYPTO_CONFIGURATION_ENABLE; + + err = cqhci_crypto_cfg_entry_write_key(&cfg, key->raw, + host->crypto_cap_array[cap_idx]); + if (err) + return err; + + cqhci_program_key(host, &cfg, slot); + + memzero_explicit(&cfg, sizeof(cfg)); + + return 0; +} + +static int cqhci_crypto_keyslot_evict(struct keyslot_manager *ksm, + const struct blk_crypto_key *key, + unsigned int slot) +{ + struct cqhci_host *host = keyslot_manager_private(ksm); + + if (!cqhci_is_crypto_enabled(host) || + !cqhci_keyslot_valid(host, slot)) + return -EINVAL; + + /* + * Clear the crypto cfg on the device. Clearing CFGE + * might not be sufficient, so just clear the entire cfg. + */ + cqhci_crypto_clear_keyslot(host, slot); + + return 0; +} + +/* Functions implementing eMMC v5.2 specification behaviour */ +void cqhci_crypto_enable_spec(struct cqhci_host *host) +{ + if (!cqhci_host_is_crypto_supported(host)) + return; + + host->caps |= CQHCI_CAP_CRYPTO_SUPPORT; +} +EXPORT_SYMBOL(cqhci_crypto_enable_spec); + +void cqhci_crypto_disable_spec(struct cqhci_host *host) +{ + host->caps &= ~CQHCI_CAP_CRYPTO_SUPPORT; +} +EXPORT_SYMBOL(cqhci_crypto_disable_spec); + +static const struct keyslot_mgmt_ll_ops cqhci_ksm_ops = { + .keyslot_program = cqhci_crypto_keyslot_program, + .keyslot_evict = cqhci_crypto_keyslot_evict, +}; + +enum blk_crypto_mode_num cqhci_crypto_blk_crypto_mode_num_for_alg_dusize( + enum cqhci_crypto_alg cqhci_crypto_alg, + enum cqhci_crypto_key_size key_size) +{ + /* + * Currently the only mode that eMMC and blk-crypto both support. + */ + if (cqhci_crypto_alg == CQHCI_CRYPTO_ALG_AES_XTS && + key_size == CQHCI_CRYPTO_KEY_SIZE_256) + return BLK_ENCRYPTION_MODE_AES_256_XTS; + + return BLK_ENCRYPTION_MODE_INVALID; +} + +/** + * cqhci_host_init_crypto - Read crypto capabilities, init crypto fields in host + * @host: Per adapter instance + * + * Returns 0 on success. Returns -ENODEV if such capabilities don't exist, and + * -ENOMEM upon OOM. + */ +int cqhci_host_init_crypto_spec(struct cqhci_host *host, + const struct keyslot_mgmt_ll_ops *ksm_ops) +{ + int cap_idx = 0; + int err = 0; + unsigned int crypto_modes_supported[BLK_ENCRYPTION_MODE_MAX]; + enum blk_crypto_mode_num blk_mode_num; + + /* Default to disabling crypto */ + host->caps &= ~CQHCI_CAP_CRYPTO_SUPPORT; + + if (!(cqhci_readl(host, CQHCI_CAP) & CQHCI_CAP_CS)) { + pr_err("%s no crypto capability\n", __func__); + err = -ENODEV; + goto out; + } + + /* + * Crypto Capabilities should never be 0, because the + * config_array_ptr > 04h. So we use a 0 value to indicate that + * crypto init failed, and can't be enabled. + */ + host->crypto_capabilities.reg_val = cqhci_readl(host, CQHCI_CCAP); + host->crypto_cfg_register = + (u32)host->crypto_capabilities.config_array_ptr * 0x100; + host->crypto_cap_array = + devm_kcalloc(mmc_dev(host->mmc), + host->crypto_capabilities.num_crypto_cap, + sizeof(host->crypto_cap_array[0]), GFP_KERNEL); + if (!host->crypto_cap_array) { + err = -ENOMEM; + pr_err("%s no memory cap\n", __func__); + goto out; + } + + memset(crypto_modes_supported, 0, sizeof(crypto_modes_supported)); + + /* + * Store all the capabilities now so that we don't need to repeatedly + * access the device each time we want to know its capabilities + */ + for (cap_idx = 0; cap_idx < host->crypto_capabilities.num_crypto_cap; + cap_idx++) { + host->crypto_cap_array[cap_idx].reg_val = + cpu_to_le32(cqhci_readl(host, + CQHCI_CRYPTOCAP + + cap_idx * sizeof(__le32))); + blk_mode_num = cqhci_crypto_blk_crypto_mode_num_for_alg_dusize( + host->crypto_cap_array[cap_idx].algorithm_id, + host->crypto_cap_array[cap_idx].key_size); + if (blk_mode_num == BLK_ENCRYPTION_MODE_INVALID) + continue; + crypto_modes_supported[blk_mode_num] |= + host->crypto_cap_array[cap_idx].sdus_mask * 512; + } + + cqhci_crypto_clear_all_keyslots(host); + + host->ksm = keyslot_manager_create(host->mmc->parent, + cqhci_num_keyslots(host), ksm_ops, + BLK_CRYPTO_FEATURE_STANDARD_KEYS | + BLK_CRYPTO_FEATURE_WRAPPED_KEYS, + crypto_modes_supported, + host); + + if (!host->ksm) { + err = -ENOMEM; + goto out_free_caps; + } + /* + * In case host controller supports cryptographic operations + * then, it uses 128bit task descriptor. Upper 64 bits of task + * descriptor would be used to pass crypto specific informaton. + */ + host->caps |= CQHCI_TASK_DESC_SZ_128; + + return 0; +out_free_caps: + devm_kfree(mmc_dev(host->mmc), host->crypto_cap_array); +out: + // TODO: print error? + /* Indicate that init failed by setting crypto_capabilities to 0 */ + host->crypto_capabilities.reg_val = 0; + return err; +} +EXPORT_SYMBOL(cqhci_host_init_crypto_spec); + +void cqhci_crypto_setup_rq_keyslot_manager_spec(struct cqhci_host *host, + struct request_queue *q) +{ + if (!cqhci_host_is_crypto_supported(host) || !q) + return; + + q->ksm = host->ksm; +} +EXPORT_SYMBOL(cqhci_crypto_setup_rq_keyslot_manager_spec); + +void cqhci_crypto_destroy_rq_keyslot_manager_spec(struct cqhci_host *host, + struct request_queue *q) +{ + keyslot_manager_destroy(host->ksm); +} +EXPORT_SYMBOL(cqhci_crypto_destroy_rq_keyslot_manager_spec); + +int cqhci_prepare_crypto_desc_spec(struct cqhci_host *host, + struct mmc_request *mrq, + u64 *ice_ctx) +{ + struct bio_crypt_ctx *bc; + struct mmc_queue_req *mqrq = container_of(mrq, struct mmc_queue_req, + brq.mrq); + struct request *req = mmc_queue_req_to_req(mqrq); + + if (!req->bio || + !bio_crypt_should_process(req)) { + *ice_ctx = 0; + return 0; + } + if (WARN_ON(!cqhci_is_crypto_enabled(host))) { + /* + * Upper layer asked us to do inline encryption + * but that isn't enabled, so we fail this request. + */ + return -EINVAL; + } + + bc = req->bio->bi_crypt_context; + + if (!cqhci_keyslot_valid(host, bc->bc_keyslot)) + return -EINVAL; + + if (ice_ctx) { + *ice_ctx = DATA_UNIT_NUM(bc->bc_dun[0]) | + CRYPTO_CONFIG_INDEX(bc->bc_keyslot) | + CRYPTO_ENABLE(true); + } + + return 0; +} +EXPORT_SYMBOL(cqhci_prepare_crypto_desc_spec); + +/* Crypto Variant Ops Support */ + +void cqhci_crypto_enable(struct cqhci_host *host) +{ + if (host->crypto_vops && host->crypto_vops->enable) + return host->crypto_vops->enable(host); + + return cqhci_crypto_enable_spec(host); +} + +void cqhci_crypto_disable(struct cqhci_host *host) +{ + if (host->crypto_vops && host->crypto_vops->disable) + return host->crypto_vops->disable(host); + + return cqhci_crypto_disable_spec(host); +} + +int cqhci_host_init_crypto(struct cqhci_host *host) +{ + if (host->crypto_vops && host->crypto_vops->host_init_crypto) + return host->crypto_vops->host_init_crypto(host, + &cqhci_ksm_ops); + + return cqhci_host_init_crypto_spec(host, &cqhci_ksm_ops); +} + +void cqhci_crypto_setup_rq_keyslot_manager(struct cqhci_host *host, + struct request_queue *q) +{ + if (host->crypto_vops && host->crypto_vops->setup_rq_keyslot_manager) + return host->crypto_vops->setup_rq_keyslot_manager(host, q); + + return cqhci_crypto_setup_rq_keyslot_manager_spec(host, q); +} + +void cqhci_crypto_destroy_rq_keyslot_manager(struct cqhci_host *host, + struct request_queue *q) +{ + if (host->crypto_vops && host->crypto_vops->destroy_rq_keyslot_manager) + return host->crypto_vops->destroy_rq_keyslot_manager(host, q); + + return cqhci_crypto_destroy_rq_keyslot_manager_spec(host, q); +} + +int cqhci_crypto_get_ctx(struct cqhci_host *host, + struct mmc_request *mrq, + u64 *ice_ctx) +{ + if (host->crypto_vops && host->crypto_vops->prepare_crypto_desc) + return host->crypto_vops->prepare_crypto_desc(host, mrq, + ice_ctx); + + return cqhci_prepare_crypto_desc_spec(host, mrq, ice_ctx); +} + +int cqhci_complete_crypto_desc(struct cqhci_host *host, + struct mmc_request *mrq, + u64 *ice_ctx) +{ + if (host->crypto_vops && host->crypto_vops->complete_crypto_desc) + return host->crypto_vops->complete_crypto_desc(host, mrq, + ice_ctx); + + return 0; +} + +void cqhci_crypto_debug(struct cqhci_host *host) +{ + if (host->crypto_vops && host->crypto_vops->debug) + host->crypto_vops->debug(host); +} + +void cqhci_crypto_set_vops(struct cqhci_host *host, + struct cqhci_host_crypto_variant_ops *crypto_vops) +{ + host->crypto_vops = crypto_vops; +} + +int cqhci_crypto_suspend(struct cqhci_host *host) +{ + if (host->crypto_vops && host->crypto_vops->suspend) + return host->crypto_vops->suspend(host); + + return 0; +} + +int cqhci_crypto_resume(struct cqhci_host *host) +{ + if (host->crypto_vops && host->crypto_vops->resume) + return host->crypto_vops->resume(host); + + return 0; +} + +int cqhci_crypto_reset(struct cqhci_host *host) +{ + if (host->crypto_vops && host->crypto_vops->reset) + return host->crypto_vops->reset(host); + + return 0; +} + +int cqhci_crypto_recovery_finish(struct cqhci_host *host) +{ + if (host->crypto_vops && host->crypto_vops->recovery_finish) + return host->crypto_vops->recovery_finish(host); + + /* Reset/Recovery might clear all keys, so reprogram all the keys. */ + keyslot_manager_reprogram_all_keys(host->ksm); + + return 0; +} diff --git a/drivers/mmc/host/cqhci-crypto.h b/drivers/mmc/host/cqhci-crypto.h new file mode 100644 index 000000000000..fefad902ddea --- /dev/null +++ b/drivers/mmc/host/cqhci-crypto.h @@ -0,0 +1,182 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright 2019 Google LLC + * + * Copyright (c) 2020 The Linux Foundation. All rights reserved. + * + */ + +#ifndef _CQHCI_CRYPTO_H +#define _CQHCI_CRYPTO_H + +#ifdef CONFIG_MMC_CQHCI_CRYPTO +#include +#include "cqhci.h" + +static inline int cqhci_num_keyslots(struct cqhci_host *host) +{ + return host->crypto_capabilities.config_count + 1; +} + +static inline bool cqhci_keyslot_valid(struct cqhci_host *host, + unsigned int slot) +{ + /* + * The actual number of configurations supported is (CFGC+1), so slot + * numbers range from 0 to config_count inclusive. + */ + return slot < cqhci_num_keyslots(host); +} + +static inline bool cqhci_host_is_crypto_supported(struct cqhci_host *host) +{ + return host->crypto_capabilities.reg_val != 0; +} + +static inline bool cqhci_is_crypto_enabled(struct cqhci_host *host) +{ + return host->caps & CQHCI_CAP_CRYPTO_SUPPORT; +} + +/* Functions implementing eMMC v5.2 specification behaviour */ +int cqhci_prepare_crypto_desc_spec(struct cqhci_host *host, + struct mmc_request *mrq, + u64 *ice_ctx); + +void cqhci_crypto_enable_spec(struct cqhci_host *host); + +void cqhci_crypto_disable_spec(struct cqhci_host *host); + +int cqhci_host_init_crypto_spec(struct cqhci_host *host, + const struct keyslot_mgmt_ll_ops *ksm_ops); + +void cqhci_crypto_setup_rq_keyslot_manager_spec(struct cqhci_host *host, + struct request_queue *q); + +void cqhci_crypto_destroy_rq_keyslot_manager_spec(struct cqhci_host *host, + struct request_queue *q); + +void cqhci_crypto_set_vops(struct cqhci_host *host, + struct cqhci_host_crypto_variant_ops *crypto_vops); + +/* Crypto Variant Ops Support */ + +void cqhci_crypto_enable(struct cqhci_host *host); + +void cqhci_crypto_disable(struct cqhci_host *host); + +int cqhci_host_init_crypto(struct cqhci_host *host); + +void cqhci_crypto_setup_rq_keyslot_manager(struct cqhci_host *host, + struct request_queue *q); + +void cqhci_crypto_destroy_rq_keyslot_manager(struct cqhci_host *host, + struct request_queue *q); + +int cqhci_crypto_get_ctx(struct cqhci_host *host, + struct mmc_request *mrq, + u64 *ice_ctx); + +int cqhci_complete_crypto_desc(struct cqhci_host *host, + struct mmc_request *mrq, + u64 *ice_ctx); + +void cqhci_crypto_debug(struct cqhci_host *host); + +int cqhci_crypto_suspend(struct cqhci_host *host); + +int cqhci_crypto_resume(struct cqhci_host *host); + +int cqhci_crypto_reset(struct cqhci_host *host); + +int cqhci_crypto_recovery_finish(struct cqhci_host *host); + +int cqhci_crypto_cap_find(void *host_p, enum blk_crypto_mode_num crypto_mode, + unsigned int data_unit_size); + +#else /* CONFIG_MMC_CQHCI_CRYPTO */ + +static inline bool cqhci_keyslot_valid(struct cqhci_host *host, + unsigned int slot) +{ + return false; +} + +static inline bool cqhci_host_is_crypto_supported(struct cqhci_host *host) +{ + return false; +} + +static inline bool cqhci_is_crypto_enabled(struct cqhci_host *host) +{ + return false; +} + +static inline void cqhci_crypto_enable(struct cqhci_host *host) { } + +static inline int cqhci_crypto_cap_find(void *host_p, + enum blk_crypto_mode_num crypto_mode, + unsigned int data_unit_size) +{ + return 0; +} + +static inline void cqhci_crypto_disable(struct cqhci_host *host) { } + +static inline int cqhci_host_init_crypto(struct cqhci_host *host) +{ + return 0; +} + +static inline void cqhci_crypto_setup_rq_keyslot_manager( + struct cqhci_host *host, + struct request_queue *q) { } + +static inline void +cqhci_crypto_destroy_rq_keyslot_manager(struct cqhci_host *host, + struct request_queue *q) { } + +static inline int cqhci_crypto_get_ctx(struct cqhci_host *host, + struct mmc_request *mrq, + u64 *ice_ctx) +{ + *ice_ctx = 0; + return 0; +} + +static inline int cqhci_complete_crypto_desc(struct cqhci_host *host, + struct mmc_request *mrq, + u64 *ice_ctx) +{ + return 0; +} + +static inline void cqhci_crypto_debug(struct cqhci_host *host) { } + +static inline void cqhci_crypto_set_vops(struct cqhci_host *host, + struct cqhci_host_crypto_variant_ops *crypto_vops) { } + +static inline int cqhci_crypto_suspend(struct cqhci_host *host) +{ + return 0; +} + +static inline int cqhci_crypto_resume(struct cqhci_host *host) +{ + return 0; +} + +static inline int cqhci_crypto_reset(struct cqhci_host *host) +{ + return 0; +} + +static inline int cqhci_crypto_recovery_finish(struct cqhci_host *host) +{ + return 0; +} + +#endif /* CONFIG_MMC_CQHCI_CRYPTO */ +#endif /* _CQHCI_CRYPTO_H */ + + diff --git a/drivers/mmc/host/cqhci.c b/drivers/mmc/host/cqhci.c index d97a9041fdd9..e7b6b995f5a3 100644 --- a/drivers/mmc/host/cqhci.c +++ b/drivers/mmc/host/cqhci.c @@ -18,6 +18,7 @@ #include #include "cqhci.h" +#include "cqhci-crypto.h" #define DCMD_SLOT 31 #define NUM_SLOTS 32 @@ -138,6 +139,8 @@ static void cqhci_dumpregs(struct cqhci_host *cq_host) cqhci_readl(cq_host, CQHCI_CRI), cqhci_readl(cq_host, CQHCI_CRA)); + cqhci_crypto_debug(cq_host); + if (cq_host->ops->dumpregs) cq_host->ops->dumpregs(mmc); else @@ -258,6 +261,11 @@ static void __cqhci_enable(struct cqhci_host *cq_host) if (cq_host->caps & CQHCI_TASK_DESC_SZ_128) cqcfg |= CQHCI_TASK_DESC_SZ; + if (cqhci_host_is_crypto_supported(cq_host)) { + cqhci_crypto_enable(cq_host); + cqcfg |= CQHCI_ICE_ENABLE; + } + cqhci_writel(cq_host, cqcfg, CQHCI_CFG); cqhci_writel(cq_host, lower_32_bits(cq_host->desc_dma_base), @@ -290,6 +298,9 @@ static void __cqhci_disable(struct cqhci_host *cq_host) { u32 cqcfg; + if (cqhci_host_is_crypto_supported(cq_host)) + cqhci_crypto_disable(cq_host); + cqcfg = cqhci_readl(cq_host, CQHCI_CFG); cqcfg &= ~CQHCI_ENABLE; cqhci_writel(cq_host, cqcfg, CQHCI_CFG); @@ -303,6 +314,8 @@ int cqhci_deactivate(struct mmc_host *mmc) { struct cqhci_host *cq_host = mmc->cqe_private; + cqhci_crypto_suspend(cq_host); + if (cq_host->enabled && cq_host->activated) __cqhci_disable(cq_host); @@ -561,6 +574,30 @@ static inline int cqhci_tag(struct mmc_request *mrq) return mrq->cmd ? DCMD_SLOT : mrq->tag; } +static inline +void cqhci_prep_crypto_desc(struct cqhci_host *cq_host, u64 *task_desc, + u64 ice_ctx) +{ + __le64 *ice_desc = NULL; + + if (cq_host->caps & CQHCI_CAP_CRYPTO_SUPPORT) { + /* + * Get the address of ice context for the given task descriptor. + * ice context is present in the upper 64bits of task descriptor + * ice_conext_base_address = task_desc + 8-bytes + */ + ice_desc = (__le64 *)((u8 *)task_desc + + CQHCI_TASK_DESC_ICE_PARAM_OFFSET); + memset(ice_desc, 0, CQHCI_TASK_DESC_ICE_PARAMS_SIZE); + + /* + * Assign upper 64bits data of task descritor with ice context + */ + if (ice_ctx) + *ice_desc = cpu_to_le64(ice_ctx); + } +} + static int cqhci_request(struct mmc_host *mmc, struct mmc_request *mrq) { int err = 0; @@ -569,6 +606,7 @@ static int cqhci_request(struct mmc_host *mmc, struct mmc_request *mrq) int tag = cqhci_tag(mrq); struct cqhci_host *cq_host = mmc->cqe_private; unsigned long flags; + u64 ice_ctx = 0; if (!cq_host->enabled) { pr_err("%s: cqhci: not enabled\n", mmc_hostname(mmc)); @@ -592,9 +630,17 @@ static int cqhci_request(struct mmc_host *mmc, struct mmc_request *mrq) } if (mrq->data) { + err = cqhci_crypto_get_ctx(cq_host, mrq, &ice_ctx); + if (err) { + pr_err("%s: failed to retrieve crypto ctx for tag %d\n", + mmc_hostname(mmc), tag); + goto out; + } task_desc = (__le64 __force *)get_desc(cq_host, tag); cqhci_prep_task_desc(mrq, &data, 1); *task_desc = cpu_to_le64(data); + cqhci_prep_crypto_desc(cq_host, task_desc, ice_ctx); + err = cqhci_prep_tran_desc(mrq, cq_host, tag); if (err) { pr_err("%s: cqhci: failed to setup tx desc: %d\n", @@ -628,6 +674,9 @@ out_unlock: if (err) cqhci_post_req(mmc, mrq); + if (mrq->data) + cqhci_complete_crypto_desc(cq_host, mrq, NULL); +out: return err; } @@ -635,7 +684,15 @@ out_unlock: static void cqhci_crypto_update_queue(struct mmc_host *mmc, struct request_queue *queue) { - //struct cqhci_host *cq_host = mmc->cqe_private; + struct cqhci_host *cq_host = mmc->cqe_private; + + if (cq_host->caps & CQHCI_CAP_CRYPTO_SUPPORT) { + if (queue) + cqhci_crypto_setup_rq_keyslot_manager(cq_host, queue); + else + pr_err("%s can not register keyslot manager\n", + mmc_hostname(mmc)); + } } #endif @@ -755,6 +812,8 @@ static void cqhci_finish_mrq(struct mmc_host *mmc, unsigned int tag) data = mrq->data; if (data) { + cqhci_complete_crypto_desc(cq_host, mrq, NULL); + if (data->error) data->bytes_xfered = 0; else @@ -942,6 +1001,8 @@ static void cqhci_recovery_start(struct mmc_host *mmc) pr_debug("%s: cqhci: %s\n", mmc_hostname(mmc), __func__); + cqhci_crypto_reset(cq_host); + WARN_ON(!cq_host->recovery_halt); cqhci_halt(mmc, CQHCI_START_HALT_TIMEOUT); @@ -1062,6 +1123,8 @@ static void cqhci_recovery_finish(struct mmc_host *mmc) cqhci_set_irqs(cq_host, CQHCI_IS_MASK); + cqhci_crypto_recovery_finish(cq_host); + pr_debug("%s: cqhci: recovery done\n", mmc_hostname(mmc)); } @@ -1103,6 +1166,11 @@ struct cqhci_host *cqhci_pltfm_init(struct platform_device *pdev) dev_err(&pdev->dev, "failed to remap cqhci regs\n"); return ERR_PTR(-EBUSY); } + +#ifdef CONFIG_MMC_CQHCI_CRYPTO_QTI + cq_host->pdev = pdev; +#endif /* CONFIG_MMC_CQHCI_CRYPTO_QTI */ + dev_dbg(&pdev->dev, "CMDQ ioremap: done\n"); return cq_host; @@ -1148,6 +1216,13 @@ int cqhci_init(struct cqhci_host *cq_host, struct mmc_host *mmc, spin_lock_init(&cq_host->lock); + err = cqhci_host_init_crypto(cq_host); + if (err) { + pr_err("%s: CQHCI version %u.%02u Crypto init failed err %d\n", + mmc_hostname(mmc), cqhci_ver_major(cq_host), + cqhci_ver_minor(cq_host), err); + } + init_completion(&cq_host->halt_comp); init_waitqueue_head(&cq_host->wait_queue); diff --git a/drivers/mmc/host/cqhci.h b/drivers/mmc/host/cqhci.h index 09f9f675ab51..7a9b6d00a36c 100644 --- a/drivers/mmc/host/cqhci.h +++ b/drivers/mmc/host/cqhci.h @@ -12,6 +12,7 @@ #include #include #include +#include /* registers */ /* version */ @@ -31,6 +32,7 @@ #define CQHCI_DCMD 0x00001000 #define CQHCI_TASK_DESC_SZ 0x00000100 #define CQHCI_ENABLE 0x00000001 +#define CQHCI_ICE_ENABLE 0x00000002 /* control */ #define CQHCI_CTL 0x0C @@ -147,11 +149,20 @@ #define CQHCI_DAT_LENGTH(x) (((x) & 0xFFFF) << 16) #define CQHCI_DAT_ADDR_LO(x) (((x) & 0xFFFFFFFF) << 32) #define CQHCI_DAT_ADDR_HI(x) (((x) & 0xFFFFFFFF) << 0) +#define DATA_UNIT_NUM(x) (((u64)(x) & 0xFFFFFFFF) << 0) +#define CRYPTO_CONFIG_INDEX(x) (((u64)(x) & 0xFF) << 32) +#define CRYPTO_ENABLE(x) (((u64)(x) & 0x1) << 47) + +/* ICE context is present in the upper 64bits of task descriptor */ +#define CQHCI_TASK_DESC_ICE_PARAM_OFFSET 8 +/* ICE descriptor size */ +#define CQHCI_TASK_DESC_ICE_PARAMS_SIZE 8 struct cqhci_host_ops; struct mmc_host; struct mmc_request; struct cqhci_slot; +struct cqhci_host; /* CCAP - Crypto Capability 100h */ union cqhci_crypto_capabilities { @@ -208,9 +219,38 @@ union cqhci_crypto_cfg_entry { }; }; +struct cqhci_host_crypto_variant_ops { + void (*setup_rq_keyslot_manager)(struct cqhci_host *host, + struct request_queue *q); + void (*destroy_rq_keyslot_manager)(struct cqhci_host *host, + struct request_queue *q); +#ifdef CONFIG_BLK_INLINE_ENCRYPTION + int (*host_init_crypto)(struct cqhci_host *host, + const struct keyslot_mgmt_ll_ops *ksm_ops); +#endif + void (*enable)(struct cqhci_host *host); + void (*disable)(struct cqhci_host *host); + int (*suspend)(struct cqhci_host *host); + int (*resume)(struct cqhci_host *host); + int (*debug)(struct cqhci_host *host); + int (*prepare_crypto_desc)(struct cqhci_host *host, + struct mmc_request *mrq, + u64 *ice_ctx); + int (*complete_crypto_desc)(struct cqhci_host *host, + struct mmc_request *mrq, + u64 *ice_ctx); + int (*reset)(struct cqhci_host *host); + int (*recovery_finish)(struct cqhci_host *host); + int (*program_key)(struct cqhci_host *host, + const union cqhci_crypto_cfg_entry *cfg, + int slot); + void *priv; +}; + struct cqhci_host { const struct cqhci_host_ops *ops; void __iomem *mmio; + void __iomem *icemmio; struct mmc_host *mmc; spinlock_t lock; @@ -226,6 +266,7 @@ struct cqhci_host { u32 dcmd_slot; u32 caps; #define CQHCI_TASK_DESC_SZ_128 0x1 +#define CQHCI_CAP_CRYPTO_SUPPORT 0x2 u32 quirks; #define CQHCI_QUIRK_SHORT_TXFR_DESC_SZ 0x1 @@ -262,6 +303,19 @@ struct cqhci_host { struct completion halt_comp; wait_queue_head_t wait_queue; struct cqhci_slot *slot; + const struct cqhci_host_crypto_variant_ops *crypto_vops; + +#ifdef CONFIG_MMC_CQHCI_CRYPTO + union cqhci_crypto_capabilities crypto_capabilities; + union cqhci_crypto_cap_entry *crypto_cap_array; + u32 crypto_cfg_register; +#ifdef CONFIG_BLK_INLINE_ENCRYPTION + struct keyslot_manager *ksm; +#endif /* CONFIG_BLK_INLINE_ENCRYPTION */ +#endif /* CONFIG_MMC_CQHCI_CRYPTO */ +#ifdef CONFIG_MMC_CQHCI_CRYPTO_QTI + struct platform_device *pdev; +#endif /* CONFIG_MMC_CQHCI_CRYPTO_QTI */ }; struct cqhci_host_ops { diff --git a/drivers/mmc/host/sdhci-msm.c b/drivers/mmc/host/sdhci-msm.c index a2f53866af18..265c8532246d 100644 --- a/drivers/mmc/host/sdhci-msm.c +++ b/drivers/mmc/host/sdhci-msm.c @@ -23,6 +23,7 @@ #include "sdhci-pltfm.h" #include "cqhci.h" +#include "cqhci-crypto-qti.h" #if defined(CONFIG_SDC_QTI) #include "../core/core.h" #endif @@ -3103,6 +3104,14 @@ static int sdhci_msm_cqe_add_host(struct sdhci_host *host, dma64 = host->flags & SDHCI_USE_64_BIT_DMA; + /* + * Set the vendor specific ops needed for ICE. + * Default implementation if the ops are not set. + */ +#ifdef CONFIG_MMC_CQHCI_CRYPTO_QTI + cqhci_crypto_qti_set_vops(cq_host); +#endif + ret = cqhci_init(cq_host, host->mmc, dma64); if (ret) { dev_err(&pdev->dev, "%s: CQE init: failed (%d)\n",