A-Team_Digital_Solutions/android_kernel_motorola_sm6375
Watch
2
0
Fork
You've already forked android_kernel_motorola_sm6375
 0
mirror of https://github.com/BobTheBlinker/android_kernel_motorola_sm6375.git synced 2026-10-29 19:21:31 -04:00
Code Issues Projects Releases Packages Wiki Activity

Merge fbe changes from upstream 5.4

Browse source 
b7b3af9 BACKPORT: FROMLIST: Update Inline Encryption from v5 to v6 of patch series
4f6a1f4 ANDROID: ufshcd-crypto: export cap find API
e25d82c ANDROID: block: provide key size as input to inline crypto APIs
3e8cceb ANDROID: block: add KSM op to derive software secret from wrapped key
8173487 ANDROID: fscrypt: add support for hardware-wrapped keys
8aaaa67 ANDROID: block: fix some inline crypto bugs
9f8ee7b ANDROID: block: export symbols needed for modules to use inline crypto
0faba9a FROMLIST: scsi: ufs: add program_key() variant op
403731b ANDROID: ext4, f2fs: enable direct I/O with inline encryption
1fc06ba ANDROID: block: Introduce passthrough keyslot manager
695cfeb ANDROID: dm: add support for passing through inline crypto support
85942bc ANDROID: dm: enable may_passthrough_inline_crypto on some targets
5515324 ANDROID: dm: add dm-default-key target for metadata encryption
792b182 ANDROID: gki_defconfig: enable dm-default-key
3ab1972 ANDROID: block: Prevent crypto fallback for wrapped keys
a142356 ANDROID: dm: add support for passing through derive_raw_secret
0dccdc0 ANDROID: dm: Add wrapped key support in dm-default-key

Git-repo: https://android.googlesource.com/kernel/common/+/refs/heads/android-5.4
Change-Id: Ida31c27ac24457efc1aa0343806e2f32a056167b
Signed-off-by: Gaurav Kashyap <[email protected]>
This commit is contained in:
Satya Tangirala 2019-12-17 14:26:29 -08:00 • committed by Gaurav Kashyap
commit c83b552741
50 changed files with 48070 additions and 46599 deletions
Download patch file Download diff file Expand all files Collapse all files

15
block/Kconfig
View file

@ -179,13 +179,20 @@ config BLK_SED_OPAL
config BLK_INLINE_ENCRYPTION
bool "Enable inline encryption support in block layer"
help
Build the blk-crypto subsystem. Enabling this lets the
block layer handle encryption, so users can take
advantage of inline encryption hardware if present.
config BLK_INLINE_ENCRYPTION_FALLBACK
bool "Enable crypto API fallback for blk-crypto"
depends on BLK_INLINE_ENCRYPTION
select CRYPTO
select CRYPTO_BLKCIPHER
help
Build the blk-crypto subsystem.
Enabling this lets the block layer handle encryption,
so users can take advantage of inline encryption
hardware if present.
Enabling this lets the block layer handle inline encryption
by falling back to the kernel crypto API when inline
encryption hardware is not present.
menu "Partition Types"

1
block/Makefile
View file

@ -38,3 +38,4 @@ obj-$(CONFIG_BLK_SED_OPAL) += sed-opal.o
obj-$(CONFIG_BLK_PM) += blk-pm.o
obj-$(CONFIG_BLK_INLINE_ENCRYPTION) += keyslot-manager.o bio-crypt-ctx.o \
blk-crypto.o
obj-$(CONFIG_BLK_INLINE_ENCRYPTION_FALLBACK) += blk-crypto-fallback.o

123
block/bio-crypt-ctx.c
View file

@ -5,26 +5,43 @@
#include <linux/bio.h>
#include <linux/blkdev.h>
#include <linux/slab.h>
#include <linux/keyslot-manager.h>
#include <linux/module.h>
#include <linux/slab.h>
#include "blk-crypto-internal.h"
static int num_prealloc_crypt_ctxs = 128;
module_param(num_prealloc_crypt_ctxs, int, 0444);
MODULE_PARM_DESC(num_prealloc_crypt_ctxs,
"Number of bio crypto contexts to preallocate");
static struct kmem_cache *bio_crypt_ctx_cache;
static mempool_t *bio_crypt_ctx_pool;
int bio_crypt_ctx_init(void)
int __init bio_crypt_ctx_init(void)
{
size_t i;
bio_crypt_ctx_cache = KMEM_CACHE(bio_crypt_ctx, 0);
if (!bio_crypt_ctx_cache)
return -ENOMEM;
bio_crypt_ctx_pool = mempool_create_slab_pool(
num_prealloc_crypt_ctxs,
bio_crypt_ctx_cache);
bio_crypt_ctx_pool = mempool_create_slab_pool(num_prealloc_crypt_ctxs,
bio_crypt_ctx_cache);
if (!bio_crypt_ctx_pool)
return -ENOMEM;
/* This is assumed in various places. */
BUILD_BUG_ON(BLK_ENCRYPTION_MODE_INVALID != 0);
/* Sanity check that no algorithm exceeds the defined limits. */
for (i = 0; i < BLK_ENCRYPTION_MODE_MAX; i++) {
BUG_ON(blk_crypto_modes[i].keysize > BLK_CRYPTO_MAX_KEY_SIZE);
BUG_ON(blk_crypto_modes[i].ivsize > BLK_CRYPTO_MAX_IV_SIZE);
}
return 0;
}
@ -32,51 +49,46 @@ struct bio_crypt_ctx *bio_crypt_alloc_ctx(gfp_t gfp_mask)
{
return mempool_alloc(bio_crypt_ctx_pool, gfp_mask);
}
EXPORT_SYMBOL(bio_crypt_alloc_ctx);
EXPORT_SYMBOL_GPL(bio_crypt_alloc_ctx);
void bio_crypt_free_ctx(struct bio *bio)
{
mempool_free(bio->bi_crypt_context, bio_crypt_ctx_pool);
bio->bi_crypt_context = NULL;
}
EXPORT_SYMBOL(bio_crypt_free_ctx);
int bio_crypt_clone(struct bio *dst, struct bio *src, gfp_t gfp_mask)
void bio_crypt_clone(struct bio *dst, struct bio *src, gfp_t gfp_mask)
{
const struct bio_crypt_ctx *src_bc = src->bi_crypt_context;
bio_clone_skip_dm_default_key(dst, src);
/*
* If a bio is swhandled, then it will be decrypted when bio_endio
* is called. As we only want the data to be decrypted once, copies
* of the bio must not have have a crypt context.
* If a bio is fallback_crypted, then it will be decrypted when
* bio_endio is called. As we only want the data to be decrypted once,
* copies of the bio must not have have a crypt context.
*/
if (!bio_has_crypt_ctx(src) || bio_crypt_swhandled(src))
return 0;
if (!src_bc || bio_crypt_fallback_crypted(src_bc))
return;
dst->bi_crypt_context = bio_crypt_alloc_ctx(gfp_mask);
if (!dst->bi_crypt_context)
return -ENOMEM;
*dst->bi_crypt_context = *src_bc;
*dst->bi_crypt_context = *src->bi_crypt_context;
if (bio_crypt_has_keyslot(src))
keyslot_manager_get_slot(src->bi_crypt_context->processing_ksm,
src->bi_crypt_context->keyslot);
return 0;
if (src_bc->bc_keyslot >= 0)
keyslot_manager_get_slot(src_bc->bc_ksm, src_bc->bc_keyslot);
}
EXPORT_SYMBOL(bio_crypt_clone);
EXPORT_SYMBOL_GPL(bio_crypt_clone);
bool bio_crypt_should_process(struct bio *bio, struct request_queue *q)
bool bio_crypt_should_process(struct request *rq)
{
if (!bio_has_crypt_ctx(bio))
struct bio *bio = rq->bio;
if (!bio || !bio->bi_crypt_context)
return false;
if (q->ksm != bio->bi_crypt_context->processing_ksm)
return false;
WARN_ON(!bio_crypt_has_keyslot(bio));
return true;
return rq->q->ksm == bio->bi_crypt_context->bc_ksm;
}
EXPORT_SYMBOL(bio_crypt_should_process);
EXPORT_SYMBOL_GPL(bio_crypt_should_process);
/*
* Checks that two bio crypt contexts are compatible - i.e. that
@ -87,23 +99,18 @@ bool bio_crypt_ctx_compatible(struct bio *b_1, struct bio *b_2)
struct bio_crypt_ctx *bc1 = b_1->bi_crypt_context;
struct bio_crypt_ctx *bc2 = b_2->bi_crypt_context;
if (bio_has_crypt_ctx(b_1) != bio_has_crypt_ctx(b_2))
return false;
if (!bio_has_crypt_ctx(b_1))
return true;
return bc1->keyslot == bc2->keyslot &&
bc1->data_unit_size_bits == bc2->data_unit_size_bits;
if (!bc1)
return !bc2;
return bc2 && bc1->bc_key == bc2->bc_key;
}
/*
* Checks that two bio crypt contexts are compatible, and also
* that their data_unit_nums are continuous (and can hence be merged)
* in the order b_1 followed by b_2.
*/
bool bio_crypt_ctx_back_mergeable(struct bio *b_1,
unsigned int b1_sectors,
struct bio *b_2)
bool bio_crypt_ctx_mergeable(struct bio *b_1, unsigned int b1_bytes,
struct bio *b_2)
{
struct bio_crypt_ctx *bc1 = b_1->bi_crypt_context;
struct bio_crypt_ctx *bc2 = b_2->bi_crypt_context;
@ -111,35 +118,25 @@ bool bio_crypt_ctx_back_mergeable(struct bio *b_1,
if (!bio_crypt_ctx_compatible(b_1, b_2))
return false;
return !bio_has_crypt_ctx(b_1) ||
(bc1->data_unit_num +
(b1_sectors >> (bc1->data_unit_size_bits - 9)) ==
bc2->data_unit_num);
return !bc1 || bio_crypt_dun_is_contiguous(bc1, b1_bytes, bc2->bc_dun);
}
void bio_crypt_ctx_release_keyslot(struct bio *bio)
void bio_crypt_ctx_release_keyslot(struct bio_crypt_ctx *bc)
{
struct bio_crypt_ctx *crypt_ctx = bio->bi_crypt_context;
keyslot_manager_put_slot(crypt_ctx->processing_ksm, crypt_ctx->keyslot);
bio->bi_crypt_context->processing_ksm = NULL;
bio->bi_crypt_context->keyslot = -1;
keyslot_manager_put_slot(bc->bc_ksm, bc->bc_keyslot);
bc->bc_ksm = NULL;
bc->bc_keyslot = -1;
}
int bio_crypt_ctx_acquire_keyslot(struct bio *bio, struct keyslot_manager *ksm)
int bio_crypt_ctx_acquire_keyslot(struct bio_crypt_ctx *bc,
struct keyslot_manager *ksm)
{
int slot;
enum blk_crypto_mode_num crypto_mode = bio_crypto_mode(bio);
int slot = keyslot_manager_get_slot_for_key(ksm, bc->bc_key);
if (!ksm)
return -ENOMEM;
slot = keyslot_manager_get_slot_for_key(ksm,
bio_crypt_raw_key(bio), crypto_mode,
1 << bio->bi_crypt_context->data_unit_size_bits);
if (slot < 0)
return slot;
bio_crypt_set_keyslot(bio, slot, ksm);
bc->bc_keyslot = slot;
bc->bc_ksm = ksm;
return 0;
}

8
block/bio.c
View file

@ -234,6 +234,8 @@ fallback:
void bio_uninit(struct bio *bio)
{
bio_disassociate_blkg(bio);
bio_crypt_free_ctx(bio);
}
EXPORT_SYMBOL(bio_uninit);
@ -242,7 +244,6 @@ static void bio_free(struct bio *bio)
struct bio_set *bs = bio->bi_pool;
void *p;
bio_crypt_free_ctx(bio);
bio_uninit(bio);
if (bs) {
@ -614,10 +615,7 @@ struct bio *bio_clone_fast(struct bio *bio, gfp_t gfp_mask, struct bio_set *bs)
__bio_clone_fast(b, bio);
if (bio_crypt_clone(b, bio, gfp_mask) < 0) {
bio_put(b);
return NULL;
}
bio_crypt_clone(b, bio, gfp_mask);
if (bio_integrity(bio) &&
bio_integrity_clone(b, bio, gfp_mask) < 0) {

12
block/blk-core.c
View file

@ -1112,14 +1112,11 @@ blk_qc_t direct_make_request(struct bio *bio)
{
struct request_queue *q = bio->bi_disk->queue;
bool nowait = bio->bi_opf & REQ_NOWAIT;
blk_qc_t ret;
blk_qc_t ret = BLK_QC_T_NONE;
if (!generic_make_request_checks(bio))
return BLK_QC_T_NONE;
if (blk_crypto_submit_bio(&bio))
return BLK_QC_T_NONE;
if (unlikely(blk_queue_enter(q, nowait ? BLK_MQ_REQ_NOWAIT : 0))) {
if (nowait && !blk_queue_dying(q))
bio->bi_status = BLK_STS_AGAIN;
@ -1129,7 +1126,8 @@ blk_qc_t direct_make_request(struct bio *bio)
return BLK_QC_T_NONE;
}
ret = q->make_request_fn(q, bio);
if (!blk_crypto_submit_bio(&bio))
ret = q->make_request_fn(q, bio);
blk_queue_exit(q);
return ret;
}
@ -1816,8 +1814,8 @@ int __init blk_dev_init(void)
if (bio_crypt_ctx_init() < 0)
panic("Failed to allocate mem for bio crypt ctxs\n");
if (blk_crypto_init() < 0)
panic("Failed to init blk-crypto\n");
if (blk_crypto_fallback_init() < 0)
panic("Failed to init blk-crypto-fallback\n");
return 0;
}

657
block/blk-crypto-fallback.c Normal file
View file

@ -0,0 +1,657 @@
// SPDX-License-Identifier: GPL-2.0
/*
* Copyright 2019 Google LLC
*/
/*
* Refer to Documentation/block/inline-encryption.rst for detailed explanation.
*/
#define pr_fmt(fmt) "blk-crypto-fallback: " fmt
#include <crypto/skcipher.h>
#include <linux/blk-cgroup.h>
#include <linux/blk-crypto.h>
#include <linux/crypto.h>
#include <linux/keyslot-manager.h>
#include <linux/mempool.h>
#include <linux/module.h>
#include <linux/random.h>
#include "blk-crypto-internal.h"
static unsigned int num_prealloc_bounce_pg = 32;
module_param(num_prealloc_bounce_pg, uint, 0);
MODULE_PARM_DESC(num_prealloc_bounce_pg,
"Number of preallocated bounce pages for the blk-crypto crypto API fallback");
static unsigned int blk_crypto_num_keyslots = 100;
module_param_named(num_keyslots, blk_crypto_num_keyslots, uint, 0);
MODULE_PARM_DESC(num_keyslots,
"Number of keyslots for the blk-crypto crypto API fallback");
static unsigned int num_prealloc_fallback_crypt_ctxs = 128;
module_param(num_prealloc_fallback_crypt_ctxs, uint, 0);
MODULE_PARM_DESC(num_prealloc_crypt_fallback_ctxs,
"Number of preallocated bio fallback crypto contexts for blk-crypto to use during crypto API fallback");
struct bio_fallback_crypt_ctx {
struct bio_crypt_ctx crypt_ctx;
/*
* Copy of the bvec_iter when this bio was submitted.
* We only want to en/decrypt the part of the bio as described by the
* bvec_iter upon submission because bio might be split before being
* resubmitted
*/
struct bvec_iter crypt_iter;
u64 fallback_dun[BLK_CRYPTO_DUN_ARRAY_SIZE];
};
/* The following few vars are only used during the crypto API fallback */
static struct kmem_cache *bio_fallback_crypt_ctx_cache;
static mempool_t *bio_fallback_crypt_ctx_pool;
/*
* Allocating a crypto tfm during I/O can deadlock, so we have to preallocate
* all of a mode's tfms when that mode starts being used. Since each mode may
* need all the keyslots at some point, each mode needs its own tfm for each
* keyslot; thus, a keyslot may contain tfms for multiple modes. However, to
* match the behavior of real inline encryption hardware (which only supports a
* single encryption context per keyslot), we only allow one tfm per keyslot to
* be used at a time - the rest of the unused tfms have their keys cleared.
*/
static DEFINE_MUTEX(tfms_init_lock);
static bool tfms_inited[BLK_ENCRYPTION_MODE_MAX];
struct blk_crypto_decrypt_work {
struct work_struct work;
struct bio *bio;
};
static struct blk_crypto_keyslot {
struct crypto_skcipher *tfm;
enum blk_crypto_mode_num crypto_mode;
struct crypto_skcipher *tfms[BLK_ENCRYPTION_MODE_MAX];
} *blk_crypto_keyslots;
/* The following few vars are only used during the crypto API fallback */
static struct keyslot_manager *blk_crypto_ksm;
static struct workqueue_struct *blk_crypto_wq;
static mempool_t *blk_crypto_bounce_page_pool;
static struct kmem_cache *blk_crypto_decrypt_work_cache;
bool bio_crypt_fallback_crypted(const struct bio_crypt_ctx *bc)
{
return bc && bc->bc_ksm == blk_crypto_ksm;
}
/*
* This is the key we set when evicting a keyslot. This *should* be the all 0's
* key, but AES-XTS rejects that key, so we use some random bytes instead.
*/
static u8 blank_key[BLK_CRYPTO_MAX_KEY_SIZE];
static void blk_crypto_evict_keyslot(unsigned int slot)
{
struct blk_crypto_keyslot *slotp = &blk_crypto_keyslots[slot];
enum blk_crypto_mode_num crypto_mode = slotp->crypto_mode;
int err;
WARN_ON(slotp->crypto_mode == BLK_ENCRYPTION_MODE_INVALID);
/* Clear the key in the skcipher */
err = crypto_skcipher_setkey(slotp->tfms[crypto_mode], blank_key,
blk_crypto_modes[crypto_mode].keysize);
WARN_ON(err);
slotp->crypto_mode = BLK_ENCRYPTION_MODE_INVALID;
}
static int blk_crypto_keyslot_program(struct keyslot_manager *ksm,
const struct blk_crypto_key *key,
unsigned int slot)
{
struct blk_crypto_keyslot *slotp = &blk_crypto_keyslots[slot];
const enum blk_crypto_mode_num crypto_mode = key->crypto_mode;
int err;
if (crypto_mode != slotp->crypto_mode &&
slotp->crypto_mode != BLK_ENCRYPTION_MODE_INVALID) {
blk_crypto_evict_keyslot(slot);
}
if (!slotp->tfms[crypto_mode])
return -ENOMEM;
slotp->crypto_mode = crypto_mode;
err = crypto_skcipher_setkey(slotp->tfms[crypto_mode], key->raw,
key->size);
if (err) {
blk_crypto_evict_keyslot(slot);
return err;
}
return 0;
}
static int blk_crypto_keyslot_evict(struct keyslot_manager *ksm,
const struct blk_crypto_key *key,
unsigned int slot)
{
blk_crypto_evict_keyslot(slot);
return 0;
}
/*
* The crypto API fallback KSM ops - only used for a bio when it specifies a
* blk_crypto_mode for which we failed to get a keyslot in the device's inline
* encryption hardware (which probably means the device doesn't have inline
* encryption hardware that supports that crypto mode).
*/
static const struct keyslot_mgmt_ll_ops blk_crypto_ksm_ll_ops = {
.keyslot_program = blk_crypto_keyslot_program,
.keyslot_evict = blk_crypto_keyslot_evict,
};
static void blk_crypto_encrypt_endio(struct bio *enc_bio)
{
struct bio *src_bio = enc_bio->bi_private;
int i;
for (i = 0; i < enc_bio->bi_vcnt; i++)
mempool_free(enc_bio->bi_io_vec[i].bv_page,
blk_crypto_bounce_page_pool);
src_bio->bi_status = enc_bio->bi_status;
bio_put(enc_bio);
bio_endio(src_bio);
}
static struct bio *blk_crypto_clone_bio(struct bio *bio_src)
{
struct bvec_iter iter;
struct bio_vec bv;
struct bio *bio;
bio = bio_alloc_bioset(GFP_NOIO, bio_segments(bio_src), NULL);
if (!bio)
return NULL;
bio->bi_disk = bio_src->bi_disk;
bio->bi_opf = bio_src->bi_opf;
bio->bi_ioprio = bio_src->bi_ioprio;
bio->bi_write_hint = bio_src->bi_write_hint;
bio->bi_iter.bi_sector = bio_src->bi_iter.bi_sector;
bio->bi_iter.bi_size = bio_src->bi_iter.bi_size;
bio_for_each_segment(bv, bio_src, iter)
bio->bi_io_vec[bio->bi_vcnt++] = bv;
if (bio_integrity(bio_src) &&
bio_integrity_clone(bio, bio_src, GFP_NOIO) < 0) {
bio_put(bio);
return NULL;
}
bio_clone_blkg_association(bio, bio_src);
blkcg_bio_issue_init(bio);
bio_clone_skip_dm_default_key(bio, bio_src);
return bio;
}
static int blk_crypto_alloc_cipher_req(struct bio *src_bio,
struct skcipher_request **ciph_req_ret,
struct crypto_wait *wait)
{
struct skcipher_request *ciph_req;
const struct blk_crypto_keyslot *slotp;
slotp = &blk_crypto_keyslots[src_bio->bi_crypt_context->bc_keyslot];
ciph_req = skcipher_request_alloc(slotp->tfms[slotp->crypto_mode],
GFP_NOIO);
if (!ciph_req) {
src_bio->bi_status = BLK_STS_RESOURCE;
return -ENOMEM;
}
skcipher_request_set_callback(ciph_req,
CRYPTO_TFM_REQ_MAY_BACKLOG |
CRYPTO_TFM_REQ_MAY_SLEEP,
crypto_req_done, wait);
*ciph_req_ret = ciph_req;
return 0;
}
static int blk_crypto_split_bio_if_needed(struct bio **bio_ptr)
{
struct bio *bio = *bio_ptr;
unsigned int i = 0;
unsigned int num_sectors = 0;
struct bio_vec bv;
struct bvec_iter iter;
bio_for_each_segment(bv, bio, iter) {
num_sectors += bv.bv_len >> SECTOR_SHIFT;
if (++i == BIO_MAX_PAGES)
break;
}
if (num_sectors < bio_sectors(bio)) {
struct bio *split_bio;
split_bio = bio_split(bio, num_sectors, GFP_NOIO, NULL);
if (!split_bio) {
bio->bi_status = BLK_STS_RESOURCE;
return -ENOMEM;
}
bio_chain(split_bio, bio);
generic_make_request(bio);
*bio_ptr = split_bio;
}
return 0;
}
union blk_crypto_iv {
__le64 dun[BLK_CRYPTO_DUN_ARRAY_SIZE];
u8 bytes[BLK_CRYPTO_MAX_IV_SIZE];
};
static void blk_crypto_dun_to_iv(const u64 dun[BLK_CRYPTO_DUN_ARRAY_SIZE],
union blk_crypto_iv *iv)
{
int i;
for (i = 0; i < BLK_CRYPTO_DUN_ARRAY_SIZE; i++)
iv->dun[i] = cpu_to_le64(dun[i]);
}
/*
* The crypto API fallback's encryption routine.
* Allocate a bounce bio for encryption, encrypt the input bio using crypto API,
* and replace *bio_ptr with the bounce bio. May split input bio if it's too
* large.
*/
static int blk_crypto_encrypt_bio(struct bio **bio_ptr)
{
struct bio *src_bio;
struct skcipher_request *ciph_req = NULL;
DECLARE_CRYPTO_WAIT(wait);
u64 curr_dun[BLK_CRYPTO_DUN_ARRAY_SIZE];
union blk_crypto_iv iv;
struct scatterlist src, dst;
struct bio *enc_bio;
unsigned int i, j;
int data_unit_size;
struct bio_crypt_ctx *bc;
int err = 0;
/* Split the bio if it's too big for single page bvec */
err = blk_crypto_split_bio_if_needed(bio_ptr);
if (err)
return err;
src_bio = *bio_ptr;
bc = src_bio->bi_crypt_context;
data_unit_size = bc->bc_key->data_unit_size;
/* Allocate bounce bio for encryption */
enc_bio = blk_crypto_clone_bio(src_bio);
if (!enc_bio) {
src_bio->bi_status = BLK_STS_RESOURCE;
return -ENOMEM;
}
/*
* Use the crypto API fallback keyslot manager to get a crypto_skcipher
* for the algorithm and key specified for this bio.
*/
err = bio_crypt_ctx_acquire_keyslot(bc, blk_crypto_ksm);
if (err) {
src_bio->bi_status = BLK_STS_IOERR;
goto out_put_enc_bio;
}
/* and then allocate an skcipher_request for it */
err = blk_crypto_alloc_cipher_req(src_bio, &ciph_req, &wait);
if (err)
goto out_release_keyslot;
memcpy(curr_dun, bc->bc_dun, sizeof(curr_dun));
sg_init_table(&src, 1);
sg_init_table(&dst, 1);
skcipher_request_set_crypt(ciph_req, &src, &dst, data_unit_size,
iv.bytes);
/* Encrypt each page in the bounce bio */
for (i = 0; i < enc_bio->bi_vcnt; i++) {
struct bio_vec *enc_bvec = &enc_bio->bi_io_vec[i];
struct page *plaintext_page = enc_bvec->bv_page;
struct page *ciphertext_page =
mempool_alloc(blk_crypto_bounce_page_pool, GFP_NOIO);
enc_bvec->bv_page = ciphertext_page;
if (!ciphertext_page) {
src_bio->bi_status = BLK_STS_RESOURCE;
err = -ENOMEM;
goto out_free_bounce_pages;
}
sg_set_page(&src, plaintext_page, data_unit_size,
enc_bvec->bv_offset);
sg_set_page(&dst, ciphertext_page, data_unit_size,
enc_bvec->bv_offset);
/* Encrypt each data unit in this page */
for (j = 0; j < enc_bvec->bv_len; j += data_unit_size) {
blk_crypto_dun_to_iv(curr_dun, &iv);
err = crypto_wait_req(crypto_skcipher_encrypt(ciph_req),
&wait);
if (err) {
i++;
src_bio->bi_status = BLK_STS_RESOURCE;
goto out_free_bounce_pages;
}
bio_crypt_dun_increment(curr_dun, 1);
src.offset += data_unit_size;
dst.offset += data_unit_size;
}
}
enc_bio->bi_private = src_bio;
enc_bio->bi_end_io = blk_crypto_encrypt_endio;
*bio_ptr = enc_bio;
enc_bio = NULL;
err = 0;
goto out_free_ciph_req;
out_free_bounce_pages:
while (i > 0)
mempool_free(enc_bio->bi_io_vec[--i].bv_page,
blk_crypto_bounce_page_pool);
out_free_ciph_req:
skcipher_request_free(ciph_req);
out_release_keyslot:
bio_crypt_ctx_release_keyslot(bc);
out_put_enc_bio:
if (enc_bio)
bio_put(enc_bio);
return err;
}
static void blk_crypto_free_fallback_crypt_ctx(struct bio *bio)
{
mempool_free(container_of(bio->bi_crypt_context,
struct bio_fallback_crypt_ctx,
crypt_ctx),
bio_fallback_crypt_ctx_pool);
bio->bi_crypt_context = NULL;
}
/*
* The crypto API fallback's main decryption routine.
* Decrypts input bio in place.
*/
static void blk_crypto_decrypt_bio(struct work_struct *work)
{
struct blk_crypto_decrypt_work *decrypt_work =
container_of(work, struct blk_crypto_decrypt_work, work);
struct bio *bio = decrypt_work->bio;
struct skcipher_request *ciph_req = NULL;
DECLARE_CRYPTO_WAIT(wait);
struct bio_vec bv;
struct bvec_iter iter;
u64 curr_dun[BLK_CRYPTO_DUN_ARRAY_SIZE];
union blk_crypto_iv iv;
struct scatterlist sg;
struct bio_crypt_ctx *bc = bio->bi_crypt_context;
struct bio_fallback_crypt_ctx *f_ctx =
container_of(bc, struct bio_fallback_crypt_ctx, crypt_ctx);
const int data_unit_size = bc->bc_key->data_unit_size;
unsigned int i;
int err;
/*
* Use the crypto API fallback keyslot manager to get a crypto_skcipher
* for the algorithm and key specified for this bio.
*/
if (bio_crypt_ctx_acquire_keyslot(bc, blk_crypto_ksm)) {
bio->bi_status = BLK_STS_RESOURCE;
goto out_no_keyslot;
}
/* and then allocate an skcipher_request for it */
err = blk_crypto_alloc_cipher_req(bio, &ciph_req, &wait);
if (err)
goto out;
memcpy(curr_dun, f_ctx->fallback_dun, sizeof(curr_dun));
sg_init_table(&sg, 1);
skcipher_request_set_crypt(ciph_req, &sg, &sg, data_unit_size,
iv.bytes);
/* Decrypt each segment in the bio */
__bio_for_each_segment(bv, bio, iter, f_ctx->crypt_iter) {
struct page *page = bv.bv_page;
sg_set_page(&sg, page, data_unit_size, bv.bv_offset);
/* Decrypt each data unit in the segment */
for (i = 0; i < bv.bv_len; i += data_unit_size) {
blk_crypto_dun_to_iv(curr_dun, &iv);
if (crypto_wait_req(crypto_skcipher_decrypt(ciph_req),
&wait)) {
bio->bi_status = BLK_STS_IOERR;
goto out;
}
bio_crypt_dun_increment(curr_dun, 1);
sg.offset += data_unit_size;
}
}
out:
skcipher_request_free(ciph_req);
bio_crypt_ctx_release_keyslot(bc);
out_no_keyslot:
kmem_cache_free(blk_crypto_decrypt_work_cache, decrypt_work);
blk_crypto_free_fallback_crypt_ctx(bio);
bio_endio(bio);
}
/*
* Queue bio for decryption.
* Returns true iff bio was queued for decryption.
*/
bool blk_crypto_queue_decrypt_bio(struct bio *bio)
{
struct blk_crypto_decrypt_work *decrypt_work;
/* If there was an IO error, don't queue for decrypt. */
if (bio->bi_status)
goto out;
decrypt_work = kmem_cache_zalloc(blk_crypto_decrypt_work_cache,
GFP_ATOMIC);
if (!decrypt_work) {
bio->bi_status = BLK_STS_RESOURCE;
goto out;
}
INIT_WORK(&decrypt_work->work, blk_crypto_decrypt_bio);
decrypt_work->bio = bio;
queue_work(blk_crypto_wq, &decrypt_work->work);
return true;
out:
blk_crypto_free_fallback_crypt_ctx(bio);
return false;
}
/**
* blk_crypto_start_using_mode() - Start using a crypto algorithm on a device
* @mode_num: the blk_crypto_mode we want to allocate ciphers for.
* @data_unit_size: the data unit size that will be used
* @q: the request queue for the device
*
* Upper layers must call this function to ensure that a the crypto API fallback
* has transforms for this algorithm, if they become necessary.
*
* Return: 0 on success and -err on error.
*/
int blk_crypto_start_using_mode(enum blk_crypto_mode_num mode_num,
unsigned int data_unit_size,
struct request_queue *q)
{
struct blk_crypto_keyslot *slotp;
unsigned int i;
int err = 0;
/*
* Fast path
* Ensure that updates to blk_crypto_keyslots[i].tfms[mode_num]
* for each i are visible before we try to access them.
*/
if (likely(smp_load_acquire(&tfms_inited[mode_num])))
return 0;
/*
* If the keyslot manager of the request queue supports this
* crypto mode, then we don't need to allocate this mode.
*/
if (keyslot_manager_crypto_mode_supported(q->ksm, mode_num,
data_unit_size))
return 0;
mutex_lock(&tfms_init_lock);
if (likely(tfms_inited[mode_num]))
goto out;
for (i = 0; i < blk_crypto_num_keyslots; i++) {
slotp = &blk_crypto_keyslots[i];
slotp->tfms[mode_num] = crypto_alloc_skcipher(
blk_crypto_modes[mode_num].cipher_str,
0, 0);
if (IS_ERR(slotp->tfms[mode_num])) {
err = PTR_ERR(slotp->tfms[mode_num]);
slotp->tfms[mode_num] = NULL;
goto out_free_tfms;
}
crypto_skcipher_set_flags(slotp->tfms[mode_num],
CRYPTO_TFM_REQ_FORBID_WEAK_KEYS);
}
/*
* Ensure that updates to blk_crypto_keyslots[i].tfms[mode_num]
* for each i are visible before we set tfms_inited[mode_num].
*/
smp_store_release(&tfms_inited[mode_num], true);
goto out;
out_free_tfms:
for (i = 0; i < blk_crypto_num_keyslots; i++) {
slotp = &blk_crypto_keyslots[i];
crypto_free_skcipher(slotp->tfms[mode_num]);
slotp->tfms[mode_num] = NULL;
}
out:
mutex_unlock(&tfms_init_lock);
return err;
}
EXPORT_SYMBOL_GPL(blk_crypto_start_using_mode);
int blk_crypto_fallback_evict_key(const struct blk_crypto_key *key)
{
return keyslot_manager_evict_key(blk_crypto_ksm, key);
}
int blk_crypto_fallback_submit_bio(struct bio **bio_ptr)
{
struct bio *bio = *bio_ptr;
struct bio_crypt_ctx *bc = bio->bi_crypt_context;
struct bio_fallback_crypt_ctx *f_ctx;
if (bc->bc_key->is_hw_wrapped) {
pr_warn_once("HW wrapped key cannot be used with fallback.\n");
bio->bi_status = BLK_STS_NOTSUPP;
return -EOPNOTSUPP;
}
if (!tfms_inited[bc->bc_key->crypto_mode]) {
bio->bi_status = BLK_STS_IOERR;
return -EIO;
}
if (bio_data_dir(bio) == WRITE)
return blk_crypto_encrypt_bio(bio_ptr);
/*
* Mark bio as fallback crypted and replace the bio_crypt_ctx with
* another one contained in a bio_fallback_crypt_ctx, so that the
* fallback has space to store the info it needs for decryption.
*/
bc->bc_ksm = blk_crypto_ksm;
f_ctx = mempool_alloc(bio_fallback_crypt_ctx_pool, GFP_NOIO);
f_ctx->crypt_ctx = *bc;
memcpy(f_ctx->fallback_dun, bc->bc_dun, sizeof(f_ctx->fallback_dun));
f_ctx->crypt_iter = bio->bi_iter;
bio_crypt_free_ctx(bio);
bio->bi_crypt_context = &f_ctx->crypt_ctx;
return 0;
}
int __init blk_crypto_fallback_init(void)
{
int i;
unsigned int crypto_mode_supported[BLK_ENCRYPTION_MODE_MAX];
prandom_bytes(blank_key, BLK_CRYPTO_MAX_KEY_SIZE);
/* All blk-crypto modes have a crypto API fallback. */
for (i = 0; i < BLK_ENCRYPTION_MODE_MAX; i++)
crypto_mode_supported[i] = 0xFFFFFFFF;
crypto_mode_supported[BLK_ENCRYPTION_MODE_INVALID] = 0;
blk_crypto_ksm = keyslot_manager_create(blk_crypto_num_keyslots,
&blk_crypto_ksm_ll_ops,
crypto_mode_supported, NULL);
if (!blk_crypto_ksm)
return -ENOMEM;
blk_crypto_wq = alloc_workqueue("blk_crypto_wq",
WQ_UNBOUND | WQ_HIGHPRI |
WQ_MEM_RECLAIM, num_online_cpus());
if (!blk_crypto_wq)
return -ENOMEM;
blk_crypto_keyslots = kcalloc(blk_crypto_num_keyslots,
sizeof(blk_crypto_keyslots[0]),
GFP_KERNEL);
if (!blk_crypto_keyslots)
return -ENOMEM;
blk_crypto_bounce_page_pool =
mempool_create_page_pool(num_prealloc_bounce_pg, 0);
if (!blk_crypto_bounce_page_pool)
return -ENOMEM;
blk_crypto_decrypt_work_cache = KMEM_CACHE(blk_crypto_decrypt_work,
SLAB_RECLAIM_ACCOUNT);
if (!blk_crypto_decrypt_work_cache)
return -ENOMEM;
bio_fallback_crypt_ctx_cache = KMEM_CACHE(bio_fallback_crypt_ctx, 0);
if (!bio_fallback_crypt_ctx_cache)
return -ENOMEM;
bio_fallback_crypt_ctx_pool =
mempool_create_slab_pool(num_prealloc_fallback_crypt_ctxs,
bio_fallback_crypt_ctx_cache);
if (!bio_fallback_crypt_ctx_pool)
return -ENOMEM;
return 0;
}

58
block/blk-crypto-internal.h Normal file
View file

@ -0,0 +1,58 @@
/* SPDX-License-Identifier: GPL-2.0 */
/*
* Copyright 2019 Google LLC
*/
#ifndef __LINUX_BLK_CRYPTO_INTERNAL_H
#define __LINUX_BLK_CRYPTO_INTERNAL_H
#include <linux/bio.h>
/* Represents a crypto mode supported by blk-crypto */
struct blk_crypto_mode {
const char *cipher_str; /* crypto API name (for fallback case) */
unsigned int keysize; /* key size in bytes */
unsigned int ivsize; /* iv size in bytes */
};
extern const struct blk_crypto_mode blk_crypto_modes[];
#ifdef CONFIG_BLK_INLINE_ENCRYPTION_FALLBACK
int blk_crypto_fallback_submit_bio(struct bio **bio_ptr);
bool blk_crypto_queue_decrypt_bio(struct bio *bio);
int blk_crypto_fallback_evict_key(const struct blk_crypto_key *key);
bool bio_crypt_fallback_crypted(const struct bio_crypt_ctx *bc);
#else /* CONFIG_BLK_INLINE_ENCRYPTION_FALLBACK */
static inline bool bio_crypt_fallback_crypted(const struct bio_crypt_ctx *bc)
{
return false;
}
static inline int blk_crypto_fallback_submit_bio(struct bio **bio_ptr)
{
pr_warn_once("crypto API fallback disabled; failing request\n");
(*bio_ptr)->bi_status = BLK_STS_NOTSUPP;
return -EIO;
}
static inline bool blk_crypto_queue_decrypt_bio(struct bio *bio)
{
WARN_ON(1);
return false;
}
static inline int
blk_crypto_fallback_evict_key(const struct blk_crypto_key *key)
{
return 0;
}
#endif /* CONFIG_BLK_INLINE_ENCRYPTION_FALLBACK */
#endif /* __LINUX_BLK_CRYPTO_INTERNAL_H */

756
block/blk-crypto.c
View file

@ -10,219 +10,36 @@
#define pr_fmt(fmt) "blk-crypto: " fmt
#include <linux/blk-crypto.h>
#include <linux/blkdev.h>
#include <linux/keyslot-manager.h>
#include <linux/mempool.h>
#include <linux/blk-cgroup.h>
#include <linux/crypto.h>
#include <crypto/skcipher.h>
#include <crypto/algapi.h>
#include <linux/module.h>
#include <linux/sched/mm.h>
#include <linux/random.h>
#include <linux/siphash.h>
/* Represents a crypto mode supported by blk-crypto */
struct blk_crypto_mode {
const char *cipher_str; /* crypto API name (for fallback case) */
size_t keysize; /* key size in bytes */
};
#include "blk-crypto-internal.h"
static const struct blk_crypto_mode blk_crypto_modes[] = {
const struct blk_crypto_mode blk_crypto_modes[] = {
[BLK_ENCRYPTION_MODE_AES_256_XTS] = {
.cipher_str = "xts(aes)",
.keysize = 64,
.ivsize = 16,
},
[BLK_ENCRYPTION_MODE_AES_128_CBC_ESSIV] = {
.cipher_str = "essiv(cbc(aes),sha256)",
.keysize = 16,
.ivsize = 16,
},
[BLK_ENCRYPTION_MODE_ADIANTUM] = {
.cipher_str = "adiantum(xchacha12,aes)",
.keysize = 32,
.ivsize = 32,
},
};
static unsigned int num_prealloc_bounce_pg = 32;
module_param(num_prealloc_bounce_pg, uint, 0);
MODULE_PARM_DESC(num_prealloc_bounce_pg,
"Number of preallocated bounce pages for blk-crypto to use during crypto API fallback encryption");
#define BLK_CRYPTO_MAX_KEY_SIZE 64
static int blk_crypto_num_keyslots = 100;
module_param_named(num_keyslots, blk_crypto_num_keyslots, int, 0);
MODULE_PARM_DESC(num_keyslots,
"Number of keyslots for crypto API fallback in blk-crypto.");
static struct blk_crypto_keyslot {
struct crypto_skcipher *tfm;
enum blk_crypto_mode_num crypto_mode;
u8 key[BLK_CRYPTO_MAX_KEY_SIZE];
struct crypto_skcipher *tfms[ARRAY_SIZE(blk_crypto_modes)];
} *blk_crypto_keyslots;
/*
* Allocating a crypto tfm during I/O can deadlock, so we have to preallocate
* all of a mode's tfms when that mode starts being used. Since each mode may
* need all the keyslots at some point, each mode needs its own tfm for each
* keyslot; thus, a keyslot may contain tfms for multiple modes. However, to
* match the behavior of real inline encryption hardware (which only supports a
* single encryption context per keyslot), we only allow one tfm per keyslot to
* be used at a time - the rest of the unused tfms have their keys cleared.
*/
static struct mutex tfms_lock[ARRAY_SIZE(blk_crypto_modes)];
static bool tfms_inited[ARRAY_SIZE(blk_crypto_modes)];
struct work_mem {
struct work_struct crypto_work;
struct bio *bio;
};
/* The following few vars are only used during the crypto API fallback */
static struct keyslot_manager *blk_crypto_ksm;
static struct workqueue_struct *blk_crypto_wq;
static mempool_t *blk_crypto_page_pool;
static struct kmem_cache *blk_crypto_work_mem_cache;
bool bio_crypt_swhandled(struct bio *bio)
{
return bio_has_crypt_ctx(bio) &&
bio->bi_crypt_context->processing_ksm == blk_crypto_ksm;
}
static u8 blank_key[BLK_CRYPTO_MAX_KEY_SIZE];
static void evict_keyslot(unsigned int slot)
{
struct blk_crypto_keyslot *slotp = &blk_crypto_keyslots[slot];
enum blk_crypto_mode_num crypto_mode = slotp->crypto_mode;
int err;
WARN_ON(slotp->crypto_mode == BLK_ENCRYPTION_MODE_INVALID);
/* Clear the key in the skcipher */
err = crypto_skcipher_setkey(slotp->tfms[crypto_mode], blank_key,
blk_crypto_modes[crypto_mode].keysize);
WARN_ON(err);
memzero_explicit(slotp->key, BLK_CRYPTO_MAX_KEY_SIZE);
slotp->crypto_mode = BLK_ENCRYPTION_MODE_INVALID;
}
static int blk_crypto_keyslot_program(void *priv, const u8 *key,
enum blk_crypto_mode_num crypto_mode,
unsigned int data_unit_size,
unsigned int slot)
{
struct blk_crypto_keyslot *slotp = &blk_crypto_keyslots[slot];
const struct blk_crypto_mode *mode = &blk_crypto_modes[crypto_mode];
size_t keysize = mode->keysize;
int err;
if (crypto_mode != slotp->crypto_mode &&
slotp->crypto_mode != BLK_ENCRYPTION_MODE_INVALID) {
evict_keyslot(slot);
}
if (!slotp->tfms[crypto_mode])
return -ENOMEM;
slotp->crypto_mode = crypto_mode;
err = crypto_skcipher_setkey(slotp->tfms[crypto_mode], key, keysize);
if (err) {
evict_keyslot(slot);
return err;
}
memcpy(slotp->key, key, keysize);
return 0;
}
static int blk_crypto_keyslot_evict(void *priv, const u8 *key,
enum blk_crypto_mode_num crypto_mode,
unsigned int data_unit_size,
unsigned int slot)
{
evict_keyslot(slot);
return 0;
}
static int blk_crypto_keyslot_find(void *priv,
const u8 *key,
enum blk_crypto_mode_num crypto_mode,
unsigned int data_unit_size_bytes)
{
int slot;
const size_t keysize = blk_crypto_modes[crypto_mode].keysize;
for (slot = 0; slot < blk_crypto_num_keyslots; slot++) {
if (blk_crypto_keyslots[slot].crypto_mode == crypto_mode &&
!crypto_memneq(blk_crypto_keyslots[slot].key, key, keysize))
return slot;
}
return -ENOKEY;
}
static bool blk_crypto_mode_supported(void *priv,
enum blk_crypto_mode_num crypt_mode,
unsigned int data_unit_size)
{
/* All blk_crypto_modes are required to have a crypto API fallback. */
return true;
}
/*
* The crypto API fallback KSM ops - only used for a bio when it specifies a
* blk_crypto_mode for which we failed to get a keyslot in the device's inline
* encryption hardware (which probably means the device doesn't have inline
* encryption hardware that supports that crypto mode).
*/
static const struct keyslot_mgmt_ll_ops blk_crypto_ksm_ll_ops = {
.keyslot_program = blk_crypto_keyslot_program,
.keyslot_evict = blk_crypto_keyslot_evict,
.keyslot_find = blk_crypto_keyslot_find,
.crypto_mode_supported = blk_crypto_mode_supported,
};
static void blk_crypto_encrypt_endio(struct bio *enc_bio)
{
struct bio *src_bio = enc_bio->bi_private;
int i;
for (i = 0; i < enc_bio->bi_vcnt; i++)
mempool_free(enc_bio->bi_io_vec[i].bv_page,
blk_crypto_page_pool);
src_bio->bi_status = enc_bio->bi_status;
bio_put(enc_bio);
bio_endio(src_bio);
}
static struct bio *blk_crypto_clone_bio(struct bio *bio_src)
{
struct bvec_iter iter;
struct bio_vec bv;
struct bio *bio;
bio = bio_alloc_bioset(GFP_NOIO, bio_segments(bio_src), NULL);
if (!bio)
return NULL;
bio->bi_disk = bio_src->bi_disk;
bio->bi_opf = bio_src->bi_opf;
bio->bi_ioprio = bio_src->bi_ioprio;
bio->bi_write_hint = bio_src->bi_write_hint;
bio->bi_iter.bi_sector = bio_src->bi_iter.bi_sector;
bio->bi_iter.bi_size = bio_src->bi_iter.bi_size;
bio_for_each_segment(bv, bio_src, iter)
bio->bi_io_vec[bio->bi_vcnt++] = bv;
if (bio_integrity(bio_src) &&
bio_integrity_clone(bio, bio_src, GFP_NOIO) < 0) {
bio_put(bio);
return NULL;
}
bio_clone_blkg_association(bio, bio_src);
blkcg_bio_issue_init(bio);
return bio;
}
/* Check that all I/O segments are data unit aligned */
static int bio_crypt_check_alignment(struct bio *bio)
{
int data_unit_size = 1 << bio->bi_crypt_context->data_unit_size_bits;
const unsigned int data_unit_size =
bio->bi_crypt_context->bc_key->data_unit_size;
struct bvec_iter iter;
struct bio_vec bv;
@ -233,268 +50,6 @@ static int bio_crypt_check_alignment(struct bio *bio)
return 0;
}
static int blk_crypto_alloc_cipher_req(struct bio *src_bio,
struct skcipher_request **ciph_req_ptr,
struct crypto_wait *wait)
{
int slot;
struct skcipher_request *ciph_req;
struct blk_crypto_keyslot *slotp;
slot = bio_crypt_get_keyslot(src_bio);
slotp = &blk_crypto_keyslots[slot];
ciph_req = skcipher_request_alloc(slotp->tfms[slotp->crypto_mode],
GFP_NOIO);
if (!ciph_req) {
src_bio->bi_status = BLK_STS_RESOURCE;
return -ENOMEM;
}
skcipher_request_set_callback(ciph_req,
CRYPTO_TFM_REQ_MAY_BACKLOG |
CRYPTO_TFM_REQ_MAY_SLEEP,
crypto_req_done, wait);
*ciph_req_ptr = ciph_req;
return 0;
}
static int blk_crypto_split_bio_if_needed(struct bio **bio_ptr)
{
struct bio *bio = *bio_ptr;
unsigned int i = 0;
unsigned int num_sectors = 0;
struct bio_vec bv;
struct bvec_iter iter;
bio_for_each_segment(bv, bio, iter) {
num_sectors += bv.bv_len >> SECTOR_SHIFT;
if (++i == BIO_MAX_PAGES)
break;
}
if (num_sectors < bio_sectors(bio)) {
struct bio *split_bio;
split_bio = bio_split(bio, num_sectors, GFP_NOIO, NULL);
if (!split_bio) {
bio->bi_status = BLK_STS_RESOURCE;
return -ENOMEM;
}
bio_chain(split_bio, bio);
generic_make_request(bio);
*bio_ptr = split_bio;
}
return 0;
}
/*
* The crypto API fallback's encryption routine.
* Allocate a bounce bio for encryption, encrypt the input bio using
* crypto API, and replace *bio_ptr with the bounce bio. May split input
* bio if it's too large.
*/
static int blk_crypto_encrypt_bio(struct bio **bio_ptr)
{
struct bio *src_bio;
struct skcipher_request *ciph_req = NULL;
DECLARE_CRYPTO_WAIT(wait);
int err = 0;
u64 curr_dun;
union {
__le64 dun;
u8 bytes[16];
} iv;
struct scatterlist src, dst;
struct bio *enc_bio;
struct bio_vec *enc_bvec;
int i, j;
int data_unit_size;
/* Split the bio if it's too big for single page bvec */
err = blk_crypto_split_bio_if_needed(bio_ptr);
if (err)
return err;
src_bio = *bio_ptr;
data_unit_size = 1 << src_bio->bi_crypt_context->data_unit_size_bits;
/* Allocate bounce bio for encryption */
enc_bio = blk_crypto_clone_bio(src_bio);
if (!enc_bio) {
src_bio->bi_status = BLK_STS_RESOURCE;
return -ENOMEM;
}
/*
* Use the crypto API fallback keyslot manager to get a crypto_skcipher
* for the algorithm and key specified for this bio.
*/
err = bio_crypt_ctx_acquire_keyslot(src_bio, blk_crypto_ksm);
if (err) {
src_bio->bi_status = BLK_STS_IOERR;
goto out_put_enc_bio;
}
/* and then allocate an skcipher_request for it */
err = blk_crypto_alloc_cipher_req(src_bio, &ciph_req, &wait);
if (err)
goto out_release_keyslot;
curr_dun = bio_crypt_data_unit_num(src_bio);
sg_init_table(&src, 1);
sg_init_table(&dst, 1);
skcipher_request_set_crypt(ciph_req, &src, &dst,
data_unit_size, iv.bytes);
/* Encrypt each page in the bounce bio */
for (i = 0, enc_bvec = enc_bio->bi_io_vec; i < enc_bio->bi_vcnt;
enc_bvec++, i++) {
struct page *plaintext_page = enc_bvec->bv_page;
struct page *ciphertext_page =
mempool_alloc(blk_crypto_page_pool, GFP_NOIO);
enc_bvec->bv_page = ciphertext_page;
if (!ciphertext_page) {
src_bio->bi_status = BLK_STS_RESOURCE;
err = -ENOMEM;
goto out_free_bounce_pages;
}
sg_set_page(&src, plaintext_page, data_unit_size,
enc_bvec->bv_offset);
sg_set_page(&dst, ciphertext_page, data_unit_size,
enc_bvec->bv_offset);
/* Encrypt each data unit in this page */
for (j = 0; j < enc_bvec->bv_len; j += data_unit_size) {
memset(&iv, 0, sizeof(iv));
iv.dun = cpu_to_le64(curr_dun);
err = crypto_wait_req(crypto_skcipher_encrypt(ciph_req),
&wait);
if (err) {
i++;
src_bio->bi_status = BLK_STS_RESOURCE;
goto out_free_bounce_pages;
}
curr_dun++;
src.offset += data_unit_size;
dst.offset += data_unit_size;
}
}
enc_bio->bi_private = src_bio;
enc_bio->bi_end_io = blk_crypto_encrypt_endio;
*bio_ptr = enc_bio;
enc_bio = NULL;
err = 0;
goto out_free_ciph_req;
out_free_bounce_pages:
while (i > 0)
mempool_free(enc_bio->bi_io_vec[--i].bv_page,
blk_crypto_page_pool);
out_free_ciph_req:
skcipher_request_free(ciph_req);
out_release_keyslot:
bio_crypt_ctx_release_keyslot(src_bio);
out_put_enc_bio:
if (enc_bio)
bio_put(enc_bio);
return err;
}
/*
* The crypto API fallback's main decryption routine.
* Decrypts input bio in place.
*/
static void blk_crypto_decrypt_bio(struct work_struct *w)
{
struct work_mem *work_mem =
container_of(w, struct work_mem, crypto_work);
struct bio *bio = work_mem->bio;
struct skcipher_request *ciph_req = NULL;
DECLARE_CRYPTO_WAIT(wait);
struct bio_vec bv;
struct bvec_iter iter;
u64 curr_dun;
union {
__le64 dun;
u8 bytes[16];
} iv;
struct scatterlist sg;
int data_unit_size = 1 << bio->bi_crypt_context->data_unit_size_bits;
int i;
int err;
/*
* Use the crypto API fallback keyslot manager to get a crypto_skcipher
* for the algorithm and key specified for this bio.
*/
if (bio_crypt_ctx_acquire_keyslot(bio, blk_crypto_ksm)) {
bio->bi_status = BLK_STS_RESOURCE;
goto out_no_keyslot;
}
/* and then allocate an skcipher_request for it */
err = blk_crypto_alloc_cipher_req(bio, &ciph_req, &wait);
if (err)
goto out;
curr_dun = bio_crypt_sw_data_unit_num(bio);
sg_init_table(&sg, 1);
skcipher_request_set_crypt(ciph_req, &sg, &sg, data_unit_size,
iv.bytes);
/* Decrypt each segment in the bio */
__bio_for_each_segment(bv, bio, iter,
bio->bi_crypt_context->crypt_iter) {
struct page *page = bv.bv_page;
sg_set_page(&sg, page, data_unit_size, bv.bv_offset);
/* Decrypt each data unit in the segment */
for (i = 0; i < bv.bv_len; i += data_unit_size) {
memset(&iv, 0, sizeof(iv));
iv.dun = cpu_to_le64(curr_dun);
if (crypto_wait_req(crypto_skcipher_decrypt(ciph_req),
&wait)) {
bio->bi_status = BLK_STS_IOERR;
goto out;
}
curr_dun++;
sg.offset += data_unit_size;
}
}
out:
skcipher_request_free(ciph_req);
bio_crypt_ctx_release_keyslot(bio);
out_no_keyslot:
kmem_cache_free(blk_crypto_work_mem_cache, work_mem);
bio_endio(bio);
}
/* Queue bio for decryption */
static void blk_crypto_queue_decrypt_bio(struct bio *bio)
{
struct work_mem *work_mem =
kmem_cache_zalloc(blk_crypto_work_mem_cache, GFP_ATOMIC);
if (!work_mem) {
bio->bi_status = BLK_STS_RESOURCE;
bio_endio(bio);
return;
}
INIT_WORK(&work_mem->crypto_work, blk_crypto_decrypt_bio);
work_mem->bio = bio;
queue_work(blk_crypto_wq, &work_mem->crypto_work);
}
/**
* blk_crypto_submit_bio - handle submitting bio for inline encryption
*
@ -518,20 +73,20 @@ int blk_crypto_submit_bio(struct bio **bio_ptr)
{
struct bio *bio = *bio_ptr;
struct request_queue *q;
struct bio_crypt_ctx *bc = bio->bi_crypt_context;
int err;
struct bio_crypt_ctx *crypt_ctx;
if (!bio_has_crypt_ctx(bio) || !bio_has_data(bio))
if (!bc || !bio_has_data(bio))
return 0;
/*
* When a read bio is marked for sw decryption, its bi_iter is saved
* so that when we decrypt the bio later, we know what part of it was
* marked for sw decryption (when the bio is passed down after
* When a read bio is marked for fallback decryption, its bi_iter is
* saved so that when we decrypt the bio later, we know what part of it
* was marked for fallback decryption (when the bio is passed down after
* blk_crypto_submit bio, it may be split or advanced so we cannot rely
* on the bi_iter while decrypting in blk_crypto_endio)
*/
if (bio_crypt_swhandled(bio))
if (bio_crypt_fallback_crypted(bc))
return 0;
err = bio_crypt_check_alignment(bio);
@ -540,21 +95,22 @@ int blk_crypto_submit_bio(struct bio **bio_ptr)
goto out;
}
crypt_ctx = bio->bi_crypt_context;
q = bio->bi_disk->queue;
if (bio_crypt_has_keyslot(bio)) {
if (bc->bc_ksm) {
/* Key already programmed into device? */
if (q->ksm == crypt_ctx->processing_ksm)
if (q->ksm == bc->bc_ksm)
return 0;
/* Nope, release the existing keyslot. */
bio_crypt_ctx_release_keyslot(bio);
bio_crypt_ctx_release_keyslot(bc);
}
/* Get device keyslot if supported */
if (q->ksm) {
err = bio_crypt_ctx_acquire_keyslot(bio, q->ksm);
if (keyslot_manager_crypto_mode_supported(q->ksm,
bc->bc_key->crypto_mode,
bc->bc_key->data_unit_size)) {
err = bio_crypt_ctx_acquire_keyslot(bc, q->ksm);
if (!err)
return 0;
@ -563,24 +119,10 @@ int blk_crypto_submit_bio(struct bio **bio_ptr)
}
/* Fallback to crypto API */
if (!READ_ONCE(tfms_inited[bio->bi_crypt_context->crypto_mode])) {
err = -EIO;
bio->bi_status = BLK_STS_IOERR;
err = blk_crypto_fallback_submit_bio(bio_ptr);
if (err)
goto out;
}
if (bio_data_dir(bio) == WRITE) {
/* Encrypt the data now */
err = blk_crypto_encrypt_bio(bio_ptr);
if (err)
goto out;
} else {
/* Mark bio as swhandled */
bio->bi_crypt_context->processing_ksm = blk_crypto_ksm;
bio->bi_crypt_context->crypt_iter = bio->bi_iter;
bio->bi_crypt_context->sw_data_unit_num =
bio->bi_crypt_context->data_unit_num;
}
return 0;
out:
bio_endio(*bio_ptr);
@ -590,10 +132,10 @@ out:
/**
* blk_crypto_endio - clean up bio w.r.t inline encryption during bio_endio
*
* @bio - the bio to clean up
* @bio: the bio to clean up
*
* If blk_crypto_submit_bio decided to fallback to crypto API for this
* bio, we queue the bio for decryption into a workqueue and return false,
* If blk_crypto_submit_bio decided to fallback to crypto API for this bio,
* we queue the bio for decryption into a workqueue and return false,
* and call bio_endio(bio) at a later time (after the bio has been decrypted).
*
* If the bio is not to be decrypted by the crypto API, this function releases
@ -604,195 +146,115 @@ out:
*/
bool blk_crypto_endio(struct bio *bio)
{
if (!bio_has_crypt_ctx(bio))
struct bio_crypt_ctx *bc = bio->bi_crypt_context;
if (!bc)
return true;
if (bio_crypt_swhandled(bio)) {
if (bio_crypt_fallback_crypted(bc)) {
/*
* The only bios that are swhandled when they reach here
* are those with bio_data_dir(bio) == READ, since WRITE
* bios that are encrypted by the crypto API fallback are
* handled by blk_crypto_encrypt_endio.
* The only bios who's crypto is handled by the blk-crypto
* fallback when they reach here are those with
* bio_data_dir(bio) == READ, since WRITE bios that are
* encrypted by the crypto API fallback are handled by
* blk_crypto_encrypt_endio().
*/
/* If there was an IO error, don't decrypt. */
if (bio->bi_status)
return true;
blk_crypto_queue_decrypt_bio(bio);
return false;
return !blk_crypto_queue_decrypt_bio(bio);
}
if (bio_crypt_has_keyslot(bio))
bio_crypt_ctx_release_keyslot(bio);
if (bc->bc_keyslot >= 0)
bio_crypt_ctx_release_keyslot(bc);
return true;
}
/**
* blk_crypto_start_using_mode() - Allocate skciphers for a
* mode_num for all keyslots
* @mode_num - the blk_crypto_mode we want to allocate ciphers for.
* blk_crypto_init_key() - Prepare a key for use with blk-crypto
* @blk_key: Pointer to the blk_crypto_key to initialize.
* @raw_key: Pointer to the raw key.
* @raw_key_size: Size of raw key. Must be at least the required size for the
* chosen @crypto_mode; see blk_crypto_modes[]. (It's allowed
* to be longer than the mode's actual key size, in order to
* support inline encryption hardware that accepts wrapped keys.
* @is_hw_wrapped has to be set for such keys)
* @is_hw_wrapped: Denotes @raw_key is wrapped.
* @crypto_mode: identifier for the encryption algorithm to use
* @data_unit_size: the data unit size to use for en/decryption
*
* Upper layers (filesystems) should call this function to ensure that a
* the crypto API fallback has transforms for this algorithm, if they become
* necessary.
*
* Return: 0 on success and -err on error.
* Return: The blk_crypto_key that was prepared, or an ERR_PTR() on error. When
* done using the key, it must be freed with blk_crypto_free_key().
*/
int blk_crypto_start_using_mode(enum blk_crypto_mode_num mode_num,
unsigned int data_unit_size,
struct request_queue *q)
int blk_crypto_init_key(struct blk_crypto_key *blk_key,
const u8 *raw_key, unsigned int raw_key_size,
bool is_hw_wrapped,
enum blk_crypto_mode_num crypto_mode,
unsigned int data_unit_size)
{
struct blk_crypto_keyslot *slotp;
int err = 0;
int i;
const struct blk_crypto_mode *mode;
static siphash_key_t hash_key;
/*
* Fast path
* Ensure that updates to blk_crypto_keyslots[i].tfms[mode_num]
* for each i are visible before we try to access them.
*/
if (likely(smp_load_acquire(&tfms_inited[mode_num])))
return 0;
memset(blk_key, 0, sizeof(*blk_key));
/*
* If the keyslot manager of the request queue supports this
* crypto mode, then we don't need to allocate this mode.
*/
if (keyslot_manager_crypto_mode_supported(q->ksm, mode_num,
data_unit_size)) {
return 0;
if (crypto_mode >= ARRAY_SIZE(blk_crypto_modes))
return -EINVAL;
BUILD_BUG_ON(BLK_CRYPTO_MAX_WRAPPED_KEY_SIZE < BLK_CRYPTO_MAX_KEY_SIZE);
mode = &blk_crypto_modes[crypto_mode];
if (is_hw_wrapped) {
if (raw_key_size < mode->keysize ||
raw_key_size > BLK_CRYPTO_MAX_WRAPPED_KEY_SIZE)
return -EINVAL;
} else {
if (raw_key_size != mode->keysize)
return -EINVAL;
}
mutex_lock(&tfms_lock[mode_num]);
if (likely(tfms_inited[mode_num]))
goto out;
if (!is_power_of_2(data_unit_size))
return -EINVAL;
for (i = 0; i < blk_crypto_num_keyslots; i++) {
slotp = &blk_crypto_keyslots[i];
slotp->tfms[mode_num] = crypto_alloc_skcipher(
blk_crypto_modes[mode_num].cipher_str,
0, 0);
if (IS_ERR(slotp->tfms[mode_num])) {
err = PTR_ERR(slotp->tfms[mode_num]);
slotp->tfms[mode_num] = NULL;
goto out_free_tfms;
}
crypto_skcipher_set_flags(slotp->tfms[mode_num],
CRYPTO_TFM_REQ_FORBID_WEAK_KEYS);
}
blk_key->crypto_mode = crypto_mode;
blk_key->data_unit_size = data_unit_size;
blk_key->data_unit_size_bits = ilog2(data_unit_size);
blk_key->size = raw_key_size;
blk_key->is_hw_wrapped = is_hw_wrapped;
memcpy(blk_key->raw, raw_key, raw_key_size);
/*
* Ensure that updates to blk_crypto_keyslots[i].tfms[mode_num]
* for each i are visible before we set tfms_inited[mode_num].
* The keyslot manager uses the SipHash of the key to implement O(1) key
* lookups while avoiding leaking information about the keys. It's
* precomputed here so that it only needs to be computed once per key.
*/
smp_store_release(&tfms_inited[mode_num], true);
goto out;
get_random_once(&hash_key, sizeof(hash_key));
blk_key->hash = siphash(raw_key, raw_key_size, &hash_key);
out_free_tfms:
for (i = 0; i < blk_crypto_num_keyslots; i++) {
slotp = &blk_crypto_keyslots[i];
crypto_free_skcipher(slotp->tfms[mode_num]);
slotp->tfms[mode_num] = NULL;
}
out:
mutex_unlock(&tfms_lock[mode_num]);
return err;
return 0;
}
EXPORT_SYMBOL(blk_crypto_start_using_mode);
EXPORT_SYMBOL_GPL(blk_crypto_init_key);
/**
* blk_crypto_evict_key() - Evict a key from any inline encryption hardware
* it may have been programmed into
* @q - The request queue who's keyslot manager this key might have been
* programmed into
* @key - The key to evict
* @mode - The blk_crypto_mode_num used with this key
* @data_unit_size - The data unit size used with this key
* @q: The request queue who's keyslot manager this key might have been
* programmed into
* @key: The key to evict
*
* Upper layers (filesystems) should call this function to ensure that a key
* is evicted from hardware that it might have been programmed into. This
* will call keyslot_manager_evict_key on the queue's keyslot manager, if one
* exists, and supports the crypto algorithm with the specified data unit size.
* Otherwise, it will evict the key from the blk_crypto_ksm.
* Otherwise, it will evict the key from the blk-crypto-fallback's ksm.
*
* Return: 0 on success, -err on error.
*/
int blk_crypto_evict_key(struct request_queue *q, const u8 *key,
enum blk_crypto_mode_num mode,
unsigned int data_unit_size)
int blk_crypto_evict_key(struct request_queue *q,
const struct blk_crypto_key *key)
{
struct keyslot_manager *ksm = blk_crypto_ksm;
if (q->ksm &&
keyslot_manager_crypto_mode_supported(q->ksm, key->crypto_mode,
key->data_unit_size))
return keyslot_manager_evict_key(q->ksm, key);
if (q && q->ksm && keyslot_manager_crypto_mode_supported(q->ksm, mode,
data_unit_size)) {
ksm = q->ksm;
}
return keyslot_manager_evict_key(ksm, key, mode, data_unit_size);
}
EXPORT_SYMBOL(blk_crypto_evict_key);
int __init blk_crypto_init(void)
{
int i;
int err = -ENOMEM;
prandom_bytes(blank_key, BLK_CRYPTO_MAX_KEY_SIZE);
blk_crypto_ksm = keyslot_manager_create(blk_crypto_num_keyslots,
&blk_crypto_ksm_ll_ops,
NULL);
if (!blk_crypto_ksm)
goto out;
blk_crypto_wq = alloc_workqueue("blk_crypto_wq",
WQ_UNBOUND | WQ_HIGHPRI |
WQ_MEM_RECLAIM,
num_online_cpus());
if (!blk_crypto_wq)
goto out_free_ksm;
blk_crypto_keyslots = kcalloc(blk_crypto_num_keyslots,
sizeof(*blk_crypto_keyslots),
GFP_KERNEL);
if (!blk_crypto_keyslots)
goto out_free_workqueue;
for (i = 0; i < blk_crypto_num_keyslots; i++) {
blk_crypto_keyslots[i].crypto_mode =
BLK_ENCRYPTION_MODE_INVALID;
}
for (i = 0; i < ARRAY_SIZE(blk_crypto_modes); i++)
mutex_init(&tfms_lock[i]);
blk_crypto_page_pool =
mempool_create_page_pool(num_prealloc_bounce_pg, 0);
if (!blk_crypto_page_pool)
goto out_free_keyslots;
blk_crypto_work_mem_cache = KMEM_CACHE(work_mem, SLAB_RECLAIM_ACCOUNT);
if (!blk_crypto_work_mem_cache)
goto out_free_page_pool;
return 0;
out_free_page_pool:
mempool_destroy(blk_crypto_page_pool);
blk_crypto_page_pool = NULL;
out_free_keyslots:
kzfree(blk_crypto_keyslots);
blk_crypto_keyslots = NULL;
out_free_workqueue:
destroy_workqueue(blk_crypto_wq);
blk_crypto_wq = NULL;
out_free_ksm:
keyslot_manager_destroy(blk_crypto_ksm);
blk_crypto_ksm = NULL;
out:
pr_warn("No memory for blk-crypto crypto API fallback.");
return err;
return blk_crypto_fallback_evict_key(key);
}
EXPORT_SYMBOL_GPL(blk_crypto_evict_key);

38
block/blk-merge.c
View file

@ -557,9 +557,6 @@ static inline int ll_new_hw_segment(struct request *req, struct bio *bio,
if (blk_integrity_merge_bio(req->q, req, bio) == false)
goto no_merge;
if (WARN_ON_ONCE(!bio_crypt_ctx_compatible(bio, req->bio)))
goto no_merge;
/*
* This will form the start of a new hw segment. Bump both
* counters.
@ -584,6 +581,8 @@ int ll_back_merge_fn(struct request *req, struct bio *bio, unsigned int nr_segs)
req_set_nomerge(req->q, req);
return 0;
}
if (!bio_crypt_ctx_mergeable(req->bio, blk_rq_bytes(req), bio))
return 0;
return ll_new_hw_segment(req, bio, nr_segs);
}
@ -600,6 +599,8 @@ int ll_front_merge_fn(struct request *req, struct bio *bio, unsigned int nr_segs
req_set_nomerge(req->q, req);
return 0;
}
if (!bio_crypt_ctx_mergeable(bio, bio->bi_iter.bi_size, req->bio))
return 0;
return ll_new_hw_segment(req, bio, nr_segs);
}
@ -644,6 +645,9 @@ static int ll_merge_requests_fn(struct request_queue *q, struct request *req,
if (blk_integrity_merge_rq(q, req, next) == false)
return 0;
if (!bio_crypt_ctx_mergeable(req->bio, blk_rq_bytes(req), next->bio))
return 0;
/* Merge is OK... */
req->nr_phys_segments = total_phys_segments;
return 1;
@ -714,14 +718,8 @@ static enum elv_merge blk_try_req_merge(struct request *req,
{
if (blk_discard_mergable(req))
return ELEVATOR_DISCARD_MERGE;
else if (blk_rq_pos(req) + blk_rq_sectors(req) == blk_rq_pos(next)) {
if (!bio_crypt_ctx_back_mergeable(req->bio,
blk_rq_sectors(req),
next->bio)) {
return ELEVATOR_NO_MERGE;
}
else if (blk_rq_pos(req) + blk_rq_sectors(req) == blk_rq_pos(next))
return ELEVATOR_BACK_MERGE;
}
return ELEVATOR_NO_MERGE;
}
@ -757,9 +755,6 @@ static struct request *attempt_merge(struct request_queue *q,
if (req->ioprio != next->ioprio)
return NULL;
if (!bio_crypt_ctx_compatible(req->bio, next->bio))
return NULL;
/*
* If we are allowed to merge, then append bio list
* from next to rq and release next. merge_requests_fn
@ -901,22 +896,11 @@ bool blk_rq_merge_ok(struct request *rq, struct bio *bio)
enum elv_merge blk_try_merge(struct request *rq, struct bio *bio)
{
if (blk_discard_mergable(rq)) {
if (blk_discard_mergable(rq))
return ELEVATOR_DISCARD_MERGE;
} else if (blk_rq_pos(rq) + blk_rq_sectors(rq) ==
bio->bi_iter.bi_sector) {
if (!bio_crypt_ctx_back_mergeable(rq->bio,
blk_rq_sectors(rq), bio)) {
return ELEVATOR_NO_MERGE;
}
else if (blk_rq_pos(rq) + blk_rq_sectors(rq) == bio->bi_iter.bi_sector)
return ELEVATOR_BACK_MERGE;
} else if (blk_rq_pos(rq) - bio_sectors(bio) ==
bio->bi_iter.bi_sector) {
if (!bio_crypt_ctx_back_mergeable(bio,
bio_sectors(bio), rq->bio)) {
return ELEVATOR_NO_MERGE;
}
else if (blk_rq_pos(rq) - bio_sectors(bio) == bio->bi_iter.bi_sector)
return ELEVATOR_FRONT_MERGE;
}
return ELEVATOR_NO_MERGE;
}

5
block/bounce.c
View file

@ -267,10 +267,7 @@ static struct bio *bounce_clone_bio(struct bio *bio_src, gfp_t gfp_mask,
break;
}
if (bio_crypt_clone(bio, bio_src, gfp_mask) < 0) {
bio_put(bio);
return NULL;
}
bio_crypt_clone(bio, bio_src, gfp_mask);
if (bio_integrity(bio_src) &&
bio_integrity_clone(bio, bio_src, gfp_mask) < 0) {

389
block/keyslot-manager.c
View file

@ -1,7 +1,5 @@
// SPDX-License-Identifier: GPL-2.0
/*
* keyslot-manager.c
*
* Copyright 2019 Google LLC
*/
@ -27,6 +25,7 @@
* Upper layers will call keyslot_manager_get_slot_for_key() to program a
* key into some slot in the inline encryption hardware.
*/
#include <crypto/algapi.h>
#include <linux/keyslot-manager.h>
#include <linux/atomic.h>
#include <linux/mutex.h>
@ -36,12 +35,14 @@
struct keyslot {
atomic_t slot_refs;
struct list_head idle_slot_node;
struct hlist_node hash_node;
struct blk_crypto_key key;
};
struct keyslot_manager {
unsigned int num_slots;
atomic_t num_idle_slots;
struct keyslot_mgmt_ll_ops ksm_ll_ops;
unsigned int crypto_mode_supported[BLK_ENCRYPTION_MODE_MAX];
void *ll_priv_data;
/* Protects programming and evicting keys from the device */
@ -52,16 +53,37 @@ struct keyslot_manager {
struct list_head idle_slots;
spinlock_t idle_slots_lock;
/*
* Hash table which maps key hashes to keyslots, so that we can find a
* key's keyslot in O(1) time rather than O(num_slots). Protected by
* 'lock'. A cryptographic hash function is used so that timing attacks
* can't leak information about the raw keys.
*/
struct hlist_head *slot_hashtable;
unsigned int slot_hashtable_size;
/* Per-keyslot data */
struct keyslot slots[];
};
static inline bool keyslot_manager_is_passthrough(struct keyslot_manager *ksm)
{
return ksm->num_slots == 0;
}
/**
* keyslot_manager_create() - Create a keyslot manager
* @num_slots: The number of key slots to manage.
* @ksm_ll_ops: The struct keyslot_mgmt_ll_ops for the device that this keyslot
* manager will use to perform operations like programming and
* evicting keys.
* @crypto_mode_supported: Array of size BLK_ENCRYPTION_MODE_MAX of
* bitmasks that represents whether a crypto mode
* and data unit size are supported. The i'th bit
* of crypto_mode_supported[crypto_mode] is set iff
* a data unit size of (1 << i) is supported. We
* only support data unit sizes that are powers of
* 2.
* @ll_priv_data: Private data passed as is to the functions in ksm_ll_ops.
*
* Allocate memory for and initialize a keyslot manager. Called by e.g.
@ -71,20 +93,20 @@ struct keyslot_manager {
* Return: Pointer to constructed keyslot manager or NULL on error.
*/
struct keyslot_manager *keyslot_manager_create(unsigned int num_slots,
const struct keyslot_mgmt_ll_ops *ksm_ll_ops,
void *ll_priv_data)
const struct keyslot_mgmt_ll_ops *ksm_ll_ops,
const unsigned int crypto_mode_supported[BLK_ENCRYPTION_MODE_MAX],
void *ll_priv_data)
{
struct keyslot_manager *ksm;
int slot;
unsigned int slot;
unsigned int i;
if (num_slots == 0)
return NULL;
/* Check that all ops are specified */
if (ksm_ll_ops->keyslot_program == NULL ||
ksm_ll_ops->keyslot_evict == NULL ||
ksm_ll_ops->crypto_mode_supported == NULL ||
ksm_ll_ops->keyslot_find == NULL)
ksm_ll_ops->keyslot_evict == NULL)
return NULL;
ksm = kvzalloc(struct_size(ksm, slots, num_slots), GFP_KERNEL);
@ -92,8 +114,9 @@ struct keyslot_manager *keyslot_manager_create(unsigned int num_slots,
return NULL;
ksm->num_slots = num_slots;
atomic_set(&ksm->num_idle_slots, num_slots);
ksm->ksm_ll_ops = *ksm_ll_ops;
memcpy(ksm->crypto_mode_supported, crypto_mode_supported,
sizeof(ksm->crypto_mode_supported));
ksm->ll_priv_data = ll_priv_data;
init_rwsem(&ksm->lock);
@ -108,9 +131,29 @@ struct keyslot_manager *keyslot_manager_create(unsigned int num_slots,
spin_lock_init(&ksm->idle_slots_lock);
ksm->slot_hashtable_size = roundup_pow_of_two(num_slots);
ksm->slot_hashtable = kvmalloc_array(ksm->slot_hashtable_size,
sizeof(ksm->slot_hashtable[0]),
GFP_KERNEL);
if (!ksm->slot_hashtable)
goto err_free_ksm;
for (i = 0; i < ksm->slot_hashtable_size; i++)
INIT_HLIST_HEAD(&ksm->slot_hashtable[i]);
return ksm;
err_free_ksm:
keyslot_manager_destroy(ksm);
return NULL;
}
EXPORT_SYMBOL_GPL(keyslot_manager_create);
static inline struct hlist_head *
hash_bucket_for_key(struct keyslot_manager *ksm,
const struct blk_crypto_key *key)
{
return &ksm->slot_hashtable[key->hash & (ksm->slot_hashtable_size - 1)];
}
EXPORT_SYMBOL(keyslot_manager_create);
static void remove_slot_from_lru_list(struct keyslot_manager *ksm, int slot)
{
@ -119,22 +162,33 @@ static void remove_slot_from_lru_list(struct keyslot_manager *ksm, int slot)
spin_lock_irqsave(&ksm->idle_slots_lock, flags);
list_del(&ksm->slots[slot].idle_slot_node);
spin_unlock_irqrestore(&ksm->idle_slots_lock, flags);
atomic_dec(&ksm->num_idle_slots);
}
static int find_and_grab_keyslot(struct keyslot_manager *ksm, const u8 *key,
enum blk_crypto_mode_num crypto_mode,
unsigned int data_unit_size)
static int find_keyslot(struct keyslot_manager *ksm,
const struct blk_crypto_key *key)
{
const struct hlist_head *head = hash_bucket_for_key(ksm, key);
const struct keyslot *slotp;
hlist_for_each_entry(slotp, head, hash_node) {
if (slotp->key.hash == key->hash &&
slotp->key.crypto_mode == key->crypto_mode &&
slotp->key.size == key->size &&
slotp->key.data_unit_size == key->data_unit_size &&
!crypto_memneq(slotp->key.raw, key->raw, key->size))
return slotp - ksm->slots;
}
return -ENOKEY;
}
static int find_and_grab_keyslot(struct keyslot_manager *ksm,
const struct blk_crypto_key *key)
{
int slot;
slot = ksm->ksm_ll_ops.keyslot_find(ksm->ll_priv_data, key,
crypto_mode, data_unit_size);
slot = find_keyslot(ksm, key);
if (slot < 0)
return slot;
if (WARN_ON(slot >= ksm->num_slots))
return -EINVAL;
if (atomic_inc_return(&ksm->slots[slot].slot_refs) == 1) {
/* Took first reference to this slot; remove it from LRU list */
remove_slot_from_lru_list(ksm, slot);
@ -145,37 +199,35 @@ static int find_and_grab_keyslot(struct keyslot_manager *ksm, const u8 *key,
/**
* keyslot_manager_get_slot_for_key() - Program a key into a keyslot.
* @ksm: The keyslot manager to program the key into.
* @key: Pointer to the bytes of the key to program. Must be the correct length
* for the chosen @crypto_mode; see blk_crypto_modes in blk-crypto.c.
* @crypto_mode: Identifier for the encryption algorithm to use.
* @data_unit_size: The data unit size to use for en/decryption.
* @key: Pointer to the key object to program, including the raw key, crypto
* mode, and data unit size.
*
* Get a keyslot that's been programmed with the specified key, crypto_mode, and
* data_unit_size. If one already exists, return it with incremented refcount.
* Otherwise, wait for a keyslot to become idle and program it.
* Get a keyslot that's been programmed with the specified key. If one already
* exists, return it with incremented refcount. Otherwise, wait for a keyslot
* to become idle and program it.
*
* Context: Process context. Takes and releases ksm->lock.
* Return: The keyslot on success, else a -errno value.
*/
int keyslot_manager_get_slot_for_key(struct keyslot_manager *ksm,
const u8 *key,
enum blk_crypto_mode_num crypto_mode,
unsigned int data_unit_size)
const struct blk_crypto_key *key)
{
int slot;
int err;
struct keyslot *idle_slot;
if (keyslot_manager_is_passthrough(ksm))
return 0;
down_read(&ksm->lock);
slot = find_and_grab_keyslot(ksm, key, crypto_mode, data_unit_size);
slot = find_and_grab_keyslot(ksm, key);
up_read(&ksm->lock);
if (slot != -ENOKEY)
return slot;
for (;;) {
down_write(&ksm->lock);
slot = find_and_grab_keyslot(ksm, key, crypto_mode,
data_unit_size);
slot = find_and_grab_keyslot(ksm, key);
if (slot != -ENOKEY) {
up_write(&ksm->lock);
return slot;
@ -185,42 +237,43 @@ int keyslot_manager_get_slot_for_key(struct keyslot_manager *ksm,
* If we're here, that means there wasn't a slot that was
* already programmed with the key. So try to program it.
*/
if (atomic_read(&ksm->num_idle_slots) > 0)
if (!list_empty(&ksm->idle_slots))
break;
up_write(&ksm->lock);
wait_event(ksm->idle_slots_wait_queue,
(atomic_read(&ksm->num_idle_slots) > 0));
!list_empty(&ksm->idle_slots));
}
idle_slot = list_first_entry(&ksm->idle_slots, struct keyslot,
idle_slot_node);
slot = idle_slot - ksm->slots;
err = ksm->ksm_ll_ops.keyslot_program(ksm->ll_priv_data, key,
crypto_mode,
data_unit_size,
slot);
err = ksm->ksm_ll_ops.keyslot_program(ksm, key, slot);
if (err) {
wake_up(&ksm->idle_slots_wait_queue);
up_write(&ksm->lock);
return err;
}
atomic_set(&ksm->slots[slot].slot_refs, 1);
/* Move this slot to the hash list for the new key. */
if (idle_slot->key.crypto_mode != BLK_ENCRYPTION_MODE_INVALID)
hlist_del(&idle_slot->hash_node);
hlist_add_head(&idle_slot->hash_node, hash_bucket_for_key(ksm, key));
atomic_set(&idle_slot->slot_refs, 1);
idle_slot->key = *key;
remove_slot_from_lru_list(ksm, slot);
up_write(&ksm->lock);
return slot;
}
EXPORT_SYMBOL(keyslot_manager_get_slot_for_key);
/**
* keyslot_manager_get_slot() - Increment the refcount on the specified slot.
* @ksm - The keyslot manager that we want to modify.
* @slot - The slot to increment the refcount of.
* @ksm: The keyslot manager that we want to modify.
* @slot: The slot to increment the refcount of.
*
* This function assumes that there is already an active reference to that slot
* and simply increments the refcount. This is useful when cloning a bio that
@ -231,12 +284,14 @@ EXPORT_SYMBOL(keyslot_manager_get_slot_for_key);
*/
void keyslot_manager_get_slot(struct keyslot_manager *ksm, unsigned int slot)
{
if (keyslot_manager_is_passthrough(ksm))
return;
if (WARN_ON(slot >= ksm->num_slots))
return;
WARN_ON(atomic_inc_return(&ksm->slots[slot].slot_refs) < 2);
}
EXPORT_SYMBOL(keyslot_manager_get_slot);
/**
* keyslot_manager_put_slot() - Release a reference to a slot
@ -249,6 +304,9 @@ void keyslot_manager_put_slot(struct keyslot_manager *ksm, unsigned int slot)
{
unsigned long flags;
if (keyslot_manager_is_passthrough(ksm))
return;
if (WARN_ON(slot >= ksm->num_slots))
return;
@ -257,19 +315,17 @@ void keyslot_manager_put_slot(struct keyslot_manager *ksm, unsigned int slot)
list_add_tail(&ksm->slots[slot].idle_slot_node,
&ksm->idle_slots);
spin_unlock_irqrestore(&ksm->idle_slots_lock, flags);
atomic_inc(&ksm->num_idle_slots);
wake_up(&ksm->idle_slots_wait_queue);
}
}
EXPORT_SYMBOL(keyslot_manager_put_slot);
/**
* keyslot_manager_crypto_mode_supported() - Find out if a crypto_mode/data
* unit size combination is supported
* by a ksm.
* @ksm - The keyslot manager to check
* @crypto_mode - The crypto mode to check for.
* @data_unit_size - The data_unit_size for the mode.
* @ksm: The keyslot manager to check
* @crypto_mode: The crypto mode to check for.
* @data_unit_size: The data_unit_size for the mode.
*
* Calls and returns the result of the crypto_mode_supported function specified
* by the ksm.
@ -284,69 +340,220 @@ bool keyslot_manager_crypto_mode_supported(struct keyslot_manager *ksm,
{
if (!ksm)
return false;
return ksm->ksm_ll_ops.crypto_mode_supported(ksm->ll_priv_data,
crypto_mode,
data_unit_size);
if (WARN_ON(crypto_mode >= BLK_ENCRYPTION_MODE_MAX))
return false;
if (WARN_ON(!is_power_of_2(data_unit_size)))
return false;
return ksm->crypto_mode_supported[crypto_mode] & data_unit_size;
}
EXPORT_SYMBOL(keyslot_manager_crypto_mode_supported);
bool keyslot_manager_rq_crypto_mode_supported(struct request_queue *q,
enum blk_crypto_mode_num crypto_mode,
unsigned int data_unit_size)
{
return keyslot_manager_crypto_mode_supported(q->ksm, crypto_mode,
data_unit_size);
}
EXPORT_SYMBOL(keyslot_manager_rq_crypto_mode_supported);
/**
* keyslot_manager_evict_key() - Evict a key from the lower layer device.
* @ksm - The keyslot manager to evict from
* @key - The key to evict
* @crypto_mode - The crypto algorithm the key was programmed with.
* @data_unit_size - The data_unit_size the key was programmed with.
* @ksm: The keyslot manager to evict from
* @key: The key to evict
*
* Finds the slot that the specified key, crypto_mode, data_unit_size combo
* was programmed into, and evicts that slot from the lower layer device if
* the refcount on the slot is 0. Returns -EBUSY if the refcount is not 0, and
* -errno on error.
* Find the keyslot that the specified key was programmed into, and evict that
* slot from the lower layer device if that slot is not currently in use.
*
* Context: Process context. Takes and releases ksm->lock.
* Return: 0 on success, -EBUSY if the key is still in use, or another
* -errno value on other error.
*/
int keyslot_manager_evict_key(struct keyslot_manager *ksm,
const u8 *key,
enum blk_crypto_mode_num crypto_mode,
unsigned int data_unit_size)
const struct blk_crypto_key *key)
{
int slot;
int err = 0;
int err;
struct keyslot *slotp;
if (keyslot_manager_is_passthrough(ksm)) {
if (ksm->ksm_ll_ops.keyslot_evict) {
down_write(&ksm->lock);
err = ksm->ksm_ll_ops.keyslot_evict(ksm, key, -1);
up_write(&ksm->lock);
return err;
}
return 0;
}
down_write(&ksm->lock);
slot = ksm->ksm_ll_ops.keyslot_find(ksm->ll_priv_data, key,
crypto_mode,
data_unit_size);
slot = find_keyslot(ksm, key);
if (slot < 0) {
up_write(&ksm->lock);
return slot;
err = slot;
goto out_unlock;
}
slotp = &ksm->slots[slot];
if (atomic_read(&ksm->slots[slot].slot_refs) == 0) {
err = ksm->ksm_ll_ops.keyslot_evict(ksm->ll_priv_data, key,
crypto_mode,
data_unit_size,
slot);
} else {
if (atomic_read(&slotp->slot_refs) != 0) {
err = -EBUSY;
goto out_unlock;
}
err = ksm->ksm_ll_ops.keyslot_evict(ksm, key, slot);
if (err)
goto out_unlock;
hlist_del(&slotp->hash_node);
memzero_explicit(&slotp->key, sizeof(slotp->key));
err = 0;
out_unlock:
up_write(&ksm->lock);
return err;
}
EXPORT_SYMBOL(keyslot_manager_evict_key);
/**
* keyslot_manager_reprogram_all_keys() - Re-program all keyslots.
* @ksm: The keyslot manager
*
* Re-program all keyslots that are supposed to have a key programmed. This is
* intended only for use by drivers for hardware that loses its keys on reset.
*
* Context: Process context. Takes and releases ksm->lock.
*/
void keyslot_manager_reprogram_all_keys(struct keyslot_manager *ksm)
{
unsigned int slot;
if (WARN_ON(keyslot_manager_is_passthrough(ksm)))
return;
down_write(&ksm->lock);
for (slot = 0; slot < ksm->num_slots; slot++) {
const struct keyslot *slotp = &ksm->slots[slot];
int err;
if (slotp->key.crypto_mode == BLK_ENCRYPTION_MODE_INVALID)
continue;
err = ksm->ksm_ll_ops.keyslot_program(ksm, &slotp->key, slot);
WARN_ON(err);
}
up_write(&ksm->lock);
}
EXPORT_SYMBOL_GPL(keyslot_manager_reprogram_all_keys);
/**
* keyslot_manager_private() - return the private data stored with ksm
* @ksm: The keyslot manager
*
* Returns the private data passed to the ksm when it was created.
*/
void *keyslot_manager_private(struct keyslot_manager *ksm)
{
return ksm->ll_priv_data;
}
EXPORT_SYMBOL_GPL(keyslot_manager_private);
void keyslot_manager_destroy(struct keyslot_manager *ksm)
{
kvfree(ksm);
if (ksm) {
kvfree(ksm->slot_hashtable);
memzero_explicit(ksm, struct_size(ksm, slots, ksm->num_slots));
kvfree(ksm);
}
}
EXPORT_SYMBOL(keyslot_manager_destroy);
EXPORT_SYMBOL_GPL(keyslot_manager_destroy);
/**
* keyslot_manager_create_passthrough() - Create a passthrough keyslot manager
* @ksm_ll_ops: The struct keyslot_mgmt_ll_ops
* @crypto_mode_supported: Bitmasks for supported encryption modes
* @ll_priv_data: Private data passed as is to the functions in ksm_ll_ops.
*
* Allocate memory for and initialize a passthrough keyslot manager.
* Called by e.g. storage drivers to set up a keyslot manager in their
* request_queue, when the storage driver wants to manage its keys by itself.
* This is useful for inline encryption hardware that don't have a small fixed
* number of keyslots, and for layered devices.
*
* See keyslot_manager_create() for more details about the parameters.
*
* Context: This function may sleep
* Return: Pointer to constructed keyslot manager or NULL on error.
*/
struct keyslot_manager *keyslot_manager_create_passthrough(
const struct keyslot_mgmt_ll_ops *ksm_ll_ops,
const unsigned int crypto_mode_supported[BLK_ENCRYPTION_MODE_MAX],
void *ll_priv_data)
{
struct keyslot_manager *ksm;
ksm = kzalloc(sizeof(*ksm), GFP_KERNEL);
if (!ksm)
return NULL;
ksm->ksm_ll_ops = *ksm_ll_ops;
memcpy(ksm->crypto_mode_supported, crypto_mode_supported,
sizeof(ksm->crypto_mode_supported));
ksm->ll_priv_data = ll_priv_data;
init_rwsem(&ksm->lock);
return ksm;
}
EXPORT_SYMBOL_GPL(keyslot_manager_create_passthrough);
/**
* keyslot_manager_intersect_modes() - restrict supported modes by child device
* @parent: The keyslot manager for parent device
* @child: The keyslot manager for child device, or NULL
*
* Clear any crypto mode support bits in @parent that aren't set in @child.
* If @child is NULL, then all parent bits are cleared.
*
* Only use this when setting up the keyslot manager for a layered device,
* before it's been exposed yet.
*/
void keyslot_manager_intersect_modes(struct keyslot_manager *parent,
const struct keyslot_manager *child)
{
if (child) {
unsigned int i;
for (i = 0; i < ARRAY_SIZE(child->crypto_mode_supported); i++) {
parent->crypto_mode_supported[i] &=
child->crypto_mode_supported[i];
}
} else {
memset(parent->crypto_mode_supported, 0,
sizeof(parent->crypto_mode_supported));
}
}
EXPORT_SYMBOL_GPL(keyslot_manager_intersect_modes);
/**
* keyslot_manager_derive_raw_secret() - Derive software secret from wrapped key
* @ksm: The keyslot manager
* @wrapped_key: The wrapped key
* @wrapped_key_size: Size of the wrapped key in bytes
* @secret: (output) the software secret
* @secret_size: (output) the number of secret bytes to derive
*
* Given a hardware-wrapped key, ask the hardware to derive a secret which
* software can use for cryptographic tasks other than inline encryption. The
* derived secret is guaranteed to be cryptographically isolated from the key
* with which any inline encryption with this wrapped key would actually be
* done. I.e., both will be derived from the unwrapped key.
*
* Return: 0 on success, -EOPNOTSUPP if hardware-wrapped keys are unsupported,
* or another -errno code.
*/
int keyslot_manager_derive_raw_secret(struct keyslot_manager *ksm,
const u8 *wrapped_key,
unsigned int wrapped_key_size,
u8 *secret, unsigned int secret_size)
{
int err;
down_write(&ksm->lock);
if (ksm->ksm_ll_ops.derive_raw_secret) {
err = ksm->ksm_ll_ops.derive_raw_secret(ksm, wrapped_key,
wrapped_key_size,
secret, secret_size);
} else {
err = -EOPNOTSUPP;
}
up_write(&ksm->lock);
return err;
}
EXPORT_SYMBOL_GPL(keyslot_manager_derive_raw_secret);