mirror of
https://github.com/BobTheBlinker/android_kernel_motorola_sm6375.git
synced 2026-10-29 03:01:39 -04:00
BACKPORT: FROMLIST: Update Inline Encryption from v5 to v6 of patch series
Changes v5 => v6: - Blk-crypto's kernel crypto API fallback is no longer restricted to 8-byte DUNs. It's also now separately configurable from blk-crypto, and can be disabled entirely, while still allowing the kernel to use inline encryption hardware. Further, struct bio_crypt_ctx takes up less space, and no longer contains the information needed by the crypto API fallback - the fallback allocates the required memory when necessary. - Blk-crypto now supports all file content encryption modes supported by fscrypt. - Fixed bio merging logic in blk-merge.c - Fscrypt now supports inline encryption with the direct key policy, since blk-crypto now has support for larger DUNs. - Keyslot manager now uses a hashtable to lookup which keyslot contains any particular key (thanks Eric!) - Fscrypt support for inline encryption now handles filesystems with multiple underlying block devices (thanks Eric!) - Numerous cleanups Bug: 137270441 Test: refer to I26376479ee38259b8c35732cb3a1d7e15f9b05a3 Change-Id: I13e2e327e0b4784b394cb1e7cf32a04856d95f01 Link: https://lore.kernel.org/linux-block/[email protected]/ Signed-off-by: Satya Tangirala <[email protected]>
This commit is contained in:
Satya Tangirala
Alistair Delva
parent
a32426e234
commit
b7b3af9614
35 changed files with 10735 additions and 10918 deletions
|
|
@ -10,219 +10,36 @@
|
|||
#define pr_fmt(fmt) "blk-crypto: " fmt
|
||||
|
||||
#include <linux/blk-crypto.h>
|
||||
#include <linux/blkdev.h>
|
||||
#include <linux/keyslot-manager.h>
|
||||
#include <linux/mempool.h>
|
||||
#include <linux/blk-cgroup.h>
|
||||
#include <linux/crypto.h>
|
||||
#include <crypto/skcipher.h>
|
||||
#include <crypto/algapi.h>
|
||||
#include <linux/module.h>
|
||||
#include <linux/sched/mm.h>
|
||||
#include <linux/random.h>
|
||||
#include <linux/siphash.h>
|
||||
|
||||
/* Represents a crypto mode supported by blk-crypto */
|
||||
struct blk_crypto_mode {
|
||||
const char *cipher_str; /* crypto API name (for fallback case) */
|
||||
size_t keysize; /* key size in bytes */
|
||||
};
|
||||
#include "blk-crypto-internal.h"
|
||||
|
||||
static const struct blk_crypto_mode blk_crypto_modes[] = {
|
||||
const struct blk_crypto_mode blk_crypto_modes[] = {
|
||||
[BLK_ENCRYPTION_MODE_AES_256_XTS] = {
|
||||
.cipher_str = "xts(aes)",
|
||||
.keysize = 64,
|
||||
.ivsize = 16,
|
||||
},
|
||||
[BLK_ENCRYPTION_MODE_AES_128_CBC_ESSIV] = {
|
||||
.cipher_str = "essiv(cbc(aes),sha256)",
|
||||
.keysize = 16,
|
||||
.ivsize = 16,
|
||||
},
|
||||
[BLK_ENCRYPTION_MODE_ADIANTUM] = {
|
||||
.cipher_str = "adiantum(xchacha12,aes)",
|
||||
.keysize = 32,
|
||||
.ivsize = 32,
|
||||
},
|
||||
};
|
||||
|
||||
static unsigned int num_prealloc_bounce_pg = 32;
|
||||
module_param(num_prealloc_bounce_pg, uint, 0);
|
||||
MODULE_PARM_DESC(num_prealloc_bounce_pg,
|
||||
"Number of preallocated bounce pages for blk-crypto to use during crypto API fallback encryption");
|
||||
|
||||
#define BLK_CRYPTO_MAX_KEY_SIZE 64
|
||||
static int blk_crypto_num_keyslots = 100;
|
||||
module_param_named(num_keyslots, blk_crypto_num_keyslots, int, 0);
|
||||
MODULE_PARM_DESC(num_keyslots,
|
||||
"Number of keyslots for crypto API fallback in blk-crypto.");
|
||||
|
||||
static struct blk_crypto_keyslot {
|
||||
struct crypto_skcipher *tfm;
|
||||
enum blk_crypto_mode_num crypto_mode;
|
||||
u8 key[BLK_CRYPTO_MAX_KEY_SIZE];
|
||||
struct crypto_skcipher *tfms[ARRAY_SIZE(blk_crypto_modes)];
|
||||
} *blk_crypto_keyslots;
|
||||
|
||||
/*
|
||||
* Allocating a crypto tfm during I/O can deadlock, so we have to preallocate
|
||||
* all of a mode's tfms when that mode starts being used. Since each mode may
|
||||
* need all the keyslots at some point, each mode needs its own tfm for each
|
||||
* keyslot; thus, a keyslot may contain tfms for multiple modes. However, to
|
||||
* match the behavior of real inline encryption hardware (which only supports a
|
||||
* single encryption context per keyslot), we only allow one tfm per keyslot to
|
||||
* be used at a time - the rest of the unused tfms have their keys cleared.
|
||||
*/
|
||||
static struct mutex tfms_lock[ARRAY_SIZE(blk_crypto_modes)];
|
||||
static bool tfms_inited[ARRAY_SIZE(blk_crypto_modes)];
|
||||
|
||||
struct work_mem {
|
||||
struct work_struct crypto_work;
|
||||
struct bio *bio;
|
||||
};
|
||||
|
||||
/* The following few vars are only used during the crypto API fallback */
|
||||
static struct keyslot_manager *blk_crypto_ksm;
|
||||
static struct workqueue_struct *blk_crypto_wq;
|
||||
static mempool_t *blk_crypto_page_pool;
|
||||
static struct kmem_cache *blk_crypto_work_mem_cache;
|
||||
|
||||
bool bio_crypt_swhandled(struct bio *bio)
|
||||
{
|
||||
return bio_has_crypt_ctx(bio) &&
|
||||
bio->bi_crypt_context->processing_ksm == blk_crypto_ksm;
|
||||
}
|
||||
|
||||
static u8 blank_key[BLK_CRYPTO_MAX_KEY_SIZE];
|
||||
static void evict_keyslot(unsigned int slot)
|
||||
{
|
||||
struct blk_crypto_keyslot *slotp = &blk_crypto_keyslots[slot];
|
||||
enum blk_crypto_mode_num crypto_mode = slotp->crypto_mode;
|
||||
int err;
|
||||
|
||||
WARN_ON(slotp->crypto_mode == BLK_ENCRYPTION_MODE_INVALID);
|
||||
|
||||
/* Clear the key in the skcipher */
|
||||
err = crypto_skcipher_setkey(slotp->tfms[crypto_mode], blank_key,
|
||||
blk_crypto_modes[crypto_mode].keysize);
|
||||
WARN_ON(err);
|
||||
memzero_explicit(slotp->key, BLK_CRYPTO_MAX_KEY_SIZE);
|
||||
slotp->crypto_mode = BLK_ENCRYPTION_MODE_INVALID;
|
||||
}
|
||||
|
||||
static int blk_crypto_keyslot_program(void *priv, const u8 *key,
|
||||
enum blk_crypto_mode_num crypto_mode,
|
||||
unsigned int data_unit_size,
|
||||
unsigned int slot)
|
||||
{
|
||||
struct blk_crypto_keyslot *slotp = &blk_crypto_keyslots[slot];
|
||||
const struct blk_crypto_mode *mode = &blk_crypto_modes[crypto_mode];
|
||||
size_t keysize = mode->keysize;
|
||||
int err;
|
||||
|
||||
if (crypto_mode != slotp->crypto_mode &&
|
||||
slotp->crypto_mode != BLK_ENCRYPTION_MODE_INVALID) {
|
||||
evict_keyslot(slot);
|
||||
}
|
||||
|
||||
if (!slotp->tfms[crypto_mode])
|
||||
return -ENOMEM;
|
||||
slotp->crypto_mode = crypto_mode;
|
||||
err = crypto_skcipher_setkey(slotp->tfms[crypto_mode], key, keysize);
|
||||
|
||||
if (err) {
|
||||
evict_keyslot(slot);
|
||||
return err;
|
||||
}
|
||||
|
||||
memcpy(slotp->key, key, keysize);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int blk_crypto_keyslot_evict(void *priv, const u8 *key,
|
||||
enum blk_crypto_mode_num crypto_mode,
|
||||
unsigned int data_unit_size,
|
||||
unsigned int slot)
|
||||
{
|
||||
evict_keyslot(slot);
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int blk_crypto_keyslot_find(void *priv,
|
||||
const u8 *key,
|
||||
enum blk_crypto_mode_num crypto_mode,
|
||||
unsigned int data_unit_size_bytes)
|
||||
{
|
||||
int slot;
|
||||
const size_t keysize = blk_crypto_modes[crypto_mode].keysize;
|
||||
|
||||
for (slot = 0; slot < blk_crypto_num_keyslots; slot++) {
|
||||
if (blk_crypto_keyslots[slot].crypto_mode == crypto_mode &&
|
||||
!crypto_memneq(blk_crypto_keyslots[slot].key, key, keysize))
|
||||
return slot;
|
||||
}
|
||||
|
||||
return -ENOKEY;
|
||||
}
|
||||
|
||||
static bool blk_crypto_mode_supported(void *priv,
|
||||
enum blk_crypto_mode_num crypt_mode,
|
||||
unsigned int data_unit_size)
|
||||
{
|
||||
/* All blk_crypto_modes are required to have a crypto API fallback. */
|
||||
return true;
|
||||
}
|
||||
|
||||
/*
|
||||
* The crypto API fallback KSM ops - only used for a bio when it specifies a
|
||||
* blk_crypto_mode for which we failed to get a keyslot in the device's inline
|
||||
* encryption hardware (which probably means the device doesn't have inline
|
||||
* encryption hardware that supports that crypto mode).
|
||||
*/
|
||||
static const struct keyslot_mgmt_ll_ops blk_crypto_ksm_ll_ops = {
|
||||
.keyslot_program = blk_crypto_keyslot_program,
|
||||
.keyslot_evict = blk_crypto_keyslot_evict,
|
||||
.keyslot_find = blk_crypto_keyslot_find,
|
||||
.crypto_mode_supported = blk_crypto_mode_supported,
|
||||
};
|
||||
|
||||
static void blk_crypto_encrypt_endio(struct bio *enc_bio)
|
||||
{
|
||||
struct bio *src_bio = enc_bio->bi_private;
|
||||
int i;
|
||||
|
||||
for (i = 0; i < enc_bio->bi_vcnt; i++)
|
||||
mempool_free(enc_bio->bi_io_vec[i].bv_page,
|
||||
blk_crypto_page_pool);
|
||||
|
||||
src_bio->bi_status = enc_bio->bi_status;
|
||||
|
||||
bio_put(enc_bio);
|
||||
bio_endio(src_bio);
|
||||
}
|
||||
|
||||
static struct bio *blk_crypto_clone_bio(struct bio *bio_src)
|
||||
{
|
||||
struct bvec_iter iter;
|
||||
struct bio_vec bv;
|
||||
struct bio *bio;
|
||||
|
||||
bio = bio_alloc_bioset(GFP_NOIO, bio_segments(bio_src), NULL);
|
||||
if (!bio)
|
||||
return NULL;
|
||||
bio->bi_disk = bio_src->bi_disk;
|
||||
bio->bi_opf = bio_src->bi_opf;
|
||||
bio->bi_ioprio = bio_src->bi_ioprio;
|
||||
bio->bi_write_hint = bio_src->bi_write_hint;
|
||||
bio->bi_iter.bi_sector = bio_src->bi_iter.bi_sector;
|
||||
bio->bi_iter.bi_size = bio_src->bi_iter.bi_size;
|
||||
|
||||
bio_for_each_segment(bv, bio_src, iter)
|
||||
bio->bi_io_vec[bio->bi_vcnt++] = bv;
|
||||
|
||||
if (bio_integrity(bio_src) &&
|
||||
bio_integrity_clone(bio, bio_src, GFP_NOIO) < 0) {
|
||||
bio_put(bio);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
bio_clone_blkg_association(bio, bio_src);
|
||||
blkcg_bio_issue_init(bio);
|
||||
|
||||
return bio;
|
||||
}
|
||||
|
||||
/* Check that all I/O segments are data unit aligned */
|
||||
static int bio_crypt_check_alignment(struct bio *bio)
|
||||
{
|
||||
int data_unit_size = 1 << bio->bi_crypt_context->data_unit_size_bits;
|
||||
const unsigned int data_unit_size =
|
||||
bio->bi_crypt_context->bc_key->data_unit_size;
|
||||
struct bvec_iter iter;
|
||||
struct bio_vec bv;
|
||||
|
||||
|
|
@ -233,268 +50,6 @@ static int bio_crypt_check_alignment(struct bio *bio)
|
|||
return 0;
|
||||
}
|
||||
|
||||
static int blk_crypto_alloc_cipher_req(struct bio *src_bio,
|
||||
struct skcipher_request **ciph_req_ptr,
|
||||
struct crypto_wait *wait)
|
||||
{
|
||||
int slot;
|
||||
struct skcipher_request *ciph_req;
|
||||
struct blk_crypto_keyslot *slotp;
|
||||
|
||||
slot = bio_crypt_get_keyslot(src_bio);
|
||||
slotp = &blk_crypto_keyslots[slot];
|
||||
ciph_req = skcipher_request_alloc(slotp->tfms[slotp->crypto_mode],
|
||||
GFP_NOIO);
|
||||
if (!ciph_req) {
|
||||
src_bio->bi_status = BLK_STS_RESOURCE;
|
||||
return -ENOMEM;
|
||||
}
|
||||
|
||||
skcipher_request_set_callback(ciph_req,
|
||||
CRYPTO_TFM_REQ_MAY_BACKLOG |
|
||||
CRYPTO_TFM_REQ_MAY_SLEEP,
|
||||
crypto_req_done, wait);
|
||||
*ciph_req_ptr = ciph_req;
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int blk_crypto_split_bio_if_needed(struct bio **bio_ptr)
|
||||
{
|
||||
struct bio *bio = *bio_ptr;
|
||||
unsigned int i = 0;
|
||||
unsigned int num_sectors = 0;
|
||||
struct bio_vec bv;
|
||||
struct bvec_iter iter;
|
||||
|
||||
bio_for_each_segment(bv, bio, iter) {
|
||||
num_sectors += bv.bv_len >> SECTOR_SHIFT;
|
||||
if (++i == BIO_MAX_PAGES)
|
||||
break;
|
||||
}
|
||||
if (num_sectors < bio_sectors(bio)) {
|
||||
struct bio *split_bio;
|
||||
|
||||
split_bio = bio_split(bio, num_sectors, GFP_NOIO, NULL);
|
||||
if (!split_bio) {
|
||||
bio->bi_status = BLK_STS_RESOURCE;
|
||||
return -ENOMEM;
|
||||
}
|
||||
bio_chain(split_bio, bio);
|
||||
generic_make_request(bio);
|
||||
*bio_ptr = split_bio;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
/*
|
||||
* The crypto API fallback's encryption routine.
|
||||
* Allocate a bounce bio for encryption, encrypt the input bio using
|
||||
* crypto API, and replace *bio_ptr with the bounce bio. May split input
|
||||
* bio if it's too large.
|
||||
*/
|
||||
static int blk_crypto_encrypt_bio(struct bio **bio_ptr)
|
||||
{
|
||||
struct bio *src_bio;
|
||||
struct skcipher_request *ciph_req = NULL;
|
||||
DECLARE_CRYPTO_WAIT(wait);
|
||||
int err = 0;
|
||||
u64 curr_dun;
|
||||
union {
|
||||
__le64 dun;
|
||||
u8 bytes[16];
|
||||
} iv;
|
||||
struct scatterlist src, dst;
|
||||
struct bio *enc_bio;
|
||||
struct bio_vec *enc_bvec;
|
||||
int i, j;
|
||||
int data_unit_size;
|
||||
|
||||
/* Split the bio if it's too big for single page bvec */
|
||||
err = blk_crypto_split_bio_if_needed(bio_ptr);
|
||||
if (err)
|
||||
return err;
|
||||
|
||||
src_bio = *bio_ptr;
|
||||
data_unit_size = 1 << src_bio->bi_crypt_context->data_unit_size_bits;
|
||||
|
||||
/* Allocate bounce bio for encryption */
|
||||
enc_bio = blk_crypto_clone_bio(src_bio);
|
||||
if (!enc_bio) {
|
||||
src_bio->bi_status = BLK_STS_RESOURCE;
|
||||
return -ENOMEM;
|
||||
}
|
||||
|
||||
/*
|
||||
* Use the crypto API fallback keyslot manager to get a crypto_skcipher
|
||||
* for the algorithm and key specified for this bio.
|
||||
*/
|
||||
err = bio_crypt_ctx_acquire_keyslot(src_bio, blk_crypto_ksm);
|
||||
if (err) {
|
||||
src_bio->bi_status = BLK_STS_IOERR;
|
||||
goto out_put_enc_bio;
|
||||
}
|
||||
|
||||
/* and then allocate an skcipher_request for it */
|
||||
err = blk_crypto_alloc_cipher_req(src_bio, &ciph_req, &wait);
|
||||
if (err)
|
||||
goto out_release_keyslot;
|
||||
|
||||
curr_dun = bio_crypt_data_unit_num(src_bio);
|
||||
sg_init_table(&src, 1);
|
||||
sg_init_table(&dst, 1);
|
||||
|
||||
skcipher_request_set_crypt(ciph_req, &src, &dst,
|
||||
data_unit_size, iv.bytes);
|
||||
|
||||
/* Encrypt each page in the bounce bio */
|
||||
for (i = 0, enc_bvec = enc_bio->bi_io_vec; i < enc_bio->bi_vcnt;
|
||||
enc_bvec++, i++) {
|
||||
struct page *plaintext_page = enc_bvec->bv_page;
|
||||
struct page *ciphertext_page =
|
||||
mempool_alloc(blk_crypto_page_pool, GFP_NOIO);
|
||||
|
||||
enc_bvec->bv_page = ciphertext_page;
|
||||
|
||||
if (!ciphertext_page) {
|
||||
src_bio->bi_status = BLK_STS_RESOURCE;
|
||||
err = -ENOMEM;
|
||||
goto out_free_bounce_pages;
|
||||
}
|
||||
|
||||
sg_set_page(&src, plaintext_page, data_unit_size,
|
||||
enc_bvec->bv_offset);
|
||||
sg_set_page(&dst, ciphertext_page, data_unit_size,
|
||||
enc_bvec->bv_offset);
|
||||
|
||||
/* Encrypt each data unit in this page */
|
||||
for (j = 0; j < enc_bvec->bv_len; j += data_unit_size) {
|
||||
memset(&iv, 0, sizeof(iv));
|
||||
iv.dun = cpu_to_le64(curr_dun);
|
||||
|
||||
err = crypto_wait_req(crypto_skcipher_encrypt(ciph_req),
|
||||
&wait);
|
||||
if (err) {
|
||||
i++;
|
||||
src_bio->bi_status = BLK_STS_RESOURCE;
|
||||
goto out_free_bounce_pages;
|
||||
}
|
||||
curr_dun++;
|
||||
src.offset += data_unit_size;
|
||||
dst.offset += data_unit_size;
|
||||
}
|
||||
}
|
||||
|
||||
enc_bio->bi_private = src_bio;
|
||||
enc_bio->bi_end_io = blk_crypto_encrypt_endio;
|
||||
*bio_ptr = enc_bio;
|
||||
|
||||
enc_bio = NULL;
|
||||
err = 0;
|
||||
goto out_free_ciph_req;
|
||||
|
||||
out_free_bounce_pages:
|
||||
while (i > 0)
|
||||
mempool_free(enc_bio->bi_io_vec[--i].bv_page,
|
||||
blk_crypto_page_pool);
|
||||
out_free_ciph_req:
|
||||
skcipher_request_free(ciph_req);
|
||||
out_release_keyslot:
|
||||
bio_crypt_ctx_release_keyslot(src_bio);
|
||||
out_put_enc_bio:
|
||||
if (enc_bio)
|
||||
bio_put(enc_bio);
|
||||
|
||||
return err;
|
||||
}
|
||||
|
||||
/*
|
||||
* The crypto API fallback's main decryption routine.
|
||||
* Decrypts input bio in place.
|
||||
*/
|
||||
static void blk_crypto_decrypt_bio(struct work_struct *w)
|
||||
{
|
||||
struct work_mem *work_mem =
|
||||
container_of(w, struct work_mem, crypto_work);
|
||||
struct bio *bio = work_mem->bio;
|
||||
struct skcipher_request *ciph_req = NULL;
|
||||
DECLARE_CRYPTO_WAIT(wait);
|
||||
struct bio_vec bv;
|
||||
struct bvec_iter iter;
|
||||
u64 curr_dun;
|
||||
union {
|
||||
__le64 dun;
|
||||
u8 bytes[16];
|
||||
} iv;
|
||||
struct scatterlist sg;
|
||||
int data_unit_size = 1 << bio->bi_crypt_context->data_unit_size_bits;
|
||||
int i;
|
||||
int err;
|
||||
|
||||
/*
|
||||
* Use the crypto API fallback keyslot manager to get a crypto_skcipher
|
||||
* for the algorithm and key specified for this bio.
|
||||
*/
|
||||
if (bio_crypt_ctx_acquire_keyslot(bio, blk_crypto_ksm)) {
|
||||
bio->bi_status = BLK_STS_RESOURCE;
|
||||
goto out_no_keyslot;
|
||||
}
|
||||
|
||||
/* and then allocate an skcipher_request for it */
|
||||
err = blk_crypto_alloc_cipher_req(bio, &ciph_req, &wait);
|
||||
if (err)
|
||||
goto out;
|
||||
|
||||
curr_dun = bio_crypt_sw_data_unit_num(bio);
|
||||
sg_init_table(&sg, 1);
|
||||
skcipher_request_set_crypt(ciph_req, &sg, &sg, data_unit_size,
|
||||
iv.bytes);
|
||||
|
||||
/* Decrypt each segment in the bio */
|
||||
__bio_for_each_segment(bv, bio, iter,
|
||||
bio->bi_crypt_context->crypt_iter) {
|
||||
struct page *page = bv.bv_page;
|
||||
|
||||
sg_set_page(&sg, page, data_unit_size, bv.bv_offset);
|
||||
|
||||
/* Decrypt each data unit in the segment */
|
||||
for (i = 0; i < bv.bv_len; i += data_unit_size) {
|
||||
memset(&iv, 0, sizeof(iv));
|
||||
iv.dun = cpu_to_le64(curr_dun);
|
||||
if (crypto_wait_req(crypto_skcipher_decrypt(ciph_req),
|
||||
&wait)) {
|
||||
bio->bi_status = BLK_STS_IOERR;
|
||||
goto out;
|
||||
}
|
||||
curr_dun++;
|
||||
sg.offset += data_unit_size;
|
||||
}
|
||||
}
|
||||
|
||||
out:
|
||||
skcipher_request_free(ciph_req);
|
||||
bio_crypt_ctx_release_keyslot(bio);
|
||||
out_no_keyslot:
|
||||
kmem_cache_free(blk_crypto_work_mem_cache, work_mem);
|
||||
bio_endio(bio);
|
||||
}
|
||||
|
||||
/* Queue bio for decryption */
|
||||
static void blk_crypto_queue_decrypt_bio(struct bio *bio)
|
||||
{
|
||||
struct work_mem *work_mem =
|
||||
kmem_cache_zalloc(blk_crypto_work_mem_cache, GFP_ATOMIC);
|
||||
|
||||
if (!work_mem) {
|
||||
bio->bi_status = BLK_STS_RESOURCE;
|
||||
bio_endio(bio);
|
||||
return;
|
||||
}
|
||||
|
||||
INIT_WORK(&work_mem->crypto_work, blk_crypto_decrypt_bio);
|
||||
work_mem->bio = bio;
|
||||
queue_work(blk_crypto_wq, &work_mem->crypto_work);
|
||||
}
|
||||
|
||||
/**
|
||||
* blk_crypto_submit_bio - handle submitting bio for inline encryption
|
||||
*
|
||||
|
|
@ -518,20 +73,20 @@ int blk_crypto_submit_bio(struct bio **bio_ptr)
|
|||
{
|
||||
struct bio *bio = *bio_ptr;
|
||||
struct request_queue *q;
|
||||
struct bio_crypt_ctx *bc = bio->bi_crypt_context;
|
||||
int err;
|
||||
struct bio_crypt_ctx *crypt_ctx;
|
||||
|
||||
if (!bio_has_crypt_ctx(bio) || !bio_has_data(bio))
|
||||
if (!bc || !bio_has_data(bio))
|
||||
return 0;
|
||||
|
||||
/*
|
||||
* When a read bio is marked for sw decryption, its bi_iter is saved
|
||||
* so that when we decrypt the bio later, we know what part of it was
|
||||
* marked for sw decryption (when the bio is passed down after
|
||||
* When a read bio is marked for fallback decryption, its bi_iter is
|
||||
* saved so that when we decrypt the bio later, we know what part of it
|
||||
* was marked for fallback decryption (when the bio is passed down after
|
||||
* blk_crypto_submit bio, it may be split or advanced so we cannot rely
|
||||
* on the bi_iter while decrypting in blk_crypto_endio)
|
||||
*/
|
||||
if (bio_crypt_swhandled(bio))
|
||||
if (bio_crypt_fallback_crypted(bc))
|
||||
return 0;
|
||||
|
||||
err = bio_crypt_check_alignment(bio);
|
||||
|
|
@ -540,21 +95,22 @@ int blk_crypto_submit_bio(struct bio **bio_ptr)
|
|||
goto out;
|
||||
}
|
||||
|
||||
crypt_ctx = bio->bi_crypt_context;
|
||||
q = bio->bi_disk->queue;
|
||||
|
||||
if (bio_crypt_has_keyslot(bio)) {
|
||||
if (bc->bc_ksm) {
|
||||
/* Key already programmed into device? */
|
||||
if (q->ksm == crypt_ctx->processing_ksm)
|
||||
if (q->ksm == bc->bc_ksm)
|
||||
return 0;
|
||||
|
||||
/* Nope, release the existing keyslot. */
|
||||
bio_crypt_ctx_release_keyslot(bio);
|
||||
bio_crypt_ctx_release_keyslot(bc);
|
||||
}
|
||||
|
||||
/* Get device keyslot if supported */
|
||||
if (q->ksm) {
|
||||
err = bio_crypt_ctx_acquire_keyslot(bio, q->ksm);
|
||||
if (keyslot_manager_crypto_mode_supported(q->ksm,
|
||||
bc->bc_key->crypto_mode,
|
||||
bc->bc_key->data_unit_size)) {
|
||||
err = bio_crypt_ctx_acquire_keyslot(bc, q->ksm);
|
||||
if (!err)
|
||||
return 0;
|
||||
|
||||
|
|
@ -563,24 +119,10 @@ int blk_crypto_submit_bio(struct bio **bio_ptr)
|
|||
}
|
||||
|
||||
/* Fallback to crypto API */
|
||||
if (!READ_ONCE(tfms_inited[bio->bi_crypt_context->crypto_mode])) {
|
||||
err = -EIO;
|
||||
bio->bi_status = BLK_STS_IOERR;
|
||||
err = blk_crypto_fallback_submit_bio(bio_ptr);
|
||||
if (err)
|
||||
goto out;
|
||||
}
|
||||
|
||||
if (bio_data_dir(bio) == WRITE) {
|
||||
/* Encrypt the data now */
|
||||
err = blk_crypto_encrypt_bio(bio_ptr);
|
||||
if (err)
|
||||
goto out;
|
||||
} else {
|
||||
/* Mark bio as swhandled */
|
||||
bio->bi_crypt_context->processing_ksm = blk_crypto_ksm;
|
||||
bio->bi_crypt_context->crypt_iter = bio->bi_iter;
|
||||
bio->bi_crypt_context->sw_data_unit_num =
|
||||
bio->bi_crypt_context->data_unit_num;
|
||||
}
|
||||
return 0;
|
||||
out:
|
||||
bio_endio(*bio_ptr);
|
||||
|
|
@ -590,10 +132,10 @@ out:
|
|||
/**
|
||||
* blk_crypto_endio - clean up bio w.r.t inline encryption during bio_endio
|
||||
*
|
||||
* @bio - the bio to clean up
|
||||
* @bio: the bio to clean up
|
||||
*
|
||||
* If blk_crypto_submit_bio decided to fallback to crypto API for this
|
||||
* bio, we queue the bio for decryption into a workqueue and return false,
|
||||
* If blk_crypto_submit_bio decided to fallback to crypto API for this bio,
|
||||
* we queue the bio for decryption into a workqueue and return false,
|
||||
* and call bio_endio(bio) at a later time (after the bio has been decrypted).
|
||||
*
|
||||
* If the bio is not to be decrypted by the crypto API, this function releases
|
||||
|
|
@ -604,195 +146,97 @@ out:
|
|||
*/
|
||||
bool blk_crypto_endio(struct bio *bio)
|
||||
{
|
||||
if (!bio_has_crypt_ctx(bio))
|
||||
struct bio_crypt_ctx *bc = bio->bi_crypt_context;
|
||||
|
||||
if (!bc)
|
||||
return true;
|
||||
|
||||
if (bio_crypt_swhandled(bio)) {
|
||||
if (bio_crypt_fallback_crypted(bc)) {
|
||||
/*
|
||||
* The only bios that are swhandled when they reach here
|
||||
* are those with bio_data_dir(bio) == READ, since WRITE
|
||||
* bios that are encrypted by the crypto API fallback are
|
||||
* handled by blk_crypto_encrypt_endio.
|
||||
* The only bios who's crypto is handled by the blk-crypto
|
||||
* fallback when they reach here are those with
|
||||
* bio_data_dir(bio) == READ, since WRITE bios that are
|
||||
* encrypted by the crypto API fallback are handled by
|
||||
* blk_crypto_encrypt_endio().
|
||||
*/
|
||||
|
||||
/* If there was an IO error, don't decrypt. */
|
||||
if (bio->bi_status)
|
||||
return true;
|
||||
|
||||
blk_crypto_queue_decrypt_bio(bio);
|
||||
return false;
|
||||
return !blk_crypto_queue_decrypt_bio(bio);
|
||||
}
|
||||
|
||||
if (bio_crypt_has_keyslot(bio))
|
||||
bio_crypt_ctx_release_keyslot(bio);
|
||||
if (bc->bc_keyslot >= 0)
|
||||
bio_crypt_ctx_release_keyslot(bc);
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* blk_crypto_start_using_mode() - Allocate skciphers for a
|
||||
* mode_num for all keyslots
|
||||
* @mode_num - the blk_crypto_mode we want to allocate ciphers for.
|
||||
* blk_crypto_init_key() - Prepare a key for use with blk-crypto
|
||||
* @blk_key: Pointer to the blk_crypto_key to initialize.
|
||||
* @raw_key: Pointer to the raw key. Must be the correct length for the chosen
|
||||
* @crypto_mode; see blk_crypto_modes[].
|
||||
* @crypto_mode: identifier for the encryption algorithm to use
|
||||
* @data_unit_size: the data unit size to use for en/decryption
|
||||
*
|
||||
* Upper layers (filesystems) should call this function to ensure that a
|
||||
* the crypto API fallback has transforms for this algorithm, if they become
|
||||
* necessary.
|
||||
*
|
||||
* Return: 0 on success and -err on error.
|
||||
* Return: The blk_crypto_key that was prepared, or an ERR_PTR() on error. When
|
||||
* done using the key, it must be freed with blk_crypto_free_key().
|
||||
*/
|
||||
int blk_crypto_start_using_mode(enum blk_crypto_mode_num mode_num,
|
||||
unsigned int data_unit_size,
|
||||
struct request_queue *q)
|
||||
int blk_crypto_init_key(struct blk_crypto_key *blk_key, const u8 *raw_key,
|
||||
enum blk_crypto_mode_num crypto_mode,
|
||||
unsigned int data_unit_size)
|
||||
{
|
||||
struct blk_crypto_keyslot *slotp;
|
||||
int err = 0;
|
||||
int i;
|
||||
const struct blk_crypto_mode *mode;
|
||||
static siphash_key_t hash_key;
|
||||
|
||||
memset(blk_key, 0, sizeof(*blk_key));
|
||||
|
||||
if (crypto_mode >= ARRAY_SIZE(blk_crypto_modes))
|
||||
return -EINVAL;
|
||||
|
||||
mode = &blk_crypto_modes[crypto_mode];
|
||||
if (mode->keysize == 0)
|
||||
return -EINVAL;
|
||||
|
||||
if (!is_power_of_2(data_unit_size))
|
||||
return -EINVAL;
|
||||
|
||||
blk_key->crypto_mode = crypto_mode;
|
||||
blk_key->data_unit_size = data_unit_size;
|
||||
blk_key->data_unit_size_bits = ilog2(data_unit_size);
|
||||
blk_key->size = mode->keysize;
|
||||
memcpy(blk_key->raw, raw_key, mode->keysize);
|
||||
|
||||
/*
|
||||
* Fast path
|
||||
* Ensure that updates to blk_crypto_keyslots[i].tfms[mode_num]
|
||||
* for each i are visible before we try to access them.
|
||||
* The keyslot manager uses the SipHash of the key to implement O(1) key
|
||||
* lookups while avoiding leaking information about the keys. It's
|
||||
* precomputed here so that it only needs to be computed once per key.
|
||||
*/
|
||||
if (likely(smp_load_acquire(&tfms_inited[mode_num])))
|
||||
return 0;
|
||||
get_random_once(&hash_key, sizeof(hash_key));
|
||||
blk_key->hash = siphash(raw_key, mode->keysize, &hash_key);
|
||||
|
||||
/*
|
||||
* If the keyslot manager of the request queue supports this
|
||||
* crypto mode, then we don't need to allocate this mode.
|
||||
*/
|
||||
if (keyslot_manager_crypto_mode_supported(q->ksm, mode_num,
|
||||
data_unit_size)) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
mutex_lock(&tfms_lock[mode_num]);
|
||||
if (likely(tfms_inited[mode_num]))
|
||||
goto out;
|
||||
|
||||
for (i = 0; i < blk_crypto_num_keyslots; i++) {
|
||||
slotp = &blk_crypto_keyslots[i];
|
||||
slotp->tfms[mode_num] = crypto_alloc_skcipher(
|
||||
blk_crypto_modes[mode_num].cipher_str,
|
||||
0, 0);
|
||||
if (IS_ERR(slotp->tfms[mode_num])) {
|
||||
err = PTR_ERR(slotp->tfms[mode_num]);
|
||||
slotp->tfms[mode_num] = NULL;
|
||||
goto out_free_tfms;
|
||||
}
|
||||
|
||||
crypto_skcipher_set_flags(slotp->tfms[mode_num],
|
||||
CRYPTO_TFM_REQ_FORBID_WEAK_KEYS);
|
||||
}
|
||||
|
||||
/*
|
||||
* Ensure that updates to blk_crypto_keyslots[i].tfms[mode_num]
|
||||
* for each i are visible before we set tfms_inited[mode_num].
|
||||
*/
|
||||
smp_store_release(&tfms_inited[mode_num], true);
|
||||
goto out;
|
||||
|
||||
out_free_tfms:
|
||||
for (i = 0; i < blk_crypto_num_keyslots; i++) {
|
||||
slotp = &blk_crypto_keyslots[i];
|
||||
crypto_free_skcipher(slotp->tfms[mode_num]);
|
||||
slotp->tfms[mode_num] = NULL;
|
||||
}
|
||||
out:
|
||||
mutex_unlock(&tfms_lock[mode_num]);
|
||||
return err;
|
||||
return 0;
|
||||
}
|
||||
EXPORT_SYMBOL(blk_crypto_start_using_mode);
|
||||
|
||||
/**
|
||||
* blk_crypto_evict_key() - Evict a key from any inline encryption hardware
|
||||
* it may have been programmed into
|
||||
* @q - The request queue who's keyslot manager this key might have been
|
||||
* programmed into
|
||||
* @key - The key to evict
|
||||
* @mode - The blk_crypto_mode_num used with this key
|
||||
* @data_unit_size - The data unit size used with this key
|
||||
* @q: The request queue who's keyslot manager this key might have been
|
||||
* programmed into
|
||||
* @key: The key to evict
|
||||
*
|
||||
* Upper layers (filesystems) should call this function to ensure that a key
|
||||
* is evicted from hardware that it might have been programmed into. This
|
||||
* will call keyslot_manager_evict_key on the queue's keyslot manager, if one
|
||||
* exists, and supports the crypto algorithm with the specified data unit size.
|
||||
* Otherwise, it will evict the key from the blk_crypto_ksm.
|
||||
* Otherwise, it will evict the key from the blk-crypto-fallback's ksm.
|
||||
*
|
||||
* Return: 0 on success, -err on error.
|
||||
*/
|
||||
int blk_crypto_evict_key(struct request_queue *q, const u8 *key,
|
||||
enum blk_crypto_mode_num mode,
|
||||
unsigned int data_unit_size)
|
||||
int blk_crypto_evict_key(struct request_queue *q,
|
||||
const struct blk_crypto_key *key)
|
||||
{
|
||||
struct keyslot_manager *ksm = blk_crypto_ksm;
|
||||
if (q->ksm &&
|
||||
keyslot_manager_crypto_mode_supported(q->ksm, key->crypto_mode,
|
||||
key->data_unit_size))
|
||||
return keyslot_manager_evict_key(q->ksm, key);
|
||||
|
||||
if (q && q->ksm && keyslot_manager_crypto_mode_supported(q->ksm, mode,
|
||||
data_unit_size)) {
|
||||
ksm = q->ksm;
|
||||
}
|
||||
|
||||
return keyslot_manager_evict_key(ksm, key, mode, data_unit_size);
|
||||
}
|
||||
EXPORT_SYMBOL(blk_crypto_evict_key);
|
||||
|
||||
int __init blk_crypto_init(void)
|
||||
{
|
||||
int i;
|
||||
int err = -ENOMEM;
|
||||
|
||||
prandom_bytes(blank_key, BLK_CRYPTO_MAX_KEY_SIZE);
|
||||
|
||||
blk_crypto_ksm = keyslot_manager_create(blk_crypto_num_keyslots,
|
||||
&blk_crypto_ksm_ll_ops,
|
||||
NULL);
|
||||
if (!blk_crypto_ksm)
|
||||
goto out;
|
||||
|
||||
blk_crypto_wq = alloc_workqueue("blk_crypto_wq",
|
||||
WQ_UNBOUND | WQ_HIGHPRI |
|
||||
WQ_MEM_RECLAIM,
|
||||
num_online_cpus());
|
||||
if (!blk_crypto_wq)
|
||||
goto out_free_ksm;
|
||||
|
||||
blk_crypto_keyslots = kcalloc(blk_crypto_num_keyslots,
|
||||
sizeof(*blk_crypto_keyslots),
|
||||
GFP_KERNEL);
|
||||
if (!blk_crypto_keyslots)
|
||||
goto out_free_workqueue;
|
||||
|
||||
for (i = 0; i < blk_crypto_num_keyslots; i++) {
|
||||
blk_crypto_keyslots[i].crypto_mode =
|
||||
BLK_ENCRYPTION_MODE_INVALID;
|
||||
}
|
||||
|
||||
for (i = 0; i < ARRAY_SIZE(blk_crypto_modes); i++)
|
||||
mutex_init(&tfms_lock[i]);
|
||||
|
||||
blk_crypto_page_pool =
|
||||
mempool_create_page_pool(num_prealloc_bounce_pg, 0);
|
||||
if (!blk_crypto_page_pool)
|
||||
goto out_free_keyslots;
|
||||
|
||||
blk_crypto_work_mem_cache = KMEM_CACHE(work_mem, SLAB_RECLAIM_ACCOUNT);
|
||||
if (!blk_crypto_work_mem_cache)
|
||||
goto out_free_page_pool;
|
||||
|
||||
return 0;
|
||||
|
||||
out_free_page_pool:
|
||||
mempool_destroy(blk_crypto_page_pool);
|
||||
blk_crypto_page_pool = NULL;
|
||||
out_free_keyslots:
|
||||
kzfree(blk_crypto_keyslots);
|
||||
blk_crypto_keyslots = NULL;
|
||||
out_free_workqueue:
|
||||
destroy_workqueue(blk_crypto_wq);
|
||||
blk_crypto_wq = NULL;
|
||||
out_free_ksm:
|
||||
keyslot_manager_destroy(blk_crypto_ksm);
|
||||
blk_crypto_ksm = NULL;
|
||||
out:
|
||||
pr_warn("No memory for blk-crypto crypto API fallback.");
|
||||
return err;
|
||||
return blk_crypto_fallback_evict_key(key);
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in a new issue