From d32a4878e299fcf2f69e330b5ab48c23736a1293 Mon Sep 17 00:00:00 2001 From: Chen Feng Date: Mon, 6 Feb 2017 10:51:04 +0800 Subject: [PATCH 01/89] ANDROID: reset: hisi-reboot: adb reboot bootloader Provide support for the hisi bootloader's reboot reason functionality Bug: 146450171 Signed-off-by: Chen Feng Signed-off-by: John Stultz Change-Id: Ic866ed27fdc5b9a142abe379febd720523f34964 --- drivers/power/reset/Kconfig | 2 +- drivers/power/reset/hisi-reboot.c | 59 ++++++++++++++++++++++++++++--- 2 files changed, 56 insertions(+), 5 deletions(-) diff --git a/drivers/power/reset/Kconfig b/drivers/power/reset/Kconfig index a564237278ff..13279dd4f01a 100644 --- a/drivers/power/reset/Kconfig +++ b/drivers/power/reset/Kconfig @@ -94,7 +94,7 @@ config POWER_RESET_GPIO_RESTART create a binding in your devicetree. config POWER_RESET_HISI - bool "Hisilicon power-off driver" + tristate "Hisilicon power-off driver" depends on ARCH_HISI help Reboot support for Hisilicon boards. diff --git a/drivers/power/reset/hisi-reboot.c b/drivers/power/reset/hisi-reboot.c index 0ba5fdce186f..283afb538ad3 100644 --- a/drivers/power/reset/hisi-reboot.c +++ b/drivers/power/reset/hisi-reboot.c @@ -8,26 +8,64 @@ * Author: Haojian Zhuang */ +#include #include #include #include +#include #include #include #include #include +#include +#include #include static void __iomem *base; static u32 reboot_offset; +static struct regmap *pmu_regmap; +static struct regmap *sctrl_regmap; + +#define REBOOT_REASON_BOOTLOADER (0x01) +#define REBOOT_REASON_COLDBOOT (0x00) +#define DDR_BYPASS BIT(31) + +#define RST_FLAG_MASK GENMASK(7, 0) + +#define PMU_HRST_OFFSET ((0x101) << 2) +#define SCPEREN1_OFFSET (0x170) static int hisi_restart_handler(struct notifier_block *this, unsigned long mode, void *cmd) { - writel_relaxed(0xdeadbeef, base + reboot_offset); + int ret; + char reboot_reason; + + if (!cmd || !strcmp(cmd, "bootloader")) + reboot_reason = REBOOT_REASON_BOOTLOADER; + else + reboot_reason = REBOOT_REASON_COLDBOOT; + + if (base) { + writel_relaxed(0xdeadbeef, base + reboot_offset); + } else { + ret = regmap_update_bits(pmu_regmap, PMU_HRST_OFFSET, + RST_FLAG_MASK, reboot_reason); + if (ret) + return ret; + + ret = regmap_write(sctrl_regmap, SCPEREN1_OFFSET, DDR_BYPASS); + if (ret) + return ret; + + ret = regmap_write(sctrl_regmap, reboot_offset, 0xdeadbeef); + if (ret) + return ret; + } while (1) - cpu_do_idle(); + mdelay(1); return NOTIFY_DONE; } @@ -44,8 +82,17 @@ static int hisi_reboot_probe(struct platform_device *pdev) base = of_iomap(np, 0); if (!base) { - WARN(1, "failed to map base address"); - return -ENODEV; + pmu_regmap = syscon_regmap_lookup_by_phandle(np, "pmu-regmap"); + if (!pmu_regmap) { + WARN(1, "failed to regmap pmu address"); + return -ENODEV; + } + + sctrl_regmap = syscon_regmap_lookup_by_phandle(np, "sctrl-regmap"); + if (!sctrl_regmap) { + WARN(1, "failed to regmap sctrl address"); + return -ENODEV; + } } if (of_property_read_u32(np, "reboot-offset", &reboot_offset) < 0) { @@ -66,6 +113,7 @@ static int hisi_reboot_probe(struct platform_device *pdev) static const struct of_device_id hisi_reboot_of_match[] = { { .compatible = "hisilicon,sysctrl" }, + { .compatible = "hisilicon,hi3660-reboot" }, {} }; @@ -77,3 +125,6 @@ static struct platform_driver hisi_reboot_driver = { }, }; module_platform_driver(hisi_reboot_driver); + +MODULE_DESCRIPTION("Reset driver for HiSi SoCs"); +MODULE_LICENSE("GPL v2"); From a32426e2347796e6e52b68f63abb6b86005c6401 Mon Sep 17 00:00:00 2001 From: John Stultz Date: Wed, 8 Jan 2020 17:09:23 +0000 Subject: [PATCH 02/89] ANDROID: tty: serdev: Fix broken serial console input Since commit c550a54f2302 ("ANDROID: serdev: add platform device support"), the serial console on the db845c has stopped taking input. Digging in it seems when the tty used for the console is switched to serdev via serdev_tty_port_register(), the client_ops are changed here: https://android.googlesource.com/kernel/common/+/android-mainline/drivers/tty/serdev/serdev-ttyport.c#288 The problem being, the new client_ops->receive_buf function ttyport_receive_buf() starts failing on the SERPORT_ACTIVE test here: https://android.googlesource.com/kernel/common/+/android-mainline/drivers/tty/serdev/serdev-ttyport.c#32 Which seems to be due to the fact that the tty was already opened and being used as the console when it was switched to serdev. Thus ctrl_ops->open function never gets called, which prevents the SERPORT_ACTIVE bit from being set: https://android.googlesource.com/kernel/common/+/android-mainline/drivers/tty/serdev/serdev-ttyport.c#141 Now this was at first confusing as on the HiKey960 we don't see the issue. But in the HiKey960 case it seem the check here: https://android.googlesource.com/kernel/common/+/android-mainline/drivers/tty/serdev/core.c#737 fails preventing the tty from being switched to serdev. Thus this patch tries to avoid switching the tty to serdev if the tty port's console value is true. With this, the serial console continues to function. Signed-off-by: John Stultz Signed-off-by: Alistair Delva Bug: 147453872 Change-Id: Id2747dc8c4ac633d71afabaf252d2bb69d206123 --- drivers/tty/serdev/serdev-ttyport.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/drivers/tty/serdev/serdev-ttyport.c b/drivers/tty/serdev/serdev-ttyport.c index d1cdd2ab8b4c..f9c546ac4135 100644 --- a/drivers/tty/serdev/serdev-ttyport.c +++ b/drivers/tty/serdev/serdev-ttyport.c @@ -273,6 +273,11 @@ struct device *serdev_tty_port_register(struct tty_port *port, if (!port || !drv || !parent) return ERR_PTR(-ENODEV); + if (port->console) { + /* can't convert tty's that are already in use */ + return ERR_PTR(-ENODEV); + } + ctrl = serdev_controller_alloc(parent, sizeof(struct serport)); if (!ctrl) return ERR_PTR(-ENOMEM); From b7b3af961465d23a733bb44d4a49bb78d4d5b08f Mon Sep 17 00:00:00 2001 From: Satya Tangirala Date: Tue, 17 Dec 2019 14:26:29 -0800 Subject: [PATCH 03/89] BACKPORT: FROMLIST: Update Inline Encryption from v5 to v6 of patch series Changes v5 => v6: - Blk-crypto's kernel crypto API fallback is no longer restricted to 8-byte DUNs. It's also now separately configurable from blk-crypto, and can be disabled entirely, while still allowing the kernel to use inline encryption hardware. Further, struct bio_crypt_ctx takes up less space, and no longer contains the information needed by the crypto API fallback - the fallback allocates the required memory when necessary. - Blk-crypto now supports all file content encryption modes supported by fscrypt. - Fixed bio merging logic in blk-merge.c - Fscrypt now supports inline encryption with the direct key policy, since blk-crypto now has support for larger DUNs. - Keyslot manager now uses a hashtable to lookup which keyslot contains any particular key (thanks Eric!) - Fscrypt support for inline encryption now handles filesystems with multiple underlying block devices (thanks Eric!) - Numerous cleanups Bug: 137270441 Test: refer to I26376479ee38259b8c35732cb3a1d7e15f9b05a3 Change-Id: I13e2e327e0b4784b394cb1e7cf32a04856d95f01 Link: https://lore.kernel.org/linux-block/20191218145136.172774-1-satyat@google.com/ Signed-off-by: Satya Tangirala --- Documentation/block/inline-encryption.rst | 8 +- abi_gki_aarch64.xml | 20478 ++++++++++---------- block/Kconfig | 15 +- block/Makefile | 1 + block/bio-crypt-ctx.c | 117 +- block/bio.c | 8 +- block/blk-core.c | 12 +- block/blk-crypto-fallback.c | 648 + block/blk-crypto-internal.h | 58 + block/blk-crypto.c | 748 +- block/blk-merge.c | 38 +- block/bounce.c | 5 +- block/keyslot-manager.c | 256 +- drivers/md/dm.c | 14 +- drivers/scsi/ufs/ufshcd-crypto.c | 235 +- drivers/scsi/ufs/ufshcd-crypto.h | 18 +- drivers/scsi/ufs/ufshcd.c | 10 +- drivers/scsi/ufs/ufshcd.h | 4 +- fs/buffer.c | 2 +- fs/crypto/bio.c | 7 +- fs/crypto/crypto.c | 2 +- fs/crypto/fname.c | 4 +- fs/crypto/fscrypt_private.h | 134 +- fs/crypto/inline_crypt.c | 301 +- fs/crypto/keyring.c | 6 +- fs/crypto/keysetup.c | 102 +- fs/crypto/keysetup_v1.c | 16 +- fs/ext4/page-io.c | 7 +- fs/ext4/readpage.c | 8 +- fs/f2fs/data.c | 28 +- fs/f2fs/super.c | 21 + include/linux/bio-crypt-ctx.h | 291 +- include/linux/blk-crypto.h | 39 +- include/linux/fscrypt.h | 34 +- include/linux/keyslot-manager.h | 94 +- 35 files changed, 11793 insertions(+), 11976 deletions(-) create mode 100644 block/blk-crypto-fallback.c create mode 100644 block/blk-crypto-internal.h diff --git a/Documentation/block/inline-encryption.rst b/Documentation/block/inline-encryption.rst index 202826cee95e..330106b23c09 100644 --- a/Documentation/block/inline-encryption.rst +++ b/Documentation/block/inline-encryption.rst @@ -97,7 +97,7 @@ Blk-crypto ensures that: - The bio's encryption context is programmed into a keyslot in the KSM of the request queue that the bio is being submitted to (or the crypto API fallback - KSM if the request queue doesn't have a KSM), and that the ``processing_ksm`` + KSM if the request queue doesn't have a KSM), and that the ``bc_ksm`` in the ``bi_crypt_context`` is set to this KSM - That the bio has its own individual reference to the keyslot in this KSM. @@ -107,7 +107,7 @@ Blk-crypto ensures that: ensuring that the bio has a valid reference to the keyslot when, for e.g., the crypto API fallback KSM in blk-crypto performs crypto on the device's behalf. The individual references are ensured by increasing the refcount for the - keyslot in the ``processing_ksm`` when a bio with a programmed encryption + keyslot in the ``bc_ksm`` when a bio with a programmed encryption context is cloned. @@ -120,7 +120,7 @@ been programmed into any keyslot in any KSM (for e.g. a bio from the FS). request queue the bio is being submitted to (and if this KSM does not exist, then it will program it into blk-crypto's internal KSM for crypto API fallback). The KSM that this encryption context was programmed into is stored - as the ``processing_ksm`` in the bio's ``bi_crypt_context``. + as the ``bc_ksm`` in the bio's ``bi_crypt_context``. **Case 2:** blk-crypto is given a bio whose encryption context has already been programmed into a keyslot in the *crypto API fallback* KSM. @@ -138,7 +138,7 @@ KSM). This way, when a device driver is processing a bio, it can be sure that the bio's encryption context has been programmed into some KSM (either the device driver's request queue's KSM, or blk-crypto's crypto API fallback KSM). -It then simply needs to check if the bio's processing_ksm is the device's +It then simply needs to check if the bio's ``bc_ksm`` is the device's request queue's KSM. If so, then it should proceed with IE. If not, it should simply do nothing with respect to crypto, because some other KSM (perhaps the blk-crypto crypto API fallback KSM) is handling the en/decryption. diff --git a/abi_gki_aarch64.xml b/abi_gki_aarch64.xml index 286187d01e5a..2ce464781f62 100644 --- a/abi_gki_aarch64.xml +++ b/abi_gki_aarch64.xml @@ -2170,7 +2170,7 @@ - + @@ -2196,7 +2196,7 @@ - + @@ -20952,7 +20952,45 @@ - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -20969,13 +21007,14 @@ - - - - - - - + + + + + + + + @@ -20994,12 +21033,12 @@ - + - + @@ -21027,7 +21066,7 @@ - + @@ -21075,13 +21114,13 @@ - + - - + + @@ -21108,8 +21147,8 @@ - - + + @@ -21141,223 +21180,227 @@ - - + + - + - + - + - - + + - + - + - + - + - + - + - + - - + + - - + + - + - - + + - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + - + - + - - + + - - + + - + - - + + - + - + - + - + - - + + - + - + - + - + - + - + - + - + - + - + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -21386,7 +21429,7 @@ - + @@ -21398,7 +21441,7 @@ - + @@ -21446,10 +21489,10 @@ - + - + @@ -21473,7 +21516,7 @@ - + @@ -21512,7 +21555,7 @@ - + @@ -21539,21 +21582,21 @@ - + - + - + - + - + @@ -21561,7 +21604,7 @@ - + @@ -21578,15 +21621,15 @@ - + - + - + @@ -21594,12 +21637,12 @@ - + - + @@ -21616,8 +21659,8 @@ - - + + @@ -21625,7 +21668,7 @@ - + @@ -21649,7 +21692,7 @@ - + @@ -21657,7 +21700,7 @@ - + @@ -21815,7 +21858,7 @@ - + @@ -21883,12 +21926,12 @@ - + - + - + @@ -21909,7 +21952,7 @@ - + @@ -21938,13 +21981,13 @@ - + - + - + @@ -21965,16 +22008,16 @@ - + - + - + - + @@ -21985,7 +22028,7 @@ - + @@ -21996,10 +22039,10 @@ - + - + @@ -22010,15 +22053,15 @@ - + - + - + @@ -22026,7 +22069,7 @@ - + @@ -22037,20 +22080,20 @@ - + - + - + - + - + @@ -22059,7 +22102,7 @@ - + @@ -22070,15 +22113,15 @@ - + - + - + - + @@ -22186,10 +22229,10 @@ - + - + @@ -22198,9 +22241,9 @@ - + - + @@ -22209,7 +22252,7 @@ - + @@ -22218,7 +22261,7 @@ - + @@ -22244,7 +22287,7 @@ - + @@ -22252,7 +22295,7 @@ - + @@ -22263,28 +22306,28 @@ - + - + - + - - - + + + - + @@ -22293,13 +22336,13 @@ - - + + - + @@ -22322,16 +22365,16 @@ - + - + - + - + @@ -22360,22 +22403,22 @@ - + - + - + - + - + @@ -22398,7 +22441,7 @@ - + @@ -22412,7 +22455,7 @@ - + @@ -22421,50 +22464,50 @@ - + - + - + - - + + - + - + - + - + - + - + - + - + - + @@ -22479,34 +22522,34 @@ - + - + - + - + - + - + - + @@ -22518,16 +22561,16 @@ - + - + - + - + @@ -22541,7 +22584,7 @@ - + @@ -22550,16 +22593,16 @@ - + - + - + - + @@ -22574,25 +22617,25 @@ - + - + - + - + @@ -22600,13 +22643,13 @@ - + - + - + @@ -22635,13 +22678,13 @@ - + - + @@ -22650,9 +22693,9 @@ - + - + @@ -22664,20 +22707,20 @@ - + - + - + - + @@ -22685,9 +22728,9 @@ - + - + @@ -22696,8 +22739,8 @@ - - + + @@ -22705,32 +22748,32 @@ - + - + - + - + - + - + - + @@ -22738,26 +22781,26 @@ - + - + - + - + - + - + - + - + @@ -22766,74 +22809,74 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -22845,7 +22888,7 @@ - + @@ -22878,10 +22921,10 @@ - + - + @@ -22896,7 +22939,7 @@ - + @@ -22905,25 +22948,25 @@ - + - + - + - + - + - + - + @@ -22938,7 +22981,7 @@ - + @@ -22950,10 +22993,10 @@ - + - + @@ -22962,10 +23005,10 @@ - + - + @@ -22973,7 +23016,7 @@ - + @@ -22987,7 +23030,7 @@ - + @@ -23014,7 +23057,7 @@ - + @@ -23040,15 +23083,15 @@ - + - + - + - + @@ -23056,48 +23099,48 @@ - + - + - + - + - + - + - + - + - + - + - + @@ -23112,7 +23155,7 @@ - + @@ -23123,7 +23166,7 @@ - + @@ -23134,10 +23177,10 @@ - + - + @@ -23145,27 +23188,27 @@ - + - + - + - + - + - + - + - + @@ -23183,12 +23226,12 @@ - + - + - + @@ -23197,10 +23240,10 @@ - + - + @@ -23208,12 +23251,12 @@ - + - + - + @@ -23246,7 +23289,7 @@ - + @@ -23255,7 +23298,7 @@ - + @@ -23267,30 +23310,30 @@ - + - + - + - + - + - + - + @@ -23299,13 +23342,13 @@ - + - + - + @@ -23332,10 +23375,10 @@ - + - + @@ -23344,7 +23387,7 @@ - + @@ -23377,27 +23420,27 @@ - + - + - + - + - + - + - + @@ -23408,15 +23451,15 @@ - + - + - + @@ -23424,40 +23467,40 @@ - + - + - + - + - + - + - + - + - + - + @@ -23465,7 +23508,7 @@ - + @@ -23473,7 +23516,7 @@ - + @@ -23496,33 +23539,33 @@ - + - + - + - + - + - + - + - + - + - + @@ -23531,13 +23574,13 @@ - + - + @@ -23588,7 +23631,7 @@ - + @@ -23597,10 +23640,10 @@ - + - + @@ -23615,13 +23658,13 @@ - + - + @@ -23636,7 +23679,7 @@ - + @@ -23663,7 +23706,7 @@ - + @@ -23678,13 +23721,13 @@ - + - + @@ -23708,10 +23751,10 @@ - + - + @@ -23728,19 +23771,19 @@ - + - + - + - + - + @@ -23748,19 +23791,19 @@ - + - + - + - + @@ -23775,22 +23818,22 @@ - + - + - + - + - + @@ -23814,7 +23857,7 @@ - + @@ -23826,22 +23869,22 @@ - + - - + + - + - + @@ -23849,7 +23892,7 @@ - + @@ -23857,20 +23900,20 @@ - + - - - + + + - + - + @@ -23885,15 +23928,15 @@ - + - + - + @@ -23902,7 +23945,7 @@ - + @@ -23917,108 +23960,108 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - - - - + + + + @@ -24035,7 +24078,7 @@ - + @@ -24062,7 +24105,7 @@ - + @@ -24095,10 +24138,10 @@ - - + + - + @@ -24122,10 +24165,10 @@ - + - + @@ -24149,13 +24192,13 @@ - + - + @@ -24176,7 +24219,7 @@ - + @@ -24191,10 +24234,10 @@ - + - + @@ -24202,7 +24245,7 @@ - + @@ -24211,7 +24254,7 @@ - + @@ -24222,7 +24265,7 @@ - + @@ -24235,7 +24278,7 @@ - + @@ -24252,9 +24295,9 @@ - + - + @@ -24263,13 +24306,13 @@ - + - + - + @@ -24278,13 +24321,13 @@ - + - + - + @@ -24314,7 +24357,7 @@ - + @@ -24325,13 +24368,13 @@ - + - + @@ -24343,31 +24386,31 @@ - + - + - + - + - + - + - + - + @@ -24444,16 +24487,16 @@ - + - + - + @@ -24461,27 +24504,27 @@ - + - + - + - + - + - + - + - + @@ -24582,7 +24625,7 @@ - + @@ -24590,10 +24633,10 @@ - + - + @@ -24608,8 +24651,8 @@ - - + + @@ -24626,18 +24669,18 @@ - + - + - - - + + + @@ -24651,45 +24694,57 @@ - - + + - + - + - + - - - - - - - - - - + - - - - + - + + + + + + + + + + + + + + + + + + + + + + + + - - + + + @@ -24697,174 +24752,176 @@ - + - - + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - + + + + + + + @@ -24887,7 +24944,7 @@ - + @@ -24896,315 +24953,315 @@ - + - - + + - - + + - + - + - + - + - + - - - - - - - - - - - - - - - - - - - - - - - - - - - + - + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + - + + - - - - + + + - - - + + + - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - + + + - - - - + + + + - - + + - + - - - - + + + + - - + + - - - + + + - - - - + + + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + - - + - - - + - - - + + - + + + + + - - + + + + + + + + + + + + - - - + + + + + + + + + - - + + + + + + + - - - + + + - + + + + + + + + + + + - + @@ -25212,7 +25269,7 @@ - + @@ -25220,10 +25277,9 @@ - - + @@ -25241,9 +25297,9 @@ - - - + + + @@ -25254,11 +25310,11 @@ - + - + @@ -25268,7 +25324,7 @@ - + @@ -25290,56 +25346,56 @@ - - + + - + - - + + - - - - - - - + + + + + + + - - + + - + - - + + - + - + - - + + - + - + @@ -25350,9 +25406,9 @@ - + - + @@ -25361,7 +25417,7 @@ - + @@ -25370,7 +25426,7 @@ - + @@ -25388,7 +25444,7 @@ - + @@ -25397,7 +25453,7 @@ - + @@ -25405,9 +25461,9 @@ - + - + @@ -25422,7 +25478,7 @@ - + @@ -25446,7 +25502,7 @@ - + @@ -25484,10 +25540,10 @@ - + - + @@ -25496,10 +25552,10 @@ - + - + @@ -25514,12 +25570,12 @@ - + - + @@ -25530,8 +25586,8 @@ - - + + @@ -25539,10 +25595,10 @@ - + - + @@ -25559,7 +25615,7 @@ - + @@ -25588,7 +25644,7 @@ - + @@ -25603,87 +25659,87 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -25710,10 +25766,10 @@ - + - + @@ -25724,22 +25780,22 @@ - + - + - + - + - + @@ -25750,7 +25806,7 @@ - + @@ -25761,24 +25817,24 @@ - + - + - + - + - + - + - + @@ -25786,15 +25842,15 @@ - + - + - + @@ -25806,7 +25862,7 @@ - + @@ -25824,7 +25880,7 @@ - + @@ -25850,16 +25906,16 @@ - + - + - + @@ -25873,7 +25929,7 @@ - + @@ -25887,7 +25943,7 @@ - + @@ -25922,22 +25978,22 @@ - + - + - + - + @@ -25955,7 +26011,7 @@ - + @@ -25967,7 +26023,7 @@ - + @@ -25988,7 +26044,7 @@ - + @@ -26036,25 +26092,25 @@ - + - + - + - + - + @@ -26066,19 +26122,19 @@ - + - + - + - + @@ -26093,7 +26149,7 @@ - + @@ -26102,22 +26158,22 @@ - - + + - + - - + + - + - + @@ -26128,7 +26184,7 @@ - + @@ -26145,7 +26201,7 @@ - + @@ -26170,7 +26226,7 @@ - + @@ -26302,7 +26358,7 @@ - + @@ -26310,9 +26366,9 @@ - + - + @@ -26327,7 +26383,7 @@ - + @@ -26336,7 +26392,7 @@ - + @@ -26351,23 +26407,23 @@ - + - + - + - + @@ -26377,9 +26433,9 @@ - + - + @@ -26387,7 +26443,7 @@ - + @@ -26406,10 +26462,10 @@ - + - + @@ -26435,13 +26491,13 @@ - + - + - + @@ -26455,7 +26511,7 @@ - + @@ -26463,10 +26519,10 @@ - + - + @@ -26483,7 +26539,7 @@ - + @@ -26491,10 +26547,10 @@ - + - + @@ -26505,7 +26561,7 @@ - + @@ -26513,16 +26569,16 @@ - - + + - + - + @@ -26534,7 +26590,7 @@ - + @@ -26543,7 +26599,7 @@ - + @@ -26555,19 +26611,19 @@ - + - + - + - + - + @@ -26576,20 +26632,20 @@ - + - + - + - + - + @@ -26598,24 +26654,24 @@ - + - + - - + + - + - + @@ -26644,15 +26700,15 @@ - + - + - + @@ -26681,7 +26737,7 @@ - + @@ -26689,7 +26745,7 @@ - + @@ -26697,7 +26753,7 @@ - + @@ -26705,7 +26761,7 @@ - + @@ -26713,9 +26769,9 @@ - + - + @@ -26736,17 +26792,17 @@ - + - + - + - + - + @@ -26758,9 +26814,9 @@ - + - + @@ -26772,27 +26828,13 @@ - - - - - - - - - - - - - - - - + + - + @@ -26801,19 +26843,19 @@ - + - + - + - + - + @@ -26825,8 +26867,8 @@ - - + + @@ -26837,13 +26879,13 @@ - + - + @@ -26878,7 +26920,7 @@ - + @@ -26886,7 +26928,7 @@ - + @@ -26894,66 +26936,64 @@ - - - - - - - - - - - + + + + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -26974,7 +27014,7 @@ - + @@ -26991,120 +27031,120 @@ - + - + - + - + - + - - - - - - - - - - - - - - - + - + + - - - + + + + + + - + + + + + + + + + + + - + - + - - - - - - - - - - - - - - - + - - - + + + + + + + + + + + + + + + + + - - + + - - + + - - + + - + @@ -27123,7 +27163,7 @@ - + @@ -27158,18 +27198,18 @@ - - - - + + + + - + - + @@ -27186,7 +27226,7 @@ - + @@ -27194,11 +27234,11 @@ - + - + - + @@ -27211,7 +27251,7 @@ - + @@ -27222,7 +27262,7 @@ - + @@ -27233,7 +27273,7 @@ - + @@ -27241,7 +27281,7 @@ - + @@ -27251,18 +27291,18 @@ - + - + - + @@ -27271,15 +27311,15 @@ - + - + - + @@ -27290,7 +27330,7 @@ - + @@ -27298,55 +27338,55 @@ - - - - - - + + + + + + - + - + - + - + - + - + - + - + - - + + - - + + - + @@ -27360,8 +27400,8 @@ - - + + @@ -27372,17 +27412,19 @@ - + - + + + - + @@ -27408,7 +27450,7 @@ - + @@ -27422,68 +27464,68 @@ - - + + - + - + - - + + - - + + - + - - + + - + - + - - + + - - + + - + - + @@ -27513,22 +27555,22 @@ - + - + - - - + + + - + @@ -27539,7 +27581,7 @@ - + @@ -27558,25 +27600,25 @@ - - + + - + - + - - + + @@ -27590,15 +27632,15 @@ - + - + - + @@ -27623,17 +27665,17 @@ - - - + + + - + - + @@ -27641,7 +27683,7 @@ - + @@ -27656,16 +27698,16 @@ - + - + - + - + @@ -27680,42 +27722,42 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -27728,7 +27770,7 @@ - + @@ -27736,26 +27778,26 @@ - + - - - - - - - - - + + + + - - - - - - - + + + + + + + + + + + + @@ -27790,65 +27832,65 @@ - - - + + + - - - - + + + + - - - + + + - - - - + + + + - - - + + + - - + + - + - - + + - - - + + + - - + + - - + + @@ -27933,7 +27975,7 @@ - + @@ -27941,9 +27983,9 @@ - - - + + + @@ -27960,7 +28002,7 @@ - + @@ -27968,7 +28010,7 @@ - + @@ -27982,7 +28024,7 @@ - + @@ -28017,9 +28059,9 @@ - + - + @@ -28028,7 +28070,7 @@ - + @@ -28036,12 +28078,12 @@ - + - + - + @@ -28072,7 +28114,7 @@ - + @@ -28080,26 +28122,26 @@ - - + + - + - + - + - + - + @@ -28112,7 +28154,7 @@ - + @@ -28123,7 +28165,7 @@ - + @@ -28134,7 +28176,7 @@ - + @@ -28142,13 +28184,13 @@ - + - + @@ -28162,22 +28204,22 @@ - - + + - + - + - - + + @@ -28197,7 +28239,7 @@ - + @@ -28208,19 +28250,19 @@ - + - - - - + + + + - + @@ -28262,22 +28304,22 @@ - + - + - + - + @@ -28291,7 +28333,7 @@ - + @@ -28345,7 +28387,7 @@ - + @@ -28354,18 +28396,18 @@ - + - + - - + + @@ -28378,15 +28420,15 @@ - + - - - - + + + + @@ -28400,8 +28442,8 @@ - - + + @@ -28415,14 +28457,14 @@ - + - + - + @@ -28512,16 +28554,16 @@ - + - + - + @@ -28560,8 +28602,8 @@ - - + + @@ -28571,7 +28613,7 @@ - + @@ -28588,21 +28630,29 @@ - + - - - + + + + + + + + + + + - - - + + + @@ -28612,13 +28662,13 @@ - + - - + + @@ -28664,9 +28714,9 @@ - + - + @@ -28681,10 +28731,10 @@ - + - + @@ -28702,36 +28752,36 @@ - + - + - + - + - + - + - + @@ -28746,8 +28796,8 @@ - - + + @@ -28766,44 +28816,44 @@ - + - + - + - + - - - + + + - + - + - + - + - + @@ -28812,15 +28862,15 @@ - + - + - + @@ -28829,7 +28879,7 @@ - + @@ -28839,7 +28889,7 @@ - + @@ -28847,13 +28897,13 @@ - + - + @@ -28864,13 +28914,13 @@ - + - + - + @@ -28884,19 +28934,19 @@ - + - + - + - + @@ -28906,23 +28956,23 @@ - + - + - + - + - + - + @@ -28936,7 +28986,7 @@ - + @@ -28956,33 +29006,33 @@ - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + - - + + + + + - - + + @@ -29013,85 +29063,85 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -29102,7 +29152,7 @@ - + @@ -29117,7 +29167,7 @@ - + @@ -29128,25 +29178,25 @@ - - - + + + - - + + - + - + - + @@ -29155,183 +29205,39 @@ - + - + - + - + - + - + - + - + - - - - - - - - - - - - - - - - - - - - - - - - - - - - + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + @@ -29342,39 +29248,7 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + @@ -29385,10 +29259,10 @@ - + - + @@ -29402,23 +29276,7 @@ - - - - - - - - - - - - - - - - - + @@ -29429,24 +29287,24 @@ - + - + - + - + - + - + @@ -29458,7 +29316,7 @@ - + @@ -29470,7 +29328,7 @@ - + @@ -29478,73 +29336,79 @@ - + - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - - + + + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + @@ -29555,190 +29419,188 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - + - + - + - + - + - + - + - - + + - + - - - - + + + + - - + + - + - - + + - - + + - - + + - - - + + + - - + + - + - - - - + + + + - - + + - + - - - + + + - - - - - - + + - - + - - + + + - - + + + + + + - + - - - - - - - - + + + + + + + + - + @@ -29761,7 +29623,7 @@ - + @@ -29775,9 +29637,9 @@ - + - + @@ -29787,10 +29649,10 @@ - + - + @@ -29808,7 +29670,7 @@ - + @@ -29823,19 +29685,19 @@ - + - + - + - + - + @@ -29913,7 +29775,7 @@ - + @@ -29946,7 +29808,7 @@ - + @@ -30003,7 +29865,7 @@ - + @@ -30012,65 +29874,65 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -30081,21 +29943,21 @@ - + - + - + - + @@ -30106,7 +29968,7 @@ - + @@ -30119,7 +29981,7 @@ - + @@ -30132,13 +29994,13 @@ - + - + @@ -30156,7 +30018,7 @@ - + @@ -30186,8 +30048,8 @@ - - + + @@ -30195,8 +30057,8 @@ - - + + @@ -30204,108 +30066,108 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -30341,16 +30203,16 @@ - + - + - + - + @@ -30362,10 +30224,10 @@ - + - + @@ -30379,27 +30241,27 @@ - + - + - + - + - + - + - + - + @@ -30420,10 +30282,10 @@ - + - + @@ -30435,7 +30297,7 @@ - + @@ -30446,13 +30308,13 @@ - + - + - + @@ -30460,8 +30322,8 @@ - - + + @@ -30472,7 +30334,7 @@ - + @@ -30486,7 +30348,7 @@ - + @@ -30497,7 +30359,7 @@ - + @@ -30511,7 +30373,7 @@ - + @@ -30525,10 +30387,10 @@ - + - + @@ -30555,7 +30417,7 @@ - + @@ -30570,29 +30432,29 @@ - + - + - + - + - + - + - + - + @@ -30615,31 +30477,31 @@ - + - + - + - + - + - + - + - + - + @@ -30660,7 +30522,7 @@ - + @@ -30698,10 +30560,10 @@ - + - + @@ -30725,20 +30587,20 @@ - + - + - + - - + + @@ -30746,7 +30608,7 @@ - + @@ -30754,26 +30616,26 @@ - + - + - + - + - + - + - + - + @@ -30802,20 +30664,20 @@ - + - + - + - + - + @@ -30824,18 +30686,18 @@ - + - + - + - + - + @@ -30844,10 +30706,10 @@ - + - + @@ -30855,7 +30717,7 @@ - + @@ -30863,22 +30725,22 @@ - + - + - + - + - + - + @@ -30890,27 +30752,27 @@ - + - + - + - + - + - + - + @@ -30921,7 +30783,7 @@ - + @@ -30930,7 +30792,7 @@ - + @@ -30959,20 +30821,6 @@ - - - - - - - - - - - - - - @@ -31015,13 +30863,13 @@ - + - + @@ -31104,7 +30952,7 @@ - + @@ -31113,13 +30961,13 @@ - + - + - + @@ -31295,7 +31143,7 @@ - + @@ -31304,12 +31152,12 @@ - + - + @@ -31342,51 +31190,51 @@ - + - + - - - + + + - - + + - - - - - - - - - - - - + + + + + + + + + + + + - - - - - - - - - + + + + + + + + + - - - - - + + + + + @@ -31399,45 +31247,45 @@ - - - - - - + + + + + + - - - - - - - - + + + + + + + + - - - + + + - - - - + + + + - + - - - - - - - - - - - + + + + + + + + + + + @@ -31445,7 +31293,7 @@ - + @@ -31459,40 +31307,40 @@ - + - + - + - + - + - + - + @@ -31501,7 +31349,7 @@ - + @@ -31512,7 +31360,7 @@ - + @@ -31529,16 +31377,16 @@ - + - + - + @@ -31546,7 +31394,7 @@ - + @@ -31554,19 +31402,19 @@ - + - + - + @@ -31575,7 +31423,7 @@ - + @@ -31583,50 +31431,50 @@ - + - + - - + + - + - + - + - + - + - + @@ -31634,7 +31482,7 @@ - + @@ -31642,66 +31490,66 @@ - + - + - + - + - - + + - - + + - + - + - + - + - + @@ -31715,16 +31563,16 @@ - + - + - + @@ -31732,7 +31580,7 @@ - + @@ -31755,7 +31603,7 @@ - + @@ -31763,32 +31611,32 @@ - + - + - + - + - + - + - + - + @@ -31802,7 +31650,7 @@ - + @@ -31931,7 +31779,7 @@ - + @@ -32571,158 +32419,147 @@ - + - + - + - + - - + + - - + + - + - + - + - + - + - - + + - + - - - - - - - - - - - - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - - + + @@ -32731,17 +32568,17 @@ - - - + + + - - + + @@ -32750,7 +32587,7 @@ - + @@ -32764,35 +32601,35 @@ - + - + - + - + - + - - + + - + @@ -32812,13 +32649,13 @@ - + - + - + @@ -32830,7 +32667,7 @@ - + @@ -32838,59 +32675,59 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -32902,16 +32739,16 @@ - - + + - + - + - + @@ -32931,190 +32768,190 @@ - + - + - + - + - + - + - + - + - + - + - + - - - - + + + + - + - + - + - + - + - + - + - + - + - + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - - - - + + - - + + - - + + - + - - + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - - - - - - - + - - + + - - + + - - + + - - + + - - + + - - - + + + - - + + - - + + @@ -33138,80 +32975,80 @@ - + - + - + - + - + - + - + - + - + - + - + - - + + - - - - - + + + + + - - + + - + - + - + @@ -33220,7 +33057,7 @@ - + @@ -33271,7 +33108,7 @@ - + @@ -33280,7 +33117,7 @@ - + @@ -33340,7 +33177,7 @@ - + @@ -33349,7 +33186,7 @@ - + @@ -33361,7 +33198,7 @@ - + @@ -33475,13 +33312,13 @@ - + - + @@ -33493,16 +33330,16 @@ - + - + - + @@ -33517,12 +33354,12 @@ - + - + @@ -33531,13 +33368,13 @@ - + - + @@ -33546,10 +33383,10 @@ - + - + @@ -33579,7 +33416,7 @@ - + @@ -33597,33 +33434,33 @@ - + - + - + - + - + - + - - + + - + - + @@ -33632,7 +33469,7 @@ - + @@ -33640,34 +33477,34 @@ - + - + - + - + - + - + - + - + - + - + @@ -33676,10 +33513,10 @@ - + - + @@ -33699,37 +33536,37 @@ - + - + - + - + - + - + - + - + - - + + - + @@ -33737,99 +33574,99 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + - + - - + + - - + + - - + + - - + + - - - + + + - - + + - - + + + + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - - + @@ -33844,102 +33681,102 @@ - + - + - + - + - + - + - + - + - + - + - - - - - - - + + + + + + + - + - + - + - + - + - + - + - + - - + + - - + + - - + + - - + + - - + + @@ -33954,81 +33791,81 @@ - - + + - + - + - + - + - + - - - + + + - + - + - + - + - + - + - + - + - + - + - + - + - + - - + + @@ -34052,12 +33889,12 @@ - - + + - + @@ -34083,22 +33920,22 @@ - + - + - + - + @@ -34106,20 +33943,20 @@ - + - + - + - + - + @@ -34140,13 +33977,13 @@ - + - + - + @@ -34161,16 +33998,16 @@ - + - + - + @@ -34182,7 +34019,7 @@ - + @@ -34194,7 +34031,7 @@ - + @@ -34208,7 +34045,7 @@ - + @@ -34223,13 +34060,13 @@ - + - + @@ -34253,19 +34090,19 @@ - + - + - + - + @@ -34358,50 +34195,50 @@ - + - + - + - + - + - + - + - + - + - + - + @@ -34442,16 +34279,16 @@ - + - + - + - + @@ -34496,7 +34333,7 @@ - + @@ -34516,8 +34353,8 @@ - - + + @@ -34525,113 +34362,113 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -34645,12 +34482,12 @@ - + - + - + @@ -34659,26 +34496,26 @@ - + - + - + - + - + - + @@ -34687,193 +34524,193 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - + - + - + - + - + - + - + - + + + + + + + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - - - - - - + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - + - + - + - + - - + + - - + + - + - - - - - + + + + + - + - + - + - + @@ -34885,7 +34722,7 @@ - + @@ -34893,7 +34730,7 @@ - + @@ -34902,10 +34739,10 @@ - + - + @@ -34926,7 +34763,7 @@ - + @@ -34934,7 +34771,7 @@ - + @@ -34945,13 +34782,13 @@ - + - + - + @@ -34987,24 +34824,24 @@ - + - + - + - + - + - + @@ -35016,7 +34853,7 @@ - + @@ -35024,109 +34861,109 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -35134,7 +34971,7 @@ - + @@ -35143,14 +34980,14 @@ - + - + @@ -35173,21 +35010,21 @@ - + - + - + - + - + @@ -35223,16 +35060,16 @@ - + - + - + - + @@ -35277,10 +35114,10 @@ - - + + - + @@ -35289,16 +35126,16 @@ - + - + - + @@ -35319,7 +35156,7 @@ - + @@ -35330,18 +35167,18 @@ - + - + - + - + @@ -35350,43 +35187,43 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -35401,63 +35238,63 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -35475,10 +35312,10 @@ - + - + @@ -35487,278 +35324,278 @@ - + - + - + - + - + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - + - + - + - + - + - + - + - + - - + + - - - + + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - - + + + - - - + + + - - - + + + - - + + - - + + + + + + + + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - - - - - - + - - + + - - + + - - + + + + + + + + + + + + + + + + + + + + - + + - - - - - - - - - - - - - - - - - - - - + - - + + - - + + - + - + - + - + @@ -35771,21 +35608,21 @@ - - + + - - + + - + - + @@ -35793,37 +35630,37 @@ - + - + - - - - + + + + - + - + - + - + @@ -35832,13 +35669,13 @@ - + - + - + @@ -35856,10 +35693,10 @@ - + - + @@ -35898,7 +35735,7 @@ - + @@ -35922,112 +35759,112 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -36036,9 +35873,9 @@ - + - + @@ -36050,7 +35887,7 @@ - + @@ -36064,10 +35901,10 @@ - + - + @@ -36075,12 +35912,12 @@ - + - + - + @@ -36092,110 +35929,110 @@ - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + - - - - - - - - - + + + + + - + - + - + - + - + - - + + - - - + + + - - - + + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + @@ -36204,13 +36041,13 @@ - + - + - + @@ -36237,9 +36074,9 @@ - + - + @@ -36254,44 +36091,44 @@ - + - + - + - + - + - + - + - - - - - - + + + + + + - + - + @@ -36301,65 +36138,65 @@ - + - - - + + + - - + + - + - - + + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -36383,7 +36220,7 @@ - + @@ -36392,7 +36229,7 @@ - + @@ -36404,21 +36241,21 @@ - + - + - + - + - + - + @@ -36472,9 +36309,9 @@ - + - + @@ -36486,10 +36323,10 @@ - + - + @@ -36498,12 +36335,12 @@ - + - + - + @@ -36524,15 +36361,15 @@ - + - + - + @@ -36543,19 +36380,19 @@ - + - + - + - + @@ -36579,13 +36416,13 @@ - + - + - + @@ -36603,7 +36440,7 @@ - + @@ -36633,7 +36470,7 @@ - + @@ -36672,10 +36509,10 @@ - + - + @@ -36683,10 +36520,10 @@ - + - + @@ -36713,7 +36550,7 @@ - + @@ -36722,161 +36559,161 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - - - - + + + + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -36890,7 +36727,7 @@ - + @@ -36898,13 +36735,13 @@ - + - + @@ -36942,7 +36779,7 @@ - + @@ -36974,7 +36811,7 @@ - + @@ -36998,7 +36835,7 @@ - + @@ -37015,15 +36852,15 @@ - + - + - + - + @@ -37037,410 +36874,410 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + - + - + - - + + - + - + - + - - + + - - + + - - + + - - - + + + - - + + - - + + - - - + + + - - + + - - + + - - + + - - + + - - + + - - + + - - - + + + - - - + + + - - - + + + - - + + - - + + - - + + - - - + + + - + - + - + - - + + - - + + - + - - + + - - + + - - + + - - + + + + + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - - - + - - + + - - + + - - + + - - - + + + - - + + - - + + - + - - + + - + - + - + - + @@ -37494,7 +37331,7 @@ - + @@ -37503,19 +37340,19 @@ - + - + - + - + - + @@ -37524,7 +37361,7 @@ - + @@ -37535,55 +37372,55 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -37616,13 +37453,13 @@ - + - + - + @@ -37631,10 +37468,10 @@ - + - + @@ -37649,7 +37486,7 @@ - + @@ -37676,28 +37513,28 @@ - + - + - + - + - + @@ -37708,10 +37545,10 @@ - + - + @@ -37728,18 +37565,18 @@ - + - + - + - + - + @@ -37748,16 +37585,16 @@ - + - + - + @@ -37768,7 +37605,7 @@ - + @@ -37803,27 +37640,27 @@ - + - + - + - + - + - + @@ -37838,91 +37675,91 @@ - + - - - + + + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -37931,16 +37768,16 @@ - + - + - + - + @@ -37955,10 +37792,10 @@ - + - + @@ -37979,27 +37816,27 @@ - + - + - + - + - + - + @@ -38008,7 +37845,7 @@ - + @@ -38017,7 +37854,7 @@ - + @@ -38031,10 +37868,10 @@ - + - + @@ -38043,16 +37880,16 @@ - + - + - + - + @@ -38076,85 +37913,85 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - - + + @@ -38165,13 +38002,13 @@ - - + + - + @@ -38181,46 +38018,46 @@ - + - + - + - + - + - + - + - + - + - + - - + + @@ -38231,24 +38068,24 @@ - - - + + + - + - + - + @@ -38271,15 +38108,15 @@ - + - + - + @@ -38287,7 +38124,7 @@ - + @@ -38295,7 +38132,7 @@ - + @@ -38316,10 +38153,10 @@ - + - + @@ -38333,9 +38170,9 @@ - + - + @@ -38344,10 +38181,10 @@ - + - + @@ -38368,7 +38205,7 @@ - + @@ -38380,27 +38217,27 @@ - + - + - + - + - + - + @@ -38409,10 +38246,10 @@ - + - + @@ -38420,23 +38257,23 @@ - + - + - + - + - + - + @@ -38447,89 +38284,89 @@ - + - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - - + - + @@ -38540,297 +38377,297 @@ - + - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + - + - + - - + + - + - - + + - + - - + + - + - - + + - + - - + + - + - + - - + + - + + + + + + - - - + + + + + + + + + + + + + + + + + + + + + + - - - - + + - - + - - - - + + - - - + - + - + + - - + + - + + - - - - - - - - - - - - - - - - - - - - + - - + + - + - - + + - - - + + + - - + + - - + + - - + + - - + + - + + + + + + + + + + + + + + + + + + - + + - + + - + + - + + - - + + + - - + + - - - + + + - - + + + + - - - - - - - - - - - - - - - - - - - - - - - - - + - + - - + + @@ -38853,28 +38690,28 @@ - + - + - + - + - + - + - - + + @@ -38891,7 +38728,7 @@ - + @@ -38906,10 +38743,10 @@ - + - + @@ -38920,13 +38757,13 @@ - + - + - + @@ -38935,32 +38772,32 @@ - + - + - + - + - + - + - + @@ -38977,121 +38814,121 @@ - + - + - + - + - + - + - + - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + - + - + - + - + - + - + - + - + - + - - + + - + - + - - + + - - + + - - + + - + @@ -39103,7 +38940,7 @@ - + @@ -39132,35 +38969,35 @@ - + - + - + - + - + - + - + - + - + @@ -39178,159 +39015,159 @@ - - - - - - + + + + + + - + - + - - + + - - + + - - + + - - + + - - + + - - + + - + - + - + - + - + - + - - + + - + - + - + - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + - + - - + + - + - + - + - + - + - + - + - + @@ -39360,73 +39197,73 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -39435,7 +39272,7 @@ - + @@ -39498,49 +39335,49 @@ - + - + - + - + - + - + - + - + - + - + - + - + @@ -39554,22 +39391,22 @@ - + - + - + - + @@ -39617,7 +39454,7 @@ - + @@ -39635,16 +39472,16 @@ - + - + - + @@ -39653,27 +39490,27 @@ - + - + - + - + - + - + - + @@ -39700,13 +39537,13 @@ - + - + @@ -39748,7 +39585,7 @@ - + @@ -39772,169 +39609,169 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -39961,7 +39798,7 @@ - + @@ -39970,26 +39807,26 @@ - + - + - + - + - + - + - + @@ -40001,59 +39838,59 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -40077,10 +39914,10 @@ - + - + @@ -40088,32 +39925,32 @@ - + - + - + - + - + - + - + @@ -40125,16 +39962,16 @@ - + - + - + @@ -40145,7 +39982,7 @@ - + @@ -40166,7 +40003,7 @@ - + @@ -40174,18 +40011,18 @@ - + - + - + - + - + @@ -40199,7 +40036,7 @@ - + @@ -40216,19 +40053,19 @@ - + - + - + - + - + @@ -40271,7 +40108,7 @@ - + @@ -40279,7 +40116,7 @@ - + @@ -40381,13 +40218,13 @@ - + - + @@ -40396,42 +40233,42 @@ - + - + - + - + - + - + - + - + - + - + - + - + @@ -40458,9 +40295,9 @@ - + - + @@ -40469,41 +40306,41 @@ - + - + - + - + - + - + - + - + - + - + - + - + @@ -40512,18 +40349,18 @@ - + - + - + - + @@ -40532,16 +40369,16 @@ - + - + - + - + @@ -40556,10 +40393,10 @@ - + - + @@ -40568,22 +40405,22 @@ - + - + - + - + - + @@ -40595,7 +40432,7 @@ - + @@ -40610,12 +40447,12 @@ - + - + - + @@ -40624,16 +40461,16 @@ - + - + - + @@ -40642,110 +40479,110 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -40775,7 +40612,7 @@ - + @@ -40787,50 +40624,50 @@ - + - + - + - + - + - + - + - - + + - + - + - + - + @@ -40845,7 +40682,7 @@ - + @@ -40859,12 +40696,12 @@ - + - + - + @@ -40873,17 +40710,17 @@ - + - + - - - + + + @@ -40900,109 +40737,109 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -41041,22 +40878,22 @@ - + - + - + - + - + - + @@ -41071,68 +40908,68 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -41146,7 +40983,7 @@ - + @@ -41161,7 +40998,7 @@ - + @@ -41169,12 +41006,12 @@ - + - + - + @@ -41183,21 +41020,21 @@ - + - + - + - + - + @@ -41217,7 +41054,7 @@ - + @@ -41237,34 +41074,34 @@ - + - + - + - + - + - + - + - + @@ -41276,7 +41113,7 @@ - + @@ -41294,13 +41131,13 @@ - + - + @@ -41326,16 +41163,16 @@ - + - + - + - + @@ -41386,10 +41223,10 @@ - + - + @@ -41400,7 +41237,7 @@ - + @@ -41444,27 +41281,27 @@ - + - + - + - + - + - + - + - + @@ -41530,7 +41367,7 @@ - + @@ -41556,7 +41393,7 @@ - + @@ -41600,7 +41437,7 @@ - + @@ -41629,13 +41466,13 @@ - + - + - + @@ -41643,90 +41480,90 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -41746,7 +41583,7 @@ - + @@ -41766,7 +41603,7 @@ - + @@ -41792,10 +41629,10 @@ - + - + @@ -41809,13 +41646,13 @@ - + - + - + @@ -41823,7 +41660,7 @@ - + @@ -41837,7 +41674,7 @@ - + @@ -41845,15 +41682,15 @@ - + - + - + @@ -41861,19 +41698,19 @@ - + - + - + @@ -41884,7 +41721,7 @@ - + @@ -41920,13 +41757,13 @@ - + - + @@ -41935,9 +41772,9 @@ - + - + @@ -41952,7 +41789,7 @@ - + @@ -41960,10 +41797,10 @@ - + - + @@ -41971,77 +41808,77 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -42059,7 +41896,7 @@ - + @@ -42068,18 +41905,18 @@ - + - + - + - + @@ -42097,45 +41934,45 @@ - + - + - + - + - + - + - + - + - + - + - + - + @@ -42149,116 +41986,116 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -42282,7 +42119,7 @@ - + @@ -42293,7 +42130,7 @@ - + @@ -42307,37 +42144,37 @@ - - + + - + - + - + - + - + - + - + - + - + - + @@ -42375,43 +42212,43 @@ - + - + - + - + - + - + - + - + - + @@ -42441,33 +42278,33 @@ - + - + - + - + - + - + - + - + - + - + @@ -42481,7 +42318,7 @@ - + @@ -42489,21 +42326,21 @@ - + - + - + - + - + - + @@ -42514,39 +42351,39 @@ - + - + - + - + - + - + - + - + - + - + - + - + @@ -42566,12 +42403,12 @@ - + - + @@ -42580,20 +42417,20 @@ - + - + - + - + - + @@ -42610,1000 +42447,929 @@ - + - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + - - - - - + + + - + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - + + + + + + + + + + + - + - + - + - - + + - - - + + + + + + + + + + + + + + + - - + + + + + + - - - - - - + + + - - + + + + - - - - - - - + + - - - - + + + - - + + + + + + + + + + + + + - - - + + + - - - + + + - - - + + + - - - - + + + - - - - + + + - - - - + + + - - - - + + + + + + - + + - + + + + - - - - - - - - - - - - - - - - - - - - - - - - - + + - - - + + + - - + + - - + + - - + + - - + + - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - + + + + + - - + + + + + + + - - - - + + + + + - - - - - - + + + + + + - - - - - + + + + + - - - - - - - + + + - - - - - + + + - - - - - - + + + + - - - - - + + + + + + - - - + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + - - + + - - - - + + + + - - - + + + - + + - - + + - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - + + - + + - - - - + + + + - - - - - - - - - - - - + + + - - + + - - + + - - - - + - - - + + - - - - - - - - - - - - - - - - - - - - + - - + + - - + + - - - + + + - - + + - - - + + + - - + + - + + + + + + + + + + + + + + + + + + + + + + + + + + - - - + + + + - - - + + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - + - - + + - - + + - - + + + + + + + + + + - - + + + + + + + + + + + + + - + - + - - - + + - - + - + + - + + - - + - + - - + + - - + + + - + + + - - - - - - - - - - - - - - - - - - - - - - + - - + + - - + + - - - + + + - - + + - - + + + + + + + + + + + + + + + + + + + + + + - - + - - + + - - + + - - - - - - - - - - + - - - + + + + - - - - - - - - - - - - - + + - - + + - - + + - - + + - - + + - - - + + + - + - - + + - + - + - + - - + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + - + @@ -43618,50 +43384,99 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -43670,24 +43485,24 @@ - + - + - + - + - + - + @@ -43696,7 +43511,7 @@ - + @@ -43704,36 +43519,36 @@ - + - + - + - + - + - + - + - + - + - + @@ -43744,7 +43559,7 @@ - + @@ -43767,19 +43582,19 @@ - + - + - + @@ -43824,11 +43639,34 @@ - + - - + + + + + + + + + + + + + + + + + + + + + + + + + @@ -43839,9 +43677,9 @@ - + - + @@ -43859,9 +43697,9 @@ - + - + @@ -43873,238 +43711,128 @@ - + - + - + - - - - - - - - - - - - - - - - - - - - - - - - - - - + - - - - - - - - - - - - - - - - - - - - - - - - + - + - + - + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + - + - + - + - + - + - + - + - + - + - + - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + - - + + - - + + - - + + - - + + - - + + - + - - + + - - + + - - - + + + @@ -44113,102 +43841,102 @@ - + - - + + - - + + - + - - - + + + - + - - + + - - - + + + - - + + - - + + - + - - + + - - + + - - - + + + - + - + - + - + - - - + + + - + - + - + - + @@ -44223,7 +43951,7 @@ - + @@ -44234,7 +43962,7 @@ - + @@ -44264,7 +43992,7 @@ - + @@ -44272,13 +44000,13 @@ - + - + @@ -44287,27 +44015,27 @@ - + - + - + - + - + - + - + @@ -44316,7 +44044,7 @@ - + @@ -44327,7 +44055,7 @@ - + @@ -44345,7 +44073,7 @@ - + @@ -44354,71 +44082,71 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -44439,1374 +44167,7 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + @@ -45964,12 +44325,12 @@ - + - + - + @@ -45982,7 +44343,7 @@ - + @@ -45993,7 +44354,7 @@ - + @@ -46004,7 +44365,7 @@ - + @@ -46012,13 +44373,13 @@ - + - + @@ -46032,15 +44393,15 @@ - + - + - + @@ -46051,7 +44412,7 @@ - + @@ -46059,23 +44420,1396 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - + + - + - - + + - + @@ -46087,29 +45821,29 @@ - - - + + + - - - + + + - + - + @@ -46117,7 +45851,7 @@ - + @@ -46125,7 +45859,7 @@ - + @@ -46136,36 +45870,37 @@ - - - - + + + + + - - + + - - + + - - + + - + - - + + - + @@ -46183,20 +45918,20 @@ - + - + - + - + - - + + @@ -46221,7 +45956,7 @@ - + @@ -46231,7 +45966,7 @@ - + @@ -46240,11 +45975,11 @@ - - - + + + - + @@ -46262,8 +45997,8 @@ - - + + @@ -46285,7 +46020,7 @@ - + @@ -46378,15 +46113,15 @@ - - + + - + @@ -46394,76 +46129,76 @@ - + - + - + - + - + - + - + - + - - + + - - + + - - + + - + - + - - + + @@ -46480,22 +46215,22 @@ - + - + - - - + + + - - + + - + @@ -46515,11 +46250,11 @@ - + - + @@ -46533,13 +46268,13 @@ - + - + @@ -46563,7 +46298,7 @@ - + @@ -46571,7 +46306,7 @@ - + @@ -46579,15 +46314,15 @@ - + - - - - + + + + @@ -46607,7 +46342,7 @@ - + @@ -46662,15 +46397,15 @@ - + - + - + @@ -46683,7 +46418,7 @@ - + @@ -46692,8 +46427,8 @@ - - + + @@ -46704,33 +46439,33 @@ - + - - - + + + - + - + - + - + - + @@ -46741,10 +46476,10 @@ - + - + @@ -46753,7 +46488,7 @@ - + @@ -46762,37 +46497,37 @@ - + - + - + - + - + - + - + - + - + - + @@ -46885,22 +46620,22 @@ - + - + - + - + - + @@ -46909,7 +46644,7 @@ - + @@ -46921,7 +46656,7 @@ - + @@ -46931,9 +46666,9 @@ - + - + @@ -46954,7 +46689,7 @@ - + @@ -46995,13 +46730,13 @@ - + - + - + @@ -47016,20 +46751,20 @@ - + - + - + - + - + @@ -47038,7 +46773,7 @@ - + @@ -47053,7 +46788,7 @@ - + @@ -47067,7 +46802,7 @@ - + @@ -47079,7 +46814,7 @@ - + @@ -47093,10 +46828,10 @@ - + - + @@ -47104,13 +46839,13 @@ - + - + @@ -47118,7 +46853,7 @@ - + @@ -47133,13 +46868,13 @@ - + - + - + @@ -47154,27 +46889,27 @@ - + - + - + - + - + - + - + - + @@ -47182,13 +46917,13 @@ - + - + @@ -47202,7 +46937,7 @@ - + @@ -47216,7 +46951,7 @@ - + @@ -47225,10 +46960,10 @@ - + - + @@ -47245,16 +46980,16 @@ - + - + - + - + @@ -47271,7 +47006,7 @@ - + @@ -47282,21 +47017,21 @@ - + - + - + - + - + @@ -47305,7 +47040,7 @@ - + @@ -47313,7 +47048,7 @@ - + @@ -47331,7 +47066,7 @@ - + @@ -47357,24 +47092,24 @@ - + - + - + - + - + @@ -47410,9 +47145,9 @@ - + - + @@ -47421,13 +47156,13 @@ - + - + @@ -47456,14 +47191,14 @@ - + - + @@ -47471,16 +47206,16 @@ - + - + - + @@ -47488,7 +47223,7 @@ - + @@ -47502,54 +47237,54 @@ - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + - - + + - + - - - - - - + + + + + + - + - + - + @@ -47561,13 +47296,13 @@ - + - + @@ -47576,19 +47311,19 @@ - + - + - + - + @@ -47642,37 +47377,37 @@ - + - + - + - + - + - + - + - + - + @@ -47695,13 +47430,13 @@ - + - + - + @@ -47758,13 +47493,13 @@ - + - + - + @@ -47781,8 +47516,8 @@ - - + + @@ -47796,7 +47531,7 @@ - + @@ -47807,140 +47542,140 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -47951,52 +47686,52 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -48008,40 +47743,40 @@ - + - + - + - + - + - + - + - + - + - + - + @@ -48049,42 +47784,42 @@ - + - + - + - + - + - + - + - + - + - + - + - + @@ -48092,7 +47827,7 @@ - + @@ -48104,48 +47839,48 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -48166,12 +47901,12 @@ - + - + - + @@ -48248,15 +47983,15 @@ - + - + - + @@ -48273,15 +48008,15 @@ - + - + - + - + @@ -48289,13 +48024,13 @@ - + - + @@ -48321,7 +48056,7 @@ - + @@ -48329,13 +48064,13 @@ - + - + @@ -48346,15 +48081,15 @@ - + - + - + - + @@ -48363,13 +48098,13 @@ - + - + @@ -48380,7 +48115,7 @@ - + @@ -48389,372 +48124,372 @@ - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - + - - + + - - + + - - + + - - + + - + - + - + - - + + - + - + - + - + - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - + + + + + + + - + + - - + + - - - - + + + - - + + - + + + - - + + + - - + + + - - - + + + + + - - + + + + + + - - - - - - + + + + - - + - - - + + - - - + + - - + + - - - + - - - + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + - - + + - - + + - - + + - - + + - - - + + + - + - + - - + + - - + + - + - - + + - - + + - - + + - - - + + + - - - + + + - - - + + + - - + + - - + + @@ -48762,13 +48497,13 @@ - + - + @@ -48795,48 +48530,48 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -48871,7 +48606,7 @@ - + @@ -48889,7 +48624,7 @@ - + @@ -48909,12 +48644,12 @@ - + - + - + @@ -48926,33 +48661,33 @@ - + - + - + - + - + - + - + - + - + @@ -48967,7 +48702,7 @@ - + @@ -48975,59 +48710,59 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -49036,191 +48771,191 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - + - - + + - - + + - + - + - - + + - - - + + + - - + + - - - + + + - - - + + + - - + + - - - + + + - - + + - - + + - - + + - - - + + + - - - - - + + + + + - - + + - + - - + + - - - + + + - - + + - - - + + + - - - + + + - - + + - + - + - + - + - + - + - + - + @@ -49231,7 +48966,7 @@ - + @@ -49276,16 +49011,16 @@ - + - + - + - + @@ -49297,10 +49032,10 @@ - + - + @@ -49315,19 +49050,19 @@ - + - + - + - + - + @@ -49357,16 +49092,16 @@ - + - + - + @@ -49380,7 +49115,7 @@ - + @@ -49397,24 +49132,24 @@ - + - + - + - + - + - + - + @@ -49429,13 +49164,13 @@ - + - + - + @@ -49446,16 +49181,16 @@ - + - + - + - + @@ -49463,7 +49198,7 @@ - + @@ -49471,32 +49206,32 @@ - + - + - + - + - + - + - + - + - + @@ -49510,7 +49245,7 @@ - + @@ -49518,10 +49253,10 @@ - + - + @@ -49538,16 +49273,16 @@ - + - + - + @@ -49567,7 +49302,7 @@ - + @@ -49575,7 +49310,7 @@ - + @@ -49595,12 +49330,12 @@ - + - + - + @@ -49609,7 +49344,7 @@ - + @@ -49629,7 +49364,7 @@ - + @@ -49640,10 +49375,10 @@ - + - + @@ -49652,30 +49387,30 @@ - + - + - + - + - + - + - + - + @@ -49687,7 +49422,7 @@ - + @@ -49696,7 +49431,7 @@ - + @@ -49707,9 +49442,9 @@ - + - + @@ -49724,89 +49459,89 @@ - + - + - + - + - + - + - + - + - + - + - + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - + - + - + - + @@ -49815,104 +49550,104 @@ - + - + - + - - + + - - - + + + - - - - + + + + - - + + - - - + + + - - + + - - + + - - + + - - - - + + + + - - + + - - - + + + - - + + - - + + - - + + - - + + - - - + + + - - + + @@ -49921,14 +49656,14 @@ - + - + @@ -49957,19 +49692,19 @@ - + - + - + - + @@ -49980,13 +49715,13 @@ - + - + @@ -50013,10 +49748,10 @@ - + - + @@ -50046,7 +49781,7 @@ - + @@ -50058,36 +49793,36 @@ - + - + - + - + - + - + - + - + - + - + - + @@ -50095,10 +49830,10 @@ - + - + @@ -50106,90 +49841,90 @@ - + - + - - - - - - - - - - - - - - + + + + + + + + + + + + + + - + - + - + - + - + - + - + - + - + - + - + - + - - + + - + - - + + @@ -50200,64 +49935,64 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -50266,7 +50001,7 @@ - + @@ -50283,10 +50018,10 @@ - + - + @@ -50310,7 +50045,7 @@ - + @@ -50322,19 +50057,19 @@ - + - + - + - + - + @@ -50379,22 +50114,22 @@ - + - + - + - + - + @@ -50405,30 +50140,30 @@ - + - + - + - + - + - + - + - + @@ -50440,37 +50175,37 @@ - + - + - + - + - + - + - + - + - + - + @@ -50481,13 +50216,13 @@ - + - + @@ -50495,18 +50230,18 @@ - + - + - + @@ -50515,7 +50250,7 @@ - + @@ -50530,32 +50265,32 @@ - + - + - + - + - + - + - + - + - + @@ -50564,13 +50299,13 @@ - + - + - + @@ -50612,13 +50347,13 @@ - + - + - + @@ -50627,7 +50362,7 @@ - + @@ -50639,77 +50374,77 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -50717,9 +50452,9 @@ - + - + @@ -50728,7 +50463,7 @@ - + @@ -50755,9 +50490,9 @@ - + - + @@ -50769,27 +50504,27 @@ - + - + - + - + - + - + - + @@ -50800,220 +50535,220 @@ - + - + - + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - + - + - - + + - - + + - - + + - + - - - + + + - - - + + + - - - + + + - - + + - - + + - - + + - - - + + + - - + + - - + + - - + + - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + - - + + - - + + - - + + - - + + - + - + - + - + @@ -51022,22 +50757,22 @@ - + - + - + - + - + - + @@ -51067,7 +50802,7 @@ - + @@ -51079,7 +50814,7 @@ - + @@ -51094,14 +50829,14 @@ - + - + @@ -51109,15 +50844,15 @@ - + - + - + @@ -51132,7 +50867,7 @@ - + @@ -51146,96 +50881,96 @@ - - - - - - - - - - - + + + + + + + + + + + - - + + - + - - - + + + - - + + - - - + + + - - - + + + - - + + - - + + - - - + + + - + - - + + - + - + - + - + - + - + @@ -51246,20 +50981,20 @@ - + - + - - + + @@ -51270,7 +51005,7 @@ - + @@ -51281,7 +51016,7 @@ - + @@ -51293,7 +51028,7 @@ - + @@ -51307,10 +51042,10 @@ - + - + @@ -51318,34 +51053,34 @@ - + - + - + - + - + - - + + - + - + @@ -51353,21 +51088,21 @@ - + - + - + - + - + - + @@ -51387,7 +51122,7 @@ - + @@ -51407,7 +51142,7 @@ - + @@ -51424,9 +51159,9 @@ - + - + @@ -51438,32 +51173,37 @@ - - - + + + + + + + + + - - + + - + - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + @@ -51471,166 +51211,161 @@ - - - - - - + - + - + - - + + - + - + - + - + - - + + - - + + - - - + + + - - + + - - + + - - + + - - + + - + - + - + - + - + - + - + - - + + - - + + - - + + - - + + - + - + + - - + - - + + - - + + - - - + + + - - + + - - + + - + - + @@ -51638,7 +51373,7 @@ - + @@ -51646,53 +51381,53 @@ - + - + - - - + + + - + - + - + - + - + - + - - + + - + @@ -51700,7 +51435,7 @@ - + @@ -51708,7 +51443,7 @@ - + @@ -51719,7 +51454,7 @@ - + @@ -51733,12 +51468,12 @@ - + - + @@ -51746,7 +51481,7 @@ - + @@ -51754,10 +51489,10 @@ - + - + @@ -51765,7 +51500,7 @@ - + @@ -51800,22 +51535,22 @@ - + - + - + - + - + - + @@ -51836,10 +51571,10 @@ - + - + @@ -51848,7 +51583,7 @@ - + @@ -51856,12 +51591,12 @@ - + - + - + @@ -51894,8 +51629,8 @@ - - + + @@ -51906,7 +51641,7 @@ - + @@ -51932,7 +51667,7 @@ - + @@ -51946,7 +51681,7 @@ - + @@ -51954,7 +51689,7 @@ - + @@ -51965,7 +51700,7 @@ - + @@ -51982,12 +51717,12 @@ - + - + - + @@ -52017,9 +51752,9 @@ - + - + @@ -52028,12 +51763,12 @@ - + - + @@ -52048,13 +51783,13 @@ - + - + - + @@ -52066,55 +51801,55 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -52129,16 +51864,16 @@ - + - + - + @@ -52150,28 +51885,28 @@ - + - + - + - + - + - + - + @@ -52188,15 +51923,15 @@ - + - + - + - + @@ -52208,7 +51943,7 @@ - + @@ -52234,7 +51969,7 @@ - + @@ -52246,10 +51981,10 @@ - + - + @@ -52263,7 +51998,7 @@ - + @@ -52274,7 +52009,7 @@ - + @@ -52288,7 +52023,7 @@ - + @@ -52299,10 +52034,10 @@ - + - + @@ -52313,7 +52048,7 @@ - + @@ -52336,24 +52071,24 @@ - + - + - + - + - + - + - + @@ -52364,7 +52099,7 @@ - + @@ -52378,7 +52113,7 @@ - + @@ -52392,25 +52127,25 @@ - + - + - + - + - + - + - + @@ -52421,7 +52156,7 @@ - + @@ -52432,16 +52167,16 @@ - + - + - + - + @@ -52450,7 +52185,7 @@ - + @@ -52461,7 +52196,7 @@ - + @@ -52502,15 +52237,15 @@ - + - + - + - + @@ -52531,7 +52266,7 @@ - + @@ -52539,15 +52274,15 @@ - + - + - + @@ -52555,21 +52290,21 @@ - + - + - + - + - + @@ -52589,10 +52324,10 @@ - + - + @@ -52601,16 +52336,16 @@ - + - + - + - + @@ -52618,9 +52353,9 @@ - + - + @@ -52644,7 +52379,7 @@ - + @@ -52656,10 +52391,10 @@ - + - + @@ -52667,14 +52402,14 @@ - + - + - + - + @@ -52695,10 +52430,10 @@ - + - + @@ -52707,24 +52442,24 @@ - + - + - + - + - + @@ -52733,13 +52468,13 @@ - + - + @@ -52747,10 +52482,10 @@ - + - + @@ -52789,7 +52524,7 @@ - + @@ -52816,13 +52551,13 @@ - + - + @@ -52840,21 +52575,21 @@ - + - + - + - + - + @@ -52887,9 +52622,9 @@ - + - + @@ -52907,7 +52642,7 @@ - + @@ -52915,7 +52650,7 @@ - + @@ -52936,7 +52671,7 @@ - + @@ -52947,10 +52682,10 @@ - + - + @@ -52968,21 +52703,21 @@ - + - + - + - + - + @@ -53036,10 +52771,10 @@ - + - + @@ -53051,10 +52786,10 @@ - + - + @@ -53068,31 +52803,31 @@ - + - + - + - + - + - + - + @@ -53137,18 +52872,18 @@ - + - + - + - + - + @@ -53163,10 +52898,10 @@ - + - + @@ -53204,38 +52939,38 @@ - + - + - + - + - + - + - + - + - + @@ -53243,23 +52978,23 @@ - + - + - + - + - + - + @@ -53267,108 +53002,108 @@ - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + - + - + - + - + - + @@ -53376,11 +53111,11 @@ - + - + @@ -53388,59 +53123,59 @@ - + - + - + - - + + - - + + - + - + - + - + - + @@ -53448,7 +53183,7 @@ - + @@ -53457,12 +53192,12 @@ - + - + @@ -53470,164 +53205,164 @@ - - + + - + - + - + - + - - + + - - + + - - + + - - + + - - + + - - + + - + - - + + - - + + - - + + - + - + - + - - + + - + - - + + - + - - + + - - + + - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - + - - + + - - + + - - + + - - + + - + @@ -53637,43 +53372,43 @@ - - + + - + - + - - + + - + - + - + - + - + @@ -53682,15 +53417,15 @@ - + - + - + @@ -53698,27 +53433,27 @@ - - - - + + + + - + - + - + - + - + @@ -53726,65 +53461,65 @@ - - + + - - - + + + - + - + - + - - + + - + - + - + - + - + - + - + - - + + - + @@ -53792,24 +53527,24 @@ - + - + - + - + - + - + - + @@ -53841,21 +53576,21 @@ - + - + - - + + - + @@ -53864,7 +53599,7 @@ - + @@ -53873,19 +53608,19 @@ - - + + - + - + - + @@ -53924,13 +53659,13 @@ - + - + @@ -53972,7 +53707,7 @@ - + @@ -53984,18 +53719,18 @@ - + - + - + @@ -54004,10 +53739,10 @@ - + - + @@ -54016,10 +53751,10 @@ - + - + @@ -54028,7 +53763,7 @@ - + @@ -54036,19 +53771,19 @@ - + - + - + - + @@ -54057,13 +53792,13 @@ - + - + @@ -54072,25 +53807,25 @@ - + - + - + - + - + @@ -54102,7 +53837,7 @@ - + @@ -54111,109 +53846,109 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -54221,15 +53956,15 @@ - + - + - + @@ -54238,8 +53973,8 @@ - - + + @@ -54267,7 +54002,7 @@ - + @@ -54278,24 +54013,24 @@ - + - + - + - + - + - + @@ -54304,9 +54039,9 @@ - + - + @@ -54315,7 +54050,7 @@ - + @@ -54330,59 +54065,59 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -54391,148 +54126,148 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - - + + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - - + + - + @@ -54541,53 +54276,53 @@ - + - + - + - + - + - + - + - + - + - + - + - + - - + + - + - + @@ -54601,67 +54336,67 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -54669,9 +54404,9 @@ - + - + @@ -54680,7 +54415,7 @@ - + @@ -54688,19 +54423,19 @@ - + - + - + - + - + @@ -54715,108 +54450,108 @@ - + - + - + - + - + - + - + - + - + - + - + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - - - - - - - - + + + + + + - + - + @@ -54825,147 +54560,147 @@ - + - - + + - + - + - + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - - + + + - - + + - - + + - - + + - - + + - - + + - - + + - + - + - + - + - + - - + + - + - + - + - + - + - + - - + + - + - - - + + + @@ -54975,67 +54710,67 @@ - - - + + + - - + + - - + + - - + + - - + + - - + + - - + + - + - - - + + + - + - + - - + + @@ -55043,48 +54778,48 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -55093,13 +54828,13 @@ - + - + - + @@ -55110,7 +54845,7 @@ - + @@ -55124,7 +54859,7 @@ - + @@ -55138,7 +54873,7 @@ - + @@ -55152,7 +54887,7 @@ - + @@ -55166,21 +54901,21 @@ - + - + - + - + - + @@ -55197,33 +54932,33 @@ - + - + - + - + - + - + - + - - + + @@ -55234,13 +54969,13 @@ - + - + - + @@ -55249,7 +54984,7 @@ - + @@ -55282,7 +55017,7 @@ - + @@ -55291,15 +55026,15 @@ - + - + - + @@ -55344,9 +55079,9 @@ - + - + @@ -55355,7 +55090,7 @@ - + @@ -55363,10 +55098,10 @@ - + - + @@ -55374,7 +55109,7 @@ - + @@ -55382,7 +55117,7 @@ - + @@ -55393,13 +55128,13 @@ - + - + - + @@ -55408,7 +55143,7 @@ - + @@ -55422,7 +55157,7 @@ - + @@ -55433,7 +55168,7 @@ - + @@ -55441,21 +55176,21 @@ - + - + - + - + - - + + @@ -55464,26 +55199,26 @@ - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + @@ -55500,8 +55235,8 @@ - - + + @@ -55525,11 +55260,11 @@ - + - + @@ -55584,7 +55319,7 @@ - + @@ -55599,7 +55334,7 @@ - + @@ -55649,33 +55384,33 @@ - + - + - + - + - + - + - + - + @@ -55754,44 +55489,41 @@ - + - - + + - + - + - + - + - - - + + + - + - + - - - - - - - + + + + @@ -55814,7 +55546,7 @@ - + @@ -55831,11 +55563,11 @@ - + - - + + @@ -55845,21 +55577,21 @@ - + - + - + - + - + @@ -55873,45 +55605,45 @@ - + - + - + - + - - - - + + + + - - + + - + - + - + - + - + @@ -55947,7 +55679,7 @@ - + @@ -55955,7 +55687,7 @@ - + @@ -55963,9 +55695,9 @@ - + - + @@ -55974,7 +55706,7 @@ - + @@ -55982,10 +55714,10 @@ - + - + @@ -56003,125 +55735,125 @@ - + - + - + - - + + - + - + - + - - + + - + - + - + - + - + - + - + - + - + - + - + - + - - + + - - + + - + - + - + - + - + - + - + - + - + @@ -56148,13 +55880,13 @@ - + - + @@ -56220,10 +55952,10 @@ - + - + @@ -56250,7 +55982,7 @@ - + @@ -56259,13 +55991,13 @@ - + - + - + @@ -56277,10 +56009,10 @@ - + - + @@ -56289,13 +56021,13 @@ - + - + - + @@ -56334,18 +56066,18 @@ - + - + - + - + @@ -56353,9 +56085,9 @@ - + - + @@ -56379,7 +56111,7 @@ - + @@ -56387,13 +56119,13 @@ - + - + @@ -56413,12 +56145,12 @@ - + - + - + @@ -56433,7 +56165,7 @@ - + @@ -56444,7 +56176,7 @@ - + @@ -56467,19 +56199,19 @@ - + - + - + - + @@ -56493,9 +56225,9 @@ - + - + @@ -56510,7 +56242,7 @@ - + @@ -56519,7 +56251,7 @@ - + @@ -56537,10 +56269,10 @@ - + - + @@ -56551,7 +56283,7 @@ - + @@ -56562,12 +56294,12 @@ - + - + @@ -56576,7 +56308,7 @@ - + @@ -56588,7 +56320,7 @@ - + @@ -56600,16 +56332,16 @@ - + - + - + @@ -56618,10 +56350,10 @@ - + - + @@ -56645,38 +56377,38 @@ - + - + - + - + - + - + - + - + - + @@ -56684,15 +56416,15 @@ - + - + - + @@ -56700,12 +56432,12 @@ - + - + - + @@ -56732,10 +56464,10 @@ - + - + @@ -56747,13 +56479,13 @@ - + - + - + @@ -56762,7 +56494,7 @@ - + @@ -56771,13 +56503,13 @@ - + - + - + @@ -56813,7 +56545,7 @@ - + @@ -56825,7 +56557,7 @@ - + @@ -56843,18 +56575,18 @@ - + - + - + - + @@ -56871,16 +56603,16 @@ - + - + - + @@ -56888,7 +56620,7 @@ - + @@ -56914,33 +56646,33 @@ - + - + - + - + - + - + - + - + - + - + @@ -56955,13 +56687,13 @@ - + - + @@ -56970,12 +56702,12 @@ - + - + - + @@ -57008,7 +56740,7 @@ - + @@ -57017,13 +56749,13 @@ - + - + @@ -57037,15 +56769,15 @@ - + - + - + - + @@ -57054,10 +56786,10 @@ - + - + @@ -57068,15 +56800,15 @@ - + - + - + @@ -57091,7 +56823,7 @@ - + @@ -57102,12 +56834,12 @@ - + - + - + @@ -57119,15 +56851,15 @@ - + - + - + - + @@ -57136,22 +56868,22 @@ - + - + - + - + - + @@ -57165,7 +56897,7 @@ - + @@ -57179,15 +56911,15 @@ - + - + - + - + @@ -57196,7 +56928,7 @@ - + @@ -57204,74 +56936,74 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -57279,16 +57011,16 @@ - + - + - + - + @@ -57297,39 +57029,39 @@ - + - + - + - + - + - + @@ -57337,24 +57069,24 @@ - + - + - + - + - + - + - + @@ -57366,7 +57098,7 @@ - + @@ -57377,7 +57109,7 @@ - + @@ -57385,15 +57117,15 @@ - + - + - + - + @@ -57407,18 +57139,18 @@ - + - + - + - + @@ -57427,15 +57159,15 @@ - + - + - + @@ -57455,18 +57187,18 @@ - + - + - + - + @@ -57475,7 +57207,7 @@ - + @@ -57483,9 +57215,9 @@ - + - + @@ -57494,7 +57226,7 @@ - + @@ -57502,10 +57234,10 @@ - + - + @@ -57513,7 +57245,7 @@ - + @@ -57524,10 +57256,10 @@ - + - + @@ -57556,333 +57288,333 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -57890,7 +57622,7 @@ - + @@ -57899,12 +57631,12 @@ - + - + - + @@ -57919,19 +57651,19 @@ - + - + - + - + @@ -57943,22 +57675,22 @@ - + - + - + - + - + @@ -57973,10 +57705,10 @@ - + - + @@ -58029,13 +57761,13 @@ - + - + @@ -58052,7 +57784,7 @@ - + @@ -58067,7 +57799,7 @@ - + @@ -58082,7 +57814,7 @@ - + @@ -58096,7 +57828,7 @@ - + @@ -58104,48 +57836,48 @@ - + - + - + - + - + - + - + - + - + - + - + - + @@ -58156,7 +57888,7 @@ - + @@ -58191,10 +57923,10 @@ - + - + @@ -58203,7 +57935,7 @@ - + @@ -58236,7 +57968,7 @@ - + @@ -58245,10 +57977,10 @@ - + - + @@ -58257,21 +57989,21 @@ - + - + - + - + @@ -58282,7 +58014,7 @@ - + @@ -58320,16 +58052,16 @@ - + - + - + - + @@ -58347,10 +58079,10 @@ - + - + @@ -58368,13 +58100,13 @@ - + - + - + @@ -58395,7 +58127,7 @@ - + @@ -58416,7 +58148,7 @@ - + @@ -58445,7 +58177,7 @@ - + @@ -58456,7 +58188,7 @@ - + @@ -58464,7 +58196,7 @@ - + @@ -58481,10 +58213,10 @@ - + - + @@ -58522,7 +58254,7 @@ - + @@ -58557,7 +58289,7 @@ - + @@ -58637,7 +58369,7 @@ - + @@ -58646,9 +58378,9 @@ - + - + @@ -58696,7 +58428,7 @@ - + @@ -58705,12 +58437,12 @@ - + - + - + @@ -58739,9 +58471,9 @@ - + - + @@ -58756,7 +58488,7 @@ - + @@ -58764,9 +58496,9 @@ - + - + @@ -58775,7 +58507,7 @@ - + @@ -58793,28 +58525,28 @@ - + - + - + - + - + - + - + @@ -58823,23 +58555,23 @@ - + - + - + - + - + @@ -58848,7 +58580,7 @@ - + @@ -58866,9 +58598,9 @@ - + - + @@ -58880,7 +58612,7 @@ - + @@ -58889,16 +58621,16 @@ - + - + - + - + @@ -58910,7 +58642,7 @@ - + @@ -58927,9 +58659,9 @@ - + - + @@ -58944,12 +58676,12 @@ - + - + - + @@ -58964,7 +58696,7 @@ - + @@ -58976,10 +58708,10 @@ - + - + @@ -58997,22 +58729,22 @@ - + - + - + - + - + @@ -59042,24 +58774,24 @@ - + - + - + - + - + - + @@ -59068,15 +58800,15 @@ - + - + - + - + @@ -59084,7 +58816,7 @@ - + @@ -59120,21 +58852,21 @@ - + - + - + - + - + @@ -59164,7 +58896,7 @@ - + @@ -59187,9 +58919,9 @@ - + - + @@ -59213,10 +58945,10 @@ - + - + @@ -59227,7 +58959,7 @@ - + @@ -59235,7 +58967,7 @@ - + @@ -59246,7 +58978,7 @@ - + @@ -59254,45 +58986,45 @@ - + - + - + - + - + - + - + - + - + - + - + @@ -59301,7 +59033,7 @@ - + @@ -59313,18 +59045,18 @@ - + - + - + - + @@ -59332,7 +59064,7 @@ - + @@ -59340,7 +59072,7 @@ - + @@ -59348,12 +59080,12 @@ - + - + - + @@ -59374,7 +59106,7 @@ - + @@ -59398,16 +59130,16 @@ - + - + - + @@ -59422,7 +59154,7 @@ - + @@ -59430,7 +59162,7 @@ - + @@ -59438,7 +59170,7 @@ - + @@ -59452,15 +59184,15 @@ - + - + - + - + @@ -59472,12 +59204,12 @@ - + - + @@ -59509,7 +59241,7 @@ - + @@ -59526,35 +59258,35 @@ - + - + - + - + - + - + - + - + - + @@ -59584,16 +59316,16 @@ - + - + - + @@ -59605,473 +59337,473 @@ - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - + - + - + - + - + - - + + - - - + + + - - - + + + - - - + + + - - - + + + - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - + + + + + + + - - + + - - - - - - - + - + - + - + - + - + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + - - + + - - + + - + - - + + - + - - - - - - - - - - - - - - + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + - - - - - - - - - + + - - + + - + - - + + - - + + - + - - + + - + - - + + - + - - + + - - + + - - + + - - + + - + - - + + @@ -60083,143 +59815,143 @@ - - + + - - + + - - + + - - + + - + - - + + - + - - + + - - + + - + - - + + - + - - + + - + - - + + - + - - + + - + - - + + - + - - + + - - + + - - + + - - + + - - + + - + - - + + - - + + - - + + - - + + - - + + @@ -60227,230 +59959,230 @@ - - + + - - + + - + - - + + - + - - + + - - + + - + - - + + - - + + - - + + - - - + + + - - - - + + + + - - - - + + + + - - - - + + + + - - - - + + + + - - - - + + + + - - - - + + + + - - - - + + + + - - - - + + + + - - - - + + + + - - - - + + + + - - - + + + - - - + + + - - - + + + - - - + + + - + - + - - + + - - + + - - - + + + - - + + - - - + + + - - - - + + + + - - - + + + - - - + + + - - + + - - - + + + - + - + - + @@ -60462,32 +60194,32 @@ - + - + - + - - + + - - + + - - - + + + @@ -60495,16 +60227,16 @@ - + - - + + - + @@ -60519,7 +60251,7 @@ - + @@ -60528,12 +60260,12 @@ - + - + @@ -60547,7 +60279,7 @@ - + @@ -60555,7 +60287,7 @@ - + @@ -60570,13 +60302,13 @@ - + - + - + @@ -60599,17 +60331,17 @@ - + - + - + @@ -60632,40 +60364,40 @@ - + - + - + - + - + - + - + @@ -60688,9 +60420,9 @@ - + - + @@ -60698,7 +60430,7 @@ - + @@ -60730,81 +60462,87 @@ - - - + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + - + - + - + - + - - - - - - - - - - - - - + - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/block/Kconfig b/block/Kconfig index 606a67e47e68..e9ce5e9bc9c6 100644 --- a/block/Kconfig +++ b/block/Kconfig @@ -179,13 +179,20 @@ config BLK_SED_OPAL config BLK_INLINE_ENCRYPTION bool "Enable inline encryption support in block layer" + help + Build the blk-crypto subsystem. Enabling this lets the + block layer handle encryption, so users can take + advantage of inline encryption hardware if present. + +config BLK_INLINE_ENCRYPTION_FALLBACK + bool "Enable crypto API fallback for blk-crypto" + depends on BLK_INLINE_ENCRYPTION select CRYPTO select CRYPTO_BLKCIPHER help - Build the blk-crypto subsystem. - Enabling this lets the block layer handle encryption, - so users can take advantage of inline encryption - hardware if present. + Enabling this lets the block layer handle inline encryption + by falling back to the kernel crypto API when inline + encryption hardware is not present. menu "Partition Types" diff --git a/block/Makefile b/block/Makefile index 8932c7e4fd07..9bfd4c9f9e72 100644 --- a/block/Makefile +++ b/block/Makefile @@ -38,3 +38,4 @@ obj-$(CONFIG_BLK_SED_OPAL) += sed-opal.o obj-$(CONFIG_BLK_PM) += blk-pm.o obj-$(CONFIG_BLK_INLINE_ENCRYPTION) += keyslot-manager.o bio-crypt-ctx.o \ blk-crypto.o +obj-$(CONFIG_BLK_INLINE_ENCRYPTION_FALLBACK) += blk-crypto-fallback.o \ No newline at end of file diff --git a/block/bio-crypt-ctx.c b/block/bio-crypt-ctx.c index 0f7641b875e9..75982dabc7a3 100644 --- a/block/bio-crypt-ctx.c +++ b/block/bio-crypt-ctx.c @@ -5,26 +5,43 @@ #include #include -#include #include +#include +#include + +#include "blk-crypto-internal.h" static int num_prealloc_crypt_ctxs = 128; + +module_param(num_prealloc_crypt_ctxs, int, 0444); +MODULE_PARM_DESC(num_prealloc_crypt_ctxs, + "Number of bio crypto contexts to preallocate"); + static struct kmem_cache *bio_crypt_ctx_cache; static mempool_t *bio_crypt_ctx_pool; -int bio_crypt_ctx_init(void) +int __init bio_crypt_ctx_init(void) { + size_t i; + bio_crypt_ctx_cache = KMEM_CACHE(bio_crypt_ctx, 0); if (!bio_crypt_ctx_cache) return -ENOMEM; - bio_crypt_ctx_pool = mempool_create_slab_pool( - num_prealloc_crypt_ctxs, - bio_crypt_ctx_cache); - + bio_crypt_ctx_pool = mempool_create_slab_pool(num_prealloc_crypt_ctxs, + bio_crypt_ctx_cache); if (!bio_crypt_ctx_pool) return -ENOMEM; + /* This is assumed in various places. */ + BUILD_BUG_ON(BLK_ENCRYPTION_MODE_INVALID != 0); + + /* Sanity check that no algorithm exceeds the defined limits. */ + for (i = 0; i < BLK_ENCRYPTION_MODE_MAX; i++) { + BUG_ON(blk_crypto_modes[i].keysize > BLK_CRYPTO_MAX_KEY_SIZE); + BUG_ON(blk_crypto_modes[i].ivsize > BLK_CRYPTO_MAX_IV_SIZE); + } + return 0; } @@ -32,51 +49,43 @@ struct bio_crypt_ctx *bio_crypt_alloc_ctx(gfp_t gfp_mask) { return mempool_alloc(bio_crypt_ctx_pool, gfp_mask); } -EXPORT_SYMBOL(bio_crypt_alloc_ctx); void bio_crypt_free_ctx(struct bio *bio) { mempool_free(bio->bi_crypt_context, bio_crypt_ctx_pool); bio->bi_crypt_context = NULL; } -EXPORT_SYMBOL(bio_crypt_free_ctx); -int bio_crypt_clone(struct bio *dst, struct bio *src, gfp_t gfp_mask) +void bio_crypt_clone(struct bio *dst, struct bio *src, gfp_t gfp_mask) { + const struct bio_crypt_ctx *src_bc = src->bi_crypt_context; + /* - * If a bio is swhandled, then it will be decrypted when bio_endio - * is called. As we only want the data to be decrypted once, copies - * of the bio must not have have a crypt context. + * If a bio is fallback_crypted, then it will be decrypted when + * bio_endio is called. As we only want the data to be decrypted once, + * copies of the bio must not have have a crypt context. */ - if (!bio_has_crypt_ctx(src) || bio_crypt_swhandled(src)) - return 0; + if (!src_bc || bio_crypt_fallback_crypted(src_bc)) + return; dst->bi_crypt_context = bio_crypt_alloc_ctx(gfp_mask); - if (!dst->bi_crypt_context) - return -ENOMEM; + *dst->bi_crypt_context = *src_bc; - *dst->bi_crypt_context = *src->bi_crypt_context; - - if (bio_crypt_has_keyslot(src)) - keyslot_manager_get_slot(src->bi_crypt_context->processing_ksm, - src->bi_crypt_context->keyslot); - - return 0; + if (src_bc->bc_keyslot >= 0) + keyslot_manager_get_slot(src_bc->bc_ksm, src_bc->bc_keyslot); } -EXPORT_SYMBOL(bio_crypt_clone); +EXPORT_SYMBOL_GPL(bio_crypt_clone); -bool bio_crypt_should_process(struct bio *bio, struct request_queue *q) +bool bio_crypt_should_process(struct request *rq) { - if (!bio_has_crypt_ctx(bio)) + struct bio *bio = rq->bio; + + if (!bio || !bio->bi_crypt_context) return false; - if (q->ksm != bio->bi_crypt_context->processing_ksm) - return false; - - WARN_ON(!bio_crypt_has_keyslot(bio)); - return true; + return rq->q->ksm == bio->bi_crypt_context->bc_ksm; } -EXPORT_SYMBOL(bio_crypt_should_process); +EXPORT_SYMBOL_GPL(bio_crypt_should_process); /* * Checks that two bio crypt contexts are compatible - i.e. that @@ -87,23 +96,19 @@ bool bio_crypt_ctx_compatible(struct bio *b_1, struct bio *b_2) struct bio_crypt_ctx *bc1 = b_1->bi_crypt_context; struct bio_crypt_ctx *bc2 = b_2->bi_crypt_context; - if (bio_has_crypt_ctx(b_1) != bio_has_crypt_ctx(b_2)) + if (bc1 != bc2) return false; - if (!bio_has_crypt_ctx(b_1)) - return true; - - return bc1->keyslot == bc2->keyslot && - bc1->data_unit_size_bits == bc2->data_unit_size_bits; + return !bc1 || bc1->bc_key == bc2->bc_key; } /* * Checks that two bio crypt contexts are compatible, and also * that their data_unit_nums are continuous (and can hence be merged) + * in the order b_1 followed by b_2. */ -bool bio_crypt_ctx_back_mergeable(struct bio *b_1, - unsigned int b1_sectors, - struct bio *b_2) +bool bio_crypt_ctx_mergeable(struct bio *b_1, unsigned int b1_bytes, + struct bio *b_2) { struct bio_crypt_ctx *bc1 = b_1->bi_crypt_context; struct bio_crypt_ctx *bc2 = b_2->bi_crypt_context; @@ -111,35 +116,25 @@ bool bio_crypt_ctx_back_mergeable(struct bio *b_1, if (!bio_crypt_ctx_compatible(b_1, b_2)) return false; - return !bio_has_crypt_ctx(b_1) || - (bc1->data_unit_num + - (b1_sectors >> (bc1->data_unit_size_bits - 9)) == - bc2->data_unit_num); + return !bc1 || bio_crypt_dun_is_contiguous(bc1, b1_bytes, bc2->bc_dun); } -void bio_crypt_ctx_release_keyslot(struct bio *bio) +void bio_crypt_ctx_release_keyslot(struct bio_crypt_ctx *bc) { - struct bio_crypt_ctx *crypt_ctx = bio->bi_crypt_context; - - keyslot_manager_put_slot(crypt_ctx->processing_ksm, crypt_ctx->keyslot); - bio->bi_crypt_context->processing_ksm = NULL; - bio->bi_crypt_context->keyslot = -1; + keyslot_manager_put_slot(bc->bc_ksm, bc->bc_keyslot); + bc->bc_ksm = NULL; + bc->bc_keyslot = -1; } -int bio_crypt_ctx_acquire_keyslot(struct bio *bio, struct keyslot_manager *ksm) +int bio_crypt_ctx_acquire_keyslot(struct bio_crypt_ctx *bc, + struct keyslot_manager *ksm) { - int slot; - enum blk_crypto_mode_num crypto_mode = bio_crypto_mode(bio); + int slot = keyslot_manager_get_slot_for_key(ksm, bc->bc_key); - if (!ksm) - return -ENOMEM; - - slot = keyslot_manager_get_slot_for_key(ksm, - bio_crypt_raw_key(bio), crypto_mode, - 1 << bio->bi_crypt_context->data_unit_size_bits); if (slot < 0) return slot; - bio_crypt_set_keyslot(bio, slot, ksm); + bc->bc_keyslot = slot; + bc->bc_ksm = ksm; return 0; } diff --git a/block/bio.c b/block/bio.c index 2d6a840bb67f..d2f552ae0753 100644 --- a/block/bio.c +++ b/block/bio.c @@ -234,6 +234,8 @@ fallback: void bio_uninit(struct bio *bio) { bio_disassociate_blkg(bio); + + bio_crypt_free_ctx(bio); } EXPORT_SYMBOL(bio_uninit); @@ -242,7 +244,6 @@ static void bio_free(struct bio *bio) struct bio_set *bs = bio->bi_pool; void *p; - bio_crypt_free_ctx(bio); bio_uninit(bio); if (bs) { @@ -653,10 +654,7 @@ struct bio *bio_clone_fast(struct bio *bio, gfp_t gfp_mask, struct bio_set *bs) __bio_clone_fast(b, bio); - if (bio_crypt_clone(b, bio, gfp_mask) < 0) { - bio_put(b); - return NULL; - } + bio_crypt_clone(b, bio, gfp_mask); if (bio_integrity(bio) && bio_integrity_clone(b, bio, gfp_mask) < 0) { diff --git a/block/blk-core.c b/block/blk-core.c index 808adf0bb44b..9e10ad79df34 100644 --- a/block/blk-core.c +++ b/block/blk-core.c @@ -1115,14 +1115,11 @@ blk_qc_t direct_make_request(struct bio *bio) { struct request_queue *q = bio->bi_disk->queue; bool nowait = bio->bi_opf & REQ_NOWAIT; - blk_qc_t ret; + blk_qc_t ret = BLK_QC_T_NONE; if (!generic_make_request_checks(bio)) return BLK_QC_T_NONE; - if (blk_crypto_submit_bio(&bio)) - return BLK_QC_T_NONE; - if (unlikely(blk_queue_enter(q, nowait ? BLK_MQ_REQ_NOWAIT : 0))) { if (nowait && !blk_queue_dying(q)) bio->bi_status = BLK_STS_AGAIN; @@ -1132,7 +1129,8 @@ blk_qc_t direct_make_request(struct bio *bio) return BLK_QC_T_NONE; } - ret = q->make_request_fn(q, bio); + if (!blk_crypto_submit_bio(&bio)) + ret = q->make_request_fn(q, bio); blk_queue_exit(q); return ret; } @@ -1819,8 +1817,8 @@ int __init blk_dev_init(void) if (bio_crypt_ctx_init() < 0) panic("Failed to allocate mem for bio crypt ctxs\n"); - if (blk_crypto_init() < 0) - panic("Failed to init blk-crypto\n"); + if (blk_crypto_fallback_init() < 0) + panic("Failed to init blk-crypto-fallback\n"); return 0; } diff --git a/block/blk-crypto-fallback.c b/block/blk-crypto-fallback.c new file mode 100644 index 000000000000..9773bd6a5186 --- /dev/null +++ b/block/blk-crypto-fallback.c @@ -0,0 +1,648 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright 2019 Google LLC + */ + +/* + * Refer to Documentation/block/inline-encryption.rst for detailed explanation. + */ + +#define pr_fmt(fmt) "blk-crypto-fallback: " fmt + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "blk-crypto-internal.h" + +static unsigned int num_prealloc_bounce_pg = 32; +module_param(num_prealloc_bounce_pg, uint, 0); +MODULE_PARM_DESC(num_prealloc_bounce_pg, + "Number of preallocated bounce pages for the blk-crypto crypto API fallback"); + +static unsigned int blk_crypto_num_keyslots = 100; +module_param_named(num_keyslots, blk_crypto_num_keyslots, uint, 0); +MODULE_PARM_DESC(num_keyslots, + "Number of keyslots for the blk-crypto crypto API fallback"); + +static unsigned int num_prealloc_fallback_crypt_ctxs = 128; +module_param(num_prealloc_fallback_crypt_ctxs, uint, 0); +MODULE_PARM_DESC(num_prealloc_crypt_fallback_ctxs, + "Number of preallocated bio fallback crypto contexts for blk-crypto to use during crypto API fallback"); + +struct bio_fallback_crypt_ctx { + struct bio_crypt_ctx crypt_ctx; + /* + * Copy of the bvec_iter when this bio was submitted. + * We only want to en/decrypt the part of the bio as described by the + * bvec_iter upon submission because bio might be split before being + * resubmitted + */ + struct bvec_iter crypt_iter; + u64 fallback_dun[BLK_CRYPTO_DUN_ARRAY_SIZE]; +}; + +/* The following few vars are only used during the crypto API fallback */ +static struct kmem_cache *bio_fallback_crypt_ctx_cache; +static mempool_t *bio_fallback_crypt_ctx_pool; + +/* + * Allocating a crypto tfm during I/O can deadlock, so we have to preallocate + * all of a mode's tfms when that mode starts being used. Since each mode may + * need all the keyslots at some point, each mode needs its own tfm for each + * keyslot; thus, a keyslot may contain tfms for multiple modes. However, to + * match the behavior of real inline encryption hardware (which only supports a + * single encryption context per keyslot), we only allow one tfm per keyslot to + * be used at a time - the rest of the unused tfms have their keys cleared. + */ +static DEFINE_MUTEX(tfms_init_lock); +static bool tfms_inited[BLK_ENCRYPTION_MODE_MAX]; + +struct blk_crypto_decrypt_work { + struct work_struct work; + struct bio *bio; +}; + +static struct blk_crypto_keyslot { + struct crypto_skcipher *tfm; + enum blk_crypto_mode_num crypto_mode; + struct crypto_skcipher *tfms[BLK_ENCRYPTION_MODE_MAX]; +} *blk_crypto_keyslots; + +/* The following few vars are only used during the crypto API fallback */ +static struct keyslot_manager *blk_crypto_ksm; +static struct workqueue_struct *blk_crypto_wq; +static mempool_t *blk_crypto_bounce_page_pool; +static struct kmem_cache *blk_crypto_decrypt_work_cache; + +bool bio_crypt_fallback_crypted(const struct bio_crypt_ctx *bc) +{ + return bc && bc->bc_ksm == blk_crypto_ksm; +} + +/* + * This is the key we set when evicting a keyslot. This *should* be the all 0's + * key, but AES-XTS rejects that key, so we use some random bytes instead. + */ +static u8 blank_key[BLK_CRYPTO_MAX_KEY_SIZE]; + +static void blk_crypto_evict_keyslot(unsigned int slot) +{ + struct blk_crypto_keyslot *slotp = &blk_crypto_keyslots[slot]; + enum blk_crypto_mode_num crypto_mode = slotp->crypto_mode; + int err; + + WARN_ON(slotp->crypto_mode == BLK_ENCRYPTION_MODE_INVALID); + + /* Clear the key in the skcipher */ + err = crypto_skcipher_setkey(slotp->tfms[crypto_mode], blank_key, + blk_crypto_modes[crypto_mode].keysize); + WARN_ON(err); + slotp->crypto_mode = BLK_ENCRYPTION_MODE_INVALID; +} + +static int blk_crypto_keyslot_program(struct keyslot_manager *ksm, + const struct blk_crypto_key *key, + unsigned int slot) +{ + struct blk_crypto_keyslot *slotp = &blk_crypto_keyslots[slot]; + const enum blk_crypto_mode_num crypto_mode = key->crypto_mode; + int err; + + if (crypto_mode != slotp->crypto_mode && + slotp->crypto_mode != BLK_ENCRYPTION_MODE_INVALID) { + blk_crypto_evict_keyslot(slot); + } + + if (!slotp->tfms[crypto_mode]) + return -ENOMEM; + slotp->crypto_mode = crypto_mode; + err = crypto_skcipher_setkey(slotp->tfms[crypto_mode], key->raw, + key->size); + if (err) { + blk_crypto_evict_keyslot(slot); + return err; + } + return 0; +} + +static int blk_crypto_keyslot_evict(struct keyslot_manager *ksm, + const struct blk_crypto_key *key, + unsigned int slot) +{ + blk_crypto_evict_keyslot(slot); + return 0; +} + +/* + * The crypto API fallback KSM ops - only used for a bio when it specifies a + * blk_crypto_mode for which we failed to get a keyslot in the device's inline + * encryption hardware (which probably means the device doesn't have inline + * encryption hardware that supports that crypto mode). + */ +static const struct keyslot_mgmt_ll_ops blk_crypto_ksm_ll_ops = { + .keyslot_program = blk_crypto_keyslot_program, + .keyslot_evict = blk_crypto_keyslot_evict, +}; + +static void blk_crypto_encrypt_endio(struct bio *enc_bio) +{ + struct bio *src_bio = enc_bio->bi_private; + int i; + + for (i = 0; i < enc_bio->bi_vcnt; i++) + mempool_free(enc_bio->bi_io_vec[i].bv_page, + blk_crypto_bounce_page_pool); + + src_bio->bi_status = enc_bio->bi_status; + + bio_put(enc_bio); + bio_endio(src_bio); +} + +static struct bio *blk_crypto_clone_bio(struct bio *bio_src) +{ + struct bvec_iter iter; + struct bio_vec bv; + struct bio *bio; + + bio = bio_alloc_bioset(GFP_NOIO, bio_segments(bio_src), NULL); + if (!bio) + return NULL; + bio->bi_disk = bio_src->bi_disk; + bio->bi_opf = bio_src->bi_opf; + bio->bi_ioprio = bio_src->bi_ioprio; + bio->bi_write_hint = bio_src->bi_write_hint; + bio->bi_iter.bi_sector = bio_src->bi_iter.bi_sector; + bio->bi_iter.bi_size = bio_src->bi_iter.bi_size; + + bio_for_each_segment(bv, bio_src, iter) + bio->bi_io_vec[bio->bi_vcnt++] = bv; + + if (bio_integrity(bio_src) && + bio_integrity_clone(bio, bio_src, GFP_NOIO) < 0) { + bio_put(bio); + return NULL; + } + + bio_clone_blkg_association(bio, bio_src); + blkcg_bio_issue_init(bio); + + return bio; +} + +static int blk_crypto_alloc_cipher_req(struct bio *src_bio, + struct skcipher_request **ciph_req_ret, + struct crypto_wait *wait) +{ + struct skcipher_request *ciph_req; + const struct blk_crypto_keyslot *slotp; + + slotp = &blk_crypto_keyslots[src_bio->bi_crypt_context->bc_keyslot]; + ciph_req = skcipher_request_alloc(slotp->tfms[slotp->crypto_mode], + GFP_NOIO); + if (!ciph_req) { + src_bio->bi_status = BLK_STS_RESOURCE; + return -ENOMEM; + } + + skcipher_request_set_callback(ciph_req, + CRYPTO_TFM_REQ_MAY_BACKLOG | + CRYPTO_TFM_REQ_MAY_SLEEP, + crypto_req_done, wait); + *ciph_req_ret = ciph_req; + return 0; +} + +static int blk_crypto_split_bio_if_needed(struct bio **bio_ptr) +{ + struct bio *bio = *bio_ptr; + unsigned int i = 0; + unsigned int num_sectors = 0; + struct bio_vec bv; + struct bvec_iter iter; + + bio_for_each_segment(bv, bio, iter) { + num_sectors += bv.bv_len >> SECTOR_SHIFT; + if (++i == BIO_MAX_PAGES) + break; + } + if (num_sectors < bio_sectors(bio)) { + struct bio *split_bio; + + split_bio = bio_split(bio, num_sectors, GFP_NOIO, NULL); + if (!split_bio) { + bio->bi_status = BLK_STS_RESOURCE; + return -ENOMEM; + } + bio_chain(split_bio, bio); + generic_make_request(bio); + *bio_ptr = split_bio; + } + return 0; +} + +union blk_crypto_iv { + __le64 dun[BLK_CRYPTO_DUN_ARRAY_SIZE]; + u8 bytes[BLK_CRYPTO_MAX_IV_SIZE]; +}; + +static void blk_crypto_dun_to_iv(const u64 dun[BLK_CRYPTO_DUN_ARRAY_SIZE], + union blk_crypto_iv *iv) +{ + int i; + + for (i = 0; i < BLK_CRYPTO_DUN_ARRAY_SIZE; i++) + iv->dun[i] = cpu_to_le64(dun[i]); +} + +/* + * The crypto API fallback's encryption routine. + * Allocate a bounce bio for encryption, encrypt the input bio using crypto API, + * and replace *bio_ptr with the bounce bio. May split input bio if it's too + * large. + */ +static int blk_crypto_encrypt_bio(struct bio **bio_ptr) +{ + struct bio *src_bio; + struct skcipher_request *ciph_req = NULL; + DECLARE_CRYPTO_WAIT(wait); + u64 curr_dun[BLK_CRYPTO_DUN_ARRAY_SIZE]; + union blk_crypto_iv iv; + struct scatterlist src, dst; + struct bio *enc_bio; + unsigned int i, j; + int data_unit_size; + struct bio_crypt_ctx *bc; + int err = 0; + + /* Split the bio if it's too big for single page bvec */ + err = blk_crypto_split_bio_if_needed(bio_ptr); + if (err) + return err; + + src_bio = *bio_ptr; + bc = src_bio->bi_crypt_context; + data_unit_size = bc->bc_key->data_unit_size; + + /* Allocate bounce bio for encryption */ + enc_bio = blk_crypto_clone_bio(src_bio); + if (!enc_bio) { + src_bio->bi_status = BLK_STS_RESOURCE; + return -ENOMEM; + } + + /* + * Use the crypto API fallback keyslot manager to get a crypto_skcipher + * for the algorithm and key specified for this bio. + */ + err = bio_crypt_ctx_acquire_keyslot(bc, blk_crypto_ksm); + if (err) { + src_bio->bi_status = BLK_STS_IOERR; + goto out_put_enc_bio; + } + + /* and then allocate an skcipher_request for it */ + err = blk_crypto_alloc_cipher_req(src_bio, &ciph_req, &wait); + if (err) + goto out_release_keyslot; + + memcpy(curr_dun, bc->bc_dun, sizeof(curr_dun)); + sg_init_table(&src, 1); + sg_init_table(&dst, 1); + + skcipher_request_set_crypt(ciph_req, &src, &dst, data_unit_size, + iv.bytes); + + /* Encrypt each page in the bounce bio */ + for (i = 0; i < enc_bio->bi_vcnt; i++) { + struct bio_vec *enc_bvec = &enc_bio->bi_io_vec[i]; + struct page *plaintext_page = enc_bvec->bv_page; + struct page *ciphertext_page = + mempool_alloc(blk_crypto_bounce_page_pool, GFP_NOIO); + + enc_bvec->bv_page = ciphertext_page; + + if (!ciphertext_page) { + src_bio->bi_status = BLK_STS_RESOURCE; + err = -ENOMEM; + goto out_free_bounce_pages; + } + + sg_set_page(&src, plaintext_page, data_unit_size, + enc_bvec->bv_offset); + sg_set_page(&dst, ciphertext_page, data_unit_size, + enc_bvec->bv_offset); + + /* Encrypt each data unit in this page */ + for (j = 0; j < enc_bvec->bv_len; j += data_unit_size) { + blk_crypto_dun_to_iv(curr_dun, &iv); + err = crypto_wait_req(crypto_skcipher_encrypt(ciph_req), + &wait); + if (err) { + i++; + src_bio->bi_status = BLK_STS_RESOURCE; + goto out_free_bounce_pages; + } + bio_crypt_dun_increment(curr_dun, 1); + src.offset += data_unit_size; + dst.offset += data_unit_size; + } + } + + enc_bio->bi_private = src_bio; + enc_bio->bi_end_io = blk_crypto_encrypt_endio; + *bio_ptr = enc_bio; + + enc_bio = NULL; + err = 0; + goto out_free_ciph_req; + +out_free_bounce_pages: + while (i > 0) + mempool_free(enc_bio->bi_io_vec[--i].bv_page, + blk_crypto_bounce_page_pool); +out_free_ciph_req: + skcipher_request_free(ciph_req); +out_release_keyslot: + bio_crypt_ctx_release_keyslot(bc); +out_put_enc_bio: + if (enc_bio) + bio_put(enc_bio); + + return err; +} + +static void blk_crypto_free_fallback_crypt_ctx(struct bio *bio) +{ + mempool_free(container_of(bio->bi_crypt_context, + struct bio_fallback_crypt_ctx, + crypt_ctx), + bio_fallback_crypt_ctx_pool); + bio->bi_crypt_context = NULL; +} + +/* + * The crypto API fallback's main decryption routine. + * Decrypts input bio in place. + */ +static void blk_crypto_decrypt_bio(struct work_struct *work) +{ + struct blk_crypto_decrypt_work *decrypt_work = + container_of(work, struct blk_crypto_decrypt_work, work); + struct bio *bio = decrypt_work->bio; + struct skcipher_request *ciph_req = NULL; + DECLARE_CRYPTO_WAIT(wait); + struct bio_vec bv; + struct bvec_iter iter; + u64 curr_dun[BLK_CRYPTO_DUN_ARRAY_SIZE]; + union blk_crypto_iv iv; + struct scatterlist sg; + struct bio_crypt_ctx *bc = bio->bi_crypt_context; + struct bio_fallback_crypt_ctx *f_ctx = + container_of(bc, struct bio_fallback_crypt_ctx, crypt_ctx); + const int data_unit_size = bc->bc_key->data_unit_size; + unsigned int i; + int err; + + /* + * Use the crypto API fallback keyslot manager to get a crypto_skcipher + * for the algorithm and key specified for this bio. + */ + if (bio_crypt_ctx_acquire_keyslot(bc, blk_crypto_ksm)) { + bio->bi_status = BLK_STS_RESOURCE; + goto out_no_keyslot; + } + + /* and then allocate an skcipher_request for it */ + err = blk_crypto_alloc_cipher_req(bio, &ciph_req, &wait); + if (err) + goto out; + + memcpy(curr_dun, f_ctx->fallback_dun, sizeof(curr_dun)); + sg_init_table(&sg, 1); + skcipher_request_set_crypt(ciph_req, &sg, &sg, data_unit_size, + iv.bytes); + + /* Decrypt each segment in the bio */ + __bio_for_each_segment(bv, bio, iter, f_ctx->crypt_iter) { + struct page *page = bv.bv_page; + + sg_set_page(&sg, page, data_unit_size, bv.bv_offset); + + /* Decrypt each data unit in the segment */ + for (i = 0; i < bv.bv_len; i += data_unit_size) { + blk_crypto_dun_to_iv(curr_dun, &iv); + if (crypto_wait_req(crypto_skcipher_decrypt(ciph_req), + &wait)) { + bio->bi_status = BLK_STS_IOERR; + goto out; + } + bio_crypt_dun_increment(curr_dun, 1); + sg.offset += data_unit_size; + } + } + +out: + skcipher_request_free(ciph_req); + bio_crypt_ctx_release_keyslot(bc); +out_no_keyslot: + kmem_cache_free(blk_crypto_decrypt_work_cache, decrypt_work); + blk_crypto_free_fallback_crypt_ctx(bio); + bio_endio(bio); +} + +/* + * Queue bio for decryption. + * Returns true iff bio was queued for decryption. + */ +bool blk_crypto_queue_decrypt_bio(struct bio *bio) +{ + struct blk_crypto_decrypt_work *decrypt_work; + + /* If there was an IO error, don't queue for decrypt. */ + if (bio->bi_status) + goto out; + + decrypt_work = kmem_cache_zalloc(blk_crypto_decrypt_work_cache, + GFP_ATOMIC); + if (!decrypt_work) { + bio->bi_status = BLK_STS_RESOURCE; + goto out; + } + + INIT_WORK(&decrypt_work->work, blk_crypto_decrypt_bio); + decrypt_work->bio = bio; + queue_work(blk_crypto_wq, &decrypt_work->work); + + return true; +out: + blk_crypto_free_fallback_crypt_ctx(bio); + return false; +} + +/** + * blk_crypto_start_using_mode() - Start using a crypto algorithm on a device + * @mode_num: the blk_crypto_mode we want to allocate ciphers for. + * @data_unit_size: the data unit size that will be used + * @q: the request queue for the device + * + * Upper layers must call this function to ensure that a the crypto API fallback + * has transforms for this algorithm, if they become necessary. + * + * Return: 0 on success and -err on error. + */ +int blk_crypto_start_using_mode(enum blk_crypto_mode_num mode_num, + unsigned int data_unit_size, + struct request_queue *q) +{ + struct blk_crypto_keyslot *slotp; + unsigned int i; + int err = 0; + + /* + * Fast path + * Ensure that updates to blk_crypto_keyslots[i].tfms[mode_num] + * for each i are visible before we try to access them. + */ + if (likely(smp_load_acquire(&tfms_inited[mode_num]))) + return 0; + + /* + * If the keyslot manager of the request queue supports this + * crypto mode, then we don't need to allocate this mode. + */ + if (keyslot_manager_crypto_mode_supported(q->ksm, mode_num, + data_unit_size)) + return 0; + + mutex_lock(&tfms_init_lock); + if (likely(tfms_inited[mode_num])) + goto out; + + for (i = 0; i < blk_crypto_num_keyslots; i++) { + slotp = &blk_crypto_keyslots[i]; + slotp->tfms[mode_num] = crypto_alloc_skcipher( + blk_crypto_modes[mode_num].cipher_str, + 0, 0); + if (IS_ERR(slotp->tfms[mode_num])) { + err = PTR_ERR(slotp->tfms[mode_num]); + slotp->tfms[mode_num] = NULL; + goto out_free_tfms; + } + + crypto_skcipher_set_flags(slotp->tfms[mode_num], + CRYPTO_TFM_REQ_FORBID_WEAK_KEYS); + } + + /* + * Ensure that updates to blk_crypto_keyslots[i].tfms[mode_num] + * for each i are visible before we set tfms_inited[mode_num]. + */ + smp_store_release(&tfms_inited[mode_num], true); + goto out; + +out_free_tfms: + for (i = 0; i < blk_crypto_num_keyslots; i++) { + slotp = &blk_crypto_keyslots[i]; + crypto_free_skcipher(slotp->tfms[mode_num]); + slotp->tfms[mode_num] = NULL; + } +out: + mutex_unlock(&tfms_init_lock); + return err; +} + +int blk_crypto_fallback_evict_key(const struct blk_crypto_key *key) +{ + return keyslot_manager_evict_key(blk_crypto_ksm, key); +} + +int blk_crypto_fallback_submit_bio(struct bio **bio_ptr) +{ + struct bio *bio = *bio_ptr; + struct bio_crypt_ctx *bc = bio->bi_crypt_context; + struct bio_fallback_crypt_ctx *f_ctx; + + if (WARN_ON_ONCE(!tfms_inited[bc->bc_key->crypto_mode])) { + bio->bi_status = BLK_STS_IOERR; + return -EIO; + } + + if (bio_data_dir(bio) == WRITE) + return blk_crypto_encrypt_bio(bio_ptr); + + /* + * Mark bio as fallback crypted and replace the bio_crypt_ctx with + * another one contained in a bio_fallback_crypt_ctx, so that the + * fallback has space to store the info it needs for decryption. + */ + bc->bc_ksm = blk_crypto_ksm; + f_ctx = mempool_alloc(bio_fallback_crypt_ctx_pool, GFP_NOIO); + f_ctx->crypt_ctx = *bc; + memcpy(f_ctx->fallback_dun, bc->bc_dun, sizeof(f_ctx->fallback_dun)); + f_ctx->crypt_iter = bio->bi_iter; + + bio_crypt_free_ctx(bio); + bio->bi_crypt_context = &f_ctx->crypt_ctx; + + return 0; +} + +int __init blk_crypto_fallback_init(void) +{ + int i; + unsigned int crypto_mode_supported[BLK_ENCRYPTION_MODE_MAX]; + + prandom_bytes(blank_key, BLK_CRYPTO_MAX_KEY_SIZE); + + /* All blk-crypto modes have a crypto API fallback. */ + for (i = 0; i < BLK_ENCRYPTION_MODE_MAX; i++) + crypto_mode_supported[i] = 0xFFFFFFFF; + crypto_mode_supported[BLK_ENCRYPTION_MODE_INVALID] = 0; + + blk_crypto_ksm = keyslot_manager_create(blk_crypto_num_keyslots, + &blk_crypto_ksm_ll_ops, + crypto_mode_supported, NULL); + if (!blk_crypto_ksm) + return -ENOMEM; + + blk_crypto_wq = alloc_workqueue("blk_crypto_wq", + WQ_UNBOUND | WQ_HIGHPRI | + WQ_MEM_RECLAIM, num_online_cpus()); + if (!blk_crypto_wq) + return -ENOMEM; + + blk_crypto_keyslots = kcalloc(blk_crypto_num_keyslots, + sizeof(blk_crypto_keyslots[0]), + GFP_KERNEL); + if (!blk_crypto_keyslots) + return -ENOMEM; + + blk_crypto_bounce_page_pool = + mempool_create_page_pool(num_prealloc_bounce_pg, 0); + if (!blk_crypto_bounce_page_pool) + return -ENOMEM; + + blk_crypto_decrypt_work_cache = KMEM_CACHE(blk_crypto_decrypt_work, + SLAB_RECLAIM_ACCOUNT); + if (!blk_crypto_decrypt_work_cache) + return -ENOMEM; + + bio_fallback_crypt_ctx_cache = KMEM_CACHE(bio_fallback_crypt_ctx, 0); + if (!bio_fallback_crypt_ctx_cache) + return -ENOMEM; + + bio_fallback_crypt_ctx_pool = + mempool_create_slab_pool(num_prealloc_fallback_crypt_ctxs, + bio_fallback_crypt_ctx_cache); + if (!bio_fallback_crypt_ctx_pool) + return -ENOMEM; + + return 0; +} diff --git a/block/blk-crypto-internal.h b/block/blk-crypto-internal.h new file mode 100644 index 000000000000..43351eecc97a --- /dev/null +++ b/block/blk-crypto-internal.h @@ -0,0 +1,58 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright 2019 Google LLC + */ + +#ifndef __LINUX_BLK_CRYPTO_INTERNAL_H +#define __LINUX_BLK_CRYPTO_INTERNAL_H + +#include + +/* Represents a crypto mode supported by blk-crypto */ +struct blk_crypto_mode { + const char *cipher_str; /* crypto API name (for fallback case) */ + unsigned int keysize; /* key size in bytes */ + unsigned int ivsize; /* iv size in bytes */ +}; + +extern const struct blk_crypto_mode blk_crypto_modes[]; + +#ifdef CONFIG_BLK_INLINE_ENCRYPTION_FALLBACK + +int blk_crypto_fallback_submit_bio(struct bio **bio_ptr); + +bool blk_crypto_queue_decrypt_bio(struct bio *bio); + +int blk_crypto_fallback_evict_key(const struct blk_crypto_key *key); + +bool bio_crypt_fallback_crypted(const struct bio_crypt_ctx *bc); + +#else /* CONFIG_BLK_INLINE_ENCRYPTION_FALLBACK */ + +static inline bool bio_crypt_fallback_crypted(const struct bio_crypt_ctx *bc) +{ + return false; +} + +static inline int blk_crypto_fallback_submit_bio(struct bio **bio_ptr) +{ + pr_warn_once("blk-crypto crypto API fallback disabled; failing request"); + (*bio_ptr)->bi_status = BLK_STS_NOTSUPP; + return -EIO; +} + +static inline bool blk_crypto_queue_decrypt_bio(struct bio *bio) +{ + WARN_ON(1); + return false; +} + +static inline int +blk_crypto_fallback_evict_key(const struct blk_crypto_key *key) +{ + return 0; +} + +#endif /* CONFIG_BLK_INLINE_ENCRYPTION_FALLBACK */ + +#endif /* __LINUX_BLK_CRYPTO_INTERNAL_H */ diff --git a/block/blk-crypto.c b/block/blk-crypto.c index 89649655bf4b..2c47e8eec865 100644 --- a/block/blk-crypto.c +++ b/block/blk-crypto.c @@ -10,219 +10,36 @@ #define pr_fmt(fmt) "blk-crypto: " fmt #include +#include #include -#include -#include -#include -#include -#include -#include -#include +#include +#include -/* Represents a crypto mode supported by blk-crypto */ -struct blk_crypto_mode { - const char *cipher_str; /* crypto API name (for fallback case) */ - size_t keysize; /* key size in bytes */ -}; +#include "blk-crypto-internal.h" -static const struct blk_crypto_mode blk_crypto_modes[] = { +const struct blk_crypto_mode blk_crypto_modes[] = { [BLK_ENCRYPTION_MODE_AES_256_XTS] = { .cipher_str = "xts(aes)", .keysize = 64, + .ivsize = 16, + }, + [BLK_ENCRYPTION_MODE_AES_128_CBC_ESSIV] = { + .cipher_str = "essiv(cbc(aes),sha256)", + .keysize = 16, + .ivsize = 16, + }, + [BLK_ENCRYPTION_MODE_ADIANTUM] = { + .cipher_str = "adiantum(xchacha12,aes)", + .keysize = 32, + .ivsize = 32, }, }; -static unsigned int num_prealloc_bounce_pg = 32; -module_param(num_prealloc_bounce_pg, uint, 0); -MODULE_PARM_DESC(num_prealloc_bounce_pg, - "Number of preallocated bounce pages for blk-crypto to use during crypto API fallback encryption"); - -#define BLK_CRYPTO_MAX_KEY_SIZE 64 -static int blk_crypto_num_keyslots = 100; -module_param_named(num_keyslots, blk_crypto_num_keyslots, int, 0); -MODULE_PARM_DESC(num_keyslots, - "Number of keyslots for crypto API fallback in blk-crypto."); - -static struct blk_crypto_keyslot { - struct crypto_skcipher *tfm; - enum blk_crypto_mode_num crypto_mode; - u8 key[BLK_CRYPTO_MAX_KEY_SIZE]; - struct crypto_skcipher *tfms[ARRAY_SIZE(blk_crypto_modes)]; -} *blk_crypto_keyslots; - -/* - * Allocating a crypto tfm during I/O can deadlock, so we have to preallocate - * all of a mode's tfms when that mode starts being used. Since each mode may - * need all the keyslots at some point, each mode needs its own tfm for each - * keyslot; thus, a keyslot may contain tfms for multiple modes. However, to - * match the behavior of real inline encryption hardware (which only supports a - * single encryption context per keyslot), we only allow one tfm per keyslot to - * be used at a time - the rest of the unused tfms have their keys cleared. - */ -static struct mutex tfms_lock[ARRAY_SIZE(blk_crypto_modes)]; -static bool tfms_inited[ARRAY_SIZE(blk_crypto_modes)]; - -struct work_mem { - struct work_struct crypto_work; - struct bio *bio; -}; - -/* The following few vars are only used during the crypto API fallback */ -static struct keyslot_manager *blk_crypto_ksm; -static struct workqueue_struct *blk_crypto_wq; -static mempool_t *blk_crypto_page_pool; -static struct kmem_cache *blk_crypto_work_mem_cache; - -bool bio_crypt_swhandled(struct bio *bio) -{ - return bio_has_crypt_ctx(bio) && - bio->bi_crypt_context->processing_ksm == blk_crypto_ksm; -} - -static u8 blank_key[BLK_CRYPTO_MAX_KEY_SIZE]; -static void evict_keyslot(unsigned int slot) -{ - struct blk_crypto_keyslot *slotp = &blk_crypto_keyslots[slot]; - enum blk_crypto_mode_num crypto_mode = slotp->crypto_mode; - int err; - - WARN_ON(slotp->crypto_mode == BLK_ENCRYPTION_MODE_INVALID); - - /* Clear the key in the skcipher */ - err = crypto_skcipher_setkey(slotp->tfms[crypto_mode], blank_key, - blk_crypto_modes[crypto_mode].keysize); - WARN_ON(err); - memzero_explicit(slotp->key, BLK_CRYPTO_MAX_KEY_SIZE); - slotp->crypto_mode = BLK_ENCRYPTION_MODE_INVALID; -} - -static int blk_crypto_keyslot_program(void *priv, const u8 *key, - enum blk_crypto_mode_num crypto_mode, - unsigned int data_unit_size, - unsigned int slot) -{ - struct blk_crypto_keyslot *slotp = &blk_crypto_keyslots[slot]; - const struct blk_crypto_mode *mode = &blk_crypto_modes[crypto_mode]; - size_t keysize = mode->keysize; - int err; - - if (crypto_mode != slotp->crypto_mode && - slotp->crypto_mode != BLK_ENCRYPTION_MODE_INVALID) { - evict_keyslot(slot); - } - - if (!slotp->tfms[crypto_mode]) - return -ENOMEM; - slotp->crypto_mode = crypto_mode; - err = crypto_skcipher_setkey(slotp->tfms[crypto_mode], key, keysize); - - if (err) { - evict_keyslot(slot); - return err; - } - - memcpy(slotp->key, key, keysize); - - return 0; -} - -static int blk_crypto_keyslot_evict(void *priv, const u8 *key, - enum blk_crypto_mode_num crypto_mode, - unsigned int data_unit_size, - unsigned int slot) -{ - evict_keyslot(slot); - return 0; -} - -static int blk_crypto_keyslot_find(void *priv, - const u8 *key, - enum blk_crypto_mode_num crypto_mode, - unsigned int data_unit_size_bytes) -{ - int slot; - const size_t keysize = blk_crypto_modes[crypto_mode].keysize; - - for (slot = 0; slot < blk_crypto_num_keyslots; slot++) { - if (blk_crypto_keyslots[slot].crypto_mode == crypto_mode && - !crypto_memneq(blk_crypto_keyslots[slot].key, key, keysize)) - return slot; - } - - return -ENOKEY; -} - -static bool blk_crypto_mode_supported(void *priv, - enum blk_crypto_mode_num crypt_mode, - unsigned int data_unit_size) -{ - /* All blk_crypto_modes are required to have a crypto API fallback. */ - return true; -} - -/* - * The crypto API fallback KSM ops - only used for a bio when it specifies a - * blk_crypto_mode for which we failed to get a keyslot in the device's inline - * encryption hardware (which probably means the device doesn't have inline - * encryption hardware that supports that crypto mode). - */ -static const struct keyslot_mgmt_ll_ops blk_crypto_ksm_ll_ops = { - .keyslot_program = blk_crypto_keyslot_program, - .keyslot_evict = blk_crypto_keyslot_evict, - .keyslot_find = blk_crypto_keyslot_find, - .crypto_mode_supported = blk_crypto_mode_supported, -}; - -static void blk_crypto_encrypt_endio(struct bio *enc_bio) -{ - struct bio *src_bio = enc_bio->bi_private; - int i; - - for (i = 0; i < enc_bio->bi_vcnt; i++) - mempool_free(enc_bio->bi_io_vec[i].bv_page, - blk_crypto_page_pool); - - src_bio->bi_status = enc_bio->bi_status; - - bio_put(enc_bio); - bio_endio(src_bio); -} - -static struct bio *blk_crypto_clone_bio(struct bio *bio_src) -{ - struct bvec_iter iter; - struct bio_vec bv; - struct bio *bio; - - bio = bio_alloc_bioset(GFP_NOIO, bio_segments(bio_src), NULL); - if (!bio) - return NULL; - bio->bi_disk = bio_src->bi_disk; - bio->bi_opf = bio_src->bi_opf; - bio->bi_ioprio = bio_src->bi_ioprio; - bio->bi_write_hint = bio_src->bi_write_hint; - bio->bi_iter.bi_sector = bio_src->bi_iter.bi_sector; - bio->bi_iter.bi_size = bio_src->bi_iter.bi_size; - - bio_for_each_segment(bv, bio_src, iter) - bio->bi_io_vec[bio->bi_vcnt++] = bv; - - if (bio_integrity(bio_src) && - bio_integrity_clone(bio, bio_src, GFP_NOIO) < 0) { - bio_put(bio); - return NULL; - } - - bio_clone_blkg_association(bio, bio_src); - blkcg_bio_issue_init(bio); - - return bio; -} - /* Check that all I/O segments are data unit aligned */ static int bio_crypt_check_alignment(struct bio *bio) { - int data_unit_size = 1 << bio->bi_crypt_context->data_unit_size_bits; + const unsigned int data_unit_size = + bio->bi_crypt_context->bc_key->data_unit_size; struct bvec_iter iter; struct bio_vec bv; @@ -233,268 +50,6 @@ static int bio_crypt_check_alignment(struct bio *bio) return 0; } -static int blk_crypto_alloc_cipher_req(struct bio *src_bio, - struct skcipher_request **ciph_req_ptr, - struct crypto_wait *wait) -{ - int slot; - struct skcipher_request *ciph_req; - struct blk_crypto_keyslot *slotp; - - slot = bio_crypt_get_keyslot(src_bio); - slotp = &blk_crypto_keyslots[slot]; - ciph_req = skcipher_request_alloc(slotp->tfms[slotp->crypto_mode], - GFP_NOIO); - if (!ciph_req) { - src_bio->bi_status = BLK_STS_RESOURCE; - return -ENOMEM; - } - - skcipher_request_set_callback(ciph_req, - CRYPTO_TFM_REQ_MAY_BACKLOG | - CRYPTO_TFM_REQ_MAY_SLEEP, - crypto_req_done, wait); - *ciph_req_ptr = ciph_req; - return 0; -} - -static int blk_crypto_split_bio_if_needed(struct bio **bio_ptr) -{ - struct bio *bio = *bio_ptr; - unsigned int i = 0; - unsigned int num_sectors = 0; - struct bio_vec bv; - struct bvec_iter iter; - - bio_for_each_segment(bv, bio, iter) { - num_sectors += bv.bv_len >> SECTOR_SHIFT; - if (++i == BIO_MAX_PAGES) - break; - } - if (num_sectors < bio_sectors(bio)) { - struct bio *split_bio; - - split_bio = bio_split(bio, num_sectors, GFP_NOIO, NULL); - if (!split_bio) { - bio->bi_status = BLK_STS_RESOURCE; - return -ENOMEM; - } - bio_chain(split_bio, bio); - generic_make_request(bio); - *bio_ptr = split_bio; - } - return 0; -} - -/* - * The crypto API fallback's encryption routine. - * Allocate a bounce bio for encryption, encrypt the input bio using - * crypto API, and replace *bio_ptr with the bounce bio. May split input - * bio if it's too large. - */ -static int blk_crypto_encrypt_bio(struct bio **bio_ptr) -{ - struct bio *src_bio; - struct skcipher_request *ciph_req = NULL; - DECLARE_CRYPTO_WAIT(wait); - int err = 0; - u64 curr_dun; - union { - __le64 dun; - u8 bytes[16]; - } iv; - struct scatterlist src, dst; - struct bio *enc_bio; - struct bio_vec *enc_bvec; - int i, j; - int data_unit_size; - - /* Split the bio if it's too big for single page bvec */ - err = blk_crypto_split_bio_if_needed(bio_ptr); - if (err) - return err; - - src_bio = *bio_ptr; - data_unit_size = 1 << src_bio->bi_crypt_context->data_unit_size_bits; - - /* Allocate bounce bio for encryption */ - enc_bio = blk_crypto_clone_bio(src_bio); - if (!enc_bio) { - src_bio->bi_status = BLK_STS_RESOURCE; - return -ENOMEM; - } - - /* - * Use the crypto API fallback keyslot manager to get a crypto_skcipher - * for the algorithm and key specified for this bio. - */ - err = bio_crypt_ctx_acquire_keyslot(src_bio, blk_crypto_ksm); - if (err) { - src_bio->bi_status = BLK_STS_IOERR; - goto out_put_enc_bio; - } - - /* and then allocate an skcipher_request for it */ - err = blk_crypto_alloc_cipher_req(src_bio, &ciph_req, &wait); - if (err) - goto out_release_keyslot; - - curr_dun = bio_crypt_data_unit_num(src_bio); - sg_init_table(&src, 1); - sg_init_table(&dst, 1); - - skcipher_request_set_crypt(ciph_req, &src, &dst, - data_unit_size, iv.bytes); - - /* Encrypt each page in the bounce bio */ - for (i = 0, enc_bvec = enc_bio->bi_io_vec; i < enc_bio->bi_vcnt; - enc_bvec++, i++) { - struct page *plaintext_page = enc_bvec->bv_page; - struct page *ciphertext_page = - mempool_alloc(blk_crypto_page_pool, GFP_NOIO); - - enc_bvec->bv_page = ciphertext_page; - - if (!ciphertext_page) { - src_bio->bi_status = BLK_STS_RESOURCE; - err = -ENOMEM; - goto out_free_bounce_pages; - } - - sg_set_page(&src, plaintext_page, data_unit_size, - enc_bvec->bv_offset); - sg_set_page(&dst, ciphertext_page, data_unit_size, - enc_bvec->bv_offset); - - /* Encrypt each data unit in this page */ - for (j = 0; j < enc_bvec->bv_len; j += data_unit_size) { - memset(&iv, 0, sizeof(iv)); - iv.dun = cpu_to_le64(curr_dun); - - err = crypto_wait_req(crypto_skcipher_encrypt(ciph_req), - &wait); - if (err) { - i++; - src_bio->bi_status = BLK_STS_RESOURCE; - goto out_free_bounce_pages; - } - curr_dun++; - src.offset += data_unit_size; - dst.offset += data_unit_size; - } - } - - enc_bio->bi_private = src_bio; - enc_bio->bi_end_io = blk_crypto_encrypt_endio; - *bio_ptr = enc_bio; - - enc_bio = NULL; - err = 0; - goto out_free_ciph_req; - -out_free_bounce_pages: - while (i > 0) - mempool_free(enc_bio->bi_io_vec[--i].bv_page, - blk_crypto_page_pool); -out_free_ciph_req: - skcipher_request_free(ciph_req); -out_release_keyslot: - bio_crypt_ctx_release_keyslot(src_bio); -out_put_enc_bio: - if (enc_bio) - bio_put(enc_bio); - - return err; -} - -/* - * The crypto API fallback's main decryption routine. - * Decrypts input bio in place. - */ -static void blk_crypto_decrypt_bio(struct work_struct *w) -{ - struct work_mem *work_mem = - container_of(w, struct work_mem, crypto_work); - struct bio *bio = work_mem->bio; - struct skcipher_request *ciph_req = NULL; - DECLARE_CRYPTO_WAIT(wait); - struct bio_vec bv; - struct bvec_iter iter; - u64 curr_dun; - union { - __le64 dun; - u8 bytes[16]; - } iv; - struct scatterlist sg; - int data_unit_size = 1 << bio->bi_crypt_context->data_unit_size_bits; - int i; - int err; - - /* - * Use the crypto API fallback keyslot manager to get a crypto_skcipher - * for the algorithm and key specified for this bio. - */ - if (bio_crypt_ctx_acquire_keyslot(bio, blk_crypto_ksm)) { - bio->bi_status = BLK_STS_RESOURCE; - goto out_no_keyslot; - } - - /* and then allocate an skcipher_request for it */ - err = blk_crypto_alloc_cipher_req(bio, &ciph_req, &wait); - if (err) - goto out; - - curr_dun = bio_crypt_sw_data_unit_num(bio); - sg_init_table(&sg, 1); - skcipher_request_set_crypt(ciph_req, &sg, &sg, data_unit_size, - iv.bytes); - - /* Decrypt each segment in the bio */ - __bio_for_each_segment(bv, bio, iter, - bio->bi_crypt_context->crypt_iter) { - struct page *page = bv.bv_page; - - sg_set_page(&sg, page, data_unit_size, bv.bv_offset); - - /* Decrypt each data unit in the segment */ - for (i = 0; i < bv.bv_len; i += data_unit_size) { - memset(&iv, 0, sizeof(iv)); - iv.dun = cpu_to_le64(curr_dun); - if (crypto_wait_req(crypto_skcipher_decrypt(ciph_req), - &wait)) { - bio->bi_status = BLK_STS_IOERR; - goto out; - } - curr_dun++; - sg.offset += data_unit_size; - } - } - -out: - skcipher_request_free(ciph_req); - bio_crypt_ctx_release_keyslot(bio); -out_no_keyslot: - kmem_cache_free(blk_crypto_work_mem_cache, work_mem); - bio_endio(bio); -} - -/* Queue bio for decryption */ -static void blk_crypto_queue_decrypt_bio(struct bio *bio) -{ - struct work_mem *work_mem = - kmem_cache_zalloc(blk_crypto_work_mem_cache, GFP_ATOMIC); - - if (!work_mem) { - bio->bi_status = BLK_STS_RESOURCE; - bio_endio(bio); - return; - } - - INIT_WORK(&work_mem->crypto_work, blk_crypto_decrypt_bio); - work_mem->bio = bio; - queue_work(blk_crypto_wq, &work_mem->crypto_work); -} - /** * blk_crypto_submit_bio - handle submitting bio for inline encryption * @@ -518,20 +73,20 @@ int blk_crypto_submit_bio(struct bio **bio_ptr) { struct bio *bio = *bio_ptr; struct request_queue *q; + struct bio_crypt_ctx *bc = bio->bi_crypt_context; int err; - struct bio_crypt_ctx *crypt_ctx; - if (!bio_has_crypt_ctx(bio) || !bio_has_data(bio)) + if (!bc || !bio_has_data(bio)) return 0; /* - * When a read bio is marked for sw decryption, its bi_iter is saved - * so that when we decrypt the bio later, we know what part of it was - * marked for sw decryption (when the bio is passed down after + * When a read bio is marked for fallback decryption, its bi_iter is + * saved so that when we decrypt the bio later, we know what part of it + * was marked for fallback decryption (when the bio is passed down after * blk_crypto_submit bio, it may be split or advanced so we cannot rely * on the bi_iter while decrypting in blk_crypto_endio) */ - if (bio_crypt_swhandled(bio)) + if (bio_crypt_fallback_crypted(bc)) return 0; err = bio_crypt_check_alignment(bio); @@ -540,21 +95,22 @@ int blk_crypto_submit_bio(struct bio **bio_ptr) goto out; } - crypt_ctx = bio->bi_crypt_context; q = bio->bi_disk->queue; - if (bio_crypt_has_keyslot(bio)) { + if (bc->bc_ksm) { /* Key already programmed into device? */ - if (q->ksm == crypt_ctx->processing_ksm) + if (q->ksm == bc->bc_ksm) return 0; /* Nope, release the existing keyslot. */ - bio_crypt_ctx_release_keyslot(bio); + bio_crypt_ctx_release_keyslot(bc); } /* Get device keyslot if supported */ - if (q->ksm) { - err = bio_crypt_ctx_acquire_keyslot(bio, q->ksm); + if (keyslot_manager_crypto_mode_supported(q->ksm, + bc->bc_key->crypto_mode, + bc->bc_key->data_unit_size)) { + err = bio_crypt_ctx_acquire_keyslot(bc, q->ksm); if (!err) return 0; @@ -563,24 +119,10 @@ int blk_crypto_submit_bio(struct bio **bio_ptr) } /* Fallback to crypto API */ - if (!READ_ONCE(tfms_inited[bio->bi_crypt_context->crypto_mode])) { - err = -EIO; - bio->bi_status = BLK_STS_IOERR; + err = blk_crypto_fallback_submit_bio(bio_ptr); + if (err) goto out; - } - if (bio_data_dir(bio) == WRITE) { - /* Encrypt the data now */ - err = blk_crypto_encrypt_bio(bio_ptr); - if (err) - goto out; - } else { - /* Mark bio as swhandled */ - bio->bi_crypt_context->processing_ksm = blk_crypto_ksm; - bio->bi_crypt_context->crypt_iter = bio->bi_iter; - bio->bi_crypt_context->sw_data_unit_num = - bio->bi_crypt_context->data_unit_num; - } return 0; out: bio_endio(*bio_ptr); @@ -590,10 +132,10 @@ out: /** * blk_crypto_endio - clean up bio w.r.t inline encryption during bio_endio * - * @bio - the bio to clean up + * @bio: the bio to clean up * - * If blk_crypto_submit_bio decided to fallback to crypto API for this - * bio, we queue the bio for decryption into a workqueue and return false, + * If blk_crypto_submit_bio decided to fallback to crypto API for this bio, + * we queue the bio for decryption into a workqueue and return false, * and call bio_endio(bio) at a later time (after the bio has been decrypted). * * If the bio is not to be decrypted by the crypto API, this function releases @@ -604,195 +146,97 @@ out: */ bool blk_crypto_endio(struct bio *bio) { - if (!bio_has_crypt_ctx(bio)) + struct bio_crypt_ctx *bc = bio->bi_crypt_context; + + if (!bc) return true; - if (bio_crypt_swhandled(bio)) { + if (bio_crypt_fallback_crypted(bc)) { /* - * The only bios that are swhandled when they reach here - * are those with bio_data_dir(bio) == READ, since WRITE - * bios that are encrypted by the crypto API fallback are - * handled by blk_crypto_encrypt_endio. + * The only bios who's crypto is handled by the blk-crypto + * fallback when they reach here are those with + * bio_data_dir(bio) == READ, since WRITE bios that are + * encrypted by the crypto API fallback are handled by + * blk_crypto_encrypt_endio(). */ - - /* If there was an IO error, don't decrypt. */ - if (bio->bi_status) - return true; - - blk_crypto_queue_decrypt_bio(bio); - return false; + return !blk_crypto_queue_decrypt_bio(bio); } - if (bio_crypt_has_keyslot(bio)) - bio_crypt_ctx_release_keyslot(bio); + if (bc->bc_keyslot >= 0) + bio_crypt_ctx_release_keyslot(bc); return true; } /** - * blk_crypto_start_using_mode() - Allocate skciphers for a - * mode_num for all keyslots - * @mode_num - the blk_crypto_mode we want to allocate ciphers for. + * blk_crypto_init_key() - Prepare a key for use with blk-crypto + * @blk_key: Pointer to the blk_crypto_key to initialize. + * @raw_key: Pointer to the raw key. Must be the correct length for the chosen + * @crypto_mode; see blk_crypto_modes[]. + * @crypto_mode: identifier for the encryption algorithm to use + * @data_unit_size: the data unit size to use for en/decryption * - * Upper layers (filesystems) should call this function to ensure that a - * the crypto API fallback has transforms for this algorithm, if they become - * necessary. - * - * Return: 0 on success and -err on error. + * Return: The blk_crypto_key that was prepared, or an ERR_PTR() on error. When + * done using the key, it must be freed with blk_crypto_free_key(). */ -int blk_crypto_start_using_mode(enum blk_crypto_mode_num mode_num, - unsigned int data_unit_size, - struct request_queue *q) +int blk_crypto_init_key(struct blk_crypto_key *blk_key, const u8 *raw_key, + enum blk_crypto_mode_num crypto_mode, + unsigned int data_unit_size) { - struct blk_crypto_keyslot *slotp; - int err = 0; - int i; + const struct blk_crypto_mode *mode; + static siphash_key_t hash_key; + + memset(blk_key, 0, sizeof(*blk_key)); + + if (crypto_mode >= ARRAY_SIZE(blk_crypto_modes)) + return -EINVAL; + + mode = &blk_crypto_modes[crypto_mode]; + if (mode->keysize == 0) + return -EINVAL; + + if (!is_power_of_2(data_unit_size)) + return -EINVAL; + + blk_key->crypto_mode = crypto_mode; + blk_key->data_unit_size = data_unit_size; + blk_key->data_unit_size_bits = ilog2(data_unit_size); + blk_key->size = mode->keysize; + memcpy(blk_key->raw, raw_key, mode->keysize); /* - * Fast path - * Ensure that updates to blk_crypto_keyslots[i].tfms[mode_num] - * for each i are visible before we try to access them. + * The keyslot manager uses the SipHash of the key to implement O(1) key + * lookups while avoiding leaking information about the keys. It's + * precomputed here so that it only needs to be computed once per key. */ - if (likely(smp_load_acquire(&tfms_inited[mode_num]))) - return 0; + get_random_once(&hash_key, sizeof(hash_key)); + blk_key->hash = siphash(raw_key, mode->keysize, &hash_key); - /* - * If the keyslot manager of the request queue supports this - * crypto mode, then we don't need to allocate this mode. - */ - if (keyslot_manager_crypto_mode_supported(q->ksm, mode_num, - data_unit_size)) { - return 0; - } - - mutex_lock(&tfms_lock[mode_num]); - if (likely(tfms_inited[mode_num])) - goto out; - - for (i = 0; i < blk_crypto_num_keyslots; i++) { - slotp = &blk_crypto_keyslots[i]; - slotp->tfms[mode_num] = crypto_alloc_skcipher( - blk_crypto_modes[mode_num].cipher_str, - 0, 0); - if (IS_ERR(slotp->tfms[mode_num])) { - err = PTR_ERR(slotp->tfms[mode_num]); - slotp->tfms[mode_num] = NULL; - goto out_free_tfms; - } - - crypto_skcipher_set_flags(slotp->tfms[mode_num], - CRYPTO_TFM_REQ_FORBID_WEAK_KEYS); - } - - /* - * Ensure that updates to blk_crypto_keyslots[i].tfms[mode_num] - * for each i are visible before we set tfms_inited[mode_num]. - */ - smp_store_release(&tfms_inited[mode_num], true); - goto out; - -out_free_tfms: - for (i = 0; i < blk_crypto_num_keyslots; i++) { - slotp = &blk_crypto_keyslots[i]; - crypto_free_skcipher(slotp->tfms[mode_num]); - slotp->tfms[mode_num] = NULL; - } -out: - mutex_unlock(&tfms_lock[mode_num]); - return err; + return 0; } -EXPORT_SYMBOL(blk_crypto_start_using_mode); /** * blk_crypto_evict_key() - Evict a key from any inline encryption hardware * it may have been programmed into - * @q - The request queue who's keyslot manager this key might have been - * programmed into - * @key - The key to evict - * @mode - The blk_crypto_mode_num used with this key - * @data_unit_size - The data unit size used with this key + * @q: The request queue who's keyslot manager this key might have been + * programmed into + * @key: The key to evict * * Upper layers (filesystems) should call this function to ensure that a key * is evicted from hardware that it might have been programmed into. This * will call keyslot_manager_evict_key on the queue's keyslot manager, if one * exists, and supports the crypto algorithm with the specified data unit size. - * Otherwise, it will evict the key from the blk_crypto_ksm. + * Otherwise, it will evict the key from the blk-crypto-fallback's ksm. * * Return: 0 on success, -err on error. */ -int blk_crypto_evict_key(struct request_queue *q, const u8 *key, - enum blk_crypto_mode_num mode, - unsigned int data_unit_size) +int blk_crypto_evict_key(struct request_queue *q, + const struct blk_crypto_key *key) { - struct keyslot_manager *ksm = blk_crypto_ksm; + if (q->ksm && + keyslot_manager_crypto_mode_supported(q->ksm, key->crypto_mode, + key->data_unit_size)) + return keyslot_manager_evict_key(q->ksm, key); - if (q && q->ksm && keyslot_manager_crypto_mode_supported(q->ksm, mode, - data_unit_size)) { - ksm = q->ksm; - } - - return keyslot_manager_evict_key(ksm, key, mode, data_unit_size); -} -EXPORT_SYMBOL(blk_crypto_evict_key); - -int __init blk_crypto_init(void) -{ - int i; - int err = -ENOMEM; - - prandom_bytes(blank_key, BLK_CRYPTO_MAX_KEY_SIZE); - - blk_crypto_ksm = keyslot_manager_create(blk_crypto_num_keyslots, - &blk_crypto_ksm_ll_ops, - NULL); - if (!blk_crypto_ksm) - goto out; - - blk_crypto_wq = alloc_workqueue("blk_crypto_wq", - WQ_UNBOUND | WQ_HIGHPRI | - WQ_MEM_RECLAIM, - num_online_cpus()); - if (!blk_crypto_wq) - goto out_free_ksm; - - blk_crypto_keyslots = kcalloc(blk_crypto_num_keyslots, - sizeof(*blk_crypto_keyslots), - GFP_KERNEL); - if (!blk_crypto_keyslots) - goto out_free_workqueue; - - for (i = 0; i < blk_crypto_num_keyslots; i++) { - blk_crypto_keyslots[i].crypto_mode = - BLK_ENCRYPTION_MODE_INVALID; - } - - for (i = 0; i < ARRAY_SIZE(blk_crypto_modes); i++) - mutex_init(&tfms_lock[i]); - - blk_crypto_page_pool = - mempool_create_page_pool(num_prealloc_bounce_pg, 0); - if (!blk_crypto_page_pool) - goto out_free_keyslots; - - blk_crypto_work_mem_cache = KMEM_CACHE(work_mem, SLAB_RECLAIM_ACCOUNT); - if (!blk_crypto_work_mem_cache) - goto out_free_page_pool; - - return 0; - -out_free_page_pool: - mempool_destroy(blk_crypto_page_pool); - blk_crypto_page_pool = NULL; -out_free_keyslots: - kzfree(blk_crypto_keyslots); - blk_crypto_keyslots = NULL; -out_free_workqueue: - destroy_workqueue(blk_crypto_wq); - blk_crypto_wq = NULL; -out_free_ksm: - keyslot_manager_destroy(blk_crypto_ksm); - blk_crypto_ksm = NULL; -out: - pr_warn("No memory for blk-crypto crypto API fallback."); - return err; + return blk_crypto_fallback_evict_key(key); } diff --git a/block/blk-merge.c b/block/blk-merge.c index c97c02a20c6a..50dce8cf4570 100644 --- a/block/blk-merge.c +++ b/block/blk-merge.c @@ -557,9 +557,6 @@ static inline int ll_new_hw_segment(struct request *req, struct bio *bio, if (blk_integrity_merge_bio(req->q, req, bio) == false) goto no_merge; - if (WARN_ON_ONCE(!bio_crypt_ctx_compatible(bio, req->bio))) - goto no_merge; - /* * This will form the start of a new hw segment. Bump both * counters. @@ -584,6 +581,8 @@ int ll_back_merge_fn(struct request *req, struct bio *bio, unsigned int nr_segs) req_set_nomerge(req->q, req); return 0; } + if (!bio_crypt_ctx_mergeable(req->bio, blk_rq_bytes(req), bio)) + return 0; return ll_new_hw_segment(req, bio, nr_segs); } @@ -600,6 +599,8 @@ int ll_front_merge_fn(struct request *req, struct bio *bio, unsigned int nr_segs req_set_nomerge(req->q, req); return 0; } + if (!bio_crypt_ctx_mergeable(bio, bio->bi_iter.bi_size, req->bio)) + return 0; return ll_new_hw_segment(req, bio, nr_segs); } @@ -644,6 +645,9 @@ static int ll_merge_requests_fn(struct request_queue *q, struct request *req, if (blk_integrity_merge_rq(q, req, next) == false) return 0; + if (!bio_crypt_ctx_mergeable(req->bio, blk_rq_bytes(req), next->bio)) + return 0; + /* Merge is OK... */ req->nr_phys_segments = total_phys_segments; return 1; @@ -714,14 +718,8 @@ static enum elv_merge blk_try_req_merge(struct request *req, { if (blk_discard_mergable(req)) return ELEVATOR_DISCARD_MERGE; - else if (blk_rq_pos(req) + blk_rq_sectors(req) == blk_rq_pos(next)) { - if (!bio_crypt_ctx_back_mergeable(req->bio, - blk_rq_sectors(req), - next->bio)) { - return ELEVATOR_NO_MERGE; - } + else if (blk_rq_pos(req) + blk_rq_sectors(req) == blk_rq_pos(next)) return ELEVATOR_BACK_MERGE; - } return ELEVATOR_NO_MERGE; } @@ -757,9 +755,6 @@ static struct request *attempt_merge(struct request_queue *q, if (req->ioprio != next->ioprio) return NULL; - if (!bio_crypt_ctx_compatible(req->bio, next->bio)) - return NULL; - /* * If we are allowed to merge, then append bio list * from next to rq and release next. merge_requests_fn @@ -901,22 +896,11 @@ bool blk_rq_merge_ok(struct request *rq, struct bio *bio) enum elv_merge blk_try_merge(struct request *rq, struct bio *bio) { - if (blk_discard_mergable(rq)) { + if (blk_discard_mergable(rq)) return ELEVATOR_DISCARD_MERGE; - } else if (blk_rq_pos(rq) + blk_rq_sectors(rq) == - bio->bi_iter.bi_sector) { - if (!bio_crypt_ctx_back_mergeable(rq->bio, - blk_rq_sectors(rq), bio)) { - return ELEVATOR_NO_MERGE; - } + else if (blk_rq_pos(rq) + blk_rq_sectors(rq) == bio->bi_iter.bi_sector) return ELEVATOR_BACK_MERGE; - } else if (blk_rq_pos(rq) - bio_sectors(bio) == - bio->bi_iter.bi_sector) { - if (!bio_crypt_ctx_back_mergeable(bio, - bio_sectors(bio), rq->bio)) { - return ELEVATOR_NO_MERGE; - } + else if (blk_rq_pos(rq) - bio_sectors(bio) == bio->bi_iter.bi_sector) return ELEVATOR_FRONT_MERGE; - } return ELEVATOR_NO_MERGE; } diff --git a/block/bounce.c b/block/bounce.c index 6f9a2359b22a..aa57ccc6ced3 100644 --- a/block/bounce.c +++ b/block/bounce.c @@ -267,10 +267,7 @@ static struct bio *bounce_clone_bio(struct bio *bio_src, gfp_t gfp_mask, break; } - if (bio_crypt_clone(bio, bio_src, gfp_mask) < 0) { - bio_put(bio); - return NULL; - } + bio_crypt_clone(bio, bio_src, gfp_mask); if (bio_integrity(bio_src) && bio_integrity_clone(bio, bio_src, gfp_mask) < 0) { diff --git a/block/keyslot-manager.c b/block/keyslot-manager.c index 020931fc9f7d..cdf0607019d4 100644 --- a/block/keyslot-manager.c +++ b/block/keyslot-manager.c @@ -1,7 +1,5 @@ // SPDX-License-Identifier: GPL-2.0 /* - * keyslot-manager.c - * * Copyright 2019 Google LLC */ @@ -27,6 +25,7 @@ * Upper layers will call keyslot_manager_get_slot_for_key() to program a * key into some slot in the inline encryption hardware. */ +#include #include #include #include @@ -36,12 +35,14 @@ struct keyslot { atomic_t slot_refs; struct list_head idle_slot_node; + struct hlist_node hash_node; + struct blk_crypto_key key; }; struct keyslot_manager { unsigned int num_slots; - atomic_t num_idle_slots; struct keyslot_mgmt_ll_ops ksm_ll_ops; + unsigned int crypto_mode_supported[BLK_ENCRYPTION_MODE_MAX]; void *ll_priv_data; /* Protects programming and evicting keys from the device */ @@ -52,6 +53,15 @@ struct keyslot_manager { struct list_head idle_slots; spinlock_t idle_slots_lock; + /* + * Hash table which maps key hashes to keyslots, so that we can find a + * key's keyslot in O(1) time rather than O(num_slots). Protected by + * 'lock'. A cryptographic hash function is used so that timing attacks + * can't leak information about the raw keys. + */ + struct hlist_head *slot_hashtable; + unsigned int slot_hashtable_size; + /* Per-keyslot data */ struct keyslot slots[]; }; @@ -62,6 +72,13 @@ struct keyslot_manager { * @ksm_ll_ops: The struct keyslot_mgmt_ll_ops for the device that this keyslot * manager will use to perform operations like programming and * evicting keys. + * @crypto_mode_supported: Array of size BLK_ENCRYPTION_MODE_MAX of + * bitmasks that represents whether a crypto mode + * and data unit size are supported. The i'th bit + * of crypto_mode_supported[crypto_mode] is set iff + * a data unit size of (1 << i) is supported. We + * only support data unit sizes that are powers of + * 2. * @ll_priv_data: Private data passed as is to the functions in ksm_ll_ops. * * Allocate memory for and initialize a keyslot manager. Called by e.g. @@ -71,20 +88,20 @@ struct keyslot_manager { * Return: Pointer to constructed keyslot manager or NULL on error. */ struct keyslot_manager *keyslot_manager_create(unsigned int num_slots, - const struct keyslot_mgmt_ll_ops *ksm_ll_ops, - void *ll_priv_data) + const struct keyslot_mgmt_ll_ops *ksm_ll_ops, + const unsigned int crypto_mode_supported[BLK_ENCRYPTION_MODE_MAX], + void *ll_priv_data) { struct keyslot_manager *ksm; - int slot; + unsigned int slot; + unsigned int i; if (num_slots == 0) return NULL; /* Check that all ops are specified */ if (ksm_ll_ops->keyslot_program == NULL || - ksm_ll_ops->keyslot_evict == NULL || - ksm_ll_ops->crypto_mode_supported == NULL || - ksm_ll_ops->keyslot_find == NULL) + ksm_ll_ops->keyslot_evict == NULL) return NULL; ksm = kvzalloc(struct_size(ksm, slots, num_slots), GFP_KERNEL); @@ -92,8 +109,9 @@ struct keyslot_manager *keyslot_manager_create(unsigned int num_slots, return NULL; ksm->num_slots = num_slots; - atomic_set(&ksm->num_idle_slots, num_slots); ksm->ksm_ll_ops = *ksm_ll_ops; + memcpy(ksm->crypto_mode_supported, crypto_mode_supported, + sizeof(ksm->crypto_mode_supported)); ksm->ll_priv_data = ll_priv_data; init_rwsem(&ksm->lock); @@ -108,9 +126,29 @@ struct keyslot_manager *keyslot_manager_create(unsigned int num_slots, spin_lock_init(&ksm->idle_slots_lock); + ksm->slot_hashtable_size = roundup_pow_of_two(num_slots); + ksm->slot_hashtable = kvmalloc_array(ksm->slot_hashtable_size, + sizeof(ksm->slot_hashtable[0]), + GFP_KERNEL); + if (!ksm->slot_hashtable) + goto err_free_ksm; + for (i = 0; i < ksm->slot_hashtable_size; i++) + INIT_HLIST_HEAD(&ksm->slot_hashtable[i]); + return ksm; + +err_free_ksm: + keyslot_manager_destroy(ksm); + return NULL; +} +EXPORT_SYMBOL_GPL(keyslot_manager_create); + +static inline struct hlist_head * +hash_bucket_for_key(struct keyslot_manager *ksm, + const struct blk_crypto_key *key) +{ + return &ksm->slot_hashtable[key->hash & (ksm->slot_hashtable_size - 1)]; } -EXPORT_SYMBOL(keyslot_manager_create); static void remove_slot_from_lru_list(struct keyslot_manager *ksm, int slot) { @@ -119,22 +157,32 @@ static void remove_slot_from_lru_list(struct keyslot_manager *ksm, int slot) spin_lock_irqsave(&ksm->idle_slots_lock, flags); list_del(&ksm->slots[slot].idle_slot_node); spin_unlock_irqrestore(&ksm->idle_slots_lock, flags); - - atomic_dec(&ksm->num_idle_slots); } -static int find_and_grab_keyslot(struct keyslot_manager *ksm, const u8 *key, - enum blk_crypto_mode_num crypto_mode, - unsigned int data_unit_size) +static int find_keyslot(struct keyslot_manager *ksm, + const struct blk_crypto_key *key) +{ + const struct hlist_head *head = hash_bucket_for_key(ksm, key); + const struct keyslot *slotp; + + hlist_for_each_entry(slotp, head, hash_node) { + if (slotp->key.hash == key->hash && + slotp->key.crypto_mode == key->crypto_mode && + slotp->key.data_unit_size == key->data_unit_size && + !crypto_memneq(slotp->key.raw, key->raw, key->size)) + return slotp - ksm->slots; + } + return -ENOKEY; +} + +static int find_and_grab_keyslot(struct keyslot_manager *ksm, + const struct blk_crypto_key *key) { int slot; - slot = ksm->ksm_ll_ops.keyslot_find(ksm->ll_priv_data, key, - crypto_mode, data_unit_size); + slot = find_keyslot(ksm, key); if (slot < 0) return slot; - if (WARN_ON(slot >= ksm->num_slots)) - return -EINVAL; if (atomic_inc_return(&ksm->slots[slot].slot_refs) == 1) { /* Took first reference to this slot; remove it from LRU list */ remove_slot_from_lru_list(ksm, slot); @@ -145,37 +193,32 @@ static int find_and_grab_keyslot(struct keyslot_manager *ksm, const u8 *key, /** * keyslot_manager_get_slot_for_key() - Program a key into a keyslot. * @ksm: The keyslot manager to program the key into. - * @key: Pointer to the bytes of the key to program. Must be the correct length - * for the chosen @crypto_mode; see blk_crypto_modes in blk-crypto.c. - * @crypto_mode: Identifier for the encryption algorithm to use. - * @data_unit_size: The data unit size to use for en/decryption. + * @key: Pointer to the key object to program, including the raw key, crypto + * mode, and data unit size. * - * Get a keyslot that's been programmed with the specified key, crypto_mode, and - * data_unit_size. If one already exists, return it with incremented refcount. - * Otherwise, wait for a keyslot to become idle and program it. + * Get a keyslot that's been programmed with the specified key. If one already + * exists, return it with incremented refcount. Otherwise, wait for a keyslot + * to become idle and program it. * * Context: Process context. Takes and releases ksm->lock. * Return: The keyslot on success, else a -errno value. */ int keyslot_manager_get_slot_for_key(struct keyslot_manager *ksm, - const u8 *key, - enum blk_crypto_mode_num crypto_mode, - unsigned int data_unit_size) + const struct blk_crypto_key *key) { int slot; int err; struct keyslot *idle_slot; down_read(&ksm->lock); - slot = find_and_grab_keyslot(ksm, key, crypto_mode, data_unit_size); + slot = find_and_grab_keyslot(ksm, key); up_read(&ksm->lock); if (slot != -ENOKEY) return slot; for (;;) { down_write(&ksm->lock); - slot = find_and_grab_keyslot(ksm, key, crypto_mode, - data_unit_size); + slot = find_and_grab_keyslot(ksm, key); if (slot != -ENOKEY) { up_write(&ksm->lock); return slot; @@ -185,42 +228,43 @@ int keyslot_manager_get_slot_for_key(struct keyslot_manager *ksm, * If we're here, that means there wasn't a slot that was * already programmed with the key. So try to program it. */ - if (atomic_read(&ksm->num_idle_slots) > 0) + if (!list_empty(&ksm->idle_slots)) break; up_write(&ksm->lock); wait_event(ksm->idle_slots_wait_queue, - (atomic_read(&ksm->num_idle_slots) > 0)); + !list_empty(&ksm->idle_slots)); } idle_slot = list_first_entry(&ksm->idle_slots, struct keyslot, idle_slot_node); slot = idle_slot - ksm->slots; - err = ksm->ksm_ll_ops.keyslot_program(ksm->ll_priv_data, key, - crypto_mode, - data_unit_size, - slot); - + err = ksm->ksm_ll_ops.keyslot_program(ksm, key, slot); if (err) { wake_up(&ksm->idle_slots_wait_queue); up_write(&ksm->lock); return err; } - atomic_set(&ksm->slots[slot].slot_refs, 1); + /* Move this slot to the hash list for the new key. */ + if (idle_slot->key.crypto_mode != BLK_ENCRYPTION_MODE_INVALID) + hlist_del(&idle_slot->hash_node); + hlist_add_head(&idle_slot->hash_node, hash_bucket_for_key(ksm, key)); + + atomic_set(&idle_slot->slot_refs, 1); + idle_slot->key = *key; + remove_slot_from_lru_list(ksm, slot); up_write(&ksm->lock); return slot; - } -EXPORT_SYMBOL(keyslot_manager_get_slot_for_key); /** * keyslot_manager_get_slot() - Increment the refcount on the specified slot. - * @ksm - The keyslot manager that we want to modify. - * @slot - The slot to increment the refcount of. + * @ksm: The keyslot manager that we want to modify. + * @slot: The slot to increment the refcount of. * * This function assumes that there is already an active reference to that slot * and simply increments the refcount. This is useful when cloning a bio that @@ -236,7 +280,6 @@ void keyslot_manager_get_slot(struct keyslot_manager *ksm, unsigned int slot) WARN_ON(atomic_inc_return(&ksm->slots[slot].slot_refs) < 2); } -EXPORT_SYMBOL(keyslot_manager_get_slot); /** * keyslot_manager_put_slot() - Release a reference to a slot @@ -257,19 +300,17 @@ void keyslot_manager_put_slot(struct keyslot_manager *ksm, unsigned int slot) list_add_tail(&ksm->slots[slot].idle_slot_node, &ksm->idle_slots); spin_unlock_irqrestore(&ksm->idle_slots_lock, flags); - atomic_inc(&ksm->num_idle_slots); wake_up(&ksm->idle_slots_wait_queue); } } -EXPORT_SYMBOL(keyslot_manager_put_slot); /** * keyslot_manager_crypto_mode_supported() - Find out if a crypto_mode/data * unit size combination is supported * by a ksm. - * @ksm - The keyslot manager to check - * @crypto_mode - The crypto mode to check for. - * @data_unit_size - The data_unit_size for the mode. + * @ksm: The keyslot manager to check + * @crypto_mode: The crypto mode to check for. + * @data_unit_size: The data_unit_size for the mode. * * Calls and returns the result of the crypto_mode_supported function specified * by the ksm. @@ -284,69 +325,102 @@ bool keyslot_manager_crypto_mode_supported(struct keyslot_manager *ksm, { if (!ksm) return false; - return ksm->ksm_ll_ops.crypto_mode_supported(ksm->ll_priv_data, - crypto_mode, - data_unit_size); + if (WARN_ON(crypto_mode >= BLK_ENCRYPTION_MODE_MAX)) + return false; + if (WARN_ON(!is_power_of_2(data_unit_size))) + return false; + return ksm->crypto_mode_supported[crypto_mode] & data_unit_size; } -EXPORT_SYMBOL(keyslot_manager_crypto_mode_supported); - -bool keyslot_manager_rq_crypto_mode_supported(struct request_queue *q, - enum blk_crypto_mode_num crypto_mode, - unsigned int data_unit_size) -{ - return keyslot_manager_crypto_mode_supported(q->ksm, crypto_mode, - data_unit_size); -} -EXPORT_SYMBOL(keyslot_manager_rq_crypto_mode_supported); /** * keyslot_manager_evict_key() - Evict a key from the lower layer device. - * @ksm - The keyslot manager to evict from - * @key - The key to evict - * @crypto_mode - The crypto algorithm the key was programmed with. - * @data_unit_size - The data_unit_size the key was programmed with. + * @ksm: The keyslot manager to evict from + * @key: The key to evict * - * Finds the slot that the specified key, crypto_mode, data_unit_size combo - * was programmed into, and evicts that slot from the lower layer device if - * the refcount on the slot is 0. Returns -EBUSY if the refcount is not 0, and - * -errno on error. + * Find the keyslot that the specified key was programmed into, and evict that + * slot from the lower layer device if that slot is not currently in use. * * Context: Process context. Takes and releases ksm->lock. + * Return: 0 on success, -EBUSY if the key is still in use, or another + * -errno value on other error. */ int keyslot_manager_evict_key(struct keyslot_manager *ksm, - const u8 *key, - enum blk_crypto_mode_num crypto_mode, - unsigned int data_unit_size) + const struct blk_crypto_key *key) { int slot; - int err = 0; + int err; + struct keyslot *slotp; down_write(&ksm->lock); - slot = ksm->ksm_ll_ops.keyslot_find(ksm->ll_priv_data, key, - crypto_mode, - data_unit_size); - + slot = find_keyslot(ksm, key); if (slot < 0) { - up_write(&ksm->lock); - return slot; + err = slot; + goto out_unlock; } + slotp = &ksm->slots[slot]; - if (atomic_read(&ksm->slots[slot].slot_refs) == 0) { - err = ksm->ksm_ll_ops.keyslot_evict(ksm->ll_priv_data, key, - crypto_mode, - data_unit_size, - slot); - } else { + if (atomic_read(&slotp->slot_refs) != 0) { err = -EBUSY; + goto out_unlock; } + err = ksm->ksm_ll_ops.keyslot_evict(ksm, key, slot); + if (err) + goto out_unlock; + hlist_del(&slotp->hash_node); + memzero_explicit(&slotp->key, sizeof(slotp->key)); + err = 0; +out_unlock: up_write(&ksm->lock); return err; } -EXPORT_SYMBOL(keyslot_manager_evict_key); + +/** + * keyslot_manager_reprogram_all_keys() - Re-program all keyslots. + * @ksm: The keyslot manager + * + * Re-program all keyslots that are supposed to have a key programmed. This is + * intended only for use by drivers for hardware that loses its keys on reset. + * + * Context: Process context. Takes and releases ksm->lock. + */ +void keyslot_manager_reprogram_all_keys(struct keyslot_manager *ksm) +{ + unsigned int slot; + + down_write(&ksm->lock); + for (slot = 0; slot < ksm->num_slots; slot++) { + const struct keyslot *slotp = &ksm->slots[slot]; + int err; + + if (slotp->key.crypto_mode == BLK_ENCRYPTION_MODE_INVALID) + continue; + + err = ksm->ksm_ll_ops.keyslot_program(ksm, &slotp->key, slot); + WARN_ON(err); + } + up_write(&ksm->lock); +} +EXPORT_SYMBOL_GPL(keyslot_manager_reprogram_all_keys); + +/** + * keyslot_manager_private() - return the private data stored with ksm + * @ksm: The keyslot manager + * + * Returns the private data passed to the ksm when it was created. + */ +void *keyslot_manager_private(struct keyslot_manager *ksm) +{ + return ksm->ll_priv_data; +} +EXPORT_SYMBOL_GPL(keyslot_manager_private); void keyslot_manager_destroy(struct keyslot_manager *ksm) { - kvfree(ksm); + if (ksm) { + kvfree(ksm->slot_hashtable); + memzero_explicit(ksm, struct_size(ksm, slots, ksm->num_slots)); + kvfree(ksm); + } } -EXPORT_SYMBOL(keyslot_manager_destroy); +EXPORT_SYMBOL_GPL(keyslot_manager_destroy); diff --git a/drivers/md/dm.c b/drivers/md/dm.c index 67c24294d7c8..59fe829aac9c 100644 --- a/drivers/md/dm.c +++ b/drivers/md/dm.c @@ -1322,15 +1322,13 @@ static int clone_bio(struct dm_target_io *tio, struct bio *bio, sector_t sector, unsigned len) { struct bio *clone = &tio->clone; - int ret; __bio_clone_fast(clone, bio); - ret = bio_crypt_clone(clone, bio, GFP_NOIO); - if (ret < 0) - return ret; + bio_crypt_clone(clone, bio, GFP_NOIO); if (bio_integrity(bio)) { + int r; if (unlikely(!dm_target_has_integrity(tio->ti->type) && !dm_target_passes_integrity(tio->ti->type))) { DMWARN("%s: the target %s doesn't support integrity data.", @@ -1339,11 +1337,9 @@ static int clone_bio(struct dm_target_io *tio, struct bio *bio, return -EIO; } - ret = bio_integrity_clone(clone, bio, GFP_NOIO); - if (ret < 0) { - bio_crypt_free_ctx(clone); - return ret; - } + r = bio_integrity_clone(clone, bio, GFP_NOIO); + if (r < 0) + return r; } bio_advance(clone, to_bytes(sector - clone->bi_iter.bi_sector)); diff --git a/drivers/scsi/ufs/ufshcd-crypto.c b/drivers/scsi/ufs/ufshcd-crypto.c index 75d834180f8b..1cfd9a327049 100644 --- a/drivers/scsi/ufs/ufshcd-crypto.c +++ b/drivers/scsi/ufs/ufshcd-crypto.c @@ -3,8 +3,7 @@ * Copyright 2019 Google LLC */ -#include - +#include #include "ufshcd.h" #include "ufshcd-crypto.h" @@ -25,19 +24,23 @@ static u8 get_data_unit_size_mask(unsigned int data_unit_size) static size_t get_keysize_bytes(enum ufs_crypto_key_size size) { switch (size) { - case UFS_CRYPTO_KEY_SIZE_128: return 16; - case UFS_CRYPTO_KEY_SIZE_192: return 24; - case UFS_CRYPTO_KEY_SIZE_256: return 32; - case UFS_CRYPTO_KEY_SIZE_512: return 64; - default: return 0; + case UFS_CRYPTO_KEY_SIZE_128: + return 16; + case UFS_CRYPTO_KEY_SIZE_192: + return 24; + case UFS_CRYPTO_KEY_SIZE_256: + return 32; + case UFS_CRYPTO_KEY_SIZE_512: + return 64; + default: + return 0; } } -static int ufshcd_crypto_cap_find(void *hba_p, +static int ufshcd_crypto_cap_find(struct ufs_hba *hba, enum blk_crypto_mode_num crypto_mode, unsigned int data_unit_size) { - struct ufs_hba *hba = hba_p; enum ufs_crypto_alg ufs_alg; u8 data_unit_mask; int cap_idx; @@ -52,7 +55,8 @@ static int ufshcd_crypto_cap_find(void *hba_p, ufs_alg = UFS_CRYPTO_ALG_AES_XTS; ufs_key_size = UFS_CRYPTO_KEY_SIZE_256; break; - default: return -EINVAL; + default: + return -EINVAL; } data_unit_mask = get_data_unit_size_mask(data_unit_size); @@ -101,8 +105,10 @@ static int ufshcd_crypto_cfg_entry_write_key(union ufs_crypto_cfg_entry *cfg, memcpy(cfg->crypto_key + UFS_CRYPTO_KEY_MAX_SIZE/2, key + key_size_bytes/2, key_size_bytes/2); return 0; - case UFS_CRYPTO_ALG_BITLOCKER_AES_CBC: // fallthrough - case UFS_CRYPTO_ALG_AES_ECB: // fallthrough + case UFS_CRYPTO_ALG_BITLOCKER_AES_CBC: + /* fall through */ + case UFS_CRYPTO_ALG_AES_ECB: + /* fall through */ case UFS_CRYPTO_ALG_ESSIV_AES_CBC: memcpy(cfg->crypto_key, key, key_size_bytes); return 0; @@ -111,13 +117,15 @@ static int ufshcd_crypto_cfg_entry_write_key(union ufs_crypto_cfg_entry *cfg, return -EINVAL; } -static void program_key(struct ufs_hba *hba, - const union ufs_crypto_cfg_entry *cfg, - int slot) +static void ufshcd_program_key(struct ufs_hba *hba, + const union ufs_crypto_cfg_entry *cfg, + int slot) { int i; u32 slot_offset = hba->crypto_cfg_register + slot * sizeof(*cfg); + pm_runtime_get_sync(hba->dev); + ufshcd_hold(hba, false); /* Clear the dword 16 */ ufshcd_writel(hba, 0, slot_offset + 16 * sizeof(cfg->reg_val[0])); /* Ensure that CFGE is cleared before programming the key */ @@ -137,29 +145,45 @@ static void program_key(struct ufs_hba *hba, ufshcd_writel(hba, le32_to_cpu(cfg->reg_val[16]), slot_offset + 16 * sizeof(cfg->reg_val[0])); wmb(); + ufshcd_release(hba); + pm_runtime_put_sync(hba->dev); } -static int ufshcd_crypto_keyslot_program(void *hba_p, const u8 *key, - enum blk_crypto_mode_num crypto_mode, - unsigned int data_unit_size, +static void ufshcd_clear_keyslot(struct ufs_hba *hba, int slot) +{ + union ufs_crypto_cfg_entry cfg = { 0 }; + + ufshcd_program_key(hba, &cfg, slot); +} + +/* Clear all keyslots at driver init time */ +static void ufshcd_clear_all_keyslots(struct ufs_hba *hba) +{ + int slot; + + for (slot = 0; slot < ufshcd_num_keyslots(hba); slot++) + ufshcd_clear_keyslot(hba, slot); +} + +static int ufshcd_crypto_keyslot_program(struct keyslot_manager *ksm, + const struct blk_crypto_key *key, unsigned int slot) { - struct ufs_hba *hba = hba_p; + struct ufs_hba *hba = keyslot_manager_private(ksm); int err = 0; u8 data_unit_mask; union ufs_crypto_cfg_entry cfg; - union ufs_crypto_cfg_entry *cfg_arr = hba->crypto_cfgs; int cap_idx; - cap_idx = ufshcd_crypto_cap_find(hba_p, crypto_mode, - data_unit_size); + cap_idx = ufshcd_crypto_cap_find(hba, key->crypto_mode, + key->data_unit_size); if (!ufshcd_is_crypto_enabled(hba) || !ufshcd_keyslot_valid(hba, slot) || !ufshcd_cap_idx_valid(hba, cap_idx)) return -EINVAL; - data_unit_mask = get_data_unit_size_mask(data_unit_size); + data_unit_mask = get_data_unit_size_mask(key->data_unit_size); if (!(data_unit_mask & hba->crypto_cap_array[cap_idx].sdus_mask)) return -EINVAL; @@ -169,134 +193,74 @@ static int ufshcd_crypto_keyslot_program(void *hba_p, const u8 *key, cfg.crypto_cap_idx = cap_idx; cfg.config_enable |= UFS_CRYPTO_CONFIGURATION_ENABLE; - err = ufshcd_crypto_cfg_entry_write_key(&cfg, key, - hba->crypto_cap_array[cap_idx]); + err = ufshcd_crypto_cfg_entry_write_key(&cfg, key->raw, + hba->crypto_cap_array[cap_idx]); if (err) return err; - program_key(hba, &cfg, slot); + ufshcd_program_key(hba, &cfg, slot); - memcpy(&cfg_arr[slot], &cfg, sizeof(cfg)); memzero_explicit(&cfg, sizeof(cfg)); - return 0; } -static int ufshcd_crypto_keyslot_find(void *hba_p, - const u8 *key, - enum blk_crypto_mode_num crypto_mode, - unsigned int data_unit_size) -{ - struct ufs_hba *hba = hba_p; - int err = 0; - int slot; - u8 data_unit_mask; - union ufs_crypto_cfg_entry cfg; - union ufs_crypto_cfg_entry *cfg_arr = hba->crypto_cfgs; - int cap_idx; - - cap_idx = ufshcd_crypto_cap_find(hba_p, crypto_mode, - data_unit_size); - - if (!ufshcd_is_crypto_enabled(hba) || - !ufshcd_cap_idx_valid(hba, cap_idx)) - return -EINVAL; - - data_unit_mask = get_data_unit_size_mask(data_unit_size); - - if (!(data_unit_mask & hba->crypto_cap_array[cap_idx].sdus_mask)) - return -EINVAL; - - memset(&cfg, 0, sizeof(cfg)); - err = ufshcd_crypto_cfg_entry_write_key(&cfg, key, - hba->crypto_cap_array[cap_idx]); - - if (err) - return -EINVAL; - - for (slot = 0; slot < NUM_KEYSLOTS(hba); slot++) { - if ((cfg_arr[slot].config_enable & - UFS_CRYPTO_CONFIGURATION_ENABLE) && - data_unit_mask == cfg_arr[slot].data_unit_size && - cap_idx == cfg_arr[slot].crypto_cap_idx && - !crypto_memneq(&cfg.crypto_key, cfg_arr[slot].crypto_key, - UFS_CRYPTO_KEY_MAX_SIZE)) { - memzero_explicit(&cfg, sizeof(cfg)); - return slot; - } - } - - memzero_explicit(&cfg, sizeof(cfg)); - return -ENOKEY; -} - -static int ufshcd_crypto_keyslot_evict(void *hba_p, const u8 *key, - enum blk_crypto_mode_num crypto_mode, - unsigned int data_unit_size, +static int ufshcd_crypto_keyslot_evict(struct keyslot_manager *ksm, + const struct blk_crypto_key *key, unsigned int slot) { - struct ufs_hba *hba = hba_p; - int i = 0; - u32 reg_base; - union ufs_crypto_cfg_entry *cfg_arr = hba->crypto_cfgs; + struct ufs_hba *hba = keyslot_manager_private(ksm); if (!ufshcd_is_crypto_enabled(hba) || !ufshcd_keyslot_valid(hba, slot)) return -EINVAL; - memset(&cfg_arr[slot], 0, sizeof(cfg_arr[slot])); - reg_base = hba->crypto_cfg_register + slot * sizeof(cfg_arr[0]); - /* * Clear the crypto cfg on the device. Clearing CFGE * might not be sufficient, so just clear the entire cfg. */ - for (i = 0; i < sizeof(cfg_arr[0]); i += sizeof(__le32)) - ufshcd_writel(hba, 0, reg_base + i); - wmb(); + ufshcd_clear_keyslot(hba, slot); return 0; } -static bool ufshcd_crypto_mode_supported(void *hba_p, - enum blk_crypto_mode_num crypto_mode, - unsigned int data_unit_size) -{ - return ufshcd_crypto_cap_find(hba_p, crypto_mode, data_unit_size) >= 0; -} - /* Functions implementing UFSHCI v2.1 specification behaviour */ void ufshcd_crypto_enable_spec(struct ufs_hba *hba) { - union ufs_crypto_cfg_entry *cfg_arr = hba->crypto_cfgs; - int slot; - if (!ufshcd_hba_is_crypto_supported(hba)) return; hba->caps |= UFSHCD_CAP_CRYPTO; - /* - * Reset might clear all keys, so reprogram all the keys. - * Also serves to clear keys on driver init. - */ - for (slot = 0; slot < NUM_KEYSLOTS(hba); slot++) - program_key(hba, &cfg_arr[slot], slot); + + /* Reset might clear all keys, so reprogram all the keys. */ + keyslot_manager_reprogram_all_keys(hba->ksm); } -EXPORT_SYMBOL(ufshcd_crypto_enable_spec); +EXPORT_SYMBOL_GPL(ufshcd_crypto_enable_spec); void ufshcd_crypto_disable_spec(struct ufs_hba *hba) { hba->caps &= ~UFSHCD_CAP_CRYPTO; } -EXPORT_SYMBOL(ufshcd_crypto_disable_spec); +EXPORT_SYMBOL_GPL(ufshcd_crypto_disable_spec); static const struct keyslot_mgmt_ll_ops ufshcd_ksm_ops = { .keyslot_program = ufshcd_crypto_keyslot_program, .keyslot_evict = ufshcd_crypto_keyslot_evict, - .keyslot_find = ufshcd_crypto_keyslot_find, - .crypto_mode_supported = ufshcd_crypto_mode_supported, }; +enum blk_crypto_mode_num ufshcd_blk_crypto_mode_num_for_alg_dusize( + enum ufs_crypto_alg ufs_crypto_alg, + enum ufs_crypto_key_size key_size) +{ + /* + * This is currently the only mode that UFS and blk-crypto both support. + */ + if (ufs_crypto_alg == UFS_CRYPTO_ALG_AES_XTS && + key_size == UFS_CRYPTO_KEY_SIZE_256) + return BLK_ENCRYPTION_MODE_AES_256_XTS; + + return BLK_ENCRYPTION_MODE_INVALID; +} + /** * ufshcd_hba_init_crypto - Read crypto capabilities, init crypto fields in hba * @hba: Per adapter instance @@ -308,6 +272,8 @@ int ufshcd_hba_init_crypto_spec(struct ufs_hba *hba, { int cap_idx = 0; int err = 0; + unsigned int crypto_modes_supported[BLK_ENCRYPTION_MODE_MAX]; + enum blk_crypto_mode_num blk_mode_num; /* Default to disabling crypto */ hba->caps &= ~UFSHCD_CAP_CRYPTO; @@ -336,16 +302,7 @@ int ufshcd_hba_init_crypto_spec(struct ufs_hba *hba, goto out; } - hba->crypto_cfgs = - devm_kcalloc(hba->dev, - NUM_KEYSLOTS(hba), - sizeof(hba->crypto_cfgs[0]), - GFP_KERNEL); - if (!hba->crypto_cfgs) { - err = -ENOMEM; - goto out_free_cfg_mem; - } - + memset(crypto_modes_supported, 0, sizeof(crypto_modes_supported)); /* * Store all the capabilities now so that we don't need to repeatedly * access the device each time we want to know its capabilities @@ -356,26 +313,35 @@ int ufshcd_hba_init_crypto_spec(struct ufs_hba *hba, cpu_to_le32(ufshcd_readl(hba, REG_UFS_CRYPTOCAP + cap_idx * sizeof(__le32))); + blk_mode_num = ufshcd_blk_crypto_mode_num_for_alg_dusize( + hba->crypto_cap_array[cap_idx].algorithm_id, + hba->crypto_cap_array[cap_idx].key_size); + if (blk_mode_num == BLK_ENCRYPTION_MODE_INVALID) + continue; + crypto_modes_supported[blk_mode_num] |= + hba->crypto_cap_array[cap_idx].sdus_mask * 512; } - hba->ksm = keyslot_manager_create(NUM_KEYSLOTS(hba), ksm_ops, hba); + ufshcd_clear_all_keyslots(hba); + + hba->ksm = keyslot_manager_create(ufshcd_num_keyslots(hba), ksm_ops, + crypto_modes_supported, hba); if (!hba->ksm) { err = -ENOMEM; - goto out_free_crypto_cfgs; + goto out_free_caps; } return 0; -out_free_crypto_cfgs: - devm_kfree(hba->dev, hba->crypto_cfgs); -out_free_cfg_mem: + +out_free_caps: devm_kfree(hba->dev, hba->crypto_cap_array); out: /* Indicate that init failed by setting crypto_capabilities to 0 */ hba->crypto_capabilities.reg_val = 0; return err; } -EXPORT_SYMBOL(ufshcd_hba_init_crypto_spec); +EXPORT_SYMBOL_GPL(ufshcd_hba_init_crypto_spec); void ufshcd_crypto_setup_rq_keyslot_manager_spec(struct ufs_hba *hba, struct request_queue *q) @@ -385,26 +351,26 @@ void ufshcd_crypto_setup_rq_keyslot_manager_spec(struct ufs_hba *hba, q->ksm = hba->ksm; } -EXPORT_SYMBOL(ufshcd_crypto_setup_rq_keyslot_manager_spec); +EXPORT_SYMBOL_GPL(ufshcd_crypto_setup_rq_keyslot_manager_spec); void ufshcd_crypto_destroy_rq_keyslot_manager_spec(struct ufs_hba *hba, struct request_queue *q) { keyslot_manager_destroy(hba->ksm); } -EXPORT_SYMBOL(ufshcd_crypto_destroy_rq_keyslot_manager_spec); +EXPORT_SYMBOL_GPL(ufshcd_crypto_destroy_rq_keyslot_manager_spec); int ufshcd_prepare_lrbp_crypto_spec(struct ufs_hba *hba, struct scsi_cmnd *cmd, struct ufshcd_lrb *lrbp) { - int key_slot; + struct bio_crypt_ctx *bc; - if (!cmd->request->bio || - !bio_crypt_should_process(cmd->request->bio, cmd->request->q)) { + if (!bio_crypt_should_process(cmd->request)) { lrbp->crypto_enable = false; return 0; } + bc = cmd->request->bio->bi_crypt_context; if (WARN_ON(!ufshcd_is_crypto_enabled(hba))) { /* @@ -413,17 +379,16 @@ int ufshcd_prepare_lrbp_crypto_spec(struct ufs_hba *hba, */ return -EINVAL; } - key_slot = bio_crypt_get_keyslot(cmd->request->bio); - if (!ufshcd_keyslot_valid(hba, key_slot)) + if (!ufshcd_keyslot_valid(hba, bc->bc_keyslot)) return -EINVAL; lrbp->crypto_enable = true; - lrbp->crypto_key_slot = key_slot; - lrbp->data_unit_num = bio_crypt_data_unit_num(cmd->request->bio); + lrbp->crypto_key_slot = bc->bc_keyslot; + lrbp->data_unit_num = bc->bc_dun[0]; return 0; } -EXPORT_SYMBOL(ufshcd_prepare_lrbp_crypto_spec); +EXPORT_SYMBOL_GPL(ufshcd_prepare_lrbp_crypto_spec); /* Crypto Variant Ops Support */ diff --git a/drivers/scsi/ufs/ufshcd-crypto.h b/drivers/scsi/ufs/ufshcd-crypto.h index 3c03d0e23e87..24a587c7a94e 100644 --- a/drivers/scsi/ufs/ufshcd-crypto.h +++ b/drivers/scsi/ufs/ufshcd-crypto.h @@ -11,7 +11,10 @@ #include "ufshcd.h" #include "ufshci.h" -#define NUM_KEYSLOTS(hba) (hba->crypto_capabilities.config_count + 1) +static inline int ufshcd_num_keyslots(struct ufs_hba *hba) +{ + return hba->crypto_capabilities.config_count + 1; +} static inline bool ufshcd_keyslot_valid(struct ufs_hba *hba, unsigned int slot) { @@ -19,7 +22,7 @@ static inline bool ufshcd_keyslot_valid(struct ufs_hba *hba, unsigned int slot) * The actual number of configurations supported is (CFGC+1), so slot * numbers range from 0 to config_count inclusive. */ - return slot < NUM_KEYSLOTS(hba); + return slot < ufshcd_num_keyslots(hba); } static inline bool ufshcd_hba_is_crypto_supported(struct ufs_hba *hba) @@ -51,6 +54,11 @@ void ufshcd_crypto_setup_rq_keyslot_manager_spec(struct ufs_hba *hba, void ufshcd_crypto_destroy_rq_keyslot_manager_spec(struct ufs_hba *hba, struct request_queue *q); +static inline bool ufshcd_lrbp_crypto_enabled(struct ufshcd_lrb *lrbp) +{ + return lrbp->crypto_enable; +} + /* Crypto Variant Ops Support */ void ufshcd_crypto_enable(struct ufs_hba *hba); @@ -118,10 +126,14 @@ static inline int ufshcd_prepare_lrbp_crypto(struct ufs_hba *hba, struct scsi_cmnd *cmd, struct ufshcd_lrb *lrbp) { - lrbp->crypto_enable = false; return 0; } +static inline bool ufshcd_lrbp_crypto_enabled(struct ufshcd_lrb *lrbp) +{ + return false; +} + static inline int ufshcd_complete_lrbp_crypto(struct ufs_hba *hba, struct scsi_cmnd *cmd, struct ufshcd_lrb *lrbp) diff --git a/drivers/scsi/ufs/ufshcd.c b/drivers/scsi/ufs/ufshcd.c index 9216198da200..308be8685e86 100644 --- a/drivers/scsi/ufs/ufshcd.c +++ b/drivers/scsi/ufs/ufshcd.c @@ -2246,13 +2246,15 @@ static void ufshcd_prepare_req_desc_hdr(struct ufshcd_lrb *lrbp, dword_0 |= UTP_REQ_DESC_INT_CMD; /* Transfer request descriptor header fields */ - if (lrbp->crypto_enable) { + if (ufshcd_lrbp_crypto_enabled(lrbp)) { +#if IS_ENABLED(CONFIG_SCSI_UFS_CRYPTO) dword_0 |= UTP_REQ_DESC_CRYPTO_ENABLE_CMD; dword_0 |= lrbp->crypto_key_slot; req_desc->header.dword_1 = - cpu_to_le32((u32)lrbp->data_unit_num); + cpu_to_le32(lower_32_bits(lrbp->data_unit_num)); req_desc->header.dword_3 = - cpu_to_le32((u32)(lrbp->data_unit_num >> 32)); + cpu_to_le32(upper_32_bits(lrbp->data_unit_num)); +#endif /* CONFIG_SCSI_UFS_CRYPTO */ } else { /* dword_1 and dword_3 are reserved, hence they are set to 0 */ req_desc->header.dword_1 = 0; @@ -2555,7 +2557,9 @@ static int ufshcd_compose_dev_cmd(struct ufs_hba *hba, lrbp->task_tag = tag; lrbp->lun = 0; /* device management cmd is not specific to any LUN */ lrbp->intr_cmd = true; /* No interrupt aggregation */ +#if IS_ENABLED(CONFIG_SCSI_UFS_CRYPTO) lrbp->crypto_enable = false; /* No crypto operations */ +#endif hba->dev_cmd.type = cmd_type; return ufshcd_comp_devman_upiu(hba, lrbp); diff --git a/drivers/scsi/ufs/ufshcd.h b/drivers/scsi/ufs/ufshcd.h index 6fe75302df59..9f1452299803 100644 --- a/drivers/scsi/ufs/ufshcd.h +++ b/drivers/scsi/ufs/ufshcd.h @@ -194,9 +194,11 @@ struct ufshcd_lrb { bool intr_cmd; ktime_t issue_time_stamp; ktime_t compl_time_stamp; +#if IS_ENABLED(CONFIG_SCSI_UFS_CRYPTO) bool crypto_enable; u8 crypto_key_slot; u64 data_unit_num; +#endif /* CONFIG_SCSI_UFS_CRYPTO */ bool req_abort_skip; }; @@ -546,7 +548,6 @@ struct ufs_stats { * @crypto_capabilities: Content of crypto capabilities register (0x100) * @crypto_cap_array: Array of crypto capabilities * @crypto_cfg_register: Start of the crypto cfg array - * @crypto_cfgs: Array of crypto configurations (i.e. config for each slot) * @ksm: the keyslot manager tied to this hba */ struct ufs_hba { @@ -777,7 +778,6 @@ struct ufs_hba { union ufs_crypto_capabilities crypto_capabilities; union ufs_crypto_cap_entry *crypto_cap_array; u32 crypto_cfg_register; - union ufs_crypto_cfg_entry *crypto_cfgs; struct keyslot_manager *ksm; #endif /* CONFIG_SCSI_UFS_CRYPTO */ }; diff --git a/fs/buffer.c b/fs/buffer.c index 2e64fa95a1ac..32141c922fed 100644 --- a/fs/buffer.c +++ b/fs/buffer.c @@ -3046,7 +3046,7 @@ static int submit_bh_wbc(int op, int op_flags, struct buffer_head *bh, */ bio = bio_alloc(GFP_NOIO, 1); - fscrypt_set_bio_crypt_ctx_bh(bio, bh, GFP_NOIO | __GFP_NOFAIL); + fscrypt_set_bio_crypt_ctx_bh(bio, bh, GFP_NOIO); bio->bi_iter.bi_sector = bh->b_blocknr * (bh->b_size >> 9); bio_set_dev(bio, bh->b_bdev); diff --git a/fs/crypto/bio.c b/fs/crypto/bio.c index 956798debf71..d28d8e803554 100644 --- a/fs/crypto/bio.c +++ b/fs/crypto/bio.c @@ -73,11 +73,8 @@ int fscrypt_zeroout_range(const struct inode *inode, pgoff_t lblk, err = -ENOMEM; goto errout; } - err = fscrypt_set_bio_crypt_ctx(bio, inode, lblk, GFP_NOIO); - if (err) { - bio_put(bio); - goto errout; - } + fscrypt_set_bio_crypt_ctx(bio, inode, lblk, GFP_NOIO); + bio_set_dev(bio, inode->i_sb->s_bdev); bio->bi_iter.bi_sector = pblk << (blockbits - 9); bio_set_op_attrs(bio, REQ_OP_WRITE, 0); diff --git a/fs/crypto/crypto.c b/fs/crypto/crypto.c index 3719efa546c6..7549d76a0aba 100644 --- a/fs/crypto/crypto.c +++ b/fs/crypto/crypto.c @@ -97,7 +97,7 @@ int fscrypt_crypt_block(const struct inode *inode, fscrypt_direction_t rw, DECLARE_CRYPTO_WAIT(wait); struct scatterlist dst, src; struct fscrypt_info *ci = inode->i_crypt_info; - struct crypto_skcipher *tfm = ci->ci_ctfm; + struct crypto_skcipher *tfm = ci->ci_key.tfm; int res = 0; if (WARN_ON_ONCE(len <= 0)) diff --git a/fs/crypto/fname.c b/fs/crypto/fname.c index 3da3707c10e3..3aafddaab703 100644 --- a/fs/crypto/fname.c +++ b/fs/crypto/fname.c @@ -40,7 +40,7 @@ int fname_encrypt(struct inode *inode, const struct qstr *iname, struct skcipher_request *req = NULL; DECLARE_CRYPTO_WAIT(wait); struct fscrypt_info *ci = inode->i_crypt_info; - struct crypto_skcipher *tfm = ci->ci_ctfm; + struct crypto_skcipher *tfm = ci->ci_key.tfm; union fscrypt_iv iv; struct scatterlist sg; int res; @@ -93,7 +93,7 @@ static int fname_decrypt(struct inode *inode, DECLARE_CRYPTO_WAIT(wait); struct scatterlist src_sg, dst_sg; struct fscrypt_info *ci = inode->i_crypt_info; - struct crypto_skcipher *tfm = ci->ci_ctfm; + struct crypto_skcipher *tfm = ci->ci_key.tfm; union fscrypt_iv iv; int res; diff --git a/fs/crypto/fscrypt_private.h b/fs/crypto/fscrypt_private.h index c731bd4245c5..19b7a8ea5954 100644 --- a/fs/crypto/fscrypt_private.h +++ b/fs/crypto/fscrypt_private.h @@ -15,8 +15,6 @@ #include #include -struct fscrypt_master_key; - #define CONST_STRLEN(str) (sizeof(str) - 1) #define FS_KEY_DERIVATION_NONCE_SIZE 16 @@ -154,6 +152,20 @@ struct fscrypt_symlink_data { char encrypted_path[1]; } __packed; +/** + * struct fscrypt_prepared_key - a key prepared for actual encryption/decryption + * @tfm: crypto API transform object + * @blk_key: key for blk-crypto + * + * Normally only one of the fields will be non-NULL. + */ +struct fscrypt_prepared_key { + struct crypto_skcipher *tfm; +#ifdef CONFIG_FS_ENCRYPTION_INLINE_CRYPT + struct fscrypt_blk_crypto_key *blk_key; +#endif +}; + /* * fscrypt_info - the "encryption key" for an inode * @@ -163,20 +175,20 @@ struct fscrypt_symlink_data { */ struct fscrypt_info { - /* The actual crypto transform used for encryption and decryption */ - struct crypto_skcipher *ci_ctfm; - -#ifdef CONFIG_FS_ENCRYPTION_INLINE_CRYPT - /* - * The raw key for inline encryption, if this file is using inline - * encryption rather than the traditional filesystem layer encryption. - */ - const u8 *ci_inline_crypt_key; -#endif + /* The key in a form prepared for actual encryption/decryption */ + struct fscrypt_prepared_key ci_key; /* True if the key should be freed when this fscrypt_info is freed */ bool ci_owns_key; +#ifdef CONFIG_FS_ENCRYPTION_INLINE_CRYPT + /* + * True if this inode will use inline encryption (blk-crypto) instead of + * the traditional filesystem-layer encryption. + */ + bool ci_inlinecrypt; +#endif + /* * Encryption mode used for this inode. It corresponds to either the * contents or filenames encryption mode, depending on the inode type. @@ -201,7 +213,7 @@ struct fscrypt_info { /* * If non-NULL, then encryption is done using the master key directly - * and ci_ctfm will equal ci_direct_key->dk_ctfm. + * and ci_key will equal ci_direct_key->dk_key. */ struct fscrypt_direct_key *ci_direct_key; @@ -265,6 +277,7 @@ union fscrypt_iv { u8 nonce[FS_KEY_DERIVATION_NONCE_SIZE]; }; u8 raw[FSCRYPT_MAX_IV_SIZE]; + __le64 dun[FSCRYPT_MAX_IV_SIZE / sizeof(__le64)]; }; void fscrypt_generate_iv(union fscrypt_iv *iv, u64 lblk_num, @@ -306,49 +319,71 @@ extern void fscrypt_destroy_hkdf(struct fscrypt_hkdf *hkdf); /* inline_crypt.c */ #ifdef CONFIG_FS_ENCRYPTION_INLINE_CRYPT -extern bool fscrypt_should_use_inline_encryption(const struct fscrypt_info *ci); +extern void fscrypt_select_encryption_impl(struct fscrypt_info *ci); -extern int fscrypt_set_inline_crypt_key(struct fscrypt_info *ci, - const u8 *derived_key); +static inline bool +fscrypt_using_inline_encryption(const struct fscrypt_info *ci) +{ + return ci->ci_inlinecrypt; +} -extern void fscrypt_free_inline_crypt_key(struct fscrypt_info *ci); +extern int fscrypt_prepare_inline_crypt_key( + struct fscrypt_prepared_key *prep_key, + const u8 *raw_key, + const struct fscrypt_info *ci); -extern int fscrypt_setup_per_mode_inline_crypt_key( - struct fscrypt_info *ci, - struct fscrypt_master_key *mk); +extern void fscrypt_destroy_inline_crypt_key( + struct fscrypt_prepared_key *prep_key); -extern void fscrypt_evict_inline_crypt_keys(struct fscrypt_master_key *mk); +/* + * Check whether the crypto transform or blk-crypto key has been allocated in + * @prep_key, depending on which encryption implementation the file will use. + */ +static inline bool +fscrypt_is_key_prepared(struct fscrypt_prepared_key *prep_key, + const struct fscrypt_info *ci) +{ + /* + * The READ_ONCE() here pairs with the smp_store_release() in + * fscrypt_prepare_key(). (This only matters for the per-mode keys, + * which are shared by multiple inodes.) + */ + if (fscrypt_using_inline_encryption(ci)) + return READ_ONCE(prep_key->blk_key) != NULL; + return READ_ONCE(prep_key->tfm) != NULL; +} #else /* CONFIG_FS_ENCRYPTION_INLINE_CRYPT */ -static inline bool fscrypt_should_use_inline_encryption( +static inline void fscrypt_select_encryption_impl(struct fscrypt_info *ci) +{ +} + +static inline bool fscrypt_using_inline_encryption( const struct fscrypt_info *ci) { return false; } -static inline int fscrypt_set_inline_crypt_key(struct fscrypt_info *ci, - const u8 *derived_key) +static inline int +fscrypt_prepare_inline_crypt_key(struct fscrypt_prepared_key *prep_key, + const u8 *raw_key, + const struct fscrypt_info *ci) { WARN_ON(1); return -EOPNOTSUPP; } -static inline void fscrypt_free_inline_crypt_key(struct fscrypt_info *ci) +static inline void +fscrypt_destroy_inline_crypt_key(struct fscrypt_prepared_key *prep_key) { } -static inline int fscrypt_setup_per_mode_inline_crypt_key( - struct fscrypt_info *ci, - struct fscrypt_master_key *mk) -{ - WARN_ON(1); - return -EOPNOTSUPP; -} - -static inline void fscrypt_evict_inline_crypt_keys( - struct fscrypt_master_key *mk) +static inline bool +fscrypt_is_key_prepared(struct fscrypt_prepared_key *prep_key, + const struct fscrypt_info *ci) { + return READ_ONCE(prep_key->tfm) != NULL; } #endif /* !CONFIG_FS_ENCRYPTION_INLINE_CRYPT */ @@ -441,25 +476,12 @@ struct fscrypt_master_key { struct list_head mk_decrypted_inodes; spinlock_t mk_decrypted_inodes_lock; - /* Crypto API transforms for DIRECT_KEY policies, allocated on-demand */ - struct crypto_skcipher *mk_direct_tfms[__FSCRYPT_MODE_MAX + 1]; + /* Per-mode keys for DIRECT_KEY policies, allocated on-demand */ + struct fscrypt_prepared_key mk_direct_keys[__FSCRYPT_MODE_MAX + 1]; - /* - * Crypto API transforms for filesystem-layer implementation of - * IV_INO_LBLK_64 policies, allocated on-demand. - */ - struct crypto_skcipher *mk_iv_ino_lblk_64_tfms[__FSCRYPT_MODE_MAX + 1]; + /* Per-mode keys for IV_INO_LBLK_64 policies, allocated on-demand */ + struct fscrypt_prepared_key mk_iv_ino_lblk_64_keys[__FSCRYPT_MODE_MAX + 1]; -#ifdef CONFIG_FS_ENCRYPTION_INLINE_CRYPT - /* Raw keys for IV_INO_LBLK_64 policies, allocated on-demand */ - u8 *mk_iv_ino_lblk_64_raw_keys[__FSCRYPT_MODE_MAX + 1]; - - /* The data unit size being used for inline encryption */ - unsigned int mk_data_unit_size; - - /* The filesystem's block device */ - struct block_device *mk_bdev; -#endif } __randomize_layout; static inline bool @@ -526,9 +548,11 @@ fscrypt_mode_supports_direct_key(const struct fscrypt_mode *mode) return mode->ivsize >= offsetofend(union fscrypt_iv, nonce); } -extern struct crypto_skcipher * -fscrypt_allocate_skcipher(struct fscrypt_mode *mode, const u8 *raw_key, - const struct inode *inode); +extern int fscrypt_prepare_key(struct fscrypt_prepared_key *prep_key, + const u8 *raw_key, + const struct fscrypt_info *ci); + +extern void fscrypt_destroy_prepared_key(struct fscrypt_prepared_key *prep_key); extern int fscrypt_set_derived_key(struct fscrypt_info *ci, const u8 *derived_key); diff --git a/fs/crypto/inline_crypt.c b/fs/crypto/inline_crypt.c index e41c6d66ff0d..aa04e8a7d674 100644 --- a/fs/crypto/inline_crypt.c +++ b/fs/crypto/inline_crypt.c @@ -15,187 +15,126 @@ #include #include #include -#include #include "fscrypt_private.h" -/* Return true iff inline encryption should be used for this file */ -bool fscrypt_should_use_inline_encryption(const struct fscrypt_info *ci) +struct fscrypt_blk_crypto_key { + struct blk_crypto_key base; + int num_devs; + struct request_queue *devs[]; +}; + +/* Enable inline encryption for this file if supported. */ +void fscrypt_select_encryption_impl(struct fscrypt_info *ci) { const struct inode *inode = ci->ci_inode; struct super_block *sb = inode->i_sb; /* The file must need contents encryption, not filenames encryption */ if (!S_ISREG(inode->i_mode)) - return false; + return; /* blk-crypto must implement the needed encryption algorithm */ if (ci->ci_mode->blk_crypto_mode == BLK_ENCRYPTION_MODE_INVALID) - return false; - - /* DIRECT_KEY needs a 24+ byte IV, so it can't work with 8-byte DUNs */ - if (fscrypt_is_direct_key_policy(&ci->ci_policy)) - return false; + return; /* The filesystem must be mounted with -o inlinecrypt */ if (!sb->s_cop->inline_crypt_enabled || !sb->s_cop->inline_crypt_enabled(sb)) - return false; + return; - return true; + ci->ci_inlinecrypt = true; } -/* Set a per-file inline encryption key (for passing to blk-crypto) */ -int fscrypt_set_inline_crypt_key(struct fscrypt_info *ci, const u8 *derived_key) +int fscrypt_prepare_inline_crypt_key(struct fscrypt_prepared_key *prep_key, + const u8 *raw_key, + const struct fscrypt_info *ci) { - const struct fscrypt_mode *mode = ci->ci_mode; - const struct super_block *sb = ci->ci_inode->i_sb; - - ci->ci_inline_crypt_key = kmemdup(derived_key, mode->keysize, GFP_NOFS); - if (!ci->ci_inline_crypt_key) - return -ENOMEM; - ci->ci_owns_key = true; - - return blk_crypto_start_using_mode(mode->blk_crypto_mode, - sb->s_blocksize, - sb->s_bdev->bd_queue); -} - -/* Free a per-file inline encryption key and evict it from blk-crypto */ -void fscrypt_free_inline_crypt_key(struct fscrypt_info *ci) -{ - if (ci->ci_inline_crypt_key != NULL) { - const struct fscrypt_mode *mode = ci->ci_mode; - const struct super_block *sb = ci->ci_inode->i_sb; - - blk_crypto_evict_key(sb->s_bdev->bd_queue, - ci->ci_inline_crypt_key, - mode->blk_crypto_mode, sb->s_blocksize); - kzfree(ci->ci_inline_crypt_key); - } -} - -/* - * Set up ->inline_crypt_key (for passing to blk-crypto) for inodes which use an - * IV_INO_LBLK_64 encryption policy. - * - * Return: 0 on success, -errno on failure - */ -int fscrypt_setup_per_mode_inline_crypt_key(struct fscrypt_info *ci, - struct fscrypt_master_key *mk) -{ - static DEFINE_MUTEX(inline_crypt_setup_mutex); - const struct super_block *sb = ci->ci_inode->i_sb; - struct block_device *bdev = sb->s_bdev; - const struct fscrypt_mode *mode = ci->ci_mode; - const u8 mode_num = mode - fscrypt_modes; - u8 *raw_key; - u8 hkdf_info[sizeof(mode_num) + sizeof(sb->s_uuid)]; + const struct inode *inode = ci->ci_inode; + struct super_block *sb = inode->i_sb; + enum blk_crypto_mode_num crypto_mode = ci->ci_mode->blk_crypto_mode; + int num_devs = 1; + int queue_refs = 0; + struct fscrypt_blk_crypto_key *blk_key; int err; + int i; - if (WARN_ON(mode_num > __FSCRYPT_MODE_MAX)) + if (sb->s_cop->get_num_devices) + num_devs = sb->s_cop->get_num_devices(sb); + if (WARN_ON(num_devs < 1)) return -EINVAL; - /* pairs with smp_store_release() below */ - raw_key = smp_load_acquire(&mk->mk_iv_ino_lblk_64_raw_keys[mode_num]); - if (raw_key) { - err = 0; - goto out; + blk_key = kzalloc(struct_size(blk_key, devs, num_devs), GFP_NOFS); + if (!blk_key) + return -ENOMEM; + + blk_key->num_devs = num_devs; + if (num_devs == 1) + blk_key->devs[0] = bdev_get_queue(sb->s_bdev); + else + sb->s_cop->get_devices(sb, blk_key->devs); + + err = blk_crypto_init_key(&blk_key->base, raw_key, crypto_mode, + sb->s_blocksize); + if (err) { + fscrypt_err(inode, "error %d initializing blk-crypto key", err); + goto fail; } - mutex_lock(&inline_crypt_setup_mutex); - - raw_key = mk->mk_iv_ino_lblk_64_raw_keys[mode_num]; - if (raw_key) { - err = 0; - goto out_unlock; - } - - raw_key = kmalloc(mode->keysize, GFP_NOFS); - if (!raw_key) { - err = -ENOMEM; - goto out_unlock; - } - - BUILD_BUG_ON(sizeof(mode_num) != 1); - BUILD_BUG_ON(sizeof(sb->s_uuid) != 16); - BUILD_BUG_ON(sizeof(hkdf_info) != 17); - hkdf_info[0] = mode_num; - memcpy(&hkdf_info[1], &sb->s_uuid, sizeof(sb->s_uuid)); - - err = fscrypt_hkdf_expand(&mk->mk_secret.hkdf, - HKDF_CONTEXT_IV_INO_LBLK_64_KEY, - hkdf_info, sizeof(hkdf_info), - raw_key, mode->keysize); - if (err) - goto out_unlock; - - err = blk_crypto_start_using_mode(mode->blk_crypto_mode, - sb->s_blocksize, bdev->bd_queue); - if (err) - goto out_unlock; - /* - * When a master key's first inline encryption key is set up, save a - * reference to the filesystem's block device so that the inline - * encryption keys can be evicted when the master key is destroyed. + * We have to start using blk-crypto on all the filesystem's devices. + * We also have to save all the request_queue's for later so that the + * key can be evicted from them. This is needed because some keys + * aren't destroyed until after the filesystem was already unmounted + * (namely, the per-mode keys in struct fscrypt_master_key). */ - if (!mk->mk_bdev) { - mk->mk_bdev = bdgrab(bdev); - mk->mk_data_unit_size = sb->s_blocksize; - } + for (i = 0; i < num_devs; i++) { + if (!blk_get_queue(blk_key->devs[i])) { + fscrypt_err(inode, "couldn't get request_queue"); + err = -EAGAIN; + goto fail; + } + queue_refs++; - /* pairs with smp_load_acquire() above */ - smp_store_release(&mk->mk_iv_ino_lblk_64_raw_keys[mode_num], raw_key); - err = 0; -out_unlock: - mutex_unlock(&inline_crypt_setup_mutex); -out: - if (err == 0) { - ci->ci_inline_crypt_key = raw_key; - /* - * Since each struct fscrypt_master_key belongs to a particular - * filesystem (a struct super_block), there should be only one - * block device, and only one data unit size as it should equal - * the filesystem's blocksize (i.e. s_blocksize). - */ - if (WARN_ON(mk->mk_bdev != bdev)) - err = -EINVAL; - if (WARN_ON(mk->mk_data_unit_size != sb->s_blocksize)) - err = -EINVAL; - } else { - kzfree(raw_key); + err = blk_crypto_start_using_mode(crypto_mode, sb->s_blocksize, + blk_key->devs[i]); + if (err) { + fscrypt_err(inode, + "error %d starting to use blk-crypto", err); + goto fail; + } } + /* + * Pairs with READ_ONCE() in fscrypt_is_key_prepared(). (Only matters + * for the per-mode keys, which are shared by multiple inodes.) + */ + smp_store_release(&prep_key->blk_key, blk_key); + return 0; + +fail: + for (i = 0; i < queue_refs; i++) + blk_put_queue(blk_key->devs[i]); + kzfree(blk_key); return err; } -/* - * Evict per-mode inline encryption keys from blk-crypto when a master key is - * destroyed. - */ -void fscrypt_evict_inline_crypt_keys(struct fscrypt_master_key *mk) +void fscrypt_destroy_inline_crypt_key(struct fscrypt_prepared_key *prep_key) { - struct block_device *bdev = mk->mk_bdev; - size_t i; + struct fscrypt_blk_crypto_key *blk_key = prep_key->blk_key; + int i; - if (!bdev) /* No inline encryption keys? */ - return; - - for (i = 0; i < ARRAY_SIZE(mk->mk_iv_ino_lblk_64_raw_keys); i++) { - u8 *raw_key = mk->mk_iv_ino_lblk_64_raw_keys[i]; - - if (raw_key != NULL) { - blk_crypto_evict_key(bdev->bd_queue, raw_key, - fscrypt_modes[i].blk_crypto_mode, - mk->mk_data_unit_size); - kzfree(raw_key); + if (blk_key) { + for (i = 0; i < blk_key->num_devs; i++) { + blk_crypto_evict_key(blk_key->devs[i], &blk_key->base); + blk_put_queue(blk_key->devs[i]); } + kzfree(blk_key); } - bdput(bdev); } /** - * fscrypt_inode_uses_inline_crypto - test whether an inode uses inline encryption + * fscrypt_inode_uses_inline_crypto - test whether an inode uses inline + * encryption * @inode: an inode * * Return: true if the inode requires file contents encryption and if the @@ -205,12 +144,13 @@ void fscrypt_evict_inline_crypt_keys(struct fscrypt_master_key *mk) bool fscrypt_inode_uses_inline_crypto(const struct inode *inode) { return IS_ENCRYPTED(inode) && S_ISREG(inode->i_mode) && - inode->i_crypt_info->ci_inline_crypt_key != NULL; + inode->i_crypt_info->ci_inlinecrypt; } EXPORT_SYMBOL_GPL(fscrypt_inode_uses_inline_crypto); /** - * fscrypt_inode_uses_fs_layer_crypto - test whether an inode uses fs-layer encryption + * fscrypt_inode_uses_fs_layer_crypto - test whether an inode uses fs-layer + * encryption * @inode: an inode * * Return: true if the inode requires file contents encryption and if the @@ -220,22 +160,22 @@ EXPORT_SYMBOL_GPL(fscrypt_inode_uses_inline_crypto); bool fscrypt_inode_uses_fs_layer_crypto(const struct inode *inode) { return IS_ENCRYPTED(inode) && S_ISREG(inode->i_mode) && - inode->i_crypt_info->ci_inline_crypt_key == NULL; + !inode->i_crypt_info->ci_inlinecrypt; } EXPORT_SYMBOL_GPL(fscrypt_inode_uses_fs_layer_crypto); -static inline u64 fscrypt_generate_dun(const struct fscrypt_info *ci, - u64 lblk_num) +static void fscrypt_generate_dun(const struct fscrypt_info *ci, u64 lblk_num, + u64 dun[BLK_CRYPTO_DUN_ARRAY_SIZE]) { union fscrypt_iv iv; + int i; fscrypt_generate_iv(&iv, lblk_num, ci); - /* - * fscrypt_should_use_inline_encryption() ensures we never get here if - * more than the first 8 bytes of the IV are nonzero. - */ - BUG_ON(memchr_inv(&iv.raw[8], 0, ci->ci_mode->ivsize - 8)); - return le64_to_cpu(iv.lblk_num); + + BUILD_BUG_ON(FSCRYPT_MAX_IV_SIZE > BLK_CRYPTO_MAX_IV_SIZE); + memset(dun, 0, BLK_CRYPTO_MAX_IV_SIZE); + for (i = 0; i < ci->ci_mode->ivsize/sizeof(dun[0]); i++) + dun[i] = le64_to_cpu(iv.dun[i]); } /** @@ -243,7 +183,8 @@ static inline u64 fscrypt_generate_dun(const struct fscrypt_info *ci, * @bio: a bio which will eventually be submitted to the file * @inode: the file's inode * @first_lblk: the first file logical block number in the I/O - * @gfp_mask: memory allocation flags + * @gfp_mask: memory allocation flags - these must be a waiting mask so that + * bio_crypt_set_ctx can't fail. * * If the contents of the file should be encrypted (or decrypted) with inline * encryption, then assign the appropriate encryption context to the bio. @@ -252,24 +193,18 @@ static inline u64 fscrypt_generate_dun(const struct fscrypt_info *ci, * otherwise fscrypt_mergeable_bio() won't work as intended. * * The encryption context will be freed automatically when the bio is freed. - * - * Return: 0 on success, -errno on failure. If __GFP_NOFAIL is specified, this - * is guaranteed to succeed. */ -int fscrypt_set_bio_crypt_ctx(struct bio *bio, const struct inode *inode, - u64 first_lblk, gfp_t gfp_mask) +void fscrypt_set_bio_crypt_ctx(struct bio *bio, const struct inode *inode, + u64 first_lblk, gfp_t gfp_mask) { const struct fscrypt_info *ci = inode->i_crypt_info; - u64 dun; + u64 dun[BLK_CRYPTO_DUN_ARRAY_SIZE]; if (!fscrypt_inode_uses_inline_crypto(inode)) - return 0; + return; - dun = fscrypt_generate_dun(ci, first_lblk); - - return bio_crypt_set_ctx(bio, ci->ci_inline_crypt_key, - ci->ci_mode->blk_crypto_mode, - dun, inode->i_blkbits, gfp_mask); + fscrypt_generate_dun(ci, first_lblk, dun); + bio_crypt_set_ctx(bio, &ci->ci_key.blk_key->base, dun, gfp_mask); } EXPORT_SYMBOL_GPL(fscrypt_set_bio_crypt_ctx); @@ -298,27 +233,24 @@ static bool bh_get_inode_and_lblk_num(const struct buffer_head *bh, } /** - * fscrypt_set_bio_crypt_ctx_bh - prepare a file contents bio for inline encryption + * fscrypt_set_bio_crypt_ctx_bh - prepare a file contents bio for inline + * encryption * @bio: a bio which will eventually be submitted to the file * @first_bh: the first buffer_head for which I/O will be submitted * @gfp_mask: memory allocation flags * * Same as fscrypt_set_bio_crypt_ctx(), except this takes a buffer_head instead * of an inode and block number directly. - * - * Return: 0 on success, -errno on failure */ -int fscrypt_set_bio_crypt_ctx_bh(struct bio *bio, +void fscrypt_set_bio_crypt_ctx_bh(struct bio *bio, const struct buffer_head *first_bh, gfp_t gfp_mask) { const struct inode *inode; u64 first_lblk; - if (!bh_get_inode_and_lblk_num(first_bh, &inode, &first_lblk)) - return 0; - - return fscrypt_set_bio_crypt_ctx(bio, inode, first_lblk, gfp_mask); + if (bh_get_inode_and_lblk_num(first_bh, &inode, &first_lblk)) + fscrypt_set_bio_crypt_ctx(bio, inode, first_lblk, gfp_mask); } EXPORT_SYMBOL_GPL(fscrypt_set_bio_crypt_ctx_bh); @@ -342,27 +274,24 @@ EXPORT_SYMBOL_GPL(fscrypt_set_bio_crypt_ctx_bh); bool fscrypt_mergeable_bio(struct bio *bio, const struct inode *inode, u64 next_lblk) { - const struct bio_crypt_ctx *bc; - const u8 *next_key; - u64 next_dun; + const struct bio_crypt_ctx *bc = bio->bi_crypt_context; + u64 next_dun[BLK_CRYPTO_DUN_ARRAY_SIZE]; - if (bio_has_crypt_ctx(bio) != fscrypt_inode_uses_inline_crypto(inode)) + if (!!bc != fscrypt_inode_uses_inline_crypto(inode)) return false; - if (!bio_has_crypt_ctx(bio)) + if (!bc) return true; - bc = bio->bi_crypt_context; - next_key = inode->i_crypt_info->ci_inline_crypt_key; - next_dun = fscrypt_generate_dun(inode->i_crypt_info, next_lblk); /* * Comparing the key pointers is good enough, as all I/O for each key * uses the same pointer. I.e., there's currently no need to support * merging requests where the keys are the same but the pointers differ. */ - return next_key == bc->raw_key && - next_dun == bc->data_unit_num + - (bio_sectors(bio) >> - (bc->data_unit_size_bits - SECTOR_SHIFT)); + if (bc->bc_key != &inode->i_crypt_info->ci_key.blk_key->base) + return false; + + fscrypt_generate_dun(inode->i_crypt_info, next_lblk, next_dun); + return bio_crypt_dun_is_contiguous(bc, bio->bi_iter.bi_size, next_dun); } EXPORT_SYMBOL_GPL(fscrypt_mergeable_bio); @@ -383,7 +312,7 @@ bool fscrypt_mergeable_bio_bh(struct bio *bio, u64 next_lblk; if (!bh_get_inode_and_lblk_num(next_bh, &inode, &next_lblk)) - return !bio_has_crypt_ctx(bio); + return !bio->bi_crypt_context; return fscrypt_mergeable_bio(bio, inode, next_lblk); } diff --git a/fs/crypto/keyring.c b/fs/crypto/keyring.c index 05cc274e5c13..e6b0f0da1eaa 100644 --- a/fs/crypto/keyring.c +++ b/fs/crypto/keyring.c @@ -44,12 +44,10 @@ static void free_master_key(struct fscrypt_master_key *mk) wipe_master_key_secret(&mk->mk_secret); for (i = 0; i <= __FSCRYPT_MODE_MAX; i++) { - crypto_free_skcipher(mk->mk_direct_tfms[i]); - crypto_free_skcipher(mk->mk_iv_ino_lblk_64_tfms[i]); + fscrypt_destroy_prepared_key(&mk->mk_direct_keys[i]); + fscrypt_destroy_prepared_key(&mk->mk_iv_ino_lblk_64_keys[i]); } - fscrypt_evict_inline_crypt_keys(mk); - key_put(mk->mk_users); kzfree(mk); } diff --git a/fs/crypto/keysetup.c b/fs/crypto/keysetup.c index 8070dad9a541..f7e909d690b2 100644 --- a/fs/crypto/keysetup.c +++ b/fs/crypto/keysetup.c @@ -32,6 +32,7 @@ struct fscrypt_mode fscrypt_modes[] = { .cipher_str = "essiv(cbc(aes),sha256)", .keysize = 16, .ivsize = 16, + .blk_crypto_mode = BLK_ENCRYPTION_MODE_AES_128_CBC_ESSIV, }, [FSCRYPT_MODE_AES_128_CTS] = { .friendly_name = "AES-128-CTS-CBC", @@ -44,6 +45,7 @@ struct fscrypt_mode fscrypt_modes[] = { .cipher_str = "adiantum(xchacha12,aes)", .keysize = 32, .ivsize = 32, + .blk_crypto_mode = BLK_ENCRYPTION_MODE_ADIANTUM, }, }; @@ -63,9 +65,9 @@ select_encryption_mode(const union fscrypt_policy *policy, } /* Create a symmetric cipher object for the given encryption mode and key */ -struct crypto_skcipher *fscrypt_allocate_skcipher(struct fscrypt_mode *mode, - const u8 *raw_key, - const struct inode *inode) +static struct crypto_skcipher * +fscrypt_allocate_skcipher(struct fscrypt_mode *mode, const u8 *raw_key, + const struct inode *inode) { struct crypto_skcipher *tfm; int err; @@ -107,33 +109,55 @@ err_free_tfm: return ERR_PTR(err); } -/* Given the per-file key, set up the file's crypto transform object */ -int fscrypt_set_derived_key(struct fscrypt_info *ci, const u8 *derived_key) +/* + * Prepare the crypto transform object or blk-crypto key in @prep_key, given the + * raw key, encryption mode, and flag indicating which encryption implementation + * (fs-layer or blk-crypto) will be used. + */ +int fscrypt_prepare_key(struct fscrypt_prepared_key *prep_key, + const u8 *raw_key, const struct fscrypt_info *ci) { struct crypto_skcipher *tfm; - if (fscrypt_should_use_inline_encryption(ci)) - return fscrypt_set_inline_crypt_key(ci, derived_key); + if (fscrypt_using_inline_encryption(ci)) + return fscrypt_prepare_inline_crypt_key(prep_key, raw_key, ci); - tfm = fscrypt_allocate_skcipher(ci->ci_mode, derived_key, ci->ci_inode); + tfm = fscrypt_allocate_skcipher(ci->ci_mode, raw_key, ci->ci_inode); if (IS_ERR(tfm)) return PTR_ERR(tfm); - - ci->ci_ctfm = tfm; - ci->ci_owns_key = true; + /* + * Pairs with READ_ONCE() in fscrypt_is_key_prepared(). (Only matters + * for the per-mode keys, which are shared by multiple inodes.) + */ + smp_store_release(&prep_key->tfm, tfm); return 0; } +/* Destroy a crypto transform object and/or blk-crypto key. */ +void fscrypt_destroy_prepared_key(struct fscrypt_prepared_key *prep_key) +{ + crypto_free_skcipher(prep_key->tfm); + fscrypt_destroy_inline_crypt_key(prep_key); +} + +/* Given the per-file key, set up the file's crypto transform object */ +int fscrypt_set_derived_key(struct fscrypt_info *ci, const u8 *derived_key) +{ + ci->ci_owns_key = true; + return fscrypt_prepare_key(&ci->ci_key, derived_key, ci); +} + static int setup_per_mode_key(struct fscrypt_info *ci, struct fscrypt_master_key *mk, - struct crypto_skcipher **tfms, + struct fscrypt_prepared_key *keys, u8 hkdf_context, bool include_fs_uuid) { + static DEFINE_MUTEX(mode_key_setup_mutex); const struct inode *inode = ci->ci_inode; const struct super_block *sb = inode->i_sb; struct fscrypt_mode *mode = ci->ci_mode; const u8 mode_num = mode - fscrypt_modes; - struct crypto_skcipher *tfm, *prev_tfm; + struct fscrypt_prepared_key *prep_key; u8 mode_key[FSCRYPT_MAX_KEY_SIZE]; u8 hkdf_info[sizeof(mode_num) + sizeof(sb->s_uuid)]; unsigned int hkdf_infolen = 0; @@ -142,10 +166,16 @@ static int setup_per_mode_key(struct fscrypt_info *ci, if (WARN_ON(mode_num > __FSCRYPT_MODE_MAX)) return -EINVAL; - /* pairs with cmpxchg() below */ - tfm = READ_ONCE(tfms[mode_num]); - if (likely(tfm != NULL)) - goto done; + prep_key = &keys[mode_num]; + if (fscrypt_is_key_prepared(prep_key, ci)) { + ci->ci_key = *prep_key; + return 0; + } + + mutex_lock(&mode_key_setup_mutex); + + if (fscrypt_is_key_prepared(prep_key, ci)) + goto done_unlock; BUILD_BUG_ON(sizeof(mode_num) != 1); BUILD_BUG_ON(sizeof(sb->s_uuid) != 16); @@ -160,21 +190,17 @@ static int setup_per_mode_key(struct fscrypt_info *ci, hkdf_context, hkdf_info, hkdf_infolen, mode_key, mode->keysize); if (err) - return err; - tfm = fscrypt_allocate_skcipher(mode, mode_key, inode); + goto out_unlock; + err = fscrypt_prepare_key(prep_key, mode_key, ci); memzero_explicit(mode_key, mode->keysize); - if (IS_ERR(tfm)) - return PTR_ERR(tfm); - - /* pairs with READ_ONCE() above */ - prev_tfm = cmpxchg(&tfms[mode_num], NULL, tfm); - if (prev_tfm != NULL) { - crypto_free_skcipher(tfm); - tfm = prev_tfm; - } -done: - ci->ci_ctfm = tfm; - return 0; + if (err) + goto out_unlock; +done_unlock: + ci->ci_key = *prep_key; + err = 0; +out_unlock: + mutex_unlock(&mode_key_setup_mutex); + return err; } static int fscrypt_setup_v2_file_key(struct fscrypt_info *ci, @@ -198,7 +224,7 @@ static int fscrypt_setup_v2_file_key(struct fscrypt_info *ci, ci->ci_mode->friendly_name); return -EINVAL; } - return setup_per_mode_key(ci, mk, mk->mk_direct_tfms, + return setup_per_mode_key(ci, mk, mk->mk_direct_keys, HKDF_CONTEXT_DIRECT_KEY, false); } else if (ci->ci_policy.v2.flags & FSCRYPT_POLICY_FLAG_IV_INO_LBLK_64) { @@ -208,9 +234,7 @@ static int fscrypt_setup_v2_file_key(struct fscrypt_info *ci, * the IVs. This format is optimized for use with inline * encryption hardware compliant with the UFS or eMMC standards. */ - if (fscrypt_should_use_inline_encryption(ci)) - return fscrypt_setup_per_mode_inline_crypt_key(ci, mk); - return setup_per_mode_key(ci, mk, mk->mk_iv_ino_lblk_64_tfms, + return setup_per_mode_key(ci, mk, mk->mk_iv_ino_lblk_64_keys, HKDF_CONTEXT_IV_INO_LBLK_64_KEY, true); } @@ -245,6 +269,8 @@ static int setup_file_encryption_key(struct fscrypt_info *ci, struct fscrypt_key_specifier mk_spec; int err; + fscrypt_select_encryption_impl(ci); + switch (ci->ci_policy.version) { case FSCRYPT_POLICY_V1: mk_spec.type = FSCRYPT_KEY_SPEC_TYPE_DESCRIPTOR; @@ -336,10 +362,8 @@ static void put_crypt_info(struct fscrypt_info *ci) if (ci->ci_direct_key) fscrypt_put_direct_key(ci->ci_direct_key); - else if (ci->ci_owns_key) { - crypto_free_skcipher(ci->ci_ctfm); - fscrypt_free_inline_crypt_key(ci); - } + else if (ci->ci_owns_key) + fscrypt_destroy_prepared_key(&ci->ci_key); key = ci->ci_master_key; if (key) { diff --git a/fs/crypto/keysetup_v1.c b/fs/crypto/keysetup_v1.c index 5298ef22aa85..423ee9b64108 100644 --- a/fs/crypto/keysetup_v1.c +++ b/fs/crypto/keysetup_v1.c @@ -146,7 +146,7 @@ struct fscrypt_direct_key { struct hlist_node dk_node; refcount_t dk_refcount; const struct fscrypt_mode *dk_mode; - struct crypto_skcipher *dk_ctfm; + struct fscrypt_prepared_key dk_key; u8 dk_descriptor[FSCRYPT_KEY_DESCRIPTOR_SIZE]; u8 dk_raw[FSCRYPT_MAX_KEY_SIZE]; }; @@ -154,7 +154,7 @@ struct fscrypt_direct_key { static void free_direct_key(struct fscrypt_direct_key *dk) { if (dk) { - crypto_free_skcipher(dk->dk_ctfm); + fscrypt_destroy_prepared_key(&dk->dk_key); kzfree(dk); } } @@ -199,6 +199,8 @@ find_or_insert_direct_key(struct fscrypt_direct_key *to_insert, continue; if (ci->ci_mode != dk->dk_mode) continue; + if (!fscrypt_is_key_prepared(&dk->dk_key, ci)) + continue; if (crypto_memneq(raw_key, dk->dk_raw, ci->ci_mode->keysize)) continue; /* using existing tfm with same (descriptor, mode, raw_key) */ @@ -231,13 +233,9 @@ fscrypt_get_direct_key(const struct fscrypt_info *ci, const u8 *raw_key) return ERR_PTR(-ENOMEM); refcount_set(&dk->dk_refcount, 1); dk->dk_mode = ci->ci_mode; - dk->dk_ctfm = fscrypt_allocate_skcipher(ci->ci_mode, raw_key, - ci->ci_inode); - if (IS_ERR(dk->dk_ctfm)) { - err = PTR_ERR(dk->dk_ctfm); - dk->dk_ctfm = NULL; + err = fscrypt_prepare_key(&dk->dk_key, raw_key, ci); + if (err) goto err_free_dk; - } memcpy(dk->dk_descriptor, ci->ci_policy.v1.master_key_descriptor, FSCRYPT_KEY_DESCRIPTOR_SIZE); memcpy(dk->dk_raw, raw_key, ci->ci_mode->keysize); @@ -274,7 +272,7 @@ static int setup_v1_file_key_direct(struct fscrypt_info *ci, if (IS_ERR(dk)) return PTR_ERR(dk); ci->ci_direct_key = dk; - ci->ci_ctfm = dk->dk_ctfm; + ci->ci_key = dk->dk_key; return 0; } diff --git a/fs/ext4/page-io.c b/fs/ext4/page-io.c index 46a4aeef8275..b55038236845 100644 --- a/fs/ext4/page-io.c +++ b/fs/ext4/page-io.c @@ -362,16 +362,11 @@ static int io_submit_init_bio(struct ext4_io_submit *io, struct buffer_head *bh) { struct bio *bio; - int err; bio = bio_alloc(GFP_NOIO, BIO_MAX_PAGES); if (!bio) return -ENOMEM; - err = fscrypt_set_bio_crypt_ctx_bh(bio, bh, GFP_NOIO); - if (err) { - bio_put(bio); - return err; - } + fscrypt_set_bio_crypt_ctx_bh(bio, bh, GFP_NOIO); bio->bi_iter.bi_sector = bh->b_blocknr * (bh->b_size >> 9); bio_set_dev(bio, bh->b_bdev); bio->bi_end_io = ext4_end_bio; diff --git a/fs/ext4/readpage.c b/fs/ext4/readpage.c index e21216e7ae24..522ccc42b7e5 100644 --- a/fs/ext4/readpage.c +++ b/fs/ext4/readpage.c @@ -406,12 +406,8 @@ int ext4_mpage_readpages(struct address_space *mapping, min_t(int, nr_pages, BIO_MAX_PAGES)); if (!bio) goto set_error_page; - if (fscrypt_set_bio_crypt_ctx(bio, inode, next_block, - GFP_KERNEL) != 0) { - bio_put(bio); - bio = NULL; - goto set_error_page; - } + fscrypt_set_bio_crypt_ctx(bio, inode, next_block, + GFP_KERNEL); ctx = get_bio_post_read_ctx(inode, bio, page->index); if (IS_ERR(ctx)) { bio_put(bio); diff --git a/fs/f2fs/data.c b/fs/f2fs/data.c index ce44cda13bfe..b494242e66b1 100644 --- a/fs/f2fs/data.c +++ b/fs/f2fs/data.c @@ -307,7 +307,7 @@ static struct bio *__bio_alloc(struct f2fs_io_info *fio, int npages) return bio; } -static int f2fs_set_bio_crypt_ctx(struct bio *bio, const struct inode *inode, +static void f2fs_set_bio_crypt_ctx(struct bio *bio, const struct inode *inode, pgoff_t first_idx, const struct f2fs_io_info *fio, gfp_t gfp_mask) @@ -316,10 +316,8 @@ static int f2fs_set_bio_crypt_ctx(struct bio *bio, const struct inode *inode, * The f2fs garbage collector sets ->encrypted_page when it wants to * read/write raw data without encryption. */ - if (fio && fio->encrypted_page) - return 0; - - return fscrypt_set_bio_crypt_ctx(bio, inode, first_idx, gfp_mask); + if (!fio || !fio->encrypted_page) + fscrypt_set_bio_crypt_ctx(bio, inode, first_idx, gfp_mask); } static bool f2fs_crypt_mergeable_bio(struct bio *bio, const struct inode *inode, @@ -507,7 +505,6 @@ int f2fs_submit_page_bio(struct f2fs_io_info *fio) struct bio *bio; struct page *page = fio->encrypted_page ? fio->encrypted_page : fio->page; - int err; if (!f2fs_is_valid_blkaddr(fio->sbi, fio->new_blkaddr, fio->is_por ? META_POR : (__is_meta_io(fio) ? @@ -520,12 +517,8 @@ int f2fs_submit_page_bio(struct f2fs_io_info *fio) /* Allocate a new bio */ bio = __bio_alloc(fio, 1); - err = f2fs_set_bio_crypt_ctx(bio, fio->page->mapping->host, - fio->page->index, fio, GFP_NOIO); - if (err) { - bio_put(bio); - return err; - } + f2fs_set_bio_crypt_ctx(bio, fio->page->mapping->host, + fio->page->index, fio, GFP_NOIO); if (bio_add_page(bio, page, PAGE_SIZE, 0) < PAGE_SIZE) { bio_put(bio); @@ -606,7 +599,7 @@ alloc_new: bio = __bio_alloc(fio, BIO_MAX_PAGES); f2fs_set_bio_crypt_ctx(bio, fio->page->mapping->host, fio->page->index, fio, - GFP_NOIO | __GFP_NOFAIL); + GFP_NOIO); bio_set_op_attrs(bio, fio->op, fio->op_flags); } @@ -690,7 +683,7 @@ alloc_new: io->bio = __bio_alloc(fio, BIO_MAX_PAGES); f2fs_set_bio_crypt_ctx(io->bio, fio->page->mapping->host, fio->page->index, fio, - GFP_NOIO | __GFP_NOFAIL); + GFP_NOIO); io->fio = *fio; } @@ -730,17 +723,12 @@ static struct bio *f2fs_grab_read_bio(struct inode *inode, block_t blkaddr, struct bio *bio; struct bio_post_read_ctx *ctx; unsigned int post_read_steps = 0; - int err; bio = f2fs_bio_alloc(sbi, min_t(int, nr_pages, BIO_MAX_PAGES), false); if (!bio) return ERR_PTR(-ENOMEM); - err = f2fs_set_bio_crypt_ctx(bio, inode, first_idx, NULL, GFP_NOFS); - if (err) { - bio_put(bio); - return ERR_PTR(err); - } + f2fs_set_bio_crypt_ctx(bio, inode, first_idx, NULL, GFP_NOFS); f2fs_target_device(sbi, blkaddr, bio); bio->bi_end_io = f2fs_read_end_io; diff --git a/fs/f2fs/super.c b/fs/f2fs/super.c index 850a2a2394d8..eecfe09c52e7 100644 --- a/fs/f2fs/super.c +++ b/fs/f2fs/super.c @@ -2339,6 +2339,25 @@ static bool f2fs_inline_crypt_enabled(struct super_block *sb) return F2FS_OPTION(F2FS_SB(sb)).inlinecrypt; } +static int f2fs_get_num_devices(struct super_block *sb) +{ + struct f2fs_sb_info *sbi = F2FS_SB(sb); + + if (f2fs_is_multi_device(sbi)) + return sbi->s_ndevs; + return 1; +} + +static void f2fs_get_devices(struct super_block *sb, + struct request_queue **devs) +{ + struct f2fs_sb_info *sbi = F2FS_SB(sb); + int i; + + for (i = 0; i < sbi->s_ndevs; i++) + devs[i] = bdev_get_queue(FDEV(i).bdev); +} + static const struct fscrypt_operations f2fs_cryptops = { .key_prefix = "f2fs:", .get_context = f2fs_get_context, @@ -2349,6 +2368,8 @@ static const struct fscrypt_operations f2fs_cryptops = { .has_stable_inodes = f2fs_has_stable_inodes, .get_ino_and_lblk_bits = f2fs_get_ino_and_lblk_bits, .inline_crypt_enabled = f2fs_inline_crypt_enabled, + .get_num_devices = f2fs_get_num_devices, + .get_devices = f2fs_get_devices, }; #endif diff --git a/include/linux/bio-crypt-ctx.h b/include/linux/bio-crypt-ctx.h index 7c389f310bab..4535df0a6349 100644 --- a/include/linux/bio-crypt-ctx.h +++ b/include/linux/bio-crypt-ctx.h @@ -6,221 +6,188 @@ #define __LINUX_BIO_CRYPT_CTX_H enum blk_crypto_mode_num { - BLK_ENCRYPTION_MODE_INVALID = 0, - BLK_ENCRYPTION_MODE_AES_256_XTS = 1, + BLK_ENCRYPTION_MODE_INVALID, + BLK_ENCRYPTION_MODE_AES_256_XTS, + BLK_ENCRYPTION_MODE_AES_128_CBC_ESSIV, + BLK_ENCRYPTION_MODE_ADIANTUM, + BLK_ENCRYPTION_MODE_MAX, }; #ifdef CONFIG_BLOCK #include #ifdef CONFIG_BLK_INLINE_ENCRYPTION -struct bio_crypt_ctx { - int keyslot; - const u8 *raw_key; + +#define BLK_CRYPTO_MAX_KEY_SIZE 64 + +/** + * struct blk_crypto_key - an inline encryption key + * @crypto_mode: encryption algorithm this key is for + * @data_unit_size: the data unit size for all encryption/decryptions with this + * key. This is the size in bytes of each individual plaintext and + * ciphertext. This is always a power of 2. It might be e.g. the + * filesystem block size or the disk sector size. + * @data_unit_size_bits: log2 of data_unit_size + * @size: size of this key in bytes (determined by @crypto_mode) + * @hash: hash of this key, for keyslot manager use only + * @raw: the raw bytes of this key. Only the first @size bytes are used. + * + * A blk_crypto_key is immutable once created, and many bios can reference it at + * the same time. It must not be freed until all bios using it have completed. + */ +struct blk_crypto_key { enum blk_crypto_mode_num crypto_mode; - u64 data_unit_num; + unsigned int data_unit_size; unsigned int data_unit_size_bits; + unsigned int size; + unsigned int hash; + u8 raw[BLK_CRYPTO_MAX_KEY_SIZE]; +}; + +#define BLK_CRYPTO_MAX_IV_SIZE 32 +#define BLK_CRYPTO_DUN_ARRAY_SIZE (BLK_CRYPTO_MAX_IV_SIZE/sizeof(u64)) + +/** + * struct bio_crypt_ctx - an inline encryption context + * @bc_key: the key, algorithm, and data unit size to use + * @bc_keyslot: the keyslot that has been assigned for this key in @bc_ksm, + * or -1 if no keyslot has been assigned yet. + * @bc_dun: the data unit number (starting IV) to use + * @bc_ksm: the keyslot manager into which the key has been programmed with + * @bc_keyslot, or NULL if this key hasn't yet been programmed. + * + * A bio_crypt_ctx specifies that the contents of the bio will be encrypted (for + * write requests) or decrypted (for read requests) inline by the storage device + * or controller, or by the crypto API fallback. + */ +struct bio_crypt_ctx { + const struct blk_crypto_key *bc_key; + int bc_keyslot; + + /* Data unit number */ + u64 bc_dun[BLK_CRYPTO_DUN_ARRAY_SIZE]; /* * The keyslot manager where the key has been programmed * with keyslot. */ - struct keyslot_manager *processing_ksm; - - /* - * Copy of the bvec_iter when this bio was submitted. - * We only want to en/decrypt the part of the bio - * as described by the bvec_iter upon submission because - * bio might be split before being resubmitted - */ - struct bvec_iter crypt_iter; - u64 sw_data_unit_num; + struct keyslot_manager *bc_ksm; }; -extern int bio_crypt_clone(struct bio *dst, struct bio *src, - gfp_t gfp_mask); +int bio_crypt_ctx_init(void); + +struct bio_crypt_ctx *bio_crypt_alloc_ctx(gfp_t gfp_mask); + +void bio_crypt_free_ctx(struct bio *bio); static inline bool bio_has_crypt_ctx(struct bio *bio) { return bio->bi_crypt_context; } -static inline void bio_crypt_advance(struct bio *bio, unsigned int bytes) +void bio_crypt_clone(struct bio *dst, struct bio *src, gfp_t gfp_mask); + +static inline void bio_crypt_set_ctx(struct bio *bio, + const struct blk_crypto_key *key, + u64 dun[BLK_CRYPTO_DUN_ARRAY_SIZE], + gfp_t gfp_mask) { - if (bio_has_crypt_ctx(bio)) { - bio->bi_crypt_context->data_unit_num += - bytes >> bio->bi_crypt_context->data_unit_size_bits; + struct bio_crypt_ctx *bc = bio_crypt_alloc_ctx(gfp_mask); + + bc->bc_key = key; + memcpy(bc->bc_dun, dun, sizeof(bc->bc_dun)); + bc->bc_ksm = NULL; + bc->bc_keyslot = -1; + + bio->bi_crypt_context = bc; +} + +void bio_crypt_ctx_release_keyslot(struct bio_crypt_ctx *bc); + +int bio_crypt_ctx_acquire_keyslot(struct bio_crypt_ctx *bc, + struct keyslot_manager *ksm); + +struct request; +bool bio_crypt_should_process(struct request *rq); + +static inline bool bio_crypt_dun_is_contiguous(const struct bio_crypt_ctx *bc, + unsigned int bytes, + u64 next_dun[BLK_CRYPTO_DUN_ARRAY_SIZE]) +{ + int i = 0; + unsigned int inc = bytes >> bc->bc_key->data_unit_size_bits; + + while (inc && i < BLK_CRYPTO_DUN_ARRAY_SIZE) { + if (bc->bc_dun[i] + inc != next_dun[i]) + return false; + inc = ((bc->bc_dun[i] + inc) < inc); + i++; + } + + return true; +} + + +static inline void bio_crypt_dun_increment(u64 dun[BLK_CRYPTO_DUN_ARRAY_SIZE], + unsigned int inc) +{ + int i = 0; + + while (inc && i < BLK_CRYPTO_DUN_ARRAY_SIZE) { + dun[i] += inc; + inc = (dun[i] < inc); + i++; } } -extern bool bio_crypt_swhandled(struct bio *bio); - -static inline bool bio_crypt_has_keyslot(struct bio *bio) +static inline void bio_crypt_advance(struct bio *bio, unsigned int bytes) { - return bio->bi_crypt_context->keyslot >= 0; + struct bio_crypt_ctx *bc = bio->bi_crypt_context; + + if (!bc) + return; + + bio_crypt_dun_increment(bc->bc_dun, + bytes >> bc->bc_key->data_unit_size_bits); } -extern int bio_crypt_ctx_init(void); +bool bio_crypt_ctx_compatible(struct bio *b_1, struct bio *b_2); -extern struct bio_crypt_ctx *bio_crypt_alloc_ctx(gfp_t gfp_mask); - -extern void bio_crypt_free_ctx(struct bio *bio); - -static inline int bio_crypt_set_ctx(struct bio *bio, - const u8 *raw_key, - enum blk_crypto_mode_num crypto_mode, - u64 dun, - unsigned int dun_bits, - gfp_t gfp_mask) -{ - struct bio_crypt_ctx *crypt_ctx; - - crypt_ctx = bio_crypt_alloc_ctx(gfp_mask); - if (!crypt_ctx) - return -ENOMEM; - - crypt_ctx->raw_key = raw_key; - crypt_ctx->data_unit_num = dun; - crypt_ctx->data_unit_size_bits = dun_bits; - crypt_ctx->crypto_mode = crypto_mode; - crypt_ctx->processing_ksm = NULL; - crypt_ctx->keyslot = -1; - bio->bi_crypt_context = crypt_ctx; - - return 0; -} - -static inline void bio_set_data_unit_num(struct bio *bio, u64 dun) -{ - bio->bi_crypt_context->data_unit_num = dun; -} - -static inline int bio_crypt_get_keyslot(struct bio *bio) -{ - return bio->bi_crypt_context->keyslot; -} - -static inline void bio_crypt_set_keyslot(struct bio *bio, - unsigned int keyslot, - struct keyslot_manager *ksm) -{ - bio->bi_crypt_context->keyslot = keyslot; - bio->bi_crypt_context->processing_ksm = ksm; -} - -extern void bio_crypt_ctx_release_keyslot(struct bio *bio); - -extern int bio_crypt_ctx_acquire_keyslot(struct bio *bio, - struct keyslot_manager *ksm); - -static inline const u8 *bio_crypt_raw_key(struct bio *bio) -{ - return bio->bi_crypt_context->raw_key; -} - -static inline enum blk_crypto_mode_num bio_crypto_mode(struct bio *bio) -{ - return bio->bi_crypt_context->crypto_mode; -} - -static inline u64 bio_crypt_data_unit_num(struct bio *bio) -{ - return bio->bi_crypt_context->data_unit_num; -} - -static inline u64 bio_crypt_sw_data_unit_num(struct bio *bio) -{ - return bio->bi_crypt_context->sw_data_unit_num; -} - -extern bool bio_crypt_should_process(struct bio *bio, struct request_queue *q); - -extern bool bio_crypt_ctx_compatible(struct bio *b_1, struct bio *b_2); - -extern bool bio_crypt_ctx_back_mergeable(struct bio *b_1, - unsigned int b1_sectors, - struct bio *b_2); +bool bio_crypt_ctx_mergeable(struct bio *b_1, unsigned int b1_bytes, + struct bio *b_2); #else /* CONFIG_BLK_INLINE_ENCRYPTION */ -struct keyslot_manager; - static inline int bio_crypt_ctx_init(void) { return 0; } -static inline int bio_crypt_clone(struct bio *dst, struct bio *src, - gfp_t gfp_mask) -{ - return 0; -} - -static inline void bio_crypt_advance(struct bio *bio, - unsigned int bytes) { } - static inline bool bio_has_crypt_ctx(struct bio *bio) { return false; } +static inline void bio_crypt_clone(struct bio *dst, struct bio *src, + gfp_t gfp_mask) { } + static inline void bio_crypt_free_ctx(struct bio *bio) { } -static inline void bio_crypt_set_ctx(struct bio *bio, - u8 *raw_key, - enum blk_crypto_mode_num crypto_mode, - u64 dun, - unsigned int dun_bits, - gfp_t gfp_mask) { } - -static inline bool bio_crypt_swhandled(struct bio *bio) -{ - return false; -} - -static inline void bio_set_data_unit_num(struct bio *bio, u64 dun) { } - -static inline bool bio_crypt_has_keyslot(struct bio *bio) -{ - return false; -} - -static inline void bio_crypt_set_keyslot(struct bio *bio, - unsigned int keyslot, - struct keyslot_manager *ksm) { } - -static inline int bio_crypt_get_keyslot(struct bio *bio) -{ - return -1; -} - -static inline u8 *bio_crypt_raw_key(struct bio *bio) -{ - return NULL; -} - -static inline u64 bio_crypt_data_unit_num(struct bio *bio) -{ - return 0; -} - -static inline bool bio_crypt_should_process(struct bio *bio, - struct request_queue *q) -{ - return false; -} +static inline void bio_crypt_advance(struct bio *bio, unsigned int bytes) { } static inline bool bio_crypt_ctx_compatible(struct bio *b_1, struct bio *b_2) { return true; } -static inline bool bio_crypt_ctx_back_mergeable(struct bio *b_1, - unsigned int b1_sectors, - struct bio *b_2) +static inline bool bio_crypt_ctx_mergeable(struct bio *b_1, + unsigned int b1_bytes, + struct bio *b_2) { return true; } #endif /* CONFIG_BLK_INLINE_ENCRYPTION */ + #endif /* CONFIG_BLOCK */ + #endif /* __LINUX_BIO_CRYPT_CTX_H */ diff --git a/include/linux/blk-crypto.h b/include/linux/blk-crypto.h index 2a07401244a6..07d32e33f0bb 100644 --- a/include/linux/blk-crypto.h +++ b/include/linux/blk-crypto.h @@ -6,32 +6,23 @@ #ifndef __LINUX_BLK_CRYPTO_H #define __LINUX_BLK_CRYPTO_H -#include #include #ifdef CONFIG_BLK_INLINE_ENCRYPTION -int blk_crypto_init(void); - int blk_crypto_submit_bio(struct bio **bio_ptr); bool blk_crypto_endio(struct bio *bio); -int blk_crypto_start_using_mode(enum blk_crypto_mode_num mode_num, - unsigned int data_unit_size, - struct request_queue *q); +int blk_crypto_init_key(struct blk_crypto_key *blk_key, const u8 *raw_key, + enum blk_crypto_mode_num crypto_mode, + unsigned int data_unit_size); -int blk_crypto_evict_key(struct request_queue *q, const u8 *key, - enum blk_crypto_mode_num mode, - unsigned int data_unit_size); +int blk_crypto_evict_key(struct request_queue *q, + const struct blk_crypto_key *key); #else /* CONFIG_BLK_INLINE_ENCRYPTION */ -static inline int blk_crypto_init(void) -{ - return 0; -} - static inline int blk_crypto_submit_bio(struct bio **bio_ptr) { return 0; @@ -42,21 +33,31 @@ static inline bool blk_crypto_endio(struct bio *bio) return true; } +#endif /* CONFIG_BLK_INLINE_ENCRYPTION */ + +#ifdef CONFIG_BLK_INLINE_ENCRYPTION_FALLBACK + +int blk_crypto_start_using_mode(enum blk_crypto_mode_num mode_num, + unsigned int data_unit_size, + struct request_queue *q); + +int blk_crypto_fallback_init(void); + +#else /* CONFIG_BLK_INLINE_ENCRYPTION_FALLBACK */ + static inline int blk_crypto_start_using_mode(enum blk_crypto_mode_num mode_num, unsigned int data_unit_size, struct request_queue *q) { - return -EOPNOTSUPP; + return 0; } -static inline int blk_crypto_evict_key(struct request_queue *q, const u8 *key, - enum blk_crypto_mode_num mode, - unsigned int data_unit_size) +static inline int blk_crypto_fallback_init(void) { return 0; } -#endif /* CONFIG_BLK_INLINE_ENCRYPTION */ +#endif /* CONFIG_BLK_INLINE_ENCRYPTION_FALLBACK */ #endif /* __LINUX_BLK_CRYPTO_H */ diff --git a/include/linux/fscrypt.h b/include/linux/fscrypt.h index 607076207f73..0b15a0a6a354 100644 --- a/include/linux/fscrypt.h +++ b/include/linux/fscrypt.h @@ -65,6 +65,9 @@ struct fscrypt_operations { void (*get_ino_and_lblk_bits)(struct super_block *sb, int *ino_bits_ret, int *lblk_bits_ret); bool (*inline_crypt_enabled)(struct super_block *sb); + int (*get_num_devices)(struct super_block *sb); + void (*get_devices)(struct super_block *sb, + struct request_queue **devs); }; static inline bool fscrypt_has_encryption_key(const struct inode *inode) @@ -550,12 +553,13 @@ extern bool fscrypt_inode_uses_inline_crypto(const struct inode *inode); extern bool fscrypt_inode_uses_fs_layer_crypto(const struct inode *inode); -extern int fscrypt_set_bio_crypt_ctx(struct bio *bio, const struct inode *inode, - u64 first_lblk, gfp_t gfp_mask); +extern void fscrypt_set_bio_crypt_ctx(struct bio *bio, + const struct inode *inode, + u64 first_lblk, gfp_t gfp_mask); -extern int fscrypt_set_bio_crypt_ctx_bh(struct bio *bio, - const struct buffer_head *first_bh, - gfp_t gfp_mask); +extern void fscrypt_set_bio_crypt_ctx_bh(struct bio *bio, + const struct buffer_head *first_bh, + gfp_t gfp_mask); extern bool fscrypt_mergeable_bio(struct bio *bio, const struct inode *inode, u64 next_lblk); @@ -574,20 +578,14 @@ static inline bool fscrypt_inode_uses_fs_layer_crypto(const struct inode *inode) return IS_ENCRYPTED(inode) && S_ISREG(inode->i_mode); } -static inline int fscrypt_set_bio_crypt_ctx(struct bio *bio, - const struct inode *inode, - u64 first_lblk, gfp_t gfp_mask) -{ - return 0; -} +static inline void fscrypt_set_bio_crypt_ctx(struct bio *bio, + const struct inode *inode, + u64 first_lblk, gfp_t gfp_mask) { } -static inline int fscrypt_set_bio_crypt_ctx_bh( - struct bio *bio, - const struct buffer_head *first_bh, - gfp_t gfp_mask) -{ - return 0; -} +static inline void fscrypt_set_bio_crypt_ctx_bh( + struct bio *bio, + const struct buffer_head *first_bh, + gfp_t gfp_mask) { } static inline bool fscrypt_mergeable_bio(struct bio *bio, const struct inode *inode, diff --git a/include/linux/keyslot-manager.h b/include/linux/keyslot-manager.h index 0777ade7907c..fbc423fe5cd5 100644 --- a/include/linux/keyslot-manager.h +++ b/include/linux/keyslot-manager.h @@ -3,96 +3,58 @@ * Copyright 2019 Google LLC */ -#include - -#ifdef CONFIG_BLOCK - #ifndef __LINUX_KEYSLOT_MANAGER_H #define __LINUX_KEYSLOT_MANAGER_H +#include + +struct keyslot_manager; + /** * struct keyslot_mgmt_ll_ops - functions to manage keyslots in hardware - * @keyslot_program: Program the specified key and algorithm into the - * specified slot in the inline encryption hardware. + * @keyslot_program: Program the specified key into the specified slot in the + * inline encryption hardware. * @keyslot_evict: Evict key from the specified keyslot in the hardware. - * The key, crypto_mode and data_unit_size are also passed - * down so that e.g. dm layers can evict keys from - * the devices that they map over. + * The key is provided so that e.g. dm layers can evict + * keys from the devices that they map over. * Returns 0 on success, -errno otherwise. - * @crypto_mode_supported: Check whether a crypto_mode and data_unit_size - * combo is supported. - * @keyslot_find: Returns the slot number that matches the key, - * or -ENOKEY if no match found, or -errno on - * error. * * This structure should be provided by storage device drivers when they set up * a keyslot manager - this structure holds the function ptrs that the keyslot * manager will use to manipulate keyslots in the hardware. */ struct keyslot_mgmt_ll_ops { - int (*keyslot_program)(void *ll_priv_data, const u8 *key, - enum blk_crypto_mode_num crypto_mode, - unsigned int data_unit_size, + int (*keyslot_program)(struct keyslot_manager *ksm, + const struct blk_crypto_key *key, unsigned int slot); - int (*keyslot_evict)(void *ll_priv_data, const u8 *key, - enum blk_crypto_mode_num crypto_mode, - unsigned int data_unit_size, + int (*keyslot_evict)(struct keyslot_manager *ksm, + const struct blk_crypto_key *key, unsigned int slot); - bool (*crypto_mode_supported)(void *ll_priv_data, - enum blk_crypto_mode_num crypto_mode, - unsigned int data_unit_size); - int (*keyslot_find)(void *ll_priv_data, const u8 *key, - enum blk_crypto_mode_num crypto_mode, - unsigned int data_unit_size); }; -#ifdef CONFIG_BLK_INLINE_ENCRYPTION -struct keyslot_manager; +struct keyslot_manager *keyslot_manager_create(unsigned int num_slots, + const struct keyslot_mgmt_ll_ops *ksm_ops, + const unsigned int crypto_mode_supported[BLK_ENCRYPTION_MODE_MAX], + void *ll_priv_data); -extern struct keyslot_manager *keyslot_manager_create(unsigned int num_slots, - const struct keyslot_mgmt_ll_ops *ksm_ops, - void *ll_priv_data); +int keyslot_manager_get_slot_for_key(struct keyslot_manager *ksm, + const struct blk_crypto_key *key); -extern int -keyslot_manager_get_slot_for_key(struct keyslot_manager *ksm, - const u8 *key, - enum blk_crypto_mode_num crypto_mode, - unsigned int data_unit_size); +void keyslot_manager_get_slot(struct keyslot_manager *ksm, unsigned int slot); -extern void keyslot_manager_get_slot(struct keyslot_manager *ksm, - unsigned int slot); +void keyslot_manager_put_slot(struct keyslot_manager *ksm, unsigned int slot); -extern void keyslot_manager_put_slot(struct keyslot_manager *ksm, - unsigned int slot); +bool keyslot_manager_crypto_mode_supported(struct keyslot_manager *ksm, + enum blk_crypto_mode_num crypto_mode, + unsigned int data_unit_size); -extern bool -keyslot_manager_crypto_mode_supported(struct keyslot_manager *ksm, - enum blk_crypto_mode_num crypto_mode, - unsigned int data_unit_size); +int keyslot_manager_evict_key(struct keyslot_manager *ksm, + const struct blk_crypto_key *key); -extern bool -keyslot_manager_rq_crypto_mode_supported(struct request_queue *q, - enum blk_crypto_mode_num crypto_mode, - unsigned int data_unit_size); +void keyslot_manager_reprogram_all_keys(struct keyslot_manager *ksm); -extern int keyslot_manager_evict_key(struct keyslot_manager *ksm, - const u8 *key, - enum blk_crypto_mode_num crypto_mode, - unsigned int data_unit_size); +void *keyslot_manager_private(struct keyslot_manager *ksm); -extern void keyslot_manager_destroy(struct keyslot_manager *ksm); - -#else /* CONFIG_BLK_INLINE_ENCRYPTION */ - -static inline bool -keyslot_manager_rq_crypto_mode_supported(struct request_queue *q, - enum blk_crypto_mode_num crypto_mode, - unsigned int data_unit_size) -{ - return false; -} -#endif /* CONFIG_BLK_INLINE_ENCRYPTION */ +void keyslot_manager_destroy(struct keyslot_manager *ksm); #endif /* __LINUX_KEYSLOT_MANAGER_H */ - -#endif /* CONFIG_BLOCK */ From 46455321f9b0f7d517d7fec8fdad96ac75760ab8 Mon Sep 17 00:00:00 2001 From: Satya Tangirala Date: Tue, 17 Dec 2019 17:32:46 -0800 Subject: [PATCH 04/89] ANDROID: gki_defconfig: Enable blk-crypto fallback Enable blk-crypto's kernel crypto API fallback, so that devices without inline encryption hardware can continue to use the inlinecrypt mount option and have file content crypto handled by the block layer. Bug: 137270441 Test: xfstests, cuttlefish boot and stress test, pixel 4 boot and stress test Change-Id: I26376479ee38259b8c35732cb3a1d7e15f9b05a3 Signed-off-by: Satya Tangirala --- abi_gki_aarch64.xml | 17984 +++++++++++++++-------------- arch/arm64/configs/gki_defconfig | 1 + arch/x86/configs/gki_defconfig | 1 + 3 files changed, 9044 insertions(+), 8942 deletions(-) diff --git a/abi_gki_aarch64.xml b/abi_gki_aarch64.xml index 2ce464781f62..8109074ed36a 100644 --- a/abi_gki_aarch64.xml +++ b/abi_gki_aarch64.xml @@ -21244,7 +21244,7 @@ - + @@ -21258,7 +21258,7 @@ - + @@ -27417,9 +27417,7 @@ - - - + @@ -27450,7 +27448,7 @@ - + @@ -27465,32 +27463,32 @@ - + - + - + - + - + - + @@ -27508,13 +27506,13 @@ - + - + @@ -27555,22 +27553,22 @@ - + - + - - - + + + - + @@ -27600,25 +27598,25 @@ - - + + - + - + - - + + @@ -27632,15 +27630,15 @@ - + - + - + @@ -27665,17 +27663,17 @@ - - - + + + - + - + @@ -27683,7 +27681,7 @@ - + @@ -27698,16 +27696,16 @@ - + - + - + - + @@ -27722,42 +27720,42 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -27770,7 +27768,7 @@ - + @@ -27778,26 +27776,26 @@ - + - - - - - + + + + + + + - - - - - - - - - - - + + + + + + + + + @@ -27832,8 +27830,8 @@ - - + + @@ -27841,56 +27839,56 @@ - - - + + + - - - + + + - - - + + + - - - + + + - - + + - + - - + + - - - + + + - - + + - - + + @@ -27975,7 +27973,7 @@ - + @@ -27983,9 +27981,9 @@ - - - + + + @@ -28002,7 +28000,7 @@ - + @@ -28010,7 +28008,7 @@ - + @@ -28024,7 +28022,7 @@ - + @@ -28059,9 +28057,9 @@ - + - + @@ -28070,7 +28068,7 @@ - + @@ -28078,10 +28076,10 @@ - + - + @@ -28114,7 +28112,7 @@ - + @@ -28122,12 +28120,12 @@ - - + + - + @@ -28137,7 +28135,7 @@ - + @@ -28184,13 +28182,13 @@ - + - + @@ -28204,22 +28202,22 @@ - - + + - + - + - - + + @@ -28239,7 +28237,7 @@ - + @@ -28253,16 +28251,16 @@ - - - - + + + + - + @@ -28310,16 +28308,16 @@ - + - + - + @@ -28333,7 +28331,7 @@ - + @@ -28387,7 +28385,7 @@ - + @@ -28396,18 +28394,18 @@ - + - + - - + + @@ -28420,15 +28418,15 @@ - + - - - - + + + + @@ -28442,8 +28440,8 @@ - - + + @@ -28457,14 +28455,14 @@ - + - + - + @@ -28554,16 +28552,16 @@ - + - + - + @@ -28602,8 +28600,8 @@ - - + + @@ -28613,7 +28611,7 @@ - + @@ -28630,13 +28628,13 @@ - + - - + + @@ -28644,15 +28642,15 @@ - - + + - - - + + + @@ -28662,13 +28660,13 @@ - + - - + + @@ -28714,9 +28712,9 @@ - + - + @@ -28731,10 +28729,10 @@ - + - + @@ -28752,36 +28750,36 @@ - + - + - + - + - + - + - + @@ -28796,8 +28794,8 @@ - - + + @@ -28816,44 +28814,44 @@ - + - + - + - + - - - + + + - + - + - + - + - + @@ -28862,15 +28860,15 @@ - + - + - + @@ -28879,7 +28877,7 @@ - + @@ -28889,7 +28887,7 @@ - + @@ -28897,13 +28895,13 @@ - + - + @@ -28914,13 +28912,13 @@ - + - + - + @@ -28937,16 +28935,16 @@ - + - + - + @@ -28956,23 +28954,23 @@ - + - + - + - + - + - + @@ -28986,7 +28984,7 @@ - + @@ -29006,33 +29004,33 @@ - - - + + + + - - - - - - + + + + + + + - - + - - - + + - - + + @@ -29063,85 +29061,85 @@ - + - + - + - + + + + + + + + + + + - - - - - - - - - - - + - + - + - + - + - + - + - + - + - + - + @@ -29152,7 +29150,7 @@ - + @@ -29167,7 +29165,7 @@ - + @@ -29178,19 +29176,19 @@ - - - + + + - - + + - + - + @@ -29205,31 +29203,31 @@ - + - + - + - + - + - + - + @@ -29237,7 +29235,7 @@ - + @@ -29248,7 +29246,7 @@ - + @@ -29259,10 +29257,10 @@ - + - + @@ -29276,7 +29274,7 @@ - + @@ -29287,10 +29285,10 @@ - + - + @@ -29302,9 +29300,9 @@ - + - + @@ -29316,7 +29314,7 @@ - + @@ -29328,7 +29326,7 @@ - + @@ -29336,7 +29334,7 @@ - + @@ -29344,49 +29342,49 @@ - + - + - + - + - + - + - + - + - + - + - + - + - - + + @@ -29394,21 +29392,21 @@ - - - - - + + + + + - - - - - - + + + + + + @@ -29419,46 +29417,46 @@ - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + - - - - - - - + + + + + + - - - - - - - - - + + + + + + + + + - - - - - - + + + + + @@ -29488,7 +29486,7 @@ - + @@ -29501,106 +29499,106 @@ - + - + - + - + - + - + - - + + - + - + - + - + - + - + - + + + + + + + + + + + + - - - - - - - - - - - - - - - - - - - + + + + + + + + - + @@ -29623,7 +29621,7 @@ - + @@ -29637,9 +29635,9 @@ - + - + @@ -29649,10 +29647,10 @@ - + - + @@ -29670,7 +29668,7 @@ - + @@ -29685,19 +29683,19 @@ - + - + - + - + - + @@ -29775,7 +29773,7 @@ - + @@ -29808,7 +29806,7 @@ - + @@ -29865,7 +29863,7 @@ - + @@ -29874,65 +29872,65 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -29943,21 +29941,21 @@ - + - + - + - + @@ -29968,7 +29966,7 @@ - + @@ -29981,7 +29979,7 @@ - + @@ -29994,13 +29992,13 @@ - + - + @@ -30018,7 +30016,7 @@ - + @@ -30048,8 +30046,8 @@ - - + + @@ -30057,8 +30055,8 @@ - - + + @@ -30066,108 +30064,108 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -30203,16 +30201,16 @@ - + - + - + - + @@ -30224,10 +30222,10 @@ - + - + @@ -30241,27 +30239,27 @@ - + - + - + - + - + - + - + - + @@ -30282,10 +30280,10 @@ - + - + @@ -30297,7 +30295,7 @@ - + @@ -30308,13 +30306,13 @@ - + - + - + @@ -30322,8 +30320,8 @@ - - + + @@ -30334,7 +30332,7 @@ - + @@ -30348,7 +30346,7 @@ - + @@ -30359,7 +30357,7 @@ - + @@ -30373,7 +30371,7 @@ - + @@ -30387,10 +30385,10 @@ - + - + @@ -30417,7 +30415,7 @@ - + @@ -30432,29 +30430,29 @@ - + - + - + - + - + - + - + - + @@ -30477,31 +30475,31 @@ - + - + - + - + - + - + - + - + - + @@ -30522,7 +30520,7 @@ - + @@ -30560,10 +30558,10 @@ - + - + @@ -30587,20 +30585,20 @@ - + - + - + - - + + @@ -30608,7 +30606,7 @@ - + @@ -30616,26 +30614,26 @@ - + - + - + - + - + - + - + - + @@ -30664,20 +30662,20 @@ - + - + - + - + - + @@ -30686,18 +30684,18 @@ - + - + - + - + - + @@ -30706,10 +30704,10 @@ - + - + @@ -30717,7 +30715,7 @@ - + @@ -30725,22 +30723,22 @@ - + - + - + - + - + - + @@ -30752,27 +30750,27 @@ - + - + - + - + - + - + - + @@ -30783,7 +30781,7 @@ - + @@ -30792,7 +30790,7 @@ - + @@ -30821,7 +30819,7 @@ - + @@ -30829,10 +30827,10 @@ - + - + @@ -30841,7 +30839,7 @@ - + @@ -30849,9 +30847,9 @@ - + - + @@ -30863,17 +30861,17 @@ - + - + - + - - + + @@ -30881,7 +30879,7 @@ - + @@ -30889,88 +30887,88 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -31128,8 +31126,8 @@ - - + + @@ -31137,13 +31135,13 @@ - + - + @@ -31151,21 +31149,21 @@ - - + + - + - + - + - + @@ -31179,483 +31177,483 @@ - + - + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - - - + + + + + + - + - - + + - + + + + + + + + + + - - - - - - - - - - - + + - - - + + + - - - + + + + + + + + + - - + + - - - - - - - + - + - + - + - + - + - - + + - + - - + + - - + + - - + + - - + + - - + + - - - + + + - - - + + + - - + + - - - - + + + + - - + + - - - + + + - - - + + + - - + + - - + + - - + + - - + + - - + + - - + + - - - + + + - - - + + + + + + + + + + + + + + + + - - - + + - - - - - - - - - - - - - - + + - - + + - - + + - - + + - + - + - + - - + + - - + + - - - + + + - - + + - - + + - - + + - + + + + + + + + + + - + + + - - + - - - + - + - - - - - - - - - + - - + + - + - + - + - + - + - + - + - + - + - + @@ -31670,16 +31668,16 @@ - - + + - + - + @@ -31707,16 +31705,16 @@ - - + + - + - + @@ -31727,15 +31725,15 @@ - + - + - + - + @@ -31755,7 +31753,7 @@ - + @@ -31763,7 +31761,7 @@ - + @@ -31779,15 +31777,15 @@ - + - + - + @@ -31819,45 +31817,45 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -31872,49 +31870,49 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -31923,14 +31921,14 @@ - + - + - + - + @@ -31948,41 +31946,41 @@ - + - + - + - - + + - + - + - + - + - + @@ -31991,36 +31989,36 @@ - + - + - + - + - + - + - + - + - + - + - + - + @@ -32029,7 +32027,7 @@ - + @@ -32040,19 +32038,19 @@ - + - + - + - + @@ -32060,29 +32058,29 @@ - + - + - + - + - + - + - + - + - + @@ -32094,15 +32092,15 @@ - + - + - + - + @@ -32114,7 +32112,7 @@ - + @@ -32122,24 +32120,24 @@ - + - + - + - + - + - + - + @@ -32153,156 +32151,156 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - + - + - + - - + + - - + + + + + + - - + + + + + + + + + - + + + - - - - + + - - - + - - - - - - - - - - + - - + + - - + + - + - + + + + + + + + + + + - - - - - - - - - - - + - - + + + + + + + + + + - - - - - - - - - + - + - + - + @@ -32316,7 +32314,7 @@ - + @@ -32328,7 +32326,7 @@ - + @@ -32342,7 +32340,7 @@ - + @@ -32388,18 +32386,18 @@ - + - + - + @@ -32477,8 +32475,8 @@ - - + + @@ -32620,16 +32618,16 @@ - + - - + + - + @@ -32649,13 +32647,13 @@ - + - + - + @@ -32667,7 +32665,7 @@ - + @@ -32675,59 +32673,59 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -32739,16 +32737,16 @@ - - + + - + - + - + @@ -32768,190 +32766,190 @@ - + - + - + - + - + - + - + - + - + - + - + - - - - + + + + - + - + - + - + - + - + - + - + - + - + - - - - + + + + + + + - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + - + + + + - - - - - - - + + - - + + - - + + + + + + + + + + + + + + + + - + + + + - - - - - - - - - - - - - - - - - - + - - + + - - + + - - + + - - + + - - + + - - + + - - - + + + - - + + - - + + @@ -32975,80 +32973,80 @@ - + - + - + - + - + - + - + - + - + - + - + - + - - - - - + + + + + - + - + - + - + @@ -33057,7 +33055,7 @@ - + @@ -33108,7 +33106,7 @@ - + @@ -33117,7 +33115,7 @@ - + @@ -33177,7 +33175,7 @@ - + @@ -33186,7 +33184,7 @@ - + @@ -33198,7 +33196,7 @@ - + @@ -33312,13 +33310,13 @@ - + - + @@ -33330,16 +33328,16 @@ - + - + - + @@ -33354,12 +33352,12 @@ - + - + @@ -33368,13 +33366,13 @@ - + - + @@ -33383,10 +33381,10 @@ - + - + @@ -33416,7 +33414,7 @@ - + @@ -33434,33 +33432,33 @@ - + - + - + - + - + - + - - + + - + - + @@ -33469,7 +33467,7 @@ - + @@ -33477,34 +33475,34 @@ - + - + - + - + - + - + - + - + - + - + @@ -33513,10 +33511,10 @@ - + - + @@ -33539,34 +33537,34 @@ - + - + - + - + - + - + - + - - + + - + @@ -33574,99 +33572,99 @@ - - - - - - - - - - - - + + + + + + + + + + + + + + - - - - - - - - - - - - - - - + + + + + + + + + + + + + - + - - + + - - + + - - + + + + + + + + + + + - - - - - - - - - - + - - + + + + + + + + + + + - - + + - - - + + - - - - - - - - - + @@ -33681,102 +33679,102 @@ - + - + - + - + - + - + - + - + - + - + - - + + + + - - - - + + - + - + - + - + - + - + - + - + - - + + - - + + - - + + - - + + - - + + @@ -33791,41 +33789,41 @@ - - + + - + - + - + - + - + - + - + @@ -33837,35 +33835,35 @@ - + - + - + - + - + - + - + - - + + @@ -33889,12 +33887,12 @@ - - + + - + @@ -33920,7 +33918,7 @@ - + @@ -33929,13 +33927,13 @@ - + - + @@ -33948,15 +33946,15 @@ - + - + - + - + @@ -33977,7 +33975,7 @@ - + @@ -33998,16 +33996,16 @@ - + - + - + @@ -34019,7 +34017,7 @@ - + @@ -34031,7 +34029,7 @@ - + @@ -34045,7 +34043,7 @@ - + @@ -34060,13 +34058,13 @@ - + - + @@ -34090,19 +34088,19 @@ - + - + - + - + @@ -34195,15 +34193,15 @@ - + - + - + @@ -34212,24 +34210,24 @@ - + - + - + - + - + @@ -34238,7 +34236,7 @@ - + @@ -34279,16 +34277,16 @@ - + - + - + - + @@ -34333,7 +34331,7 @@ - + @@ -34353,8 +34351,8 @@ - - + + @@ -34362,113 +34360,113 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -34482,12 +34480,12 @@ - + - + - + @@ -34496,26 +34494,26 @@ - + - + - + - + - + - + @@ -34524,193 +34522,193 @@ - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + - + - + - + - + - + - + - + - + - + - - + + + + + + + + + + + + + - - + - - - - - - - - - - - + - - + + + + + + + + + + + + + + + - - - - - - - - - - - - - - + - - + + - - + + - - + + - + - + - + - + + + + + + + + + + + + - - - - - - - - - - - - - - - + + + + + + + + - - - - - + - + - + @@ -34722,7 +34720,7 @@ - + @@ -34730,7 +34728,7 @@ - + @@ -34739,10 +34737,10 @@ - + - + @@ -34763,7 +34761,7 @@ - + @@ -34771,7 +34769,7 @@ - + @@ -34782,13 +34780,13 @@ - + - + - + @@ -34824,24 +34822,24 @@ - + - + - + - + - + - + @@ -34853,7 +34851,7 @@ - + @@ -34861,109 +34859,109 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -34971,7 +34969,7 @@ - + @@ -34980,14 +34978,14 @@ - + - + @@ -35010,21 +35008,21 @@ - + - + - + - + - + @@ -35069,7 +35067,7 @@ - + @@ -35114,10 +35112,10 @@ - - + + - + @@ -35126,16 +35124,16 @@ - + - + - + @@ -35156,7 +35154,7 @@ - + @@ -35167,18 +35165,18 @@ - + - + - + - + @@ -35187,43 +35185,43 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -35238,7 +35236,7 @@ - + @@ -35250,39 +35248,39 @@ - + - + - + - + - + - + - + - + - + - + - + @@ -35294,7 +35292,7 @@ - + @@ -35312,10 +35310,10 @@ - + - + @@ -35324,13 +35322,13 @@ - + - + - + @@ -35338,264 +35336,264 @@ - + - + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + - + - + - + - + - + - + - + - + - - + + - - - + + + - - + + - - + + - - + + - - + + - - + + + + + + + + + + + + - - - + + + - + + + - - + + - - - + + - - - - - - - - - - - - + - - + + + + + + + + + + + + + - - + + + - - - - - - - - - - - - - + - - + + - - + + - - + + + + + + + + + + - - - - - - - - - + - - - - - - - - - - - - - - - - + + - - - - - + + - - + + + + + + + + + + + + + + + + + + + - + @@ -35608,21 +35606,21 @@ - - + + - - + + - + - + @@ -35630,37 +35628,37 @@ - + - + - - - - + + + + - + - + - + - + @@ -35672,10 +35670,10 @@ - + - + @@ -35693,10 +35691,10 @@ - + - + @@ -35735,7 +35733,7 @@ - + @@ -35759,112 +35757,112 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -35873,9 +35871,9 @@ - + - + @@ -35887,7 +35885,7 @@ - + @@ -35901,10 +35899,10 @@ - + - + @@ -35912,12 +35910,12 @@ - + - + - + @@ -35929,110 +35927,110 @@ - - - - - - - - - + + + + + + + + + + + + + - - - - - - - - - - - - - - - - + + + + + + + + + + + + - + - + - + - + - + + + + + + + + + + - + + - - - - - - - - - - - + + + + + + + + + + + + - + + + - - - - - - - - - - - - - - + - - + + - - + + - - + + - - + + @@ -36041,13 +36039,13 @@ - + - + - + @@ -36074,9 +36072,9 @@ - + - + @@ -36097,38 +36095,38 @@ - + - + - + - + - + - + - - - - - - + + + + + + - + - + @@ -36138,65 +36136,65 @@ - + - - - + + + - - + + - + - - + + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -36220,7 +36218,7 @@ - + @@ -36229,7 +36227,7 @@ - + @@ -36241,21 +36239,21 @@ - + - + - + - + - + - + @@ -36309,9 +36307,9 @@ - + - + @@ -36323,7 +36321,7 @@ - + @@ -36335,12 +36333,12 @@ - + - + - + @@ -36361,15 +36359,15 @@ - + - + - + @@ -36380,19 +36378,19 @@ - + - + - + - + @@ -36416,13 +36414,13 @@ - + - + - + @@ -36440,7 +36438,7 @@ - + @@ -36470,7 +36468,7 @@ - + @@ -36509,10 +36507,10 @@ - + - + @@ -36520,7 +36518,7 @@ - + @@ -36550,7 +36548,7 @@ - + @@ -36559,161 +36557,161 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - - - - + + + + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -36727,7 +36725,7 @@ - + @@ -36735,13 +36733,13 @@ - + - + @@ -36779,7 +36777,7 @@ - + @@ -36811,7 +36809,7 @@ - + @@ -36835,7 +36833,7 @@ - + @@ -36852,15 +36850,15 @@ - + - + - + - + @@ -36874,46 +36872,46 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -36922,362 +36920,362 @@ - + - + - + - + - + - + - + - + - + - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - + - + - - + + - + - + - + - - + + - - + + + + + + + + + + + + + - - - + - - - - - - - - - - + + + + + + + + + + - - - - - - - - - - + - - + + - - + + - - + + - - + + + + + + + + + + + + - - + + - - - - - - - - - - - + - - + + - - + + - - - + + + - + - + - + - - + + - - + + - + - - + + - - + + - - + + - - + + + + + + + + + + + - - - - - - - - - - + - - + + - - + + - - + + - - + + - - + + - - - + + + - - + + - - + + - + - - + + - + - + - + - + @@ -37331,7 +37329,7 @@ - + @@ -37340,19 +37338,19 @@ - + - + - + - + - + @@ -37361,7 +37359,7 @@ - + @@ -37372,55 +37370,55 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -37453,13 +37451,13 @@ - + - + - + @@ -37468,10 +37466,10 @@ - + - + @@ -37486,7 +37484,7 @@ - + @@ -37513,28 +37511,28 @@ - + - + - + - + - + @@ -37545,10 +37543,10 @@ - + - + @@ -37565,18 +37563,18 @@ - + - + - + - + - + @@ -37585,16 +37583,16 @@ - + - + - + @@ -37605,7 +37603,7 @@ - + @@ -37640,27 +37638,27 @@ - + - + - + - + - + - + @@ -37675,91 +37673,91 @@ - + - - - + + + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -37768,16 +37766,16 @@ - + - + - + - + @@ -37792,10 +37790,10 @@ - + - + @@ -37816,27 +37814,27 @@ - + - + - + - + - + - + @@ -37845,7 +37843,7 @@ - + @@ -37854,7 +37852,7 @@ - + @@ -37868,10 +37866,10 @@ - + - + @@ -37880,16 +37878,16 @@ - + - + - + - + @@ -37913,85 +37911,85 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - - + + @@ -38002,13 +38000,13 @@ - - + + - + @@ -38018,46 +38016,46 @@ - + - + - + - + - + - + - + - + - + - + - - + + @@ -38068,24 +38066,24 @@ - - - + + + - + - + - + @@ -38108,15 +38106,15 @@ - + - + - + @@ -38124,7 +38122,7 @@ - + @@ -38132,7 +38130,7 @@ - + @@ -38153,10 +38151,10 @@ - + - + @@ -38170,9 +38168,9 @@ - + - + @@ -38181,10 +38179,10 @@ - + - + @@ -38205,7 +38203,7 @@ - + @@ -38217,27 +38215,27 @@ - + - + - + - + - + - + @@ -38246,10 +38244,10 @@ - + - + @@ -38257,23 +38255,23 @@ - + - + - + - + - + - + @@ -38284,89 +38282,89 @@ - + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - + + + - - - - - - - + + + + + + + - + - - - + - + @@ -38377,297 +38375,297 @@ - + - - + + - - + + - + - + + + + + + + + + + + + + + + + + + + + - - - - - - + + - + - - - - - - - - - - - - - - - - + - + - + - - + + - + - - + + - + - - + + - + + + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - - + - + - - + + - + + + + + + + + + + + + - - - - + + + + + - + + - - - - + - - - + - + + - + - - + - + + - + + - - + + - - - - - - - - - - - + - - + + - + - - + + - - - + + + - - + + - - + + - - + + - - + + - + + + + + + + + + - + + - + - - - - - - - - - - + - - + + + + + + + + + + + + + - - - + + + - - + + - - - + + + - + + + - - - - - - - - - - - - - - + - + - - + + @@ -38690,28 +38688,28 @@ - + - + - + - + - + - - + + @@ -38728,7 +38726,7 @@ - + @@ -38743,10 +38741,10 @@ - + - + @@ -38757,13 +38755,13 @@ - + - + - + @@ -38772,32 +38770,32 @@ - + - + - + - + - + - + - + @@ -38814,121 +38812,121 @@ - + - + - + - + - + - + - + - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + - + - + - + - + - + - + - + - + - + - - + + - + - + - - + + - - + + - - + + - + @@ -38940,7 +38938,7 @@ - + @@ -38969,35 +38967,35 @@ - + - + - + - + - + - + - + - + - + @@ -39015,159 +39013,159 @@ - - + + + + - - - + - + - + - - + + - - + + - - + + - - + + - - + + - - + + - + - + - + - + - + - + - - + + - + - + - + - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + - + - - + + - + - + - + - + - + - + - + - + @@ -39197,73 +39195,73 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -39272,7 +39270,7 @@ - + @@ -39335,49 +39333,49 @@ - + - + - + - + - + - + - + - + - + - + - + - + @@ -39391,22 +39389,22 @@ - + - + - + - + @@ -39454,7 +39452,7 @@ - + @@ -39472,16 +39470,16 @@ - + - + - + @@ -39490,27 +39488,27 @@ - + - + - + - + - + - + - + @@ -39585,7 +39583,7 @@ - + @@ -39609,169 +39607,169 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -39798,7 +39796,7 @@ - + @@ -39807,13 +39805,13 @@ - + - + - + @@ -39821,12 +39819,12 @@ - + - + @@ -39838,59 +39836,59 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -39914,10 +39912,10 @@ - + - + @@ -39925,32 +39923,32 @@ - + - + - + - + - + - + - + @@ -39962,16 +39960,16 @@ - + - + - + @@ -39982,7 +39980,7 @@ - + @@ -40003,7 +40001,7 @@ - + @@ -40011,18 +40009,18 @@ - + - + - + - + - + @@ -40036,7 +40034,7 @@ - + @@ -40053,19 +40051,19 @@ - + - + - + - + - + @@ -40108,7 +40106,7 @@ - + @@ -40116,7 +40114,7 @@ - + @@ -40218,13 +40216,13 @@ - + - + @@ -40233,42 +40231,42 @@ - + - + - + - + - + - + - + - + - + - + - + - + @@ -40295,9 +40293,9 @@ - + - + @@ -40306,41 +40304,41 @@ - + - + - + - + - + - + - + - + - + - + - + @@ -40349,15 +40347,15 @@ - + - + - + @@ -40369,16 +40367,16 @@ - + - + - + - + @@ -40393,10 +40391,10 @@ - + - + @@ -40405,22 +40403,22 @@ - + - + - + - + - + @@ -40432,7 +40430,7 @@ - + @@ -40447,12 +40445,12 @@ - + - + - + @@ -40461,16 +40459,16 @@ - + - + - + @@ -40479,110 +40477,110 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -40612,7 +40610,7 @@ - + @@ -40624,50 +40622,50 @@ - + - + - + - + - + - + - + - - + + - + - + - + - + @@ -40682,7 +40680,7 @@ - + @@ -40696,12 +40694,12 @@ - + - + - + @@ -40710,17 +40708,17 @@ - + - + - - - + + + @@ -40737,109 +40735,109 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -40878,22 +40876,22 @@ - + - + - + - + - + - + @@ -40908,68 +40906,68 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -40983,7 +40981,7 @@ - + @@ -40998,7 +40996,7 @@ - + @@ -41006,12 +41004,12 @@ - + - + - + @@ -41020,21 +41018,21 @@ - + - + - + - + - + @@ -41054,7 +41052,7 @@ - + @@ -41074,34 +41072,34 @@ - + - + - + - + - + - + - + - + @@ -41113,7 +41111,7 @@ - + @@ -41131,13 +41129,13 @@ - + - + @@ -41163,16 +41161,16 @@ - + - + - + - + @@ -41223,10 +41221,10 @@ - + - + @@ -41237,7 +41235,7 @@ - + @@ -41281,27 +41279,27 @@ - + - + - + - + - + - + - + - + @@ -41367,7 +41365,7 @@ - + @@ -41393,7 +41391,7 @@ - + @@ -41437,7 +41435,7 @@ - + @@ -41466,13 +41464,13 @@ - + - + - + @@ -41480,90 +41478,90 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -41583,7 +41581,7 @@ - + @@ -41603,7 +41601,7 @@ - + @@ -41629,10 +41627,10 @@ - + - + @@ -41646,13 +41644,13 @@ - + - + - + @@ -41660,7 +41658,7 @@ - + @@ -41674,7 +41672,7 @@ - + @@ -41682,15 +41680,15 @@ - + - + - + @@ -41698,19 +41696,19 @@ - + - + - + @@ -41721,7 +41719,7 @@ - + @@ -41757,13 +41755,13 @@ - + - + @@ -41772,9 +41770,9 @@ - + - + @@ -41789,7 +41787,7 @@ - + @@ -41797,10 +41795,10 @@ - + - + @@ -41808,77 +41806,77 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -41896,7 +41894,7 @@ - + @@ -41905,18 +41903,18 @@ - + - + - + - + @@ -41934,27 +41932,27 @@ - + - + - + - + - + - + - + - + @@ -41963,16 +41961,16 @@ - + - + - + @@ -41986,116 +41984,116 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -42119,7 +42117,7 @@ - + @@ -42130,7 +42128,7 @@ - + @@ -42144,37 +42142,37 @@ - - + + - + - + - + - + - + - + - + - + - + - + @@ -42212,23 +42210,23 @@ - + - + - + - + @@ -42236,19 +42234,19 @@ - + - + - + - + @@ -42278,33 +42276,33 @@ - + - + - + - + - + - + - + - + - + - + @@ -42315,10 +42313,10 @@ - + - + @@ -42326,21 +42324,21 @@ - + - + - + - + - + - + @@ -42351,39 +42349,39 @@ - + - + - + - + - + - + - + - + - + - + - + - + @@ -42403,12 +42401,12 @@ - + - + @@ -42417,20 +42415,20 @@ - + - + - + - + - + @@ -42447,929 +42445,1000 @@ - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + - - - - - - - - + + + + + + + + + - - - - - - - - - - - - - - - + + + + + + + + + + + + + + - + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - + - + - + - - + + - - - + + + + + + + - - + + + + + + - - + + - - + + - - - - - - + + + - - + + + - - - - - + + + - - - + + + + + + + + + - - - + + + - - - + + + - - - + + + - - + + - - - + + + - - - - + + + - - - - + + + - + - + + + + - + + - - - - + + + + - - + + + - - - + + - - - - - - - - - - - - - + - - + + - - + + - - + + - - + + + + + + + + + + + + + + - - - - + + + + - - + + + + + + - - - - + + + + + - - - - - - + + + + + + + - - - - + + + + - - + + + - - - + - - - - - + + + + + - - - - - - + + + - - + - - - - + + + + - - - + + + + + + - - - - + + - - - - - + - - - - - - - - - - - + + + + + + + + + + + + - - - - - - - - - - - - - + + + + + + + + + + + + + + + + - - - - - - + - - + + + + - + + + + + + + + + + + + - - - - + + - - - - - - - - - - - - + + - - + + + + - - + + + - - - - + + - - - + + - - + - - + + - - - - - - - - - - + - - + + - - + + + + + + + + + + + - - - - - - - - - - - + + - - + + - + + + + + + + + + + + - + + + + + - - - + + + + - - - - - - - + + + + - - - + + + - - - - - - - - - - - - - + - - + + - - + + - - + + - - + + + + + + + + + + - + + + - + + - - - + - - + - + + - + - - + + - - - - - - - - - - + + + + + + + + + - + + + + - - + + + - - - - - - - - - - - - - + + + + + + + + + - + + - - - - - - - - - - + + + + + + + + + + + - - + + - - + + - - + + - - + - - + + - - - - - + - - + + + + + + + + - - - - - - - - - - - - + - - + + - - + + - - + + - - + + - - - + + + - + - - + + - + - + - + - - + + - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - + @@ -43384,94 +43453,45 @@ - + - + - + - + - + - + - + - + - + - + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + @@ -43499,10 +43519,10 @@ - + - + @@ -43511,7 +43531,7 @@ - + @@ -43519,36 +43539,36 @@ - + - + - + - + - + - + - + - + - + - + @@ -43559,7 +43579,7 @@ - + @@ -43582,19 +43602,19 @@ - + - + - + @@ -43639,34 +43659,11 @@ - + - - - - - - - - - - - - - - - - - - - - - - - - - + + @@ -43677,9 +43674,9 @@ - + - + @@ -43697,9 +43694,9 @@ - + - + @@ -43711,128 +43708,238 @@ - + - + - + - + + + + + + + + + + + + + + + + + + + + + + + + + + + - + + + + + + + + + + + + + + + + + + + + + + + + - + - + - + - + - - + + + + + + + + + + + - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - + - + - + - + - + - + - + - + - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - + + - - + + - - + + - - + + - - + + - + - - + + - - + + - - - + + + @@ -43841,102 +43948,102 @@ - + - - + + - - + + - + - - - + + + - + - - + + - - - + + + - - + + - - + + - + - - + + - - + + - - - + + + - + - + - + - + - - - + + + - + - + - + - + @@ -43951,7 +44058,7 @@ - + @@ -43962,7 +44069,7 @@ - + @@ -43992,7 +44099,7 @@ - + @@ -44000,13 +44107,13 @@ - + - + @@ -44015,27 +44122,27 @@ - + - + - + - + - + - + - + @@ -44044,7 +44151,7 @@ - + @@ -44055,7 +44162,7 @@ - + @@ -44082,71 +44189,71 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -44167,7 +44274,1374 @@ - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -44325,12 +45799,12 @@ - + - + - + @@ -44343,7 +45817,7 @@ - + @@ -44354,7 +45828,7 @@ - + @@ -44365,7 +45839,7 @@ - + @@ -44373,13 +45847,13 @@ - + - + @@ -44393,15 +45867,15 @@ - + - + - + @@ -44412,7 +45886,7 @@ - + @@ -44420,1389 +45894,16 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + - + - - + + @@ -45821,29 +45922,29 @@ - - - + + + - - - + + + - + - + @@ -45851,7 +45952,7 @@ - + @@ -45859,7 +45960,7 @@ - + @@ -45870,37 +45971,36 @@ - - - - - + + + + - - + + - - + + - - + + - + - - + + - + @@ -45918,20 +46018,20 @@ - + - + - + - + - - + + @@ -45975,11 +46075,11 @@ - - - + + + - + @@ -45997,8 +46097,8 @@ - - + + @@ -46020,7 +46120,7 @@ - + @@ -46113,15 +46213,15 @@ - - + + - + @@ -46129,76 +46229,76 @@ - + - + - + - + - + - + - + - + - - + + - - + + - - + + - + - + - - + + @@ -46215,22 +46315,22 @@ - + - + - - - + + + - - + + - + @@ -46250,11 +46350,11 @@ - + - + @@ -46268,13 +46368,13 @@ - + - + @@ -46298,7 +46398,7 @@ - + @@ -46306,7 +46406,7 @@ - + @@ -46314,15 +46414,15 @@ - + - - - - + + + + @@ -46342,7 +46442,7 @@ - + @@ -46397,15 +46497,15 @@ - + - + - + @@ -46418,7 +46518,7 @@ - + @@ -46427,8 +46527,8 @@ - - + + @@ -46439,33 +46539,33 @@ - + - - - + + + - + - - - - - - - - - + + + + + + + + + @@ -46476,10 +46576,10 @@ - + - + @@ -46488,7 +46588,7 @@ - + @@ -46497,34 +46597,34 @@ - + - + - + - + - + - + - + - + - + @@ -46620,22 +46720,22 @@ - + - + - + - + - + @@ -46644,7 +46744,7 @@ - + @@ -46656,7 +46756,7 @@ - + @@ -46666,9 +46766,9 @@ - + - + @@ -46689,7 +46789,7 @@ - + @@ -46730,13 +46830,13 @@ - + - + - + @@ -46751,20 +46851,20 @@ - + - + - + - + - + @@ -46773,7 +46873,7 @@ - + @@ -46788,7 +46888,7 @@ - + @@ -46802,7 +46902,7 @@ - + @@ -46814,7 +46914,7 @@ - + @@ -46828,10 +46928,10 @@ - + - + @@ -46839,13 +46939,13 @@ - + - + @@ -46853,7 +46953,7 @@ - + @@ -46868,13 +46968,13 @@ - + - + - + @@ -46889,27 +46989,27 @@ - + - + - + - + - + - + - + - + @@ -46917,13 +47017,13 @@ - + - + @@ -46937,7 +47037,7 @@ - + @@ -46951,7 +47051,7 @@ - + @@ -46960,10 +47060,10 @@ - + - + @@ -46980,16 +47080,16 @@ - + - + - + - + @@ -47006,7 +47106,7 @@ - + @@ -47017,21 +47117,21 @@ - + - + - + - + - + @@ -47040,7 +47140,7 @@ - + @@ -47048,7 +47148,7 @@ - + @@ -47066,7 +47166,7 @@ - + @@ -47092,24 +47192,24 @@ - + - + - + - + - + @@ -47145,9 +47245,9 @@ - + - + @@ -47156,13 +47256,13 @@ - + - + @@ -47191,14 +47291,14 @@ - + - + @@ -47206,16 +47306,16 @@ - + - + - + @@ -47223,7 +47323,7 @@ - + @@ -47238,53 +47338,53 @@ - - - - - - - - - - - - - - + - - - + + + + + + + + + + + + + + + + - - + + - + - - - - - - + + + + + + - + - + - + @@ -47302,7 +47402,7 @@ - + @@ -47311,19 +47411,19 @@ - + - + - + - + @@ -47377,10 +47477,10 @@ - + - + @@ -47389,25 +47489,25 @@ - + - + - + - + - + @@ -47430,13 +47530,13 @@ - + - + - + @@ -47493,13 +47593,13 @@ - + - + - + @@ -47516,8 +47616,8 @@ - - + + @@ -47531,7 +47631,7 @@ - + @@ -47542,140 +47642,140 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -47686,28 +47786,28 @@ - + - + - + - + - + - + @@ -47719,19 +47819,19 @@ - + - + - + - + @@ -47743,40 +47843,40 @@ - + - + - + - + - + - + - + - + - + - + - + @@ -47784,42 +47884,42 @@ - + - + - + - + - + - + - + - + - + - + - + - + @@ -47827,7 +47927,7 @@ - + @@ -47839,48 +47939,48 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -47901,12 +48001,12 @@ - + - + - + @@ -47983,15 +48083,15 @@ - + - + - + @@ -48008,15 +48108,15 @@ - + - + - + - + @@ -48024,13 +48124,13 @@ - + - + @@ -48056,7 +48156,7 @@ - + @@ -48064,13 +48164,13 @@ - + - + @@ -48081,15 +48181,15 @@ - + - + - + - + @@ -48098,13 +48198,13 @@ - + - + @@ -48115,7 +48215,7 @@ - + @@ -48124,185 +48224,185 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - + + + + + - + - - + + - - + + - - + + - - + + - + - + - + - - + + - + - + - + - + - + - - - - - - - - - - - - - - - - - - - - - - - - + + - + - - + + + - + + + - + + + + + + + + + + + + + + + + + + + + - - + + - - + + @@ -48310,186 +48410,186 @@ - - + + - - - - - - - - - - - - - - - - - - - - - - - - - + + - - - + + + - - - - - + + - - - - - - + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - + - - - + + - - + + + + + + + - + + + + + + + + + + + + + + + + + - + - - - - - - - - - - - - - - - - - - + + - - + - - + - - + - - + - + + + + + + + + + + + + + + + + + + + + + - - + + @@ -48497,13 +48597,13 @@ - + - + @@ -48530,48 +48630,48 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -48606,7 +48706,7 @@ - + @@ -48624,7 +48724,7 @@ - + @@ -48644,12 +48744,12 @@ - + - + - + @@ -48661,33 +48761,33 @@ - + - + - + - + - + - + - + - + - + @@ -48702,7 +48802,7 @@ - + @@ -48710,59 +48810,59 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -48771,191 +48871,191 @@ - - - - - - - + - - - - - - + + + + + + + + - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + - + - - + + - - + + - + - + - - - - - - - - - - - - - - - - - - - - - - + - + + - - - - - - + + + + + + - - + + + - + + + + + + + + + + + + + + + + + + + + - - - + + + - - - - - - - - - - - - - - + + + + + - - - - - - - - - - - - + + + - - + + + + + + + + + + + + + + + + + + + + - - + + - + - + - + - + - + - + - + - + @@ -48966,7 +49066,7 @@ - + @@ -49011,16 +49111,16 @@ - + - + - + - + @@ -49032,10 +49132,10 @@ - + - + @@ -49050,19 +49150,19 @@ - + - + - + - + - + @@ -49092,16 +49192,16 @@ - + - + - + @@ -49115,7 +49215,7 @@ - + @@ -49132,24 +49232,24 @@ - + - + - + - + - + - + - + @@ -49164,13 +49264,13 @@ - + - + - + @@ -49181,16 +49281,16 @@ - + - + - + - + @@ -49198,7 +49298,7 @@ - + @@ -49206,32 +49306,32 @@ - + - + - + - + - + - + - + - + - + @@ -49245,7 +49345,7 @@ - + @@ -49253,10 +49353,10 @@ - + - + @@ -49273,16 +49373,16 @@ - + - + - + @@ -49302,7 +49402,7 @@ - + @@ -49310,7 +49410,7 @@ - + @@ -49330,12 +49430,12 @@ - + - + - + @@ -49344,7 +49444,7 @@ - + @@ -49364,7 +49464,7 @@ - + @@ -49375,10 +49475,10 @@ - + - + @@ -49387,30 +49487,30 @@ - + - + - + - + - + - + - + - + @@ -49422,7 +49522,7 @@ - + @@ -49431,7 +49531,7 @@ - + @@ -49442,7 +49542,7 @@ - + @@ -49459,89 +49559,89 @@ - + - + - + - + - + - + - + - + - + - + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - + - + - + - + @@ -49550,104 +49650,104 @@ - + - + - + - - - - - - - - - - - - - - - - - - - - - - - + - - + + + - - - + + + - - - - + + - - - + + - + + + + + + + + + + + + + + + + + + + + + + + + + - - - + + + - - - - - - - - - - - - - - - - - - - - + - + + + + + + + + + + + + + + + + + + + + @@ -49656,14 +49756,14 @@ - + - + @@ -49692,19 +49792,19 @@ - + - + - + - + @@ -49715,13 +49815,13 @@ - + - + @@ -49748,10 +49848,10 @@ - + - + @@ -49781,7 +49881,7 @@ - + @@ -49793,36 +49893,36 @@ - + - + - + - + - + - + - + - + - + - + - + @@ -49830,10 +49930,10 @@ - + - + @@ -49841,90 +49941,90 @@ - + - + - - - - - - - - - - - - - - + + + + + + + + + + + + + + - + - + - + - + - + - + - + - + - + - + - + - + - - + + - + - - + + @@ -49935,64 +50035,64 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -50001,7 +50101,7 @@ - + @@ -50018,10 +50118,10 @@ - + - + @@ -50045,7 +50145,7 @@ - + @@ -50057,19 +50157,19 @@ - + - + - + - + - + @@ -50114,13 +50214,13 @@ - + - + @@ -50129,7 +50229,7 @@ - + @@ -50140,30 +50240,30 @@ - + - + - + - + - + - + - + - + @@ -50175,7 +50275,7 @@ - + @@ -50189,23 +50289,23 @@ - + - + - + - + - + @@ -50216,13 +50316,13 @@ - + - + @@ -50230,12 +50330,12 @@ - + - + @@ -50250,7 +50350,7 @@ - + @@ -50265,32 +50365,32 @@ - + - + - + - + - + - + - + - + - + @@ -50299,13 +50399,13 @@ - + - + - + @@ -50347,13 +50447,13 @@ - + - + - + @@ -50374,77 +50474,77 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -50452,9 +50552,9 @@ - + - + @@ -50463,7 +50563,7 @@ - + @@ -50490,9 +50590,9 @@ - + - + @@ -50504,27 +50604,27 @@ - + - + - + - + - + - + - + @@ -50535,220 +50635,220 @@ - + - + - + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - + - + - - + + - - + + - - + + - + - - - + + + - - - - - - - - - - - - - - - - - - - - - - - - + + + - - + + + + - - + + - - + + - - - + - + + + + + + + + + + + + + + + + + + + + + + - - + + - - + + - + - - + + - + - - - + + + - - - + + + - - + + - - + + - - + + - - + + - - + + - + - + - + - + @@ -50757,22 +50857,22 @@ - + - + - + - + - + - + @@ -50802,7 +50902,7 @@ - + @@ -50814,7 +50914,7 @@ - + @@ -50829,14 +50929,14 @@ - + - + @@ -50844,15 +50944,15 @@ - + - + - + @@ -50867,7 +50967,7 @@ - + @@ -50881,96 +50981,96 @@ - - - - - - - - - - - + + + + + + + + + + + - - + + - + - - - + + + - - + + - - - + + + - - - + + + - - + + - - + + - - - + + + - + - - + + - + - + - + - + - + - + @@ -50981,20 +51081,20 @@ - + - + - - + + @@ -51005,7 +51105,7 @@ - + @@ -51016,7 +51116,7 @@ - + @@ -51028,7 +51128,7 @@ - + @@ -51042,10 +51142,10 @@ - + - + @@ -51053,34 +51153,34 @@ - + - + - + - + - + - - + + - + - + @@ -51088,21 +51188,21 @@ - + - + - + - + - + - + @@ -51122,7 +51222,7 @@ - + @@ -51142,7 +51242,7 @@ - + @@ -51159,9 +51259,9 @@ - + - + @@ -51173,34 +51273,32 @@ - - - - - - - - - - - + + + + + + + + + - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + @@ -51211,148 +51309,150 @@ + + - - - - - - - - - - - - + + + + + + + + + + + + - - - - - - - - - - - + + + + + + + + + + + - - - + + + - - + + - - + + - - + + - - + + - + - + - + - + - - - - - - - - - - + - + - + + + + + + + + + + - - + + - + - + - - + + - + - - - - - - - - - - - + - - + + - + + + + + + + + + + + @@ -51365,7 +51465,7 @@ - + @@ -51373,7 +51473,7 @@ - + @@ -51388,46 +51488,46 @@ - + - - - + + + - + - + - + - + - + - + - - + + - + @@ -51435,7 +51535,7 @@ - + @@ -51443,7 +51543,7 @@ - + @@ -51454,26 +51554,26 @@ - + - + - + - + - + - + @@ -51481,7 +51581,7 @@ - + @@ -51489,10 +51589,10 @@ - + - + @@ -51500,7 +51600,7 @@ - + @@ -51535,22 +51635,22 @@ - + - + - + - + - + - + @@ -51571,10 +51671,10 @@ - + - + @@ -51583,7 +51683,7 @@ - + @@ -51591,12 +51691,12 @@ - + - + - + @@ -51629,8 +51729,8 @@ - - + + @@ -51641,7 +51741,7 @@ - + @@ -51667,7 +51767,7 @@ - + @@ -51681,7 +51781,7 @@ - + @@ -51689,7 +51789,7 @@ - + @@ -51700,7 +51800,7 @@ - + @@ -51717,12 +51817,12 @@ - + - + - + @@ -51752,9 +51852,9 @@ - + - + @@ -51763,12 +51863,12 @@ - + - + @@ -51783,13 +51883,13 @@ - + - + - + @@ -51801,55 +51901,55 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -51864,16 +51964,16 @@ - + - + - + @@ -51885,28 +51985,28 @@ - + - + - + - + - + - + - + @@ -51923,15 +52023,15 @@ - + - + - + - + @@ -51943,7 +52043,7 @@ - + @@ -51969,7 +52069,7 @@ - + @@ -51981,10 +52081,10 @@ - + - + @@ -51998,7 +52098,7 @@ - + @@ -52009,7 +52109,7 @@ - + @@ -52023,7 +52123,7 @@ - + @@ -52034,10 +52134,10 @@ - + - + @@ -52048,7 +52148,7 @@ - + @@ -52071,24 +52171,24 @@ - + - + - + - + - + - + - + @@ -52099,7 +52199,7 @@ - + @@ -52113,7 +52213,7 @@ - + @@ -52127,25 +52227,25 @@ - + - + - + - + - + - + - + @@ -52156,7 +52256,7 @@ - + @@ -52167,16 +52267,16 @@ - + - + - + - + @@ -52185,7 +52285,7 @@ - + @@ -52196,7 +52296,7 @@ - + @@ -52237,15 +52337,15 @@ - + - + - + - + @@ -52266,7 +52366,7 @@ - + @@ -52274,15 +52374,15 @@ - + - + - + @@ -52290,21 +52390,21 @@ - + - + - + - + - + @@ -52324,10 +52424,10 @@ - + - + @@ -52336,16 +52436,16 @@ - + - + - + - + @@ -52353,9 +52453,9 @@ - + - + @@ -52379,7 +52479,7 @@ - + @@ -52391,10 +52491,10 @@ - + - + @@ -52402,14 +52502,14 @@ - + - + - + - + @@ -52430,10 +52530,10 @@ - + - + @@ -52442,24 +52542,24 @@ - + - + - + - + - + @@ -52468,13 +52568,13 @@ - + - + @@ -52482,10 +52582,10 @@ - + - + @@ -52524,7 +52624,7 @@ - + @@ -52551,13 +52651,13 @@ - + - + @@ -52575,21 +52675,21 @@ - + - + - + - + - + @@ -52622,9 +52722,9 @@ - + - + @@ -52642,7 +52742,7 @@ - + @@ -52650,7 +52750,7 @@ - + @@ -52671,7 +52771,7 @@ - + @@ -52682,10 +52782,10 @@ - + - + @@ -52703,21 +52803,21 @@ - + - + - + - + - + @@ -52771,10 +52871,10 @@ - + - + @@ -52786,10 +52886,10 @@ - + - + @@ -52803,31 +52903,31 @@ - + - + - + - + - + - + - + @@ -52872,18 +52972,18 @@ - + - + - + - + - + @@ -52898,10 +52998,10 @@ - + - + @@ -52939,38 +53039,38 @@ - + - + - + - + - + - + - + - + - + @@ -52978,23 +53078,23 @@ - + - + - + - + - + - + @@ -53002,108 +53102,108 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + - + - + - + - + - + @@ -53111,11 +53211,11 @@ - + - + @@ -53123,59 +53223,59 @@ - + - - + + - - + + - - + + - - + + - - + + - + - + - + - + @@ -53183,7 +53283,7 @@ - + @@ -53192,12 +53292,12 @@ - + - + @@ -53205,164 +53305,164 @@ - - + + - + - + - + - + - - - - - - - - - - - - - - - - - - - + - + - - + + + + + + + + + + + + + + + + + + + + - - + + - - + + - + - + - + - - + + - + - - + + - + - - + + - - + + - - - - - - - - - - - - + + - - + + - - + + - - - + + - - + + - - - - - - + + + - + + + + + + + + + + + + + + + - - + + - + @@ -53372,43 +53472,43 @@ - - + + - + - + - - + + - + - + - + - + - + @@ -53417,15 +53517,15 @@ - + - + - + @@ -53433,27 +53533,27 @@ - - - - + + + + - + - + - + - + - + @@ -53461,65 +53561,65 @@ - - + + - - - + + + - + - + - + - + - + - + - + - + - + - + - + - - + + - + @@ -53527,24 +53627,24 @@ - + - + - + - + - + - + - + @@ -53576,21 +53676,21 @@ - + - + - - + + - + @@ -53599,7 +53699,7 @@ - + @@ -53608,19 +53708,19 @@ - - + + - + - + - + @@ -53659,13 +53759,13 @@ - + - + @@ -53707,7 +53807,7 @@ - + @@ -53719,18 +53819,18 @@ - + - + - + @@ -53739,10 +53839,10 @@ - + - + @@ -53751,10 +53851,10 @@ - + - + @@ -53763,7 +53863,7 @@ - + @@ -53771,19 +53871,19 @@ - + - + - + - + @@ -53792,13 +53892,13 @@ - + - + @@ -53807,25 +53907,25 @@ - + - + - + - + - + @@ -53837,7 +53937,7 @@ - + @@ -53846,109 +53946,109 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -53956,15 +54056,15 @@ - + - + - + @@ -53973,8 +54073,8 @@ - - + + @@ -54002,7 +54102,7 @@ - + @@ -54013,24 +54113,24 @@ - + - + - + - + - + - + @@ -54039,9 +54139,9 @@ - + - + @@ -54050,7 +54150,7 @@ - + @@ -54065,59 +54165,59 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -54126,148 +54226,148 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - - + + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - - + + - + @@ -54276,53 +54376,53 @@ - + - + - + - + - + - + - + - + - + - + - + - + - - + + - + - + @@ -54336,67 +54436,67 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -54404,9 +54504,9 @@ - + - + @@ -54415,7 +54515,7 @@ - + @@ -54423,19 +54523,19 @@ - + - + - + - + - + @@ -54450,108 +54550,108 @@ - + - + - + - + - + - + - + - + - + - + - + - - - - - + - + - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - + - + @@ -54560,147 +54660,147 @@ - - - - - - - - - - - - - - - - - - - - - + + + - + + + + + + + + + + + + + + + + + + + - - + + - - + + - - + + - - + + - - - + + + - - + + - - + + - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + - - + + + + + + + + + - - + - - + + + + + + + + + + + + - + - + - + - + - + - + - - + + - + - - - + + + @@ -54710,67 +54810,67 @@ - - - + + + - - + + - - + + - - + + - - + + - - + + - - + + - + - - - + + + - + - + - - + + @@ -54778,48 +54878,48 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -54828,13 +54928,13 @@ - + - + - + @@ -54845,7 +54945,7 @@ - + @@ -54859,7 +54959,7 @@ - + @@ -54873,7 +54973,7 @@ - + @@ -54887,7 +54987,7 @@ - + @@ -54901,21 +55001,21 @@ - + - + - + - + - + @@ -54932,33 +55032,33 @@ - + - + - + - + - + - + - + - - + + @@ -54969,13 +55069,13 @@ - + - + - + @@ -54984,7 +55084,7 @@ - + @@ -55017,7 +55117,7 @@ - + @@ -55026,15 +55126,15 @@ - + - + - + @@ -55079,9 +55179,9 @@ - + - + @@ -55090,7 +55190,7 @@ - + @@ -55098,10 +55198,10 @@ - + - + @@ -55109,7 +55209,7 @@ - + @@ -55117,7 +55217,7 @@ - + @@ -55128,13 +55228,13 @@ - + - + - + @@ -55143,21 +55243,21 @@ - + - + - + - + - + @@ -55168,7 +55268,7 @@ - + @@ -55176,21 +55276,21 @@ - + - + - + - + - - + + @@ -55199,26 +55299,26 @@ - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + @@ -55235,8 +55335,8 @@ - - + + @@ -55260,11 +55360,11 @@ - + - + @@ -55319,7 +55419,7 @@ - + @@ -55334,7 +55434,7 @@ - + @@ -55384,33 +55484,33 @@ - + - + - + - + - + - + - + - + @@ -55489,41 +55589,39 @@ - + - - + + - + - + - + - + - - - + + + - + - + - - - - + + @@ -55546,7 +55644,7 @@ - + @@ -55563,11 +55661,11 @@ - + - - + + @@ -55577,73 +55675,73 @@ - + - + - + - + - + - + - + - + - + - + - + - + - - - - + + + + - - + + - + - + - + - + - + @@ -55679,7 +55777,7 @@ - + @@ -55687,7 +55785,7 @@ - + @@ -55695,9 +55793,9 @@ - + - + @@ -55706,7 +55804,7 @@ - + @@ -55714,10 +55812,10 @@ - + - + @@ -55735,125 +55833,125 @@ - + - + - + - - + + - + - + - + - - - - - + - - + + - + - + + + + + - + - + - + - + - + - + - + - + - - + + - + - + - + - + - + - + - + - + - + @@ -55880,13 +55978,13 @@ - + - + @@ -55952,10 +56050,10 @@ - + - + @@ -55982,7 +56080,7 @@ - + @@ -55991,13 +56089,13 @@ - + - + - + @@ -56009,10 +56107,10 @@ - + - + @@ -56021,13 +56119,13 @@ - + - + - + @@ -56066,18 +56164,18 @@ - + - + - + - + @@ -56085,9 +56183,9 @@ - + - + @@ -56111,7 +56209,7 @@ - + @@ -56119,13 +56217,13 @@ - + - + @@ -56145,12 +56243,12 @@ - + - + - + @@ -56165,7 +56263,7 @@ - + @@ -56176,7 +56274,7 @@ - + @@ -56199,19 +56297,19 @@ - + - + - + - + @@ -56225,9 +56323,9 @@ - + - + @@ -56242,7 +56340,7 @@ - + @@ -56251,7 +56349,7 @@ - + @@ -56269,10 +56367,10 @@ - + - + @@ -56283,7 +56381,7 @@ - + @@ -56294,12 +56392,12 @@ - + - + @@ -56308,7 +56406,7 @@ - + @@ -56320,7 +56418,7 @@ - + @@ -56332,16 +56430,16 @@ - + - + - + @@ -56350,10 +56448,10 @@ - + - + @@ -56377,38 +56475,38 @@ - + - + - + - + - + - + - + - + - + @@ -56416,15 +56514,15 @@ - + - + - + @@ -56432,12 +56530,12 @@ - + - + - + @@ -56464,10 +56562,10 @@ - + - + @@ -56479,13 +56577,13 @@ - + - + - + @@ -56494,7 +56592,7 @@ - + @@ -56503,13 +56601,13 @@ - + - + - + @@ -56545,7 +56643,7 @@ - + @@ -56557,7 +56655,7 @@ - + @@ -56575,18 +56673,18 @@ - + - + - + - + @@ -56603,16 +56701,16 @@ - + - + - + @@ -56620,7 +56718,7 @@ - + @@ -56646,33 +56744,33 @@ - + - + - + - + - + - + - + - + - + - + @@ -56687,13 +56785,13 @@ - + - + @@ -56702,12 +56800,12 @@ - + - + - + @@ -56740,7 +56838,7 @@ - + @@ -56749,13 +56847,13 @@ - + - + @@ -56769,15 +56867,15 @@ - + - + - + - + @@ -56786,10 +56884,10 @@ - + - + @@ -56800,15 +56898,15 @@ - + - + - + @@ -56823,7 +56921,7 @@ - + @@ -56834,12 +56932,12 @@ - + - + - + @@ -56851,15 +56949,15 @@ - + - + - + - + @@ -56868,22 +56966,22 @@ - + - + - + - + - + @@ -56897,7 +56995,7 @@ - + @@ -56911,15 +57009,15 @@ - + - + - + - + @@ -56928,7 +57026,7 @@ - + @@ -56936,74 +57034,74 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -57011,16 +57109,16 @@ - + - + - + - + @@ -57029,39 +57127,39 @@ - + - + - + - + - + - + @@ -57069,24 +57167,24 @@ - + - + - + - + - + - + - + @@ -57098,7 +57196,7 @@ - + @@ -57109,7 +57207,7 @@ - + @@ -57117,15 +57215,15 @@ - + - + - + - + @@ -57139,18 +57237,18 @@ - + - + - + - + @@ -57159,15 +57257,15 @@ - + - + - + @@ -57187,18 +57285,18 @@ - + - + - + - + @@ -57207,7 +57305,7 @@ - + @@ -57215,9 +57313,9 @@ - + - + @@ -57226,7 +57324,7 @@ - + @@ -57234,10 +57332,10 @@ - + - + @@ -57245,7 +57343,7 @@ - + @@ -57256,10 +57354,10 @@ - + - + @@ -57288,333 +57386,333 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -57622,7 +57720,7 @@ - + @@ -57631,12 +57729,12 @@ - + - + - + @@ -57651,19 +57749,19 @@ - + - + - + - + @@ -57675,22 +57773,22 @@ - + - + - + - + - + @@ -57705,10 +57803,10 @@ - + - + @@ -57761,13 +57859,13 @@ - + - + @@ -57784,7 +57882,7 @@ - + @@ -57799,7 +57897,7 @@ - + @@ -57814,7 +57912,7 @@ - + @@ -57828,7 +57926,7 @@ - + @@ -57836,48 +57934,48 @@ - + - + - + - + - + - + - + - + - + - + - + - + @@ -57888,7 +57986,7 @@ - + @@ -57923,10 +58021,10 @@ - + - + @@ -57935,7 +58033,7 @@ - + @@ -57968,7 +58066,7 @@ - + @@ -57977,10 +58075,10 @@ - + - + @@ -57989,21 +58087,21 @@ - + - + - + - + @@ -58014,7 +58112,7 @@ - + @@ -58052,16 +58150,16 @@ - + - + - + - + @@ -58079,10 +58177,10 @@ - + - + @@ -58100,13 +58198,13 @@ - + - + - + @@ -58127,7 +58225,7 @@ - + @@ -58148,7 +58246,7 @@ - + @@ -58177,7 +58275,7 @@ - + @@ -58188,7 +58286,7 @@ - + @@ -58196,7 +58294,7 @@ - + @@ -58213,10 +58311,10 @@ - + - + @@ -58254,7 +58352,7 @@ - + @@ -58289,7 +58387,7 @@ - + @@ -58369,7 +58467,7 @@ - + @@ -58378,9 +58476,9 @@ - + - + @@ -58428,7 +58526,7 @@ - + @@ -58437,12 +58535,12 @@ - + - + - + @@ -58471,9 +58569,9 @@ - + - + @@ -58488,7 +58586,7 @@ - + @@ -58496,9 +58594,9 @@ - + - + @@ -58507,7 +58605,7 @@ - + @@ -58525,28 +58623,28 @@ - + - + - + - + - + - + - + @@ -58555,23 +58653,23 @@ - + - + - + - + - + @@ -58580,7 +58678,7 @@ - + @@ -58598,9 +58696,9 @@ - + - + @@ -58612,7 +58710,7 @@ - + @@ -58621,16 +58719,16 @@ - + - + - + - + @@ -58642,7 +58740,7 @@ - + @@ -58659,9 +58757,9 @@ - + - + @@ -58676,12 +58774,12 @@ - + - + - + @@ -58696,7 +58794,7 @@ - + @@ -58708,10 +58806,10 @@ - + - + @@ -58729,22 +58827,22 @@ - + - + - + - + - + @@ -58774,24 +58872,24 @@ - + - + - + - + - + - + @@ -58800,15 +58898,15 @@ - + - + - + - + @@ -58816,7 +58914,7 @@ - + @@ -58852,21 +58950,21 @@ - + - + - + - + - + @@ -58896,7 +58994,7 @@ - + @@ -58919,9 +59017,9 @@ - + - + @@ -58945,10 +59043,10 @@ - + - + @@ -58959,7 +59057,7 @@ - + @@ -58967,7 +59065,7 @@ - + @@ -58978,7 +59076,7 @@ - + @@ -58986,45 +59084,45 @@ - + - + - + - + - + - + - + - + - + - + - + @@ -59033,7 +59131,7 @@ - + @@ -59045,18 +59143,18 @@ - + - + - + - + @@ -59064,7 +59162,7 @@ - + @@ -59072,7 +59170,7 @@ - + @@ -59080,12 +59178,12 @@ - + - + - + @@ -59106,7 +59204,7 @@ - + @@ -59130,16 +59228,16 @@ - + - + - + @@ -59154,7 +59252,7 @@ - + @@ -59162,7 +59260,7 @@ - + @@ -59170,7 +59268,7 @@ - + @@ -59184,15 +59282,15 @@ - + - + - + - + @@ -59204,12 +59302,12 @@ - + - + @@ -59241,7 +59339,7 @@ - + @@ -59258,35 +59356,35 @@ - + - + - + - + - + - + - + - + - + @@ -59316,16 +59414,16 @@ - + - + - + @@ -59337,621 +59435,623 @@ - - - - - - - - - - - - - - - - - - - - + + + + + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - + - + - + - + - + - - - - - - - - - - - - - - - - + - - + + - - + + - - - + + - - - + + - + - - + - + - + - + - - + - + - - - - - - - - + - - + + - + + - - + + + + + + + + + - + - + - + - + - + - + - + - + - + - + - + + + + + + + - - - - - - - + - + - + - + - + - + - + - - + - + - + - + - - + - + - - + + - + - - + - + - + - + - + - + - + - - - - + - + - - + - + - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + - - + + - + - - + + + + + + + + + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - - - - - + - - + + - + - - + - + + + + + + + + + + + + + + + + + + + + + + - - + + - - + + - - + + @@ -59959,230 +60059,230 @@ - - + + - - + + - + - - + + - + - - + + - - + + - - - - - - - - - - - - - - - - - - + - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + - - - + + + + - - + + - - - + + + + - - - - + + + - - - - + + + - - - + + + + + + + + + + + + + + + + + + + + + + + - - - + + + - - - + + + - - - + + + - - - + + + - + - + - - + + - - + + - - - + + + - - + + - - - + + + - - - - + + + + - - - + + + - - - + + + - - + + - - - + + + - + - + - + @@ -60194,32 +60294,32 @@ - + - + - + - - + + - - + + - - - + + + @@ -60227,16 +60327,16 @@ - + - - + + - + @@ -60251,7 +60351,7 @@ - + @@ -60260,12 +60360,12 @@ - + - + @@ -60279,7 +60379,7 @@ - + @@ -60287,7 +60387,7 @@ - + @@ -60302,13 +60402,13 @@ - + - + - + @@ -60331,17 +60431,17 @@ - + - + - + @@ -60383,21 +60483,21 @@ - + - + - + - + @@ -60420,9 +60520,9 @@ - + - + diff --git a/arch/arm64/configs/gki_defconfig b/arch/arm64/configs/gki_defconfig index 156624c5faeb..1e7fccfc07de 100644 --- a/arch/arm64/configs/gki_defconfig +++ b/arch/arm64/configs/gki_defconfig @@ -80,6 +80,7 @@ CONFIG_MODULES=y CONFIG_MODULE_UNLOAD=y CONFIG_MODVERSIONS=y CONFIG_BLK_INLINE_ENCRYPTION=y +CONFIG_BLK_INLINE_ENCRYPTION_FALLBACK=y CONFIG_GKI_HACKS_TO_FIX=y # CONFIG_CORE_DUMP_DEFAULT_ELF_HEADERS is not set CONFIG_BINFMT_MISC=m diff --git a/arch/x86/configs/gki_defconfig b/arch/x86/configs/gki_defconfig index 5a8f671329a2..d456c9857d6c 100644 --- a/arch/x86/configs/gki_defconfig +++ b/arch/x86/configs/gki_defconfig @@ -55,6 +55,7 @@ CONFIG_MODULES=y CONFIG_MODULE_UNLOAD=y CONFIG_MODVERSIONS=y CONFIG_BLK_INLINE_ENCRYPTION=y +CONFIG_BLK_INLINE_ENCRYPTION_FALLBACK=y CONFIG_GKI_HACKS_TO_FIX=y # CONFIG_CORE_DUMP_DEFAULT_ELF_HEADERS is not set CONFIG_BINFMT_MISC=m From 75579189e9bf156b671fe2eabfccbddc4df7a9c1 Mon Sep 17 00:00:00 2001 From: Andrey Konovalov Date: Wed, 4 Dec 2019 16:52:43 -0800 Subject: [PATCH 05/89] UPSTREAM: kcov: remote coverage support (Upstream commit eec028c9386ed1a692aa01a85b55952202b41619.) Patch series " kcov: collect coverage from usb and vhost", v3. This patchset extends kcov to allow collecting coverage from backgound kernel threads. This extension requires custom annotations for each of the places where coverage collection is desired. This patchset implements this for hub events in the USB subsystem and for vhost workers. See the first patch description for details about the kcov extension. The other two patches apply this kcov extension to USB and vhost. Examples of other subsystems that might potentially benefit from this when custom annotations are added (the list is based on process_one_work() callers for bugs recently reported by syzbot): 1. fs: writeback wb_workfn() worker, 2. net: addrconf_dad_work()/addrconf_verify_work() workers, 3. net: neigh_periodic_work() worker, 4. net/p9: p9_write_work()/p9_read_work() workers, 5. block: blk_mq_run_work_fn() worker. These patches have been used to enable coverage-guided USB fuzzing with syzkaller for the last few years, see the details here: https://github.com/google/syzkaller/blob/master/docs/linux/external_fuzzing_usb.md This patchset has been pushed to the public Linux kernel Gerrit instance: https://linux-review.googlesource.com/c/linux/kernel/git/torvalds/linux/+/1524 This patch (of 3): Add background thread coverage collection ability to kcov. With KCOV_ENABLE coverage is collected only for syscalls that are issued from the current process. With KCOV_REMOTE_ENABLE it's possible to collect coverage for arbitrary parts of the kernel code, provided that those parts are annotated with kcov_remote_start()/kcov_remote_stop(). This allows to collect coverage from two types of kernel background threads: the global ones, that are spawned during kernel boot in a limited number of instances (e.g. one USB hub_event() worker thread is spawned per USB HCD); and the local ones, that are spawned when a user interacts with some kernel interface (e.g. vhost workers). To enable collecting coverage from a global background thread, a unique global handle must be assigned and passed to the corresponding kcov_remote_start() call. Then a userspace process can pass a list of such handles to the KCOV_REMOTE_ENABLE ioctl in the handles array field of the kcov_remote_arg struct. This will attach the used kcov device to the code sections, that are referenced by those handles. Since there might be many local background threads spawned from different userspace processes, we can't use a single global handle per annotation. Instead, the userspace process passes a non-zero handle through the common_handle field of the kcov_remote_arg struct. This common handle gets saved to the kcov_handle field in the current task_struct and needs to be passed to the newly spawned threads via custom annotations. Those threads should in turn be annotated with kcov_remote_start()/kcov_remote_stop(). Internally kcov stores handles as u64 integers. The top byte of a handle is used to denote the id of a subsystem that this handle belongs to, and the lower 4 bytes are used to denote the id of a thread instance within that subsystem. A reserved value 0 is used as a subsystem id for common handles as they don't belong to a particular subsystem. The bytes 4-7 are currently reserved and must be zero. In the future the number of bytes used for the subsystem or handle ids might be increased. When a particular userspace process collects coverage by via a common handle, kcov will collect coverage for each code section that is annotated to use the common handle obtained as kcov_handle from the current task_struct. However non common handles allow to collect coverage selectively from different subsystems. Link: http://lkml.kernel.org/r/e90e315426a384207edbec1d6aa89e43008e4caf.1572366574.git.andreyknvl@google.com Signed-off-by: Andrey Konovalov Cc: Dmitry Vyukov Cc: Greg Kroah-Hartman Cc: Alan Stern Cc: "Michael S. Tsirkin" Cc: Jason Wang Cc: Arnd Bergmann Cc: Steven Rostedt Cc: David Windsor Cc: Elena Reshetova Cc: Anders Roxell Cc: Alexander Potapenko Cc: Marco Elver Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds Signed-off-by: Andrey Konovalov Bug: 147413187 Change-Id: I868c4846a412bfbae16086017e113813571df377 --- Documentation/dev-tools/kcov.rst | 129 ++++++++ include/linux/kcov.h | 23 ++ include/linux/sched.h | 8 + include/uapi/linux/kcov.h | 28 ++ kernel/kcov.c | 547 +++++++++++++++++++++++++++++-- 5 files changed, 700 insertions(+), 35 deletions(-) diff --git a/Documentation/dev-tools/kcov.rst b/Documentation/dev-tools/kcov.rst index 42b612677799..36890b026e77 100644 --- a/Documentation/dev-tools/kcov.rst +++ b/Documentation/dev-tools/kcov.rst @@ -34,6 +34,7 @@ Profiling data will only become accessible once debugfs has been mounted:: Coverage collection ------------------- + The following program demonstrates coverage collection from within a test program using kcov: @@ -128,6 +129,7 @@ only need to enable coverage (disable happens automatically on thread end). Comparison operands collection ------------------------------ + Comparison operands collection is similar to coverage collection: .. code-block:: c @@ -202,3 +204,130 @@ Comparison operands collection is similar to coverage collection: Note that the kcov modes (coverage collection or comparison operands) are mutually exclusive. + +Remote coverage collection +-------------------------- + +With KCOV_ENABLE coverage is collected only for syscalls that are issued +from the current process. With KCOV_REMOTE_ENABLE it's possible to collect +coverage for arbitrary parts of the kernel code, provided that those parts +are annotated with kcov_remote_start()/kcov_remote_stop(). + +This allows to collect coverage from two types of kernel background +threads: the global ones, that are spawned during kernel boot in a limited +number of instances (e.g. one USB hub_event() worker thread is spawned per +USB HCD); and the local ones, that are spawned when a user interacts with +some kernel interface (e.g. vhost workers). + +To enable collecting coverage from a global background thread, a unique +global handle must be assigned and passed to the corresponding +kcov_remote_start() call. Then a userspace process can pass a list of such +handles to the KCOV_REMOTE_ENABLE ioctl in the handles array field of the +kcov_remote_arg struct. This will attach the used kcov device to the code +sections, that are referenced by those handles. + +Since there might be many local background threads spawned from different +userspace processes, we can't use a single global handle per annotation. +Instead, the userspace process passes a non-zero handle through the +common_handle field of the kcov_remote_arg struct. This common handle gets +saved to the kcov_handle field in the current task_struct and needs to be +passed to the newly spawned threads via custom annotations. Those threads +should in turn be annotated with kcov_remote_start()/kcov_remote_stop(). + +Internally kcov stores handles as u64 integers. The top byte of a handle +is used to denote the id of a subsystem that this handle belongs to, and +the lower 4 bytes are used to denote the id of a thread instance within +that subsystem. A reserved value 0 is used as a subsystem id for common +handles as they don't belong to a particular subsystem. The bytes 4-7 are +currently reserved and must be zero. In the future the number of bytes +used for the subsystem or handle ids might be increased. + +When a particular userspace proccess collects coverage by via a common +handle, kcov will collect coverage for each code section that is annotated +to use the common handle obtained as kcov_handle from the current +task_struct. However non common handles allow to collect coverage +selectively from different subsystems. + +.. code-block:: c + + struct kcov_remote_arg { + unsigned trace_mode; + unsigned area_size; + unsigned num_handles; + uint64_t common_handle; + uint64_t handles[0]; + }; + + #define KCOV_INIT_TRACE _IOR('c', 1, unsigned long) + #define KCOV_DISABLE _IO('c', 101) + #define KCOV_REMOTE_ENABLE _IOW('c', 102, struct kcov_remote_arg) + + #define COVER_SIZE (64 << 10) + + #define KCOV_TRACE_PC 0 + + #define KCOV_SUBSYSTEM_COMMON (0x00ull << 56) + #define KCOV_SUBSYSTEM_USB (0x01ull << 56) + + #define KCOV_SUBSYSTEM_MASK (0xffull << 56) + #define KCOV_INSTANCE_MASK (0xffffffffull) + + static inline __u64 kcov_remote_handle(__u64 subsys, __u64 inst) + { + if (subsys & ~KCOV_SUBSYSTEM_MASK || inst & ~KCOV_INSTANCE_MASK) + return 0; + return subsys | inst; + } + + #define KCOV_COMMON_ID 0x42 + #define KCOV_USB_BUS_NUM 1 + + int main(int argc, char **argv) + { + int fd; + unsigned long *cover, n, i; + struct kcov_remote_arg *arg; + + fd = open("/sys/kernel/debug/kcov", O_RDWR); + if (fd == -1) + perror("open"), exit(1); + if (ioctl(fd, KCOV_INIT_TRACE, COVER_SIZE)) + perror("ioctl"), exit(1); + cover = (unsigned long*)mmap(NULL, COVER_SIZE * sizeof(unsigned long), + PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0); + if ((void*)cover == MAP_FAILED) + perror("mmap"), exit(1); + + /* Enable coverage collection via common handle and from USB bus #1. */ + arg = calloc(1, sizeof(*arg) + sizeof(uint64_t)); + if (!arg) + perror("calloc"), exit(1); + arg->trace_mode = KCOV_TRACE_PC; + arg->area_size = COVER_SIZE; + arg->num_handles = 1; + arg->common_handle = kcov_remote_handle(KCOV_SUBSYSTEM_COMMON, + KCOV_COMMON_ID); + arg->handles[0] = kcov_remote_handle(KCOV_SUBSYSTEM_USB, + KCOV_USB_BUS_NUM); + if (ioctl(fd, KCOV_REMOTE_ENABLE, arg)) + perror("ioctl"), free(arg), exit(1); + free(arg); + + /* + * Here the user needs to trigger execution of a kernel code section + * that is either annotated with the common handle, or to trigger some + * activity on USB bus #1. + */ + sleep(2); + + n = __atomic_load_n(&cover[0], __ATOMIC_RELAXED); + for (i = 0; i < n; i++) + printf("0x%lx\n", cover[i + 1]); + if (ioctl(fd, KCOV_DISABLE, 0)) + perror("ioctl"), exit(1); + if (munmap(cover, COVER_SIZE * sizeof(unsigned long))) + perror("munmap"), exit(1); + if (close(fd)) + perror("close"), exit(1); + return 0; + } diff --git a/include/linux/kcov.h b/include/linux/kcov.h index b76a1807028d..a10e84707d82 100644 --- a/include/linux/kcov.h +++ b/include/linux/kcov.h @@ -37,12 +37,35 @@ do { \ (t)->kcov_mode &= ~KCOV_IN_CTXSW; \ } while (0) +/* See Documentation/dev-tools/kcov.rst for usage details. */ +void kcov_remote_start(u64 handle); +void kcov_remote_stop(void); +u64 kcov_common_handle(void); + +static inline void kcov_remote_start_common(u64 id) +{ + kcov_remote_start(kcov_remote_handle(KCOV_SUBSYSTEM_COMMON, id)); +} + +static inline void kcov_remote_start_usb(u64 id) +{ + kcov_remote_start(kcov_remote_handle(KCOV_SUBSYSTEM_USB, id)); +} + #else static inline void kcov_task_init(struct task_struct *t) {} static inline void kcov_task_exit(struct task_struct *t) {} static inline void kcov_prepare_switch(struct task_struct *t) {} static inline void kcov_finish_switch(struct task_struct *t) {} +static inline void kcov_remote_start(u64 handle) {} +static inline void kcov_remote_stop(void) {} +static inline u64 kcov_common_handle(void) +{ + return 0; +} +static inline void kcov_remote_start_common(u64 id) {} +static inline void kcov_remote_start_usb(u64 id) {} #endif /* CONFIG_KCOV */ #endif /* _LINUX_KCOV_H */ diff --git a/include/linux/sched.h b/include/linux/sched.h index e2f891bb62a2..6633594ee180 100644 --- a/include/linux/sched.h +++ b/include/linux/sched.h @@ -1209,6 +1209,8 @@ struct task_struct { #endif /* CONFIG_TRACING */ #ifdef CONFIG_KCOV + /* See kernel/kcov.c for more details. */ + /* Coverage collection mode enabled for this task (0 if disabled): */ unsigned int kcov_mode; @@ -1220,6 +1222,12 @@ struct task_struct { /* KCOV descriptor wired with this task or NULL: */ struct kcov *kcov; + + /* KCOV common handle for remote coverage collection: */ + u64 kcov_handle; + + /* KCOV sequence number: */ + int kcov_sequence; #endif #ifdef CONFIG_MEMCG diff --git a/include/uapi/linux/kcov.h b/include/uapi/linux/kcov.h index 9529867717a8..409d3ad1e6e2 100644 --- a/include/uapi/linux/kcov.h +++ b/include/uapi/linux/kcov.h @@ -4,9 +4,24 @@ #include +/* + * Argument for KCOV_REMOTE_ENABLE ioctl, see Documentation/dev-tools/kcov.rst + * and the comment before kcov_remote_start() for usage details. + */ +struct kcov_remote_arg { + unsigned int trace_mode; /* KCOV_TRACE_PC or KCOV_TRACE_CMP */ + unsigned int area_size; /* Length of coverage buffer in words */ + unsigned int num_handles; /* Size of handles array */ + __u64 common_handle; + __u64 handles[0]; +}; + +#define KCOV_REMOTE_MAX_HANDLES 0x100 + #define KCOV_INIT_TRACE _IOR('c', 1, unsigned long) #define KCOV_ENABLE _IO('c', 100) #define KCOV_DISABLE _IO('c', 101) +#define KCOV_REMOTE_ENABLE _IOW('c', 102, struct kcov_remote_arg) enum { /* @@ -32,4 +47,17 @@ enum { #define KCOV_CMP_SIZE(n) ((n) << 1) #define KCOV_CMP_MASK KCOV_CMP_SIZE(3) +#define KCOV_SUBSYSTEM_COMMON (0x00ull << 56) +#define KCOV_SUBSYSTEM_USB (0x01ull << 56) + +#define KCOV_SUBSYSTEM_MASK (0xffull << 56) +#define KCOV_INSTANCE_MASK (0xffffffffull) + +static inline __u64 kcov_remote_handle(__u64 subsys, __u64 inst) +{ + if (subsys & ~KCOV_SUBSYSTEM_MASK || inst & ~KCOV_INSTANCE_MASK) + return 0; + return subsys | inst; +} + #endif /* _LINUX_KCOV_IOCTLS_H */ diff --git a/kernel/kcov.c b/kernel/kcov.c index 2ee38727844a..f50354202dbe 100644 --- a/kernel/kcov.c +++ b/kernel/kcov.c @@ -9,6 +9,7 @@ #include #include #include +#include #include #include #include @@ -21,8 +22,11 @@ #include #include #include +#include #include +#define kcov_debug(fmt, ...) pr_debug("%s: " fmt, __func__, ##__VA_ARGS__) + /* Number of 64-bit words written per one comparison: */ #define KCOV_WORDS_PER_CMP 4 @@ -44,19 +48,100 @@ struct kcov { * Reference counter. We keep one for: * - opened file descriptor * - task with enabled coverage (we can't unwire it from another task) + * - each code section for remote coverage collection */ refcount_t refcount; /* The lock protects mode, size, area and t. */ spinlock_t lock; enum kcov_mode mode; - /* Size of arena (in long's for KCOV_MODE_TRACE). */ - unsigned size; + /* Size of arena (in long's). */ + unsigned int size; /* Coverage buffer shared with user space. */ void *area; /* Task for which we collect coverage, or NULL. */ struct task_struct *t; + /* Collecting coverage from remote (background) threads. */ + bool remote; + /* Size of remote area (in long's). */ + unsigned int remote_size; + /* + * Sequence is incremented each time kcov is reenabled, used by + * kcov_remote_stop(), see the comment there. + */ + int sequence; }; +struct kcov_remote_area { + struct list_head list; + unsigned int size; +}; + +struct kcov_remote { + u64 handle; + struct kcov *kcov; + struct hlist_node hnode; +}; + +static DEFINE_SPINLOCK(kcov_remote_lock); +static DEFINE_HASHTABLE(kcov_remote_map, 4); +static struct list_head kcov_remote_areas = LIST_HEAD_INIT(kcov_remote_areas); + +/* Must be called with kcov_remote_lock locked. */ +static struct kcov_remote *kcov_remote_find(u64 handle) +{ + struct kcov_remote *remote; + + hash_for_each_possible(kcov_remote_map, remote, hnode, handle) { + if (remote->handle == handle) + return remote; + } + return NULL; +} + +static struct kcov_remote *kcov_remote_add(struct kcov *kcov, u64 handle) +{ + struct kcov_remote *remote; + + if (kcov_remote_find(handle)) + return ERR_PTR(-EEXIST); + remote = kmalloc(sizeof(*remote), GFP_ATOMIC); + if (!remote) + return ERR_PTR(-ENOMEM); + remote->handle = handle; + remote->kcov = kcov; + hash_add(kcov_remote_map, &remote->hnode, handle); + return remote; +} + +/* Must be called with kcov_remote_lock locked. */ +static struct kcov_remote_area *kcov_remote_area_get(unsigned int size) +{ + struct kcov_remote_area *area; + struct list_head *pos; + + kcov_debug("size = %u\n", size); + list_for_each(pos, &kcov_remote_areas) { + area = list_entry(pos, struct kcov_remote_area, list); + if (area->size == size) { + list_del(&area->list); + kcov_debug("rv = %px\n", area); + return area; + } + } + kcov_debug("rv = NULL\n"); + return NULL; +} + +/* Must be called with kcov_remote_lock locked. */ +static void kcov_remote_area_put(struct kcov_remote_area *area, + unsigned int size) +{ + kcov_debug("area = %px, size = %u\n", area, size); + INIT_LIST_HEAD(&area->list); + area->size = size; + list_add(&area->list, &kcov_remote_areas); +} + static notrace bool check_kcov_mode(enum kcov_mode needed_mode, struct task_struct *t) { unsigned int mode; @@ -73,7 +158,7 @@ static notrace bool check_kcov_mode(enum kcov_mode needed_mode, struct task_stru * in_interrupt() returns false (e.g. preempt_schedule_irq()). * READ_ONCE()/barrier() effectively provides load-acquire wrt * interrupts, there are paired barrier()/WRITE_ONCE() in - * kcov_ioctl_locked(). + * kcov_start(). */ barrier(); return mode == needed_mode; @@ -227,6 +312,78 @@ void notrace __sanitizer_cov_trace_switch(u64 val, u64 *cases) EXPORT_SYMBOL(__sanitizer_cov_trace_switch); #endif /* ifdef CONFIG_KCOV_ENABLE_COMPARISONS */ +static void kcov_start(struct task_struct *t, unsigned int size, + void *area, enum kcov_mode mode, int sequence) +{ + kcov_debug("t = %px, size = %u, area = %px\n", t, size, area); + /* Cache in task struct for performance. */ + t->kcov_size = size; + t->kcov_area = area; + /* See comment in check_kcov_mode(). */ + barrier(); + WRITE_ONCE(t->kcov_mode, mode); + t->kcov_sequence = sequence; +} + +static void kcov_stop(struct task_struct *t) +{ + WRITE_ONCE(t->kcov_mode, KCOV_MODE_DISABLED); + barrier(); + t->kcov_size = 0; + t->kcov_area = NULL; +} + +static void kcov_task_reset(struct task_struct *t) +{ + kcov_stop(t); + t->kcov = NULL; + t->kcov_sequence = 0; + t->kcov_handle = 0; +} + +void kcov_task_init(struct task_struct *t) +{ + kcov_task_reset(t); + t->kcov_handle = current->kcov_handle; +} + +static void kcov_reset(struct kcov *kcov) +{ + kcov->t = NULL; + kcov->mode = KCOV_MODE_INIT; + kcov->remote = false; + kcov->remote_size = 0; + kcov->sequence++; +} + +static void kcov_remote_reset(struct kcov *kcov) +{ + int bkt; + struct kcov_remote *remote; + struct hlist_node *tmp; + + spin_lock(&kcov_remote_lock); + hash_for_each_safe(kcov_remote_map, bkt, tmp, remote, hnode) { + if (remote->kcov != kcov) + continue; + kcov_debug("removing handle %llx\n", remote->handle); + hash_del(&remote->hnode); + kfree(remote); + } + /* Do reset before unlock to prevent races with kcov_remote_start(). */ + kcov_reset(kcov); + spin_unlock(&kcov_remote_lock); +} + +static void kcov_disable(struct task_struct *t, struct kcov *kcov) +{ + kcov_task_reset(t); + if (kcov->remote) + kcov_remote_reset(kcov); + else + kcov_reset(kcov); +} + static void kcov_get(struct kcov *kcov) { refcount_inc(&kcov->refcount); @@ -235,20 +392,12 @@ static void kcov_get(struct kcov *kcov) static void kcov_put(struct kcov *kcov) { if (refcount_dec_and_test(&kcov->refcount)) { + kcov_remote_reset(kcov); vfree(kcov->area); kfree(kcov); } } -void kcov_task_init(struct task_struct *t) -{ - WRITE_ONCE(t->kcov_mode, KCOV_MODE_DISABLED); - barrier(); - t->kcov_size = 0; - t->kcov_area = NULL; - t->kcov = NULL; -} - void kcov_task_exit(struct task_struct *t) { struct kcov *kcov; @@ -256,15 +405,36 @@ void kcov_task_exit(struct task_struct *t) kcov = t->kcov; if (kcov == NULL) return; + spin_lock(&kcov->lock); + kcov_debug("t = %px, kcov->t = %px\n", t, kcov->t); + /* + * For KCOV_ENABLE devices we want to make sure that t->kcov->t == t, + * which comes down to: + * WARN_ON(!kcov->remote && kcov->t != t); + * + * For KCOV_REMOTE_ENABLE devices, the exiting task is either: + * 2. A remote task between kcov_remote_start() and kcov_remote_stop(). + * In this case we should print a warning right away, since a task + * shouldn't be exiting when it's in a kcov coverage collection + * section. Here t points to the task that is collecting remote + * coverage, and t->kcov->t points to the thread that created the + * kcov device. Which means that to detect this case we need to + * check that t != t->kcov->t, and this gives us the following: + * WARN_ON(kcov->remote && kcov->t != t); + * + * 2. The task that created kcov exiting without calling KCOV_DISABLE, + * and then again we can make sure that t->kcov->t == t: + * WARN_ON(kcov->remote && kcov->t != t); + * + * By combining all three checks into one we get: + */ if (WARN_ON(kcov->t != t)) { spin_unlock(&kcov->lock); return; } /* Just to not leave dangling references behind. */ - kcov_task_init(t); - kcov->t = NULL; - kcov->mode = KCOV_MODE_INIT; + kcov_disable(t, kcov); spin_unlock(&kcov->lock); kcov_put(kcov); } @@ -313,6 +483,7 @@ static int kcov_open(struct inode *inode, struct file *filep) if (!kcov) return -ENOMEM; kcov->mode = KCOV_MODE_DISABLED; + kcov->sequence = 1; refcount_set(&kcov->refcount, 1); spin_lock_init(&kcov->lock); filep->private_data = kcov; @@ -325,6 +496,20 @@ static int kcov_close(struct inode *inode, struct file *filep) return 0; } +static int kcov_get_mode(unsigned long arg) +{ + if (arg == KCOV_TRACE_PC) + return KCOV_MODE_TRACE_PC; + else if (arg == KCOV_TRACE_CMP) +#ifdef CONFIG_KCOV_ENABLE_COMPARISONS + return KCOV_MODE_TRACE_CMP; +#else + return -ENOTSUPP; +#endif + else + return -EINVAL; +} + /* * Fault in a lazily-faulted vmalloc area before it can be used by * __santizer_cov_trace_pc(), to avoid recursion issues if any code on the @@ -340,14 +525,35 @@ static void kcov_fault_in_area(struct kcov *kcov) READ_ONCE(area[offset]); } +static inline bool kcov_check_handle(u64 handle, bool common_valid, + bool uncommon_valid, bool zero_valid) +{ + if (handle & ~(KCOV_SUBSYSTEM_MASK | KCOV_INSTANCE_MASK)) + return false; + switch (handle & KCOV_SUBSYSTEM_MASK) { + case KCOV_SUBSYSTEM_COMMON: + return (handle & KCOV_INSTANCE_MASK) ? + common_valid : zero_valid; + case KCOV_SUBSYSTEM_USB: + return uncommon_valid; + default: + return false; + } + return false; +} + static int kcov_ioctl_locked(struct kcov *kcov, unsigned int cmd, unsigned long arg) { struct task_struct *t; unsigned long size, unused; + int mode, i; + struct kcov_remote_arg *remote_arg; + struct kcov_remote *remote; switch (cmd) { case KCOV_INIT_TRACE: + kcov_debug("KCOV_INIT_TRACE\n"); /* * Enable kcov in trace mode and setup buffer size. * Must happen before anything else. @@ -366,6 +572,7 @@ static int kcov_ioctl_locked(struct kcov *kcov, unsigned int cmd, kcov->mode = KCOV_MODE_INIT; return 0; case KCOV_ENABLE: + kcov_debug("KCOV_ENABLE\n"); /* * Enable coverage for the current task. * At this point user must have been enabled trace mode, @@ -378,29 +585,20 @@ static int kcov_ioctl_locked(struct kcov *kcov, unsigned int cmd, t = current; if (kcov->t != NULL || t->kcov != NULL) return -EBUSY; - if (arg == KCOV_TRACE_PC) - kcov->mode = KCOV_MODE_TRACE_PC; - else if (arg == KCOV_TRACE_CMP) -#ifdef CONFIG_KCOV_ENABLE_COMPARISONS - kcov->mode = KCOV_MODE_TRACE_CMP; -#else - return -ENOTSUPP; -#endif - else - return -EINVAL; + mode = kcov_get_mode(arg); + if (mode < 0) + return mode; kcov_fault_in_area(kcov); - /* Cache in task struct for performance. */ - t->kcov_size = kcov->size; - t->kcov_area = kcov->area; - /* See comment in check_kcov_mode(). */ - barrier(); - WRITE_ONCE(t->kcov_mode, kcov->mode); + kcov->mode = mode; + kcov_start(t, kcov->size, kcov->area, kcov->mode, + kcov->sequence); t->kcov = kcov; kcov->t = t; - /* This is put either in kcov_task_exit() or in KCOV_DISABLE. */ + /* Put either in kcov_task_exit() or in KCOV_DISABLE. */ kcov_get(kcov); return 0; case KCOV_DISABLE: + kcov_debug("KCOV_DISABLE\n"); /* Disable coverage for the current task. */ unused = arg; if (unused != 0 || current->kcov != kcov) @@ -408,11 +606,65 @@ static int kcov_ioctl_locked(struct kcov *kcov, unsigned int cmd, t = current; if (WARN_ON(kcov->t != t)) return -EINVAL; - kcov_task_init(t); - kcov->t = NULL; - kcov->mode = KCOV_MODE_INIT; + kcov_disable(t, kcov); kcov_put(kcov); return 0; + case KCOV_REMOTE_ENABLE: + kcov_debug("KCOV_REMOTE_ENABLE\n"); + if (kcov->mode != KCOV_MODE_INIT || !kcov->area) + return -EINVAL; + t = current; + if (kcov->t != NULL || t->kcov != NULL) + return -EBUSY; + remote_arg = (struct kcov_remote_arg *)arg; + mode = kcov_get_mode(remote_arg->trace_mode); + if (mode < 0) + return mode; + if (remote_arg->area_size > LONG_MAX / sizeof(unsigned long)) + return -EINVAL; + kcov->mode = mode; + t->kcov = kcov; + kcov->t = t; + kcov->remote = true; + kcov->remote_size = remote_arg->area_size; + spin_lock(&kcov_remote_lock); + for (i = 0; i < remote_arg->num_handles; i++) { + kcov_debug("handle %llx\n", remote_arg->handles[i]); + if (!kcov_check_handle(remote_arg->handles[i], + false, true, false)) { + spin_unlock(&kcov_remote_lock); + kcov_disable(t, kcov); + return -EINVAL; + } + remote = kcov_remote_add(kcov, remote_arg->handles[i]); + if (IS_ERR(remote)) { + spin_unlock(&kcov_remote_lock); + kcov_disable(t, kcov); + return PTR_ERR(remote); + } + } + if (remote_arg->common_handle) { + kcov_debug("common handle %llx\n", + remote_arg->common_handle); + if (!kcov_check_handle(remote_arg->common_handle, + true, false, false)) { + spin_unlock(&kcov_remote_lock); + kcov_disable(t, kcov); + return -EINVAL; + } + remote = kcov_remote_add(kcov, + remote_arg->common_handle); + if (IS_ERR(remote)) { + spin_unlock(&kcov_remote_lock); + kcov_disable(t, kcov); + return PTR_ERR(remote); + } + t->kcov_handle = remote_arg->common_handle; + } + spin_unlock(&kcov_remote_lock); + /* Put either in kcov_task_exit() or in KCOV_DISABLE. */ + kcov_get(kcov); + return 0; default: return -ENOTTY; } @@ -422,11 +674,35 @@ static long kcov_ioctl(struct file *filep, unsigned int cmd, unsigned long arg) { struct kcov *kcov; int res; + struct kcov_remote_arg *remote_arg = NULL; + unsigned int remote_num_handles; + unsigned long remote_arg_size; + + if (cmd == KCOV_REMOTE_ENABLE) { + if (get_user(remote_num_handles, (unsigned __user *)(arg + + offsetof(struct kcov_remote_arg, num_handles)))) + return -EFAULT; + if (remote_num_handles > KCOV_REMOTE_MAX_HANDLES) + return -EINVAL; + remote_arg_size = struct_size(remote_arg, handles, + remote_num_handles); + remote_arg = memdup_user((void __user *)arg, remote_arg_size); + if (IS_ERR(remote_arg)) + return PTR_ERR(remote_arg); + if (remote_arg->num_handles != remote_num_handles) { + kfree(remote_arg); + return -EINVAL; + } + arg = (unsigned long)remote_arg; + } kcov = filep->private_data; spin_lock(&kcov->lock); res = kcov_ioctl_locked(kcov, cmd, arg); spin_unlock(&kcov->lock); + + kfree(remote_arg); + return res; } @@ -438,6 +714,207 @@ static const struct file_operations kcov_fops = { .release = kcov_close, }; +/* + * kcov_remote_start() and kcov_remote_stop() can be used to annotate a section + * of code in a kernel background thread to allow kcov to be used to collect + * coverage from that part of code. + * + * The handle argument of kcov_remote_start() identifies a code section that is + * used for coverage collection. A userspace process passes this handle to + * KCOV_REMOTE_ENABLE ioctl to make the used kcov device start collecting + * coverage for the code section identified by this handle. + * + * The usage of these annotations in the kernel code is different depending on + * the type of the kernel thread whose code is being annotated. + * + * For global kernel threads that are spawned in a limited number of instances + * (e.g. one USB hub_event() worker thread is spawned per USB HCD), each + * instance must be assigned a unique 4-byte instance id. The instance id is + * then combined with a 1-byte subsystem id to get a handle via + * kcov_remote_handle(subsystem_id, instance_id). + * + * For local kernel threads that are spawned from system calls handler when a + * user interacts with some kernel interface (e.g. vhost workers), a handle is + * passed from a userspace process as the common_handle field of the + * kcov_remote_arg struct (note, that the user must generate a handle by using + * kcov_remote_handle() with KCOV_SUBSYSTEM_COMMON as the subsystem id and an + * arbitrary 4-byte non-zero number as the instance id). This common handle + * then gets saved into the task_struct of the process that issued the + * KCOV_REMOTE_ENABLE ioctl. When this proccess issues system calls that spawn + * kernel threads, the common handle must be retrived via kcov_common_handle() + * and passed to the spawned threads via custom annotations. Those kernel + * threads must in turn be annotated with kcov_remote_start(common_handle) and + * kcov_remote_stop(). All of the threads that are spawned by the same process + * obtain the same handle, hence the name "common". + * + * See Documentation/dev-tools/kcov.rst for more details. + * + * Internally, this function looks up the kcov device associated with the + * provided handle, allocates an area for coverage collection, and saves the + * pointers to kcov and area into the current task_struct to allow coverage to + * be collected via __sanitizer_cov_trace_pc() + * In turns kcov_remote_stop() clears those pointers from task_struct to stop + * collecting coverage and copies all collected coverage into the kcov area. + */ +void kcov_remote_start(u64 handle) +{ + struct kcov_remote *remote; + void *area; + struct task_struct *t; + unsigned int size; + enum kcov_mode mode; + int sequence; + + if (WARN_ON(!kcov_check_handle(handle, true, true, true))) + return; + if (WARN_ON(!in_task())) + return; + t = current; + /* + * Check that kcov_remote_start is not called twice + * nor called by user tasks (with enabled kcov). + */ + if (WARN_ON(t->kcov)) + return; + + kcov_debug("handle = %llx\n", handle); + + spin_lock(&kcov_remote_lock); + remote = kcov_remote_find(handle); + if (!remote) { + kcov_debug("no remote found"); + spin_unlock(&kcov_remote_lock); + return; + } + /* Put in kcov_remote_stop(). */ + kcov_get(remote->kcov); + t->kcov = remote->kcov; + /* + * Read kcov fields before unlock to prevent races with + * KCOV_DISABLE / kcov_remote_reset(). + */ + size = remote->kcov->remote_size; + mode = remote->kcov->mode; + sequence = remote->kcov->sequence; + area = kcov_remote_area_get(size); + spin_unlock(&kcov_remote_lock); + + if (!area) { + area = vmalloc(size * sizeof(unsigned long)); + if (!area) { + t->kcov = NULL; + kcov_put(remote->kcov); + return; + } + } + /* Reset coverage size. */ + *(u64 *)area = 0; + + kcov_debug("area = %px, size = %u", area, size); + + kcov_start(t, size, area, mode, sequence); + +} +EXPORT_SYMBOL(kcov_remote_start); + +static void kcov_move_area(enum kcov_mode mode, void *dst_area, + unsigned int dst_area_size, void *src_area) +{ + u64 word_size = sizeof(unsigned long); + u64 count_size, entry_size_log; + u64 dst_len, src_len; + void *dst_entries, *src_entries; + u64 dst_occupied, dst_free, bytes_to_move, entries_moved; + + kcov_debug("%px %u <= %px %lu\n", + dst_area, dst_area_size, src_area, *(unsigned long *)src_area); + + switch (mode) { + case KCOV_MODE_TRACE_PC: + dst_len = READ_ONCE(*(unsigned long *)dst_area); + src_len = *(unsigned long *)src_area; + count_size = sizeof(unsigned long); + entry_size_log = __ilog2_u64(sizeof(unsigned long)); + break; + case KCOV_MODE_TRACE_CMP: + dst_len = READ_ONCE(*(u64 *)dst_area); + src_len = *(u64 *)src_area; + count_size = sizeof(u64); + BUILD_BUG_ON(!is_power_of_2(KCOV_WORDS_PER_CMP)); + entry_size_log = __ilog2_u64(sizeof(u64) * KCOV_WORDS_PER_CMP); + break; + default: + WARN_ON(1); + return; + } + + /* As arm can't divide u64 integers use log of entry size. */ + if (dst_len > ((dst_area_size * word_size - count_size) >> + entry_size_log)) + return; + dst_occupied = count_size + (dst_len << entry_size_log); + dst_free = dst_area_size * word_size - dst_occupied; + bytes_to_move = min(dst_free, src_len << entry_size_log); + dst_entries = dst_area + dst_occupied; + src_entries = src_area + count_size; + memcpy(dst_entries, src_entries, bytes_to_move); + entries_moved = bytes_to_move >> entry_size_log; + + switch (mode) { + case KCOV_MODE_TRACE_PC: + WRITE_ONCE(*(unsigned long *)dst_area, dst_len + entries_moved); + break; + case KCOV_MODE_TRACE_CMP: + WRITE_ONCE(*(u64 *)dst_area, dst_len + entries_moved); + break; + default: + break; + } +} + +/* See the comment before kcov_remote_start() for usage details. */ +void kcov_remote_stop(void) +{ + struct task_struct *t = current; + struct kcov *kcov = t->kcov; + void *area = t->kcov_area; + unsigned int size = t->kcov_size; + int sequence = t->kcov_sequence; + + if (!kcov) { + kcov_debug("no kcov found\n"); + return; + } + + kcov_stop(t); + t->kcov = NULL; + + spin_lock(&kcov->lock); + /* + * KCOV_DISABLE could have been called between kcov_remote_start() + * and kcov_remote_stop(), hence the check. + */ + kcov_debug("move if: %d == %d && %d\n", + sequence, kcov->sequence, (int)kcov->remote); + if (sequence == kcov->sequence && kcov->remote) + kcov_move_area(kcov->mode, kcov->area, kcov->size, area); + spin_unlock(&kcov->lock); + + spin_lock(&kcov_remote_lock); + kcov_remote_area_put(area, size); + spin_unlock(&kcov_remote_lock); + + kcov_put(kcov); +} +EXPORT_SYMBOL(kcov_remote_stop); + +/* See the comment before kcov_remote_start() for usage details. */ +u64 kcov_common_handle(void) +{ + return current->kcov_handle; +} +EXPORT_SYMBOL(kcov_common_handle); + static int __init kcov_init(void) { /* From 7db82d35eb9acc956519a3dc55c6f2a65bac57b2 Mon Sep 17 00:00:00 2001 From: Alistair Delva Date: Mon, 13 Jan 2020 15:41:53 -0800 Subject: [PATCH 06/89] ANDROID: update kernel ABI for kcov changes Leaf changes summary: 3 artifacts changed Changed leaf types summary: 3 leaf types changed Removed/Changed/Added functions summary: 0 Removed, 0 Changed, 0 Added function Removed/Changed/Added variables summary: 0 Removed, 0 Changed, 0 Added variable 'struct mem_cgroup at memcontrol.h:208:1' changed: type size changed from 22016 to 20992 (in bits) 1 data member deletion: 'deferred_split mem_cgroup::deferred_split_queue', at offset 21504 (in bits) at memcontrol.h:334:1 there are data member changes: type 'atomic_long_t[70]' of 'mem_cgroup::vmevents' changed: type name changed from 'atomic_long_t[70]' to 'atomic_long_t[56]' array type size changed from 4480 to 3584 array type subrange 1 changed length from 70 to 56 , size changed from 4480 to 3584 (in bits) (by -896 bits) 'atomic_long_t mem_cgroup::memory_events[7]' offset changed from 17664 to 16768 (in bits) (by -896 bits) 'atomic_long_t mem_cgroup::memory_events_local[7]' offset changed from 18112 to 17216 (in bits) (by -896 bits) 'unsigned long int mem_cgroup::socket_pressure' offset changed from 18560 to 17664 (in bits) (by -896 bits) 'bool mem_cgroup::tcpmem_active' offset changed from 18624 to 17728 (in bits) (by -896 bits) 'int mem_cgroup::tcpmem_pressure' offset changed from 18656 to 17760 (in bits) (by -896 bits) 'int mem_cgroup::kmemcg_id' offset changed from 18688 to 17792 (in bits) (by -896 bits) 'memcg_kmem_state mem_cgroup::kmem_state' offset changed from 18720 to 17824 (in bits) (by -896 bits) 'list_head mem_cgroup::kmem_caches' offset changed from 18752 to 17856 (in bits) (by -896 bits) 'int mem_cgroup::last_scanned_node' offset changed from 18880 to 17984 (in bits) (by -896 bits) 'list_head mem_cgroup::cgwb_list' offset changed from 18944 to 18048 (in bits) (by -896 bits) 'wb_domain mem_cgroup::cgwb_domain' offset changed from 19072 to 18176 (in bits) (by -896 bits) 'memcg_cgwb_frn mem_cgroup::cgwb_frn[4]' offset changed from 20032 to 19136 (in bits) (by -896 bits) 'list_head mem_cgroup::event_list' offset changed from 21312 to 20416 (in bits) (by -896 bits) 'spinlock_t mem_cgroup::event_list_lock' offset changed from 21440 to 20544 (in bits) (by -896 bits) 'mem_cgroup_per_node* mem_cgroup::nodeinfo[]' offset changed from 21760 to 20608 (in bits) (by -1152 bits) 17 impacted interfaces: function int __netlink_dump_start(sock*, sk_buff*, const nlmsghdr*, netlink_dump_control*) function void __sock_recv_ts_and_drops(msghdr*, sock*, sk_buff*) function void dev_add_pack(packet_type*) function void dev_remove_pack(packet_type*) function void lock_sock_nested(sock*, int) function void release_sock(sock*) function sock* sk_alloc(net*, int, gfp_t, proto*, int) function void sk_free(sock*) function void skb_free_datagram(sock*, sk_buff*) function sk_buff* skb_recv_datagram(sock*, unsigned int, int, int*) function sk_buff* sock_alloc_send_skb(sock*, unsigned long int, int, int*) function int sock_diag_register(const sock_diag_handler*) function void sock_diag_save_cookie(sock*, __u32*) function void sock_diag_unregister(const sock_diag_handler*) function unsigned long int sock_i_ino(sock*) function void sock_init_data(socket*, sock*) function int sock_queue_rcv_skb(sock*, sk_buff*) 'struct memcg_vmstats_percpu at memcontrol.h:88:1' changed: type size changed from 7168 to 6272 (in bits) there are data member changes: type 'unsigned long int[70]' of 'memcg_vmstats_percpu::events' changed: type name changed from 'unsigned long int[70]' to 'unsigned long int[56]' array type size changed from 4480 to 3584 array type subrange 1 changed length from 70 to 56 , size changed from 4480 to 3584 (in bits) (by -896 bits) 'unsigned long int memcg_vmstats_percpu::nr_page_events' offset changed from 6912 to 6016 (in bits) (by -896 bits) 'unsigned long int memcg_vmstats_percpu::targets[3]' offset changed from 6976 to 6080 (in bits) (by -896 bits) 17 impacted interfaces: function int __netlink_dump_start(sock*, sk_buff*, const nlmsghdr*, netlink_dump_control*) function void __sock_recv_ts_and_drops(msghdr*, sock*, sk_buff*) function void dev_add_pack(packet_type*) function void dev_remove_pack(packet_type*) function void lock_sock_nested(sock*, int) function void release_sock(sock*) function sock* sk_alloc(net*, int, gfp_t, proto*, int) function void sk_free(sock*) function void skb_free_datagram(sock*, sk_buff*) function sk_buff* skb_recv_datagram(sock*, unsigned int, int, int*) function sk_buff* sock_alloc_send_skb(sock*, unsigned long int, int, int*) function int sock_diag_register(const sock_diag_handler*) function void sock_diag_save_cookie(sock*, __u32*) function void sock_diag_unregister(const sock_diag_handler*) function unsigned long int sock_i_ino(sock*) function void sock_init_data(socket*, sock*) function int sock_queue_rcv_skb(sock*, sk_buff*) 'struct pglist_data at mmzone.h:702:1' changed: type size hasn't changed 1 data member deletion: 'deferred_split pglist_data::deferred_split_queue', at offset 40512 (in bits) at mmzone.h:772:1 there are data member changes: 'lruvec pglist_data::lruvec' offset changed from 40768 to 40512 (in bits) (by -256 bits) 'unsigned long int pglist_data::flags' offset changed from 41856 to 41600 (in bits) (by -256 bits) 17 impacted interfaces: function int __netlink_dump_start(sock*, sk_buff*, const nlmsghdr*, netlink_dump_control*) function void __sock_recv_ts_and_drops(msghdr*, sock*, sk_buff*) function void dev_add_pack(packet_type*) function void dev_remove_pack(packet_type*) function void lock_sock_nested(sock*, int) function void release_sock(sock*) function sock* sk_alloc(net*, int, gfp_t, proto*, int) function void sk_free(sock*) function void skb_free_datagram(sock*, sk_buff*) function sk_buff* skb_recv_datagram(sock*, unsigned int, int, int*) function sk_buff* sock_alloc_send_skb(sock*, unsigned long int, int, int*) function int sock_diag_register(const sock_diag_handler*) function void sock_diag_save_cookie(sock*, __u32*) function void sock_diag_unregister(const sock_diag_handler*) function unsigned long int sock_i_ino(sock*) function void sock_init_data(socket*, sock*) function int sock_queue_rcv_skb(sock*, sk_buff*) Bug: 147413187 Change-Id: I852d76d5c5c2c2ba99ad88e1979fa2720c6077e1 Signed-off-by: Alistair Delva --- abi_gki_aarch64.xml | 15919 +++++++++++++++++++++--------------------- 1 file changed, 7921 insertions(+), 7998 deletions(-) diff --git a/abi_gki_aarch64.xml b/abi_gki_aarch64.xml index 8109074ed36a..3ed0c80386ea 100644 --- a/abi_gki_aarch64.xml +++ b/abi_gki_aarch64.xml @@ -1163,12 +1163,12 @@ - - + + - - + + @@ -2103,7 +2103,7 @@ - + @@ -3361,27 +3361,27 @@ - - - - - - - - - - - - - - + - + + + + + + + + + + + + + + @@ -4868,7 +4868,7 @@ - + @@ -4943,7 +4943,7 @@ - + @@ -5192,7 +5192,7 @@ - + @@ -5354,43 +5354,43 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -5847,7 +5847,7 @@ - + @@ -6830,7 +6830,7 @@ - + @@ -15887,7 +15887,7 @@ - + @@ -17161,8 +17161,8 @@ - - + + @@ -17773,7 +17773,7 @@ - + @@ -18034,7 +18034,7 @@ - + @@ -20054,7 +20054,7 @@ - + @@ -20596,7 +20596,7 @@ - + @@ -20837,12 +20837,12 @@ - + - + @@ -21244,7 +21244,7 @@ - + @@ -21258,7 +21258,7 @@ - + @@ -21501,7 +21501,7 @@ - + @@ -22324,7 +22324,7 @@ - + @@ -22339,7 +22339,7 @@ - + @@ -22693,27 +22693,27 @@ - + - + - + - + - + - + - + - + @@ -22740,48 +22740,83 @@ - + - + + + + + + + + + + + + + + + - + - + - + - + - - + + - + - + + + - + - + - + + + + - + - + - + - + + + + + + + + + + + + + + + + + @@ -22789,7 +22824,7 @@ - + @@ -22812,71 +22847,71 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -22888,7 +22923,7 @@ - + @@ -22921,10 +22956,10 @@ - + - + @@ -22939,7 +22974,7 @@ - + @@ -22948,13 +22983,13 @@ - + - + - + @@ -22981,7 +23016,7 @@ - + @@ -22993,7 +23028,7 @@ - + @@ -23008,7 +23043,7 @@ - + @@ -23016,7 +23051,7 @@ - + @@ -23030,7 +23065,7 @@ - + @@ -23057,7 +23092,7 @@ - + @@ -23083,15 +23118,15 @@ - + - + - + - + @@ -23104,18 +23139,18 @@ - + - + - + @@ -23129,18 +23164,18 @@ - + - + - + @@ -23155,7 +23190,7 @@ - + @@ -23177,10 +23212,10 @@ - + - + @@ -23188,15 +23223,15 @@ - + - + - + - + @@ -23208,7 +23243,7 @@ - + @@ -23226,12 +23261,12 @@ - + - + @@ -23240,10 +23275,10 @@ - + - + @@ -23251,7 +23286,7 @@ - + @@ -23289,7 +23324,7 @@ - + @@ -23298,7 +23333,7 @@ - + @@ -23310,22 +23345,22 @@ - + - + - + - + @@ -23345,10 +23380,10 @@ - + - + @@ -23375,7 +23410,7 @@ - + @@ -23387,7 +23422,7 @@ - + @@ -23420,7 +23455,7 @@ - + @@ -23429,9 +23464,9 @@ - + - + @@ -23440,7 +23475,7 @@ - + @@ -23454,7 +23489,7 @@ - + @@ -23474,20 +23509,20 @@ - + - + - + @@ -23495,7 +23530,7 @@ - + @@ -23508,7 +23543,7 @@ - + @@ -23516,7 +23551,7 @@ - + @@ -23542,30 +23577,30 @@ - + - + - + - + - + - + - + - + - + @@ -23574,13 +23609,13 @@ - + - + @@ -23631,7 +23666,7 @@ - + @@ -23640,7 +23675,7 @@ - + @@ -23664,7 +23699,7 @@ - + @@ -23679,7 +23714,7 @@ - + @@ -23706,7 +23741,7 @@ - + @@ -23721,13 +23756,13 @@ - + - + @@ -23751,10 +23786,10 @@ - + - + @@ -23771,17 +23806,17 @@ - + - + - + - + @@ -23794,16 +23829,16 @@ - + - + - + @@ -23827,7 +23862,7 @@ - + @@ -23857,7 +23892,7 @@ - + @@ -23869,14 +23904,14 @@ - + - - + + @@ -23884,7 +23919,7 @@ - + @@ -23892,7 +23927,7 @@ - + @@ -23900,20 +23935,20 @@ - + - + - + - + - + @@ -23928,15 +23963,15 @@ - + - + - + @@ -23945,7 +23980,7 @@ - + @@ -23960,10 +23995,10 @@ - + - + @@ -23972,96 +24007,96 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - - - - + + + + @@ -24078,7 +24113,7 @@ - + @@ -24105,7 +24140,7 @@ - + @@ -24138,10 +24173,10 @@ - - + + - + @@ -24168,7 +24203,7 @@ - + @@ -24192,13 +24227,13 @@ - + - + @@ -24219,7 +24254,7 @@ - + @@ -24234,10 +24269,10 @@ - + - + @@ -24245,7 +24280,7 @@ - + @@ -24254,7 +24289,7 @@ - + @@ -24265,7 +24300,7 @@ - + @@ -24295,7 +24330,7 @@ - + @@ -24306,10 +24341,10 @@ - + - + @@ -24321,10 +24356,10 @@ - + - + @@ -24357,7 +24392,7 @@ - + @@ -24386,15 +24421,15 @@ - + - + - + @@ -24402,7 +24437,7 @@ - + @@ -24410,7 +24445,7 @@ - + @@ -24487,16 +24522,16 @@ - + - + - + @@ -24507,24 +24542,24 @@ - + - + - + - + - + - + - + @@ -24625,7 +24660,7 @@ - + @@ -24633,10 +24668,10 @@ - + - + @@ -24651,8 +24686,8 @@ - - + + @@ -24669,17 +24704,17 @@ - + - + - - + + @@ -24694,15 +24729,15 @@ - + - + - + @@ -24711,12 +24746,12 @@ - + - + - + @@ -24734,7 +24769,7 @@ - + @@ -24743,8 +24778,8 @@ - - + + @@ -24752,176 +24787,177 @@ - + - - + + - + - + - - - + + + - - - - - - - - - - - - - + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + - + + + + + + + - + - + - + - - - - - + + + + + - + + - + - - - - - - - - + + + + + - - + + - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + - - - - + + + + - - - - - - + + + + + + - - - - - + + + + + - + - - - + + + - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + - - - - + + + + + + @@ -24944,7 +24980,7 @@ - + @@ -24953,315 +24989,315 @@ - + - + - - + + - + - + - + - + - + - - - - - - - - - - - - - - - - + - + + + - - - - + + - + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - - - - - - - - + + - - - + + + + + + + + - + + + + + + + + + + + + + + + + + + - + - - - - - - - - - - - - - - + + + + + - - + + + + + + + + + + + - + - + - + - - + + - - + + - + - + - - + + - - + + - - + + - + - - + + - + - - + + - - + + - - + + - + - - - + + + - - + + - + - + - + - - - - - - - - - - - - - - - - - - - - - - - - - - - - + - + + + + + + + + + + + + - - + + + + + + + + + - - - + + - + - - + + - + + + + + + + + + + + - + @@ -25269,7 +25305,7 @@ - + @@ -25279,7 +25315,7 @@ - + @@ -25297,9 +25333,9 @@ - - - + + + @@ -25310,11 +25346,11 @@ - + - + @@ -25324,7 +25360,7 @@ - + @@ -25346,56 +25382,56 @@ - - + + - + - - + + - - - - - - - + + + + + + + - - + + - + - - + + - + - + - - + + - + - + @@ -25406,9 +25442,9 @@ - + - + @@ -25417,7 +25453,7 @@ - + @@ -25426,7 +25462,7 @@ - + @@ -25444,7 +25480,7 @@ - + @@ -25453,7 +25489,7 @@ - + @@ -25461,9 +25497,9 @@ - + - + @@ -25478,7 +25514,7 @@ - + @@ -25502,7 +25538,7 @@ - + @@ -25540,10 +25576,10 @@ - + - + @@ -25552,10 +25588,10 @@ - + - + @@ -25570,12 +25606,12 @@ - + - + @@ -25586,8 +25622,8 @@ - - + + @@ -25595,10 +25631,10 @@ - + - + @@ -25615,7 +25651,7 @@ - + @@ -25644,7 +25680,7 @@ - + @@ -25659,87 +25695,87 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -25766,10 +25802,10 @@ - + - + @@ -25780,22 +25816,22 @@ - + - + - + - + - + @@ -25806,7 +25842,7 @@ - + @@ -25817,24 +25853,24 @@ - + - + - + - + - + - + - + @@ -25842,15 +25878,15 @@ - + - + - + @@ -25862,7 +25898,7 @@ - + @@ -25880,7 +25916,7 @@ - + @@ -25906,16 +25942,16 @@ - + - + - + @@ -25929,7 +25965,7 @@ - + @@ -25943,7 +25979,7 @@ - + @@ -25978,22 +26014,22 @@ - + - + - + - + @@ -26011,7 +26047,7 @@ - + @@ -26023,7 +26059,7 @@ - + @@ -26044,7 +26080,7 @@ - + @@ -26098,19 +26134,19 @@ - + - + - + - + @@ -26122,19 +26158,19 @@ - + - + - + - + @@ -26149,7 +26185,7 @@ - + @@ -26158,22 +26194,22 @@ - - + + - - + + - + - + @@ -26184,7 +26220,7 @@ - + @@ -26201,7 +26237,7 @@ - + @@ -26226,7 +26262,7 @@ - + @@ -26358,7 +26394,7 @@ - + @@ -26366,9 +26402,9 @@ - + - + @@ -26383,7 +26419,7 @@ - + @@ -26392,7 +26428,7 @@ - + @@ -26407,23 +26443,23 @@ - + - + - + - + @@ -26433,9 +26469,9 @@ - + - + @@ -26443,7 +26479,7 @@ - + @@ -26462,10 +26498,10 @@ - + - + @@ -26491,13 +26527,13 @@ - + - + - + @@ -26511,7 +26547,7 @@ - + @@ -26519,10 +26555,10 @@ - + - + @@ -26544,13 +26580,13 @@ - + - + - + @@ -26561,7 +26597,7 @@ - + @@ -26569,16 +26605,16 @@ - - + + - + - + @@ -26590,7 +26626,7 @@ - + @@ -26599,7 +26635,7 @@ - + @@ -26611,19 +26647,19 @@ - + - + - + - + - + @@ -26632,20 +26668,20 @@ - + - + - + - + - + @@ -26654,24 +26690,24 @@ - + - + - - + + - + - + @@ -26700,15 +26736,15 @@ - + - + - + @@ -26737,7 +26773,7 @@ - + @@ -26745,7 +26781,7 @@ - + @@ -26753,7 +26789,7 @@ - + @@ -26761,7 +26797,7 @@ - + @@ -26769,9 +26805,9 @@ - + - + @@ -26792,17 +26828,17 @@ - + - + - + - + - + @@ -26814,9 +26850,9 @@ - + - + @@ -26828,13 +26864,13 @@ - - + + - + @@ -26843,19 +26879,19 @@ - + - + - + - + - + @@ -26867,8 +26903,8 @@ - - + + @@ -26879,13 +26915,13 @@ - + - + @@ -26920,7 +26956,7 @@ - + @@ -26928,7 +26964,7 @@ - + @@ -26936,64 +26972,64 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - + + + + + + + + + + @@ -27014,7 +27050,7 @@ - + @@ -27031,120 +27067,120 @@ - + - + - + - + - + - - - - - - - - - - - - - - - - - - - + - - - + + - - + + + + + + + + + + + + + + + + + + + + + - + - + - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + - - + + - - + + - - + + - + @@ -27163,7 +27199,7 @@ - + @@ -27198,18 +27234,18 @@ - - - - + + + + - + - + @@ -27226,7 +27262,7 @@ - + @@ -27234,11 +27270,11 @@ - + - + - + @@ -27251,7 +27287,7 @@ - + @@ -27262,7 +27298,7 @@ - + @@ -27273,7 +27309,7 @@ - + @@ -27281,7 +27317,7 @@ - + @@ -27291,18 +27327,18 @@ - + - + - + @@ -27311,15 +27347,15 @@ - + - + - + @@ -27330,7 +27366,7 @@ - + @@ -27338,55 +27374,55 @@ - - - - - - + + + + + + - + - + - + - + - + - + - + - + - - + + - - + + - + @@ -27400,8 +27436,8 @@ - - + + @@ -27412,12 +27448,12 @@ - + - + @@ -27448,7 +27484,7 @@ - + @@ -27462,68 +27498,68 @@ - - + + - + - + - - + + - - + + - + - - + + - + - + - - + + - - + + - + - + @@ -27553,22 +27589,22 @@ - + - + - - - + + + - + @@ -27579,7 +27615,7 @@ - + @@ -27598,25 +27634,25 @@ - - + + - + - + - - + + @@ -27630,15 +27666,15 @@ - + - + - + @@ -27663,17 +27699,17 @@ - - - + + + - + - + @@ -27681,7 +27717,7 @@ - + @@ -27696,16 +27732,16 @@ - + - + - + - + @@ -27720,42 +27756,42 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -27768,7 +27804,7 @@ - + @@ -27776,26 +27812,26 @@ - + - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + @@ -27830,65 +27866,65 @@ - - - + + + - - - - + + + + - - - + + + - - - - + + + + - - - + + + - - + + - + - - + + - - - + + + - - + + - - + + @@ -27973,7 +28009,7 @@ - + @@ -27981,9 +28017,9 @@ - - - + + + @@ -28000,7 +28036,7 @@ - + @@ -28008,7 +28044,7 @@ - + @@ -28022,7 +28058,7 @@ - + @@ -28057,9 +28093,9 @@ - + - + @@ -28068,7 +28104,7 @@ - + @@ -28076,12 +28112,12 @@ - + - + - + @@ -28112,7 +28148,7 @@ - + @@ -28120,26 +28156,26 @@ - - + + - + - + - + - + - + @@ -28152,7 +28188,7 @@ - + @@ -28163,7 +28199,7 @@ - + @@ -28174,7 +28210,7 @@ - + @@ -28182,13 +28218,13 @@ - + - + @@ -28202,22 +28238,22 @@ - - + + - + - + - - + + @@ -28237,7 +28273,7 @@ - + @@ -28248,19 +28284,19 @@ - + - - - - + + + + - + @@ -28302,22 +28338,22 @@ - + - + - + - + @@ -28331,7 +28367,7 @@ - + @@ -28385,7 +28421,7 @@ - + @@ -28394,18 +28430,18 @@ - + - + - - + + @@ -28418,15 +28454,15 @@ - + - - - - + + + + @@ -28440,8 +28476,8 @@ - - + + @@ -28455,14 +28491,14 @@ - + - + - + @@ -28552,16 +28588,16 @@ - + - + - + @@ -28600,8 +28636,8 @@ - - + + @@ -28611,7 +28647,7 @@ - + @@ -28628,13 +28664,13 @@ - + - - + + @@ -28642,15 +28678,15 @@ - - + + - - - + + + @@ -28660,13 +28696,13 @@ - + - - + + @@ -28712,9 +28748,9 @@ - + - + @@ -28729,10 +28765,10 @@ - + - + @@ -28750,36 +28786,36 @@ - + - + - + - + - + - + - + @@ -28794,8 +28830,8 @@ - - + + @@ -28814,44 +28850,44 @@ - + - + - + - + - - - + + + - + - + - + - + - + @@ -28860,15 +28896,15 @@ - + - + - + @@ -28877,7 +28913,7 @@ - + @@ -28887,7 +28923,7 @@ - + @@ -28895,13 +28931,13 @@ - + - + @@ -28912,13 +28948,13 @@ - + - + - + @@ -28935,16 +28971,16 @@ - + - + - + @@ -28954,23 +28990,23 @@ - + - + - + - + - + - + @@ -28984,7 +29020,7 @@ - + @@ -29004,33 +29040,33 @@ - - - - - + + + - - - - - - - - - - + + + + + + + + + + - - + + + + - - + + @@ -29061,85 +29097,85 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -29150,7 +29186,7 @@ - + @@ -29165,7 +29201,7 @@ - + @@ -29176,19 +29212,19 @@ - - - + + + - - + + - + - + @@ -29203,31 +29239,31 @@ - + - + - + - + - + - + - + @@ -29235,7 +29271,7 @@ - + @@ -29246,7 +29282,7 @@ - + @@ -29257,10 +29293,10 @@ - + - + @@ -29274,7 +29310,7 @@ - + @@ -29285,24 +29321,24 @@ - + - + - + - + - + - + @@ -29314,7 +29350,7 @@ - + @@ -29326,7 +29362,7 @@ - + @@ -29334,79 +29370,79 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - - + + - + - - - - - + + + + + - - - - - - + + + + + + @@ -29417,46 +29453,46 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - - - + + + + + + + + + @@ -29486,119 +29522,119 @@ - - + + - + - - - - + + + + - - + + - + - - + + - - + + - - + + - - - + + + - - + + - + - - - + + + - + - + - - - + + + - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + - + - - - - - - - - + + + + + + + + - + @@ -29621,7 +29657,7 @@ - + @@ -29635,9 +29671,9 @@ - + - + @@ -29647,10 +29683,10 @@ - + - + @@ -29668,7 +29704,7 @@ - + @@ -29683,19 +29719,19 @@ - + - + - + - + - + @@ -29773,7 +29809,7 @@ - + @@ -29806,7 +29842,7 @@ - + @@ -29863,7 +29899,7 @@ - + @@ -29872,65 +29908,65 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -29941,21 +29977,21 @@ - + - + - + - + @@ -29966,7 +30002,7 @@ - + @@ -29979,7 +30015,7 @@ - + @@ -29992,13 +30028,13 @@ - + - + @@ -30016,7 +30052,7 @@ - + @@ -30046,8 +30082,8 @@ - - + + @@ -30055,8 +30091,8 @@ - - + + @@ -30064,108 +30100,108 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -30201,16 +30237,16 @@ - + - + - + - + @@ -30222,10 +30258,10 @@ - + - + @@ -30239,27 +30275,27 @@ - + - + - + - + - + - + - + - + @@ -30280,10 +30316,10 @@ - + - + @@ -30295,7 +30331,7 @@ - + @@ -30306,13 +30342,13 @@ - + - + - + @@ -30320,8 +30356,8 @@ - - + + @@ -30332,7 +30368,7 @@ - + @@ -30346,7 +30382,7 @@ - + @@ -30357,7 +30393,7 @@ - + @@ -30371,7 +30407,7 @@ - + @@ -30385,10 +30421,10 @@ - + - + @@ -30415,7 +30451,7 @@ - + @@ -30430,29 +30466,29 @@ - + - + - + - + - + - + - + - + @@ -30475,31 +30511,31 @@ - + - + - + - + - + - + - + - + - + @@ -30520,7 +30556,7 @@ - + @@ -30558,10 +30594,10 @@ - + - + @@ -30585,20 +30621,20 @@ - + - + - + - - + + @@ -30606,7 +30642,7 @@ - + @@ -30614,26 +30650,26 @@ - + - + - + - + - + - + - + - + @@ -30662,20 +30698,20 @@ - + - + - + - + - + @@ -30684,18 +30720,18 @@ - + - + - + - + - + @@ -30704,10 +30740,10 @@ - + - + @@ -30715,7 +30751,7 @@ - + @@ -30723,22 +30759,22 @@ - + - + - + - + - + - + @@ -30750,27 +30786,27 @@ - + - + - + - + - + - + - + @@ -30781,7 +30817,7 @@ - + @@ -30790,7 +30826,7 @@ - + @@ -30819,7 +30855,7 @@ - + @@ -30827,10 +30863,10 @@ - + - + @@ -30839,7 +30875,7 @@ - + @@ -30847,9 +30883,9 @@ - + - + @@ -30861,17 +30897,17 @@ - + - + - + - - + + @@ -30879,7 +30915,7 @@ - + @@ -30887,88 +30923,88 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -31126,8 +31162,8 @@ - - + + @@ -31135,13 +31171,13 @@ - + - + @@ -31149,21 +31185,21 @@ - - + + - + - + - + - + @@ -31177,483 +31213,483 @@ - + - + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - + - - + + - + - - - - - - - - - - - - - - - - - - - - - - - - - + + - - - + - + + + + + + + + + + + + + + + + + + + + - + + + + + + + - + - + - + - - - - - - - - - - - - - - - - - - - - - - - + + + - + + + + + + + + + + + + + + + + + + + + + - - + + - - + + - - - + + + - - - + + + - - + + - - - - - - - - - - - - - - - - - - - - - - - - - - + + + - - + + - + + + - + + - + + + + + + + + + + + + + + + + + + + + + - - + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + - - + + + + + + + + + + + + + + + + + + + + + + + + + + + - - + + - - + + - - + + - + - + - + - - + + - - + + - - - + + + - - + + - - + + - - + + - + - - - - - - - - - - - - - - - - - - - - + - + + - - + + + - + + + + + + + + + + + + + + + + + + - + - + - + - + - + - + - + - + - + - + @@ -31668,16 +31704,16 @@ - - + + - + - + @@ -31705,16 +31741,16 @@ - - + + - + - + @@ -31725,15 +31761,15 @@ - + - + - + - + @@ -31753,7 +31789,7 @@ - + @@ -31761,7 +31797,7 @@ - + @@ -31777,15 +31813,15 @@ - + - + - + @@ -31817,45 +31853,45 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -31870,49 +31906,49 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -31921,14 +31957,14 @@ - + - + - + - + @@ -31946,41 +31982,41 @@ - + - + - + - - + + - + - + - + - + - + @@ -31989,36 +32025,36 @@ - + - + - + - + - + - + - + - + - + - + - + - + @@ -32027,7 +32063,7 @@ - + @@ -32038,19 +32074,19 @@ - + - + - + - + @@ -32058,29 +32094,29 @@ - + - + - + - + - + - + - + - + - + @@ -32092,15 +32128,15 @@ - + - + - + - + @@ -32112,7 +32148,7 @@ - + @@ -32120,24 +32156,24 @@ - + - + - + - + - + - + - + @@ -32151,156 +32187,156 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + - - - + + + + + + + - + - + - + - - + + - - + + - - - - - - - - - - - - - - - - - - - - - - - - + + - + - - + + + - + + + + + + + + + + + + + + + + + + + + + + - - - - - - - - - - - - - + + + + - + + + + + + + + + + - - + + - - + + - - + + - - + + - - + + - + - + - + @@ -32314,7 +32350,7 @@ - + @@ -32326,7 +32362,7 @@ - + @@ -32340,7 +32376,7 @@ - + @@ -32386,18 +32422,18 @@ - + - + - + @@ -32417,7 +32453,7 @@ - + @@ -32434,30 +32470,30 @@ - + - + - + - + - + - + @@ -32466,17 +32502,17 @@ - + - + - - + + @@ -32557,7 +32593,7 @@ - + @@ -32568,7 +32604,7 @@ - + @@ -32576,7 +32612,7 @@ - + @@ -32618,16 +32654,16 @@ - + - - + + - + @@ -32647,13 +32683,13 @@ - + - + - + @@ -32665,7 +32701,7 @@ - + @@ -32673,59 +32709,59 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -32737,16 +32773,16 @@ - - + + - + - + - + @@ -32766,190 +32802,190 @@ - + - + - + - + - + - + - + - + - + - + - + - - - - + + + + - + - + - + - + - + - + - + - + - + - + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - + + - - + + - - - - - - - - - - - - - - - - - + + + - - - - - - - - - - + + - - + + + + + + + + - - + + + + + - - + + + + + + + - - + + - + + + + + + + + + - - + + - - - + + + - - + + - - + + @@ -32973,80 +33009,80 @@ - + - + - + - + - + - + - + - + - + - + - + - + - - - - - + + + + + - + - + - + - + @@ -33055,7 +33091,7 @@ - + @@ -33106,7 +33142,7 @@ - + @@ -33115,7 +33151,7 @@ - + @@ -33175,7 +33211,7 @@ - + @@ -33184,7 +33220,7 @@ - + @@ -33196,7 +33232,7 @@ - + @@ -33310,13 +33346,13 @@ - + - + @@ -33328,16 +33364,16 @@ - + - + - + @@ -33352,12 +33388,12 @@ - + - + @@ -33366,13 +33402,13 @@ - + - + @@ -33381,10 +33417,10 @@ - + - + @@ -33414,7 +33450,7 @@ - + @@ -33432,33 +33468,33 @@ - + - + - + - + - + - + - - + + - + - + @@ -33467,7 +33503,7 @@ - + @@ -33475,34 +33511,34 @@ - + - + - + - + - + - + - + - + - + - + @@ -33511,10 +33547,10 @@ - + - + @@ -33534,37 +33570,37 @@ - + - + - + - + - + - + - + - + - - + + - + @@ -33572,99 +33608,99 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + - + - - + + - - + + - - + + - - + + - - - + + + - - + + - - + + - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + @@ -33679,102 +33715,102 @@ - + - + - + - + - + - + - + - + - + - + - - - - - - - + + + + + + + - + - + - + - + - + - + - + - + - - + + - - + + - - + + - - + + - - + + @@ -33789,81 +33825,81 @@ - - + + - + - + - + - + - + - - - + + + - + - + - + - + - + - + - + - + - + - + - + - + - + - - + + @@ -33887,12 +33923,12 @@ - - + + - + @@ -33918,7 +33954,7 @@ - + @@ -33927,13 +33963,13 @@ - + - + @@ -33946,15 +33982,15 @@ - + - + - + - + @@ -33975,13 +34011,13 @@ - + - + @@ -33996,16 +34032,16 @@ - + - + - + @@ -34017,7 +34053,7 @@ - + @@ -34029,7 +34065,7 @@ - + @@ -34043,7 +34079,7 @@ - + @@ -34058,13 +34094,13 @@ - + - + @@ -34088,19 +34124,19 @@ - + - + - + - + @@ -34193,41 +34229,41 @@ - + - + - + - + - + - + - + - + - + @@ -34236,7 +34272,7 @@ - + @@ -34277,16 +34313,16 @@ - + - + - + - + @@ -34331,7 +34367,7 @@ - + @@ -34351,8 +34387,8 @@ - - + + @@ -34360,113 +34396,113 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -34480,12 +34516,12 @@ - + - + - + @@ -34494,26 +34530,26 @@ - + - + - + - + - + - + @@ -34522,193 +34558,193 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - + - + - + - + - + - + - + - + - - - - - - - - - - - - - - - - - - - - - - - + + + - - + + - - - + + + - - - - + - - + + - - + + - - + + + - + + + + + + + + + + + + + + + + + + + + + + + - + - + - + - + - - + + - - + + - + - - - - - + + + + + - + - + - + - + @@ -34720,7 +34756,7 @@ - + @@ -34728,7 +34764,7 @@ - + @@ -34737,10 +34773,10 @@ - + - + @@ -34761,7 +34797,7 @@ - + @@ -34769,7 +34805,7 @@ - + @@ -34780,13 +34816,13 @@ - + - + - + @@ -34822,24 +34858,24 @@ - + - + - + - + - + - + @@ -34851,7 +34887,7 @@ - + @@ -34859,109 +34895,109 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -34969,7 +35005,7 @@ - + @@ -34978,14 +35014,14 @@ - + - + @@ -35008,21 +35044,21 @@ - + - + - + - + - + @@ -35061,13 +35097,13 @@ - + - + - + @@ -35112,10 +35148,10 @@ - - + + - + @@ -35124,16 +35160,16 @@ - + - + - + @@ -35154,7 +35190,7 @@ - + @@ -35165,18 +35201,18 @@ - + - + - + - + @@ -35185,43 +35221,43 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -35236,63 +35272,63 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -35310,10 +35346,10 @@ - + - + @@ -35322,278 +35358,278 @@ - + - + - + - + - + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - + - + - + - + - + - + - + - + - - - - - - - - - - - - - - - - - - - - - - - - - - + - - - + + + - - + - - - + + + - + + + + + - - - + + + - - - + + + - - - + + + - - + - + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + - - + + - - - - - - - - - - - - - - - - - - - - - + + + - - + + - - + - - + - + + - + + + - + + + + + + + + + + + + + + + + + + - + @@ -35606,21 +35642,21 @@ - - + + - - + + - + - + @@ -35628,37 +35664,37 @@ - + - + - - - - + + + + - + - + - + - + @@ -35667,13 +35703,13 @@ - + - + - + @@ -35691,10 +35727,10 @@ - + - + @@ -35733,7 +35769,7 @@ - + @@ -35757,112 +35793,112 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -35871,9 +35907,9 @@ - + - + @@ -35885,7 +35921,7 @@ - + @@ -35899,10 +35935,10 @@ - + - + @@ -35910,12 +35946,12 @@ - + - + - + @@ -35927,110 +35963,110 @@ - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + - - - - - + + + + + + + + + - + - + - + - + - + - - - - - - - - - - - - - - - - - - - - - + - + + + + + + + + + + + + + + + + + + + + + - - + + - - + + - - + + - - + + - - + + - - + + @@ -36039,13 +36075,13 @@ - + - + - + @@ -36072,9 +36108,9 @@ - + - + @@ -36089,44 +36125,44 @@ - + - + - + - + - + - + - + - - - - - - + + + + + + - + - + @@ -36136,65 +36172,65 @@ - + - - - + + + - - + + - + - - + + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -36218,7 +36254,7 @@ - + @@ -36227,7 +36263,7 @@ - + @@ -36239,21 +36275,21 @@ - + - + - + - + - + - + @@ -36307,9 +36343,9 @@ - + - + @@ -36321,7 +36357,7 @@ - + @@ -36333,12 +36369,12 @@ - + - + - + @@ -36359,15 +36395,15 @@ - + - + - + - + @@ -36378,19 +36414,19 @@ - + - + - + - + @@ -36414,13 +36450,13 @@ - + - + - + @@ -36438,7 +36474,7 @@ - + @@ -36468,7 +36504,7 @@ - + @@ -36507,10 +36543,10 @@ - + - + @@ -36518,7 +36554,7 @@ - + @@ -36548,7 +36584,7 @@ - + @@ -36557,161 +36593,161 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - - - - + + + + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -36725,7 +36761,7 @@ - + @@ -36733,13 +36769,13 @@ - + - + @@ -36777,7 +36813,7 @@ - + @@ -36809,7 +36845,7 @@ - + @@ -36833,7 +36869,7 @@ - + @@ -36850,15 +36886,15 @@ - + - + - + - + @@ -36872,46 +36908,46 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -36920,362 +36956,362 @@ - + - + - + - + - + - + - + - + - + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - + - + - - + + - + - + - + - - - - - - - - - - - - - - - - - - - - - + - + + + + + + + + + + + + + + + + + + + + + - - - + + + - - + + - - + + - - + + - - - - - - - - - - - - - - - - - - - - - - - - + + - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + - - + + + + + + + + + + + + + + + + + + + + + + + + - + - + - - + + - - + + - + - - + + - - + + - - + + - - - - - - - - - - - - - - - - - - - - + + - - + + - + + + + + + + + + + + + + + + + + + + - - + + - - + + - - - + + + - - + + - - + + - + - - + + - + - + - + - + @@ -37329,7 +37365,7 @@ - + @@ -37338,19 +37374,19 @@ - + - + - + - + - + @@ -37359,7 +37395,7 @@ - + @@ -37370,55 +37406,55 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -37451,13 +37487,13 @@ - + - + - + @@ -37466,10 +37502,10 @@ - + - + @@ -37484,7 +37520,7 @@ - + @@ -37511,28 +37547,28 @@ - + - + - + - + - + @@ -37543,10 +37579,10 @@ - + - + @@ -37563,18 +37599,18 @@ - + - + - + - + - + @@ -37583,16 +37619,16 @@ - + - + - + @@ -37603,7 +37639,7 @@ - + @@ -37638,27 +37674,27 @@ - + - + - + - + - + - + @@ -37673,91 +37709,91 @@ - + - - - + + + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -37766,16 +37802,16 @@ - + - + - + - + @@ -37790,10 +37826,10 @@ - + - + @@ -37814,27 +37850,27 @@ - + - + - + - + - + - + @@ -37843,7 +37879,7 @@ - + @@ -37852,7 +37888,7 @@ - + @@ -37866,10 +37902,10 @@ - + - + @@ -37878,16 +37914,16 @@ - + - + - + - + @@ -37911,85 +37947,85 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - - + + @@ -38000,13 +38036,13 @@ - - + + - + @@ -38016,46 +38052,46 @@ - + - + - + - + - + - + - + - + - + - + - - + + @@ -38066,24 +38102,24 @@ - - - + + + - + - + - + @@ -38106,15 +38142,15 @@ - + - + - + @@ -38122,7 +38158,7 @@ - + @@ -38130,7 +38166,7 @@ - + @@ -38151,10 +38187,10 @@ - + - + @@ -38168,9 +38204,9 @@ - + - + @@ -38179,10 +38215,10 @@ - + - + @@ -38203,7 +38239,7 @@ - + @@ -38215,27 +38251,27 @@ - + - + - + - + - + - + @@ -38244,10 +38280,10 @@ - + - + @@ -38255,23 +38291,23 @@ - + - + - + - + - + - + @@ -38282,89 +38318,89 @@ - + - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - + + + + + - + @@ -38375,297 +38411,297 @@ - + - - + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + - + + + - - + - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - + - - + + - + - - + + - + - - + + - + - - + + - + - + - - + + - - - - - - + - - - - - - - - - - - - - - - - - - - - - - + + + - - + + + + - + + - - + + + + - + + + - + - - + - - + + - - + - + + + + + + + + + + + + + + + + + + + + - - + + - + - - + + - - - + + + - - + + - - + + - - + + - - + + - + - - - - - - - - - - - - - - - - - - - + - - + - - + - - + - - - + + - - + + - - - + + + - + + + + + + + + + + + + + + + + + + + + + + + - + - + - + - - + + @@ -38688,28 +38724,28 @@ - + - + - + - + - + - + - - + + @@ -38726,7 +38762,7 @@ - + @@ -38741,10 +38777,10 @@ - + - + @@ -38755,13 +38791,13 @@ - + - + - + @@ -38770,32 +38806,32 @@ - + - + - + - + - + - + - + @@ -38812,121 +38848,121 @@ - + - + - + - + - + - + - + - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + - + - + - + - + - + - + - + - + - + - - + + - + - + - - + + - - + + - - + + - + @@ -38938,7 +38974,7 @@ - + @@ -38967,35 +39003,35 @@ - + - + - + - + - + - + - + - + - + @@ -39013,159 +39049,159 @@ - - - - - - + + + + + + - + - + - - + + - - + + - - + + - - + + - - + + - - + + - + - + - + - + - + - + - - + + - + - + - + - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + - + - - + + - + - + - + - + - + - + - + - + @@ -39195,73 +39231,73 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -39270,7 +39306,7 @@ - + @@ -39333,49 +39369,49 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -39389,22 +39425,22 @@ - + - + - + - + @@ -39452,7 +39488,7 @@ - + @@ -39470,16 +39506,16 @@ - + - + - + @@ -39488,27 +39524,27 @@ - + - + - + - + - + - + - + @@ -39535,7 +39571,7 @@ - + @@ -39583,7 +39619,7 @@ - + @@ -39607,169 +39643,169 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -39796,7 +39832,7 @@ - + @@ -39805,26 +39841,26 @@ - + - + - + - + - + - + - + @@ -39836,59 +39872,59 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -39912,10 +39948,10 @@ - + - + @@ -39923,32 +39959,32 @@ - + - + - + - + - + - + - + @@ -39960,16 +39996,16 @@ - + - + - + @@ -39980,7 +40016,7 @@ - + @@ -40001,7 +40037,7 @@ - + @@ -40009,18 +40045,18 @@ - + - + - + - + - + @@ -40034,7 +40070,7 @@ - + @@ -40051,19 +40087,19 @@ - + - + - + - + - + @@ -40106,7 +40142,7 @@ - + @@ -40114,7 +40150,7 @@ - + @@ -40216,13 +40252,13 @@ - + - + @@ -40231,42 +40267,42 @@ - + - + - + - + - + - + - + - + - + - + - + - + @@ -40293,9 +40329,9 @@ - + - + @@ -40304,41 +40340,41 @@ - + - + - + - + - + - + - + - + - + - + - + @@ -40347,18 +40383,18 @@ - + - + - + - + @@ -40367,16 +40403,16 @@ - + - + - + - + @@ -40391,10 +40427,10 @@ - + - + @@ -40403,22 +40439,22 @@ - + - + - + - + - + @@ -40430,7 +40466,7 @@ - + @@ -40445,12 +40481,12 @@ - + - + - + @@ -40459,16 +40495,16 @@ - + - + - + @@ -40477,110 +40513,110 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -40610,7 +40646,7 @@ - + @@ -40622,50 +40658,50 @@ - + - + - + - + - + - + - + - - + + - + - + - + - + @@ -40680,7 +40716,7 @@ - + @@ -40694,12 +40730,12 @@ - + - + - + @@ -40708,17 +40744,17 @@ - + - + - - - + + + @@ -40735,109 +40771,109 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -40876,22 +40912,22 @@ - + - + - + - + - + - + @@ -40906,68 +40942,68 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -40981,7 +41017,7 @@ - + @@ -40996,7 +41032,7 @@ - + @@ -41004,12 +41040,12 @@ - + - + - + @@ -41018,21 +41054,21 @@ - + - + - + - + - + @@ -41052,7 +41088,7 @@ - + @@ -41072,34 +41108,34 @@ - + - + - + - + - + - + - + - + @@ -41111,7 +41147,7 @@ - + @@ -41129,13 +41165,13 @@ - + - + @@ -41161,16 +41197,16 @@ - + - + - + - + @@ -41221,10 +41257,10 @@ - + - + @@ -41235,7 +41271,7 @@ - + @@ -41279,27 +41315,27 @@ - + - + - + - + - + - + - + - + @@ -41365,7 +41401,7 @@ - + @@ -41391,7 +41427,7 @@ - + @@ -41435,7 +41471,7 @@ - + @@ -41464,13 +41500,13 @@ - + - + - + @@ -41478,90 +41514,90 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -41581,7 +41617,7 @@ - + @@ -41601,7 +41637,7 @@ - + @@ -41627,10 +41663,10 @@ - + - + @@ -41644,13 +41680,13 @@ - + - + - + @@ -41658,7 +41694,7 @@ - + @@ -41672,7 +41708,7 @@ - + @@ -41680,15 +41716,15 @@ - + - + - + @@ -41696,19 +41732,19 @@ - + - + - + @@ -41719,7 +41755,7 @@ - + @@ -41755,13 +41791,13 @@ - + - + @@ -41770,9 +41806,9 @@ - + - + @@ -41787,7 +41823,7 @@ - + @@ -41795,10 +41831,10 @@ - + - + @@ -41806,77 +41842,77 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -41894,7 +41930,7 @@ - + @@ -41903,18 +41939,18 @@ - + - + - + - + @@ -41932,45 +41968,45 @@ - + - + - + - + - + - + - + - + - + - + - + - + @@ -41984,116 +42020,116 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -42117,7 +42153,7 @@ - + @@ -42128,7 +42164,7 @@ - + @@ -42142,37 +42178,37 @@ - - + + - + - + - + - + - + - + - + - + - + - + @@ -42210,23 +42246,23 @@ - + - + - + - + @@ -42234,19 +42270,19 @@ - + - + - + - + @@ -42276,33 +42312,33 @@ - + - + - + - + - + - + - + - + - + - + @@ -42313,10 +42349,10 @@ - + - + @@ -42324,21 +42360,21 @@ - + - + - + - + - + - + @@ -42349,39 +42385,39 @@ - + - + - + - + - + - + - + - + - + - + - + - + @@ -42401,12 +42437,12 @@ - + - + @@ -42415,20 +42451,20 @@ - + - + - + - + - + @@ -42445,925 +42481,925 @@ - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + - - - + + + + + - + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - - - - - - + + + + + + + + + + + + + + + - + - + - + - - + + - - - - - - - - - - - - - - - + + + - - - - - - + + - - - + + + + + + - - - - + + - - + + + + + + + - - - + + + + - - - - - - - - - - - - - + + - - - + + + - - - + + + - - - + + + - - - + + + + - - - + + + + - - - + + + + - - - - - - + + + + - - + - - - - + - - - + - - - + + + + - - + + - + + + + + + + + + + + + + + + + + + + + + + + + + - - + + - - + + - - + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + - - - - - - - + + - - - - - + + + + - - - - - - + + + + + + - - - - - + + + + + - - - + + + + + + + - - - + + + + + - - - - + + + + + + - - - - - - + + + + + - - + + + - - - + + + - + + + + - - - + + + + + + - - + + - - - + + + - - - - - - + - - + + + - + + - - - - + + + - - - - - + + - - - - - + + - - + + - - + - - - - + + + + - - - + + + + + + + + + + + + - - + + - - + + - + + + + - - + + + - - + + - + + + + + + + + + + + + + + + + + + + + - - + + - - - + + + - - + + - - - + + + - - + + - + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + - - - - + + + - + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - + + - - + + - - - - - - - - - - + + - - - - - - - - - - - - - + + - + - + - - + + + - + + - - + - - + - + + - + - - + + - - - - - - - - - - - - - + + + + + + + + + - + + - + + + - - + + + - - + + - - + - - + - - + + - - + + - - + + - + + - - - - - - + + + + + + - - + + - - - + - + + + + + + + + + + + + + + + + + + + + + + - - + + - - + + - - + + - - + + - - - + + + - + - - + + - + - + - + - - + + - - + + @@ -43374,7 +43410,7 @@ - + @@ -43386,9 +43422,9 @@ - + - + @@ -43427,18 +43463,18 @@ - + - + - + @@ -43453,50 +43489,50 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -43505,24 +43541,24 @@ - + - + - + - + - + - + @@ -43531,7 +43567,7 @@ - + @@ -43539,36 +43575,36 @@ - + - + - + - + - + - + - + - + - + - + @@ -43579,7 +43615,7 @@ - + @@ -43602,19 +43638,19 @@ - + - + - + @@ -43659,11 +43695,11 @@ - + - - + + @@ -43674,9 +43710,9 @@ - + - + @@ -43694,9 +43730,9 @@ - + - + @@ -43708,9 +43744,9 @@ - + - + @@ -43728,27 +43764,27 @@ - + - + - + - + - + - + - + @@ -43757,26 +43793,26 @@ - + - + - + - + - + - + - + @@ -43788,24 +43824,24 @@ - + - + - + - + - + - + @@ -43826,120 +43862,120 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - + + - - + + - - + + - - + + - - + + - + - - + + - - + + - - - + + + @@ -43948,102 +43984,102 @@ - + - - + + - - + + - + - - - + + + - + - - + + - - - + + + - - + + - - + + - + - - + + - - + + - - - + + + - + - + - + - + - - - + + + - + - + - + - + @@ -44058,7 +44094,7 @@ - + @@ -44069,7 +44105,7 @@ - + @@ -44099,7 +44135,7 @@ - + @@ -44107,13 +44143,13 @@ - + - + @@ -44122,27 +44158,27 @@ - + - + - + - + - + - + - + @@ -44151,7 +44187,7 @@ - + @@ -44162,7 +44198,7 @@ - + @@ -44189,71 +44225,71 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -44274,21 +44310,21 @@ - + - + - + - + - + @@ -44299,12 +44335,12 @@ - + - + - + @@ -44313,269 +44349,269 @@ - + - + - - + + - + - + - + - + - + - + - + - + - + - + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - + - - + + - - - - - - - - - - - - - - - - - - - - - - - - - + + + - - + + - + + - - + + + + - + + + + + + + + + + + + + + + + + + + + - - + + - - - - - - - - - - - - - - - - - - - - - - - - + + + + - + + + + + + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - - - - - - + + - - + + - + + + + + + + + + + + + + + + + + + + + + - - + + - + - - + + - - + + - + - - + + - - + + - + - - + + - - + + - - + + - + @@ -44612,127 +44648,127 @@ - - + + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -44759,7 +44795,7 @@ - + @@ -44771,25 +44807,25 @@ - + - + - + - + - + - + @@ -44797,7 +44833,7 @@ - + @@ -44820,21 +44856,21 @@ - + - + - + - + @@ -44842,82 +44878,82 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - + + - + - + - + - - + + - + - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + @@ -44927,161 +44963,161 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - + - - + + - + + + - - + + + - - - - - - - - + + + + + - - - + + - - - + + - - - - + + - - + + + + + + + + - + + + + + + + + + + + + + + + + + + + + + + + + + - - + + - + - - + + - - - + + + - - + + - - - + + + - - - + + + - - + + - + - + - + - + - + - - + + - - + + @@ -45094,12 +45130,12 @@ - + - + @@ -45114,7 +45150,7 @@ - + @@ -45132,46 +45168,46 @@ - + - + - + - + - + - + - + - + - + - + - + @@ -45186,7 +45222,7 @@ - + @@ -45196,68 +45232,68 @@ - - - - - + + + + + - - + + - + - + - + - + - - + + - - - - - - - - + + + + + + + + - - - - - + + + + + - + - + @@ -45271,50 +45307,50 @@ - - + + - + - + - - - - - - - - + + + + + + + + - + - + - + - + - + - + @@ -45377,13 +45413,13 @@ - + - + - + @@ -45392,7 +45428,7 @@ - + @@ -45403,7 +45439,7 @@ - + @@ -45411,24 +45447,24 @@ - + - + - + - - + + - + - + - + @@ -45478,7 +45514,7 @@ - + @@ -45489,32 +45525,32 @@ - + - + - + - + - + - + - + - + - + - + @@ -45534,18 +45570,18 @@ - + - + - + - + - + @@ -45556,7 +45592,7 @@ - + @@ -45565,7 +45601,7 @@ - + @@ -45588,7 +45624,7 @@ - + @@ -45602,37 +45638,37 @@ - + - - + + - - + + - - + + - + - + - + - + @@ -45641,7 +45677,7 @@ - + @@ -45799,12 +45835,12 @@ - + - + - + @@ -45817,7 +45853,7 @@ - + @@ -45828,7 +45864,7 @@ - + @@ -45839,7 +45875,7 @@ - + @@ -45847,13 +45883,13 @@ - + - + @@ -45867,15 +45903,15 @@ - + - + - + @@ -45886,7 +45922,7 @@ - + @@ -45895,22 +45931,22 @@ - - + + - + - - + + - + @@ -45922,29 +45958,29 @@ - - - + + + - - - + + + - + - + @@ -45952,7 +45988,7 @@ - + @@ -45960,7 +45996,7 @@ - + @@ -45971,36 +46007,36 @@ - - - - + + + + - - + + - - + + - - + + - + - - + + - + @@ -46018,20 +46054,20 @@ - + - + - + - + - - + + @@ -46056,7 +46092,7 @@ - + @@ -46066,7 +46102,7 @@ - + @@ -46075,11 +46111,11 @@ - - - + + + - + @@ -46097,8 +46133,8 @@ - - + + @@ -46120,7 +46156,7 @@ - + @@ -46213,15 +46249,15 @@ - - + + - + @@ -46229,76 +46265,76 @@ - + - + - + - + - + - + - + - + - - + + - - + + - - + + - + - + - - + + @@ -46315,22 +46351,22 @@ - + - + - - - + + + - - + + - + @@ -46350,11 +46386,11 @@ - + - + @@ -46368,13 +46404,13 @@ - + - + @@ -46398,7 +46434,7 @@ - + @@ -46406,7 +46442,7 @@ - + @@ -46414,15 +46450,15 @@ - + - - - - + + + + @@ -46442,7 +46478,7 @@ - + @@ -46497,15 +46533,15 @@ - + - + - + @@ -46518,7 +46554,7 @@ - + @@ -46527,8 +46563,8 @@ - - + + @@ -46539,33 +46575,33 @@ - + - - - + + + - + - - - - - - - - - + + + + + + + + + @@ -46576,10 +46612,10 @@ - + - + @@ -46588,7 +46624,7 @@ - + @@ -46597,37 +46633,37 @@ - + - + - + - + - + - + - + - + - + - + @@ -46720,22 +46756,22 @@ - + - + - + - + - + @@ -46744,7 +46780,7 @@ - + @@ -46756,7 +46792,7 @@ - + @@ -46766,9 +46802,9 @@ - + - + @@ -46789,7 +46825,7 @@ - + @@ -46830,13 +46866,13 @@ - + - + - + @@ -46851,20 +46887,20 @@ - + - + - + - + - + @@ -46873,7 +46909,7 @@ - + @@ -46888,7 +46924,7 @@ - + @@ -46902,7 +46938,7 @@ - + @@ -46914,7 +46950,7 @@ - + @@ -46928,10 +46964,10 @@ - + - + @@ -46939,13 +46975,13 @@ - + - + @@ -46953,7 +46989,7 @@ - + @@ -46968,13 +47004,13 @@ - + - + - + @@ -46989,27 +47025,27 @@ - + - + - + - + - + - + - + - + @@ -47017,13 +47053,13 @@ - + - + @@ -47037,7 +47073,7 @@ - + @@ -47051,7 +47087,7 @@ - + @@ -47060,10 +47096,10 @@ - + - + @@ -47080,16 +47116,16 @@ - + - + - + - + @@ -47106,7 +47142,7 @@ - + @@ -47117,21 +47153,21 @@ - + - + - + - + - + @@ -47140,7 +47176,7 @@ - + @@ -47148,7 +47184,7 @@ - + @@ -47166,7 +47202,7 @@ - + @@ -47192,24 +47228,24 @@ - + - + - + - + - + @@ -47245,9 +47281,9 @@ - + - + @@ -47256,13 +47292,13 @@ - + - + @@ -47291,14 +47327,14 @@ - + - + @@ -47306,16 +47342,16 @@ - + - + - + @@ -47323,7 +47359,7 @@ - + @@ -47337,54 +47373,54 @@ - - - - - - - - - - - - - - - + + - - - + + + + + + + + + + + + + + + + - - + + - + - - - - - - + + + + + + - + - + - + @@ -47396,13 +47432,13 @@ - + - + @@ -47411,19 +47447,19 @@ - + - + - + - + @@ -47477,37 +47513,37 @@ - + - + - + - + - + - + - + - + - + @@ -47530,13 +47566,13 @@ - + - + - + @@ -47593,13 +47629,13 @@ - + - + - + @@ -47616,8 +47652,8 @@ - - + + @@ -47631,7 +47667,7 @@ - + @@ -47642,140 +47678,140 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -47786,28 +47822,28 @@ - + - + - + - + - + - + @@ -47819,19 +47855,19 @@ - + - + - + - + @@ -47843,40 +47879,40 @@ - + - + - + - + - + - + - + - + - + - + - + @@ -47884,42 +47920,42 @@ - + - + - + - + - + - + - + - + - + - + - + - + @@ -47927,7 +47963,7 @@ - + @@ -47939,48 +47975,48 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -48001,12 +48037,12 @@ - + - + - + @@ -48083,15 +48119,15 @@ - + - + - + @@ -48108,15 +48144,15 @@ - + - + - + - + @@ -48124,13 +48160,13 @@ - + - + @@ -48156,7 +48192,7 @@ - + @@ -48164,13 +48200,13 @@ - + - + @@ -48181,15 +48217,15 @@ - + - + - + - + @@ -48198,13 +48234,13 @@ - + - + @@ -48215,7 +48251,7 @@ - + @@ -48224,185 +48260,185 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - + + + + + - + - - + + - - + + - - + + - - + + - + - + - + - - + + - + - + - + - + - + - - - - - - - - - - - - - - - - - - - - - - - - + + - + - - + + + - + + + - + + + + + + + + + + + + + + + + + + + + - - + + - - + + @@ -48410,186 +48446,186 @@ - - + + - - - - - - - - - - - - - - - - - - - - - - - - - + + - - - + + + - - - - - + + - - - - - - + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - + - - - + + - - + + + + + + + - + + + + + + + + + + + + + + + + + - + - - - - - - - - - - - - - - - - - - + + - - + - - + - - + - - + - + + + + + + + + + + + + + + + + + + + + + - - + + @@ -48597,13 +48633,13 @@ - + - + @@ -48630,48 +48666,48 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -48706,7 +48742,7 @@ - + @@ -48724,7 +48760,7 @@ - + @@ -48744,12 +48780,12 @@ - + - + - + @@ -48761,33 +48797,33 @@ - + - + - + - + - + - + - + - + - + @@ -48802,7 +48838,7 @@ - + @@ -48810,59 +48846,59 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -48871,191 +48907,191 @@ - - - - - - - + - - - - - - + + + + + + + + - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + - + - - + + - - + + - + - + - - - - - - - - - - - - - - - - - - - - - - + - + + - - - - - - + + + + + + - - + + + - + + + + + + + + + + + + + + + + + + + + - - - + + + - - - - - - - - - - - - - - + + + + + - - - - - - - - - - - - + + + - - + + + + + + + + + + + + + + + + + + + + - - + + - + - + - + - + - + - + - + - + @@ -49066,7 +49102,7 @@ - + @@ -49111,16 +49147,16 @@ - + - + - + - + @@ -49132,10 +49168,10 @@ - + - + @@ -49150,19 +49186,19 @@ - + - + - + - + - + @@ -49192,16 +49228,16 @@ - + - + - + @@ -49215,7 +49251,7 @@ - + @@ -49232,24 +49268,24 @@ - + - + - + - + - + - + - + @@ -49264,13 +49300,13 @@ - + - + - + @@ -49281,16 +49317,16 @@ - + - + - + - + @@ -49298,7 +49334,7 @@ - + @@ -49306,32 +49342,32 @@ - + - + - + - + - + - + - + - + - + @@ -49345,7 +49381,7 @@ - + @@ -49353,10 +49389,10 @@ - + - + @@ -49373,16 +49409,16 @@ - + - + - + @@ -49402,7 +49438,7 @@ - + @@ -49410,7 +49446,7 @@ - + @@ -49430,12 +49466,12 @@ - + - + - + @@ -49444,7 +49480,7 @@ - + @@ -49464,7 +49500,7 @@ - + @@ -49475,10 +49511,10 @@ - + - + @@ -49487,30 +49523,30 @@ - + - + - + - + - + - + - + - + @@ -49522,7 +49558,7 @@ - + @@ -49531,7 +49567,7 @@ - + @@ -49542,9 +49578,9 @@ - + - + @@ -49559,89 +49595,89 @@ - + - + - + - + - + - + - + - + - + - + - + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - + - + - + - + @@ -49650,104 +49686,104 @@ - + - + - + - - - - - - - - - - - - - - - - - - - - - - - + - - + + + - - - + + + - - - - + + - - - + + - + + + + + + + + + + + + + + + + + + + + + + + + + - - - + + + - - - - - - - - - - - - - - - - - - - - + - + + + + + + + + + + + + + + + + + + + + @@ -49756,14 +49792,14 @@ - + - + @@ -49792,19 +49828,19 @@ - + - + - + - + @@ -49815,13 +49851,13 @@ - + - + @@ -49848,10 +49884,10 @@ - + - + @@ -49881,7 +49917,7 @@ - + @@ -49893,36 +49929,36 @@ - + - + - + - + - + - + - + - + - + - + - + @@ -49930,10 +49966,10 @@ - + - + @@ -49941,90 +49977,90 @@ - + - + - - - - - - - - - - - - - - + + + + + + + + + + + + + + - + - + - + - + - + - + - + - + - + - + - + - + - - + + - + - - + + @@ -50035,64 +50071,64 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -50101,7 +50137,7 @@ - + @@ -50118,10 +50154,10 @@ - + - + @@ -50145,7 +50181,7 @@ - + @@ -50157,19 +50193,19 @@ - + - + - + - + - + @@ -50214,13 +50250,13 @@ - + - + @@ -50229,7 +50265,7 @@ - + @@ -50240,30 +50276,30 @@ - + - + - + - + - + - + - + - + @@ -50275,36 +50311,6 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - @@ -50316,7 +50322,7 @@ - + @@ -50335,13 +50341,13 @@ - + - + @@ -50390,7 +50396,7 @@ - + @@ -50462,7 +50468,7 @@ - + @@ -50482,7 +50488,7 @@ - + @@ -50554,7 +50560,7 @@ - + @@ -50592,7 +50598,7 @@ - + @@ -50609,7 +50615,7 @@ - + @@ -50640,7 +50646,7 @@ - + @@ -50648,19 +50654,18 @@ - - - - - + + + + + - - - - - - - + + + + + + @@ -50679,7 +50684,7 @@ - + @@ -50687,12 +50692,12 @@ - + - - + + - + @@ -50722,32 +50727,32 @@ - + - + - + - + - + @@ -50802,7 +50807,7 @@ - + @@ -51298,7 +51303,7 @@ - + @@ -51458,14 +51463,14 @@ - + - + @@ -51473,7 +51478,7 @@ - + @@ -51481,7 +51486,7 @@ - + @@ -51514,7 +51519,7 @@ - + @@ -51529,7 +51534,7 @@ - + @@ -51559,13 +51564,13 @@ - + - + - + @@ -52376,7 +52381,7 @@ - + @@ -52424,7 +52429,7 @@ - + @@ -52455,7 +52460,7 @@ - + @@ -52544,7 +52549,7 @@ - + @@ -53047,7 +53052,7 @@ - + @@ -53104,7 +53109,7 @@ - + @@ -53186,24 +53191,24 @@ - + - + - + - + - + @@ -53211,11 +53216,11 @@ - + - + @@ -53228,54 +53233,54 @@ - - + + - - + + - - + + - - + + - - + + - + - + - + - + @@ -53283,7 +53288,7 @@ - + @@ -53292,12 +53297,12 @@ - + - + @@ -53305,29 +53310,29 @@ - - + + - + - + - + - + @@ -53379,7 +53384,7 @@ - + @@ -53393,14 +53398,14 @@ - + - + @@ -53413,33 +53418,33 @@ - + - + - + - + - + - + @@ -53475,20 +53480,20 @@ - + - + - + @@ -53584,7 +53589,7 @@ - + @@ -53699,7 +53704,7 @@ - + @@ -53851,10 +53856,10 @@ - + - + @@ -53913,7 +53918,7 @@ - + @@ -54217,7 +54222,7 @@ - + @@ -54597,7 +54602,7 @@ - + @@ -55248,13 +55253,13 @@ - + - + - + @@ -55278,10 +55283,10 @@ - + - + @@ -55335,8 +55340,8 @@ - - + + @@ -55419,7 +55424,7 @@ - + @@ -55484,33 +55489,33 @@ - + - + - + - + - + - + - + @@ -55609,7 +55614,7 @@ - + @@ -55622,6 +55627,8 @@ + + @@ -55644,7 +55651,7 @@ - + @@ -55661,87 +55668,87 @@ - + - - - - + + + + - + - - - - + + + + - + - + - + - + - + - + - + - + - + - + - + - - - - + + + + - - + + - + - + - + - + - + @@ -55777,7 +55784,7 @@ - + @@ -55785,7 +55792,7 @@ - + @@ -55793,9 +55800,9 @@ - + - + @@ -55804,7 +55811,7 @@ - + @@ -55812,10 +55819,10 @@ - + - + @@ -55833,125 +55840,125 @@ - + - + - + - - + + - + - + - + - - - - - + - - + + - + - + + + + + - + - + - + - + - + - + - + - + - - + + - + - + - + - + - + - + - + - + - + @@ -55984,7 +55991,7 @@ - + @@ -56050,10 +56057,10 @@ - + - + @@ -56080,7 +56087,7 @@ - + @@ -56089,13 +56096,13 @@ - + - + - + @@ -56107,10 +56114,10 @@ - + - + @@ -56119,13 +56126,13 @@ - + - + - + @@ -56164,18 +56171,18 @@ - + - + - + - + @@ -56183,9 +56190,9 @@ - + - + @@ -56209,7 +56216,7 @@ - + @@ -56217,13 +56224,13 @@ - + - + @@ -56243,12 +56250,12 @@ - + - + - + @@ -56263,7 +56270,7 @@ - + @@ -56274,7 +56281,7 @@ - + @@ -56297,19 +56304,19 @@ - + - + - + - + @@ -56323,9 +56330,9 @@ - + - + @@ -56340,7 +56347,7 @@ - + @@ -56349,7 +56356,7 @@ - + @@ -56367,10 +56374,10 @@ - + - + @@ -56381,7 +56388,7 @@ - + @@ -56392,12 +56399,12 @@ - + - + @@ -56406,7 +56413,7 @@ - + @@ -56418,7 +56425,7 @@ - + @@ -56430,16 +56437,16 @@ - + - + - + @@ -56448,10 +56455,10 @@ - + - + @@ -56475,38 +56482,38 @@ - + - + - + - + - + - + - + - + - + @@ -56514,15 +56521,15 @@ - + - + - + @@ -56530,12 +56537,12 @@ - + - + - + @@ -56562,10 +56569,10 @@ - + - + @@ -56577,13 +56584,13 @@ - + - + - + @@ -56592,7 +56599,7 @@ - + @@ -56601,13 +56608,13 @@ - + - + - + @@ -56643,7 +56650,7 @@ - + @@ -56655,7 +56662,7 @@ - + @@ -56673,18 +56680,18 @@ - + - + - + - + @@ -56701,16 +56708,16 @@ - + - + - + @@ -56718,7 +56725,7 @@ - + @@ -56744,33 +56751,33 @@ - + - + - + - + - + - + - + - + - + - + @@ -56785,13 +56792,13 @@ - + - + @@ -56803,9 +56810,9 @@ - + - + @@ -56838,7 +56845,7 @@ - + @@ -56847,13 +56854,13 @@ - + - + @@ -56870,12 +56877,12 @@ - + - + - + @@ -56884,10 +56891,10 @@ - + - + @@ -56898,15 +56905,15 @@ - + - + - + @@ -56921,7 +56928,7 @@ - + @@ -56932,12 +56939,12 @@ - + - + - + @@ -56949,15 +56956,15 @@ - + - + - + - + @@ -56966,22 +56973,22 @@ - + - + - + - + - + @@ -56995,7 +57002,7 @@ - + @@ -57009,15 +57016,15 @@ - + - + - + - + @@ -57026,7 +57033,7 @@ - + @@ -57034,74 +57041,74 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -57109,16 +57116,16 @@ - + - + - + - + @@ -57127,39 +57134,39 @@ - + - + - + - + - + - + @@ -57167,24 +57174,24 @@ - + - + - + - + - + - + - + @@ -57196,7 +57203,7 @@ - + @@ -57207,7 +57214,7 @@ - + @@ -57215,15 +57222,15 @@ - + - + - + - + @@ -57237,18 +57244,18 @@ - + - + - + - + @@ -57257,15 +57264,15 @@ - + - + - + @@ -57285,18 +57292,18 @@ - + - + - + - + @@ -57305,7 +57312,7 @@ - + @@ -57313,9 +57320,9 @@ - + - + @@ -57324,7 +57331,7 @@ - + @@ -57332,10 +57339,10 @@ - + - + @@ -57343,7 +57350,7 @@ - + @@ -57354,10 +57361,10 @@ - + - + @@ -57386,333 +57393,333 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -57720,7 +57727,7 @@ - + @@ -57729,12 +57736,12 @@ - + - + - + @@ -57749,19 +57756,19 @@ - + - + - + - + @@ -57773,22 +57780,22 @@ - + - + - + - + - + @@ -57803,10 +57810,10 @@ - + - + @@ -57859,13 +57866,13 @@ - + - + @@ -57882,7 +57889,7 @@ - + @@ -57897,7 +57904,7 @@ - + @@ -57912,7 +57919,7 @@ - + @@ -57926,7 +57933,7 @@ - + @@ -57934,48 +57941,48 @@ - + - + - + - + - + - + - + - + - + - + - + - + @@ -57986,7 +57993,7 @@ - + @@ -58021,10 +58028,10 @@ - + - + @@ -58033,7 +58040,7 @@ - + @@ -58066,7 +58073,7 @@ - + @@ -58075,10 +58082,10 @@ - + - + @@ -58087,21 +58094,21 @@ - + - + - + - + @@ -58112,7 +58119,7 @@ - + @@ -58150,16 +58157,16 @@ - + - + - + - + @@ -58177,10 +58184,10 @@ - + - + @@ -58198,13 +58205,13 @@ - + - + - + @@ -58225,7 +58232,7 @@ - + @@ -58246,7 +58253,7 @@ - + @@ -58275,7 +58282,7 @@ - + @@ -58286,7 +58293,7 @@ - + @@ -58294,7 +58301,7 @@ - + @@ -58311,10 +58318,10 @@ - + - + @@ -58352,7 +58359,7 @@ - + @@ -58387,7 +58394,7 @@ - + @@ -58467,7 +58474,7 @@ - + @@ -58476,9 +58483,9 @@ - + - + @@ -58526,7 +58533,7 @@ - + @@ -58535,12 +58542,12 @@ - + - + - + @@ -58569,9 +58576,9 @@ - + - + @@ -58586,7 +58593,7 @@ - + @@ -58594,9 +58601,9 @@ - + - + @@ -58605,7 +58612,7 @@ - + @@ -58623,28 +58630,28 @@ - + - + - + - + - + - + - + @@ -58653,23 +58660,23 @@ - + - + - + - + - + @@ -58678,7 +58685,7 @@ - + @@ -58696,9 +58703,9 @@ - + - + @@ -58710,7 +58717,7 @@ - + @@ -58719,16 +58726,16 @@ - + - + - + - + @@ -58740,7 +58747,7 @@ - + @@ -58757,9 +58764,9 @@ - + - + @@ -58774,12 +58781,12 @@ - + - + - + @@ -58794,7 +58801,7 @@ - + @@ -58806,10 +58813,10 @@ - + - + @@ -58827,22 +58834,22 @@ - + - + - + - + - + @@ -58872,24 +58879,24 @@ - + - + - + - + - + - + @@ -58898,15 +58905,15 @@ - + - + - + - + @@ -58914,7 +58921,7 @@ - + @@ -58950,21 +58957,21 @@ - + - + - + - + - + @@ -58994,7 +59001,7 @@ - + @@ -59017,9 +59024,9 @@ - + - + @@ -59043,10 +59050,10 @@ - + - + @@ -59057,7 +59064,7 @@ - + @@ -59065,7 +59072,7 @@ - + @@ -59076,7 +59083,7 @@ - + @@ -59084,45 +59091,45 @@ - + - + - + - + - + - + - + - + - + - + - + @@ -59131,7 +59138,7 @@ - + @@ -59143,18 +59150,18 @@ - + - + - + - + @@ -59162,7 +59169,7 @@ - + @@ -59170,7 +59177,7 @@ - + @@ -59178,12 +59185,12 @@ - + - + - + @@ -59204,7 +59211,7 @@ - + @@ -59228,16 +59235,16 @@ - + - + - + @@ -59252,7 +59259,7 @@ - + @@ -59260,7 +59267,7 @@ - + @@ -59268,7 +59275,7 @@ - + @@ -59282,15 +59289,15 @@ - + - + - + - + @@ -59302,12 +59309,12 @@ - + - + @@ -59339,7 +59346,7 @@ - + @@ -59356,35 +59363,35 @@ - + - + - + - + - + - + - + - + - + @@ -59414,16 +59421,16 @@ - + - + - + @@ -59435,475 +59442,473 @@ - - - - - - - - - - - - - - - - - - - - - + + + + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + - - - - - - - - - - - + + + + + + + + + + + - - - - - - - - - - - + + + + + + + + + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - - - - - - + + + + + + + + + + + - + - + - + - + - + - - - - - - + - + - - + + - - + + - - + + - + - - + - + - + - + - + - + - - + - + - - - - - - - + - - + + + + + + + + + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - - + - + - + + - + - - + - + - + - + - + + + + + + + + - - + + - - - - - - - - + - - - - + - - + + + + - + + + + + + + + + @@ -59915,55 +59920,55 @@ - - + + - - + + - + - - - - - - - + - + - + + + + + + + - - + + - - + + - - + + @@ -59971,87 +59976,87 @@ - - + + - + - - + + - - + + - - + + - - + + - - + + - - + + - - - - - - - - + - - + - + + + + + + + + + - - + + - - + + - - + + @@ -60059,25 +60064,25 @@ - - - - - - - - - + - - + + + + + + + + + + @@ -60086,203 +60091,203 @@ - - + + - - + + - - - - - - - + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - + + + - - - + + + - - - - + + + - - - - + + + + - - - + + + + + + + + + + - - - + + + - - - + + + - - - + + + - - - + + + - + - + - + - + - + - - + + - - - + + + - - + + - - + + - + - + - + - + - + @@ -60294,13 +60299,13 @@ - + - + @@ -60310,14 +60315,14 @@ - + - + - + @@ -60327,11 +60332,11 @@ - + - - + + @@ -60387,7 +60392,7 @@ - + @@ -60464,26 +60469,26 @@ - + - + - + - + @@ -60530,7 +60535,7 @@ - + @@ -60562,91 +60567,9 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - From ca852080878fad60f960fa55c50e77f1295136c5 Mon Sep 17 00:00:00 2001 From: Andrey Konovalov Date: Wed, 4 Dec 2019 16:52:47 -0800 Subject: [PATCH 07/89] UPSTREAM: usb, kcov: collect coverage from hub_event (Upstream commit 95d23dc27bde0ab4b25f7ade5e2fddc08dd97d9b.) Add kcov_remote_start()/kcov_remote_stop() annotations to the hub_event() function, which is responsible for processing events on USB buses, in particular events that happen during USB device enumeration. Since hub_event() is run in a global background kernel thread (see Documentation/dev-tools/kcov.rst for details), each USB bus gets a unique global handle from the USB subsystem kcov handle range. As the result kcov can now be used to collect coverage from events that happen on a particular USB bus. [akpm@linux-foundation.org: avoid patch conflicts to make life easier for Andrew] Link: http://lkml.kernel.org/r/de4fe1c219db2d002d905dc1736e2a3bfa1db997.1572366574.git.andreyknvl@google.com Signed-off-by: Andrey Konovalov Reviewed-by: Greg Kroah-Hartman Cc: Alan Stern Cc: Alexander Potapenko Cc: Anders Roxell Cc: Arnd Bergmann Cc: David Windsor Cc: Dmitry Vyukov Cc: Elena Reshetova Cc: Jason Wang Cc: Marco Elver Cc: "Michael S. Tsirkin" Cc: Steven Rostedt Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds Signed-off-by: Andrey Konovalov Bug: 147413187 Change-Id: I4ef39331fe5b789459938e998b1f557406627ba6 --- drivers/usb/core/hub.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/drivers/usb/core/hub.c b/drivers/usb/core/hub.c index 8b23162e4239..0e26c32fdddb 100644 --- a/drivers/usb/core/hub.c +++ b/drivers/usb/core/hub.c @@ -18,6 +18,7 @@ #include #include #include +#include #include #include #include @@ -5374,6 +5375,8 @@ static void hub_event(struct work_struct *work) hub_dev = hub->intfdev; intf = to_usb_interface(hub_dev); + kcov_remote_start_usb((u64)hdev->bus->busnum); + dev_dbg(hub_dev, "state %d ports %d chg %04x evt %04x\n", hdev->state, hdev->maxchild, /* NOTE: expects max 15 ports... */ @@ -5480,6 +5483,8 @@ out_hdev_lock: /* Balance the stuff in kick_hub_wq() and allow autosuspend */ usb_autopm_put_interface(intf); kref_put(&hub->kref, hub_release); + + kcov_remote_stop(); } static const struct usb_device_id hub_id_table[] = { From d26817a9d0070a6f881ced376d042e43e4317055 Mon Sep 17 00:00:00 2001 From: Andrey Konovalov Date: Wed, 4 Dec 2019 16:52:50 -0800 Subject: [PATCH 08/89] UPSTREAM: vhost, kcov: collect coverage from vhost_worker (Upstream commit 8f6a7f96dc29cefe16ab60f06f9c3a43510b96fd.) Add kcov_remote_start()/kcov_remote_stop() annotations to the vhost_worker() function, which is responsible for processing vhost works. Since vhost_worker() threads are spawned per vhost device instance the common kcov handle is used for kcov_remote_start()/stop() annotations (see Documentation/dev-tools/kcov.rst for details). As the result kcov can now be used to collect coverage from vhost worker threads. Link: http://lkml.kernel.org/r/e49d5d154e5da6c9ada521d2b7ce10a49ce9f98b.1572366574.git.andreyknvl@google.com Signed-off-by: Andrey Konovalov Cc: Alan Stern Cc: Alexander Potapenko Cc: Anders Roxell Cc: Arnd Bergmann Cc: David Windsor Cc: Dmitry Vyukov Cc: Elena Reshetova Cc: Greg Kroah-Hartman Cc: Jason Wang Cc: Marco Elver Cc: "Michael S. Tsirkin" Cc: Steven Rostedt Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds Signed-off-by: Andrey Konovalov Bug: 147413187 Change-Id: Ie99a67ede00a839a28472877e5c3263db69d1c58 --- drivers/vhost/vhost.c | 6 ++++++ drivers/vhost/vhost.h | 1 + 2 files changed, 7 insertions(+) diff --git a/drivers/vhost/vhost.c b/drivers/vhost/vhost.c index 36ca2cf419bf..f44340b41494 100644 --- a/drivers/vhost/vhost.c +++ b/drivers/vhost/vhost.c @@ -30,6 +30,7 @@ #include #include #include +#include #include "vhost.h" @@ -357,7 +358,9 @@ static int vhost_worker(void *data) llist_for_each_entry_safe(work, work_next, node, node) { clear_bit(VHOST_WORK_QUEUED, &work->flags); __set_current_state(TASK_RUNNING); + kcov_remote_start_common(dev->kcov_handle); work->fn(work); + kcov_remote_stop(); if (need_resched()) schedule(); } @@ -546,6 +549,7 @@ long vhost_dev_set_owner(struct vhost_dev *dev) /* No owner, become one */ dev->mm = get_task_mm(current); + dev->kcov_handle = kcov_common_handle(); worker = kthread_create(vhost_worker, dev, "vhost-%d", current->pid); if (IS_ERR(worker)) { err = PTR_ERR(worker); @@ -571,6 +575,7 @@ err_worker: if (dev->mm) mmput(dev->mm); dev->mm = NULL; + dev->kcov_handle = 0; err_mm: return err; } @@ -682,6 +687,7 @@ void vhost_dev_cleanup(struct vhost_dev *dev) if (dev->worker) { kthread_stop(dev->worker); dev->worker = NULL; + dev->kcov_handle = 0; } if (dev->mm) mmput(dev->mm); diff --git a/drivers/vhost/vhost.h b/drivers/vhost/vhost.h index e9ed2722b633..a123fd70847e 100644 --- a/drivers/vhost/vhost.h +++ b/drivers/vhost/vhost.h @@ -173,6 +173,7 @@ struct vhost_dev { int iov_limit; int weight; int byte_weight; + u64 kcov_handle; }; bool vhost_exceeds_weight(struct vhost_virtqueue *vq, int pkts, int total_len); From 341464390512ed50d5e96cf8f5340dcfbebd837a Mon Sep 17 00:00:00 2001 From: Will Deacon Date: Thu, 19 Dec 2019 12:02:03 +0000 Subject: [PATCH 09/89] chardev: Avoid potential use-after-free in 'chrdev_open()' commit 68faa679b8be1a74e6663c21c3a9d25d32f1c079 upstream. 'chrdev_open()' calls 'cdev_get()' to obtain a reference to the 'struct cdev *' stashed in the 'i_cdev' field of the target inode structure. If the pointer is NULL, then it is initialised lazily by looking up the kobject in the 'cdev_map' and so the whole procedure is protected by the 'cdev_lock' spinlock to serialise initialisation of the shared pointer. Unfortunately, it is possible for the initialising thread to fail *after* installing the new pointer, for example if the subsequent '->open()' call on the file fails. In this case, 'cdev_put()' is called, the reference count on the kobject is dropped and, if nobody else has taken a reference, the release function is called which finally clears 'inode->i_cdev' from 'cdev_purge()' before potentially freeing the object. The problem here is that a racing thread can happily take the 'cdev_lock' and see the non-NULL pointer in the inode, which can result in a refcount increment from zero and a warning: | ------------[ cut here ]------------ | refcount_t: addition on 0; use-after-free. | WARNING: CPU: 2 PID: 6385 at lib/refcount.c:25 refcount_warn_saturate+0x6d/0xf0 | Modules linked in: | CPU: 2 PID: 6385 Comm: repro Not tainted 5.5.0-rc2+ #22 | Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS 1.12.0-1 04/01/2014 | RIP: 0010:refcount_warn_saturate+0x6d/0xf0 | Code: 05 55 9a 15 01 01 e8 9d aa c8 ff 0f 0b c3 80 3d 45 9a 15 01 00 75 ce 48 c7 c7 00 9c 62 b3 c6 08 | RSP: 0018:ffffb524c1b9bc70 EFLAGS: 00010282 | RAX: 0000000000000000 RBX: ffff9e9da1f71390 RCX: 0000000000000000 | RDX: ffff9e9dbbd27618 RSI: ffff9e9dbbd18798 RDI: ffff9e9dbbd18798 | RBP: 0000000000000000 R08: 000000000000095f R09: 0000000000000039 | R10: 0000000000000000 R11: ffffb524c1b9bb20 R12: ffff9e9da1e8c700 | R13: ffffffffb25ee8b0 R14: 0000000000000000 R15: ffff9e9da1e8c700 | FS: 00007f3b87d26700(0000) GS:ffff9e9dbbd00000(0000) knlGS:0000000000000000 | CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 | CR2: 00007fc16909c000 CR3: 000000012df9c000 CR4: 00000000000006e0 | DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 | DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 | Call Trace: | kobject_get+0x5c/0x60 | cdev_get+0x2b/0x60 | chrdev_open+0x55/0x220 | ? cdev_put.part.3+0x20/0x20 | do_dentry_open+0x13a/0x390 | path_openat+0x2c8/0x1470 | do_filp_open+0x93/0x100 | ? selinux_file_ioctl+0x17f/0x220 | do_sys_open+0x186/0x220 | do_syscall_64+0x48/0x150 | entry_SYSCALL_64_after_hwframe+0x44/0xa9 | RIP: 0033:0x7f3b87efcd0e | Code: 89 54 24 08 e8 a3 f4 ff ff 8b 74 24 0c 48 8b 3c 24 41 89 c0 44 8b 54 24 08 b8 01 01 00 00 89 f4 | RSP: 002b:00007f3b87d259f0 EFLAGS: 00000293 ORIG_RAX: 0000000000000101 | RAX: ffffffffffffffda RBX: 0000000000000000 RCX: 00007f3b87efcd0e | RDX: 0000000000000000 RSI: 00007f3b87d25a80 RDI: 00000000ffffff9c | RBP: 00007f3b87d25e90 R08: 0000000000000000 R09: 0000000000000000 | R10: 0000000000000000 R11: 0000000000000293 R12: 00007ffe188f504e | R13: 00007ffe188f504f R14: 00007f3b87d26700 R15: 0000000000000000 | ---[ end trace 24f53ca58db8180a ]--- Since 'cdev_get()' can already fail to obtain a reference, simply move it over to use 'kobject_get_unless_zero()' instead of 'kobject_get()', which will cause the racing thread to return -ENXIO if the initialising thread fails unexpectedly. Cc: Hillf Danton Cc: Andrew Morton Cc: Al Viro Reported-by: syzbot+82defefbbd8527e1c2cb@syzkaller.appspotmail.com Signed-off-by: Will Deacon Cc: stable Link: https://lore.kernel.org/r/20191219120203.32691-1-will@kernel.org Signed-off-by: Greg Kroah-Hartman --- fs/char_dev.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fs/char_dev.c b/fs/char_dev.c index 00dfe17871ac..c5e6eff5a381 100644 --- a/fs/char_dev.c +++ b/fs/char_dev.c @@ -352,7 +352,7 @@ static struct kobject *cdev_get(struct cdev *p) if (owner && !try_module_get(owner)) return NULL; - kobj = kobject_get(&p->kobj); + kobj = kobject_get_unless_zero(&p->kobj); if (!kobj) module_put(owner); return kobj; From 9d7e2f25a0fde3e216a97e90b6556adf0d7d3eca Mon Sep 17 00:00:00 2001 From: Russell King Date: Sun, 15 Dec 2019 16:39:05 +0000 Subject: [PATCH 10/89] i2c: fix bus recovery stop mode timing commit cf8ce8b80f8bf9669f6ec4e71e16668430febdac upstream. The I2C specification states that tsu:sto for standard mode timing must be at minimum 4us. Pictographically, this is: SCL: ____/~~~~~~~~~ SDA: _________/~~~~ ->| |<- 4us minimum We are currently waiting 2.5us between asserting SCL and SDA, which is in violation of the standard. Adjust the timings to ensure that we meet what is stipulated as the minimum timings to ensure that all devices correctly interpret the STOP bus transition. This is more important than trying to generate a square wave with even duty cycle. Signed-off-by: Russell King Signed-off-by: Wolfram Sang Signed-off-by: Greg Kroah-Hartman --- drivers/i2c/i2c-core-base.c | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/drivers/i2c/i2c-core-base.c b/drivers/i2c/i2c-core-base.c index 5f6a4985f2bc..810a942eaa8e 100644 --- a/drivers/i2c/i2c-core-base.c +++ b/drivers/i2c/i2c-core-base.c @@ -186,10 +186,11 @@ int i2c_generic_scl_recovery(struct i2c_adapter *adap) * If we can set SDA, we will always create a STOP to ensure additional * pulses will do no harm. This is achieved by letting SDA follow SCL * half a cycle later. Check the 'incomplete_write_byte' fault injector - * for details. + * for details. Note that we must honour tsu:sto, 4us, but lets use 5us + * here for simplicity. */ bri->set_scl(adap, scl); - ndelay(RECOVERY_NDELAY / 2); + ndelay(RECOVERY_NDELAY); if (bri->set_sda) bri->set_sda(adap, scl); ndelay(RECOVERY_NDELAY / 2); @@ -211,7 +212,13 @@ int i2c_generic_scl_recovery(struct i2c_adapter *adap) scl = !scl; bri->set_scl(adap, scl); /* Creating STOP again, see above */ - ndelay(RECOVERY_NDELAY / 2); + if (scl) { + /* Honour minimum tsu:sto */ + ndelay(RECOVERY_NDELAY); + } else { + /* Honour minimum tf and thd:dat */ + ndelay(RECOVERY_NDELAY / 2); + } if (bri->set_sda) bri->set_sda(adap, scl); ndelay(RECOVERY_NDELAY / 2); From d7ce45829cbfff0c4fc082d05823dac8cd241fdf Mon Sep 17 00:00:00 2001 From: Harry Pan Date: Mon, 30 Dec 2019 22:36:56 +0800 Subject: [PATCH 11/89] powercap: intel_rapl: add NULL pointer check to rapl_mmio_cpu_online() commit 3aa3c5882e4fb2274448908aaed605a3ed7dd15d upstream. RAPL MMIO support depends on the RAPL common driver. During CPU initialization rapl_mmio_cpu_online() is called via CPU hotplug to initialize the MMIO RAPL for the new CPU, but if that CPU is not present in the common RAPL driver's support list, rapl_defaults is NULL and the kernel crashes on an attempt to dereference it: [ 4.188566] BUG: kernel NULL pointer dereference, address: 0000000000000020 ...snip... [ 4.189555] RIP: 0010:rapl_add_package+0x223/0x574 [ 4.189555] Code: b5 a0 31 c0 49 8b 4d 78 48 01 d9 48 8b 0c c1 49 89 4c c6 10 48 ff c0 48 83 f8 05 75 e7 49 83 ff 03 75 15 48 8b 05 09 bc 18 01 <8b> 70 20 41 89 b6 0c 05 00 00 85 f6 75 1a 49 81 c6 18 9 [ 4.189555] RSP: 0000:ffffb3adc00b3d90 EFLAGS: 00010246 [ 4.189555] RAX: 0000000000000000 RBX: 0000000000000098 RCX: 0000000000000000 [ 4.267161] usb 1-1: New USB device found, idVendor=2109, idProduct=2812, bcdDevice= b.e0 [ 4.189555] RDX: 0000000000001000 RSI: 0000000000000000 RDI: ffff9340caafd000 [ 4.189555] RBP: ffffb3adc00b3df8 R08: ffffffffa0246e28 R09: ffff9340caafc000 [ 4.189555] R10: 000000000000024a R11: ffffffff9ff1f6f2 R12: 00000000ffffffed [ 4.189555] R13: ffff9340caa94800 R14: ffff9340caafc518 R15: 0000000000000003 [ 4.189555] FS: 0000000000000000(0000) GS:ffff9340ce200000(0000) knlGS:0000000000000000 [ 4.189555] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 4.189555] CR2: 0000000000000020 CR3: 0000000302c14001 CR4: 00000000003606f0 [ 4.189555] Call Trace: [ 4.189555] ? __switch_to_asm+0x40/0x70 [ 4.189555] rapl_mmio_cpu_online+0x47/0x64 [ 4.189555] ? rapl_mmio_write_raw+0x33/0x33 [ 4.281059] usb 1-1: New USB device strings: Mfr=1, Product=2, SerialNumber=0 [ 4.189555] cpuhp_invoke_callback+0x29f/0x66f [ 4.189555] ? __schedule+0x46d/0x6a0 [ 4.189555] cpuhp_thread_fun+0xb9/0x11c [ 4.189555] smpboot_thread_fn+0x17d/0x22f [ 4.297006] usb 1-1: Product: USB2.0 Hub [ 4.189555] ? cpu_report_death+0x43/0x43 [ 4.189555] kthread+0x137/0x13f [ 4.189555] ? cpu_report_death+0x43/0x43 [ 4.189555] ? kthread_blkcg+0x2e/0x2e [ 4.312951] usb 1-1: Manufacturer: VIA Labs, Inc. [ 4.189555] ret_from_fork+0x1f/0x40 [ 4.189555] Modules linked in: [ 4.189555] CR2: 0000000000000020 [ 4.189555] ---[ end trace 01bb812aabc791f4 ]--- To avoid that problem, check rapl_defaults NULL upfront and return an error code if it is NULL. [Note that it does not make sense to even try to allocate memory in that case, because it is not going to be used anyway.] Fixes: 555c45fe0d04 ("int340X/processor_thermal_device: add support for MMIO RAPL") Cc: 5.3+ # 5.3+ Signed-off-by: Harry Pan [ rjw: Subject & changelog ] Signed-off-by: Rafael J. Wysocki Signed-off-by: Greg Kroah-Hartman --- drivers/powercap/intel_rapl_common.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/drivers/powercap/intel_rapl_common.c b/drivers/powercap/intel_rapl_common.c index 94ddd7d659c8..925b0004a0ed 100644 --- a/drivers/powercap/intel_rapl_common.c +++ b/drivers/powercap/intel_rapl_common.c @@ -1293,6 +1293,9 @@ struct rapl_package *rapl_add_package(int cpu, struct rapl_if_priv *priv) struct cpuinfo_x86 *c = &cpu_data(cpu); int ret; + if (!rapl_defaults) + return ERR_PTR(-ENODEV); + rp = kzalloc(sizeof(struct rapl_package), GFP_KERNEL); if (!rp) return ERR_PTR(-ENOMEM); From b5c477d904eb179cbaf82b9e263b011f65ee249b Mon Sep 17 00:00:00 2001 From: Guenter Roeck Date: Thu, 26 Dec 2019 07:57:54 -0800 Subject: [PATCH 12/89] usb: chipidea: host: Disable port power only if previously enabled commit c1ffba305dbcf3fb9ca969c20a97acbddc38f8e9 upstream. On shutdown, ehci_power_off() is called unconditionally to power off each port, even if it was never called to power on the port. For chipidea, this results in a call to ehci_ci_portpower() with a request to power off ports even if the port was never powered on. This results in the following warning from the regulator code. WARNING: CPU: 0 PID: 182 at drivers/regulator/core.c:2596 _regulator_disable+0x1a8/0x210 unbalanced disables for usb_otg2_vbus Modules linked in: CPU: 0 PID: 182 Comm: init Not tainted 5.4.6 #1 Hardware name: Freescale i.MX7 Dual (Device Tree) [] (unwind_backtrace) from [] (show_stack+0x10/0x14) [] (show_stack) from [] (dump_stack+0xe0/0x10c) [] (dump_stack) from [] (__warn+0xf4/0x10c) [] (__warn) from [] (warn_slowpath_fmt+0x78/0xbc) [] (warn_slowpath_fmt) from [] (_regulator_disable+0x1a8/0x210) [] (_regulator_disable) from [] (regulator_disable+0x38/0xe8) [] (regulator_disable) from [] (ehci_ci_portpower+0x38/0xdc) [] (ehci_ci_portpower) from [] (ehci_port_power+0x50/0xa4) [] (ehci_port_power) from [] (ehci_silence_controller+0x5c/0xc4) [] (ehci_silence_controller) from [] (ehci_stop+0x3c/0xcc) [] (ehci_stop) from [] (usb_remove_hcd+0xe0/0x19c) [] (usb_remove_hcd) from [] (host_stop+0x38/0xa8) [] (host_stop) from [] (ci_hdrc_remove+0x44/0xe4) ... Keeping track of the power enable state avoids the warning and traceback. Fixes: c8679a2fb8dec ("usb: chipidea: host: add portpower override") Cc: Michael Grzeschik Cc: Peter Chen Cc: stable@vger.kernel.org Signed-off-by: Guenter Roeck Acked-by: Peter Chen Link: https://lore.kernel.org/r/20191226155754.25451-1-linux@roeck-us.net Signed-off-by: Greg Kroah-Hartman --- drivers/usb/chipidea/host.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/drivers/usb/chipidea/host.c b/drivers/usb/chipidea/host.c index b45ceb91c735..48e4a5ca1835 100644 --- a/drivers/usb/chipidea/host.c +++ b/drivers/usb/chipidea/host.c @@ -26,6 +26,7 @@ static int (*orig_bus_suspend)(struct usb_hcd *hcd); struct ehci_ci_priv { struct regulator *reg_vbus; + bool enabled; }; static int ehci_ci_portpower(struct usb_hcd *hcd, int portnum, bool enable) @@ -37,7 +38,7 @@ static int ehci_ci_portpower(struct usb_hcd *hcd, int portnum, bool enable) int ret = 0; int port = HCS_N_PORTS(ehci->hcs_params); - if (priv->reg_vbus) { + if (priv->reg_vbus && enable != priv->enabled) { if (port > 1) { dev_warn(dev, "Not support multi-port regulator control\n"); @@ -53,6 +54,7 @@ static int ehci_ci_portpower(struct usb_hcd *hcd, int portnum, bool enable) enable ? "enable" : "disable", ret); return ret; } + priv->enabled = enable; } if (enable && (ci->platdata->phy_mode == USBPHY_INTERFACE_MODE_HSIC)) { From d496377685e3e8a2fd32cf6567aab99a30585b46 Mon Sep 17 00:00:00 2001 From: Takashi Iwai Date: Sat, 4 Jan 2020 12:09:36 +0100 Subject: [PATCH 13/89] ALSA: usb-audio: Apply the sample rate quirk for Bose Companion 5 commit 51d4efab7865e6ea6a4ebcd25b3f03c019515c4c upstream. Bose Companion 5 (with USB ID 05a7:1020) doesn't seem supporting reading back the sample rate, so the existing quirk is needed. BugLink: https://bugzilla.kernel.org/show_bug.cgi?id=206063 Cc: Link: https://lore.kernel.org/r/20200104110936.14288-1-tiwai@suse.de Signed-off-by: Takashi Iwai Signed-off-by: Greg Kroah-Hartman --- sound/usb/quirks.c | 1 + 1 file changed, 1 insertion(+) diff --git a/sound/usb/quirks.c b/sound/usb/quirks.c index a81c2066499f..82184036437b 100644 --- a/sound/usb/quirks.c +++ b/sound/usb/quirks.c @@ -1397,6 +1397,7 @@ bool snd_usb_get_sample_rate_quirk(struct snd_usb_audio *chip) case USB_ID(0x04D8, 0xFEEA): /* Benchmark DAC1 Pre */ case USB_ID(0x0556, 0x0014): /* Phoenix Audio TMX320VC */ case USB_ID(0x05A3, 0x9420): /* ELP HD USB Camera */ + case USB_ID(0x05a7, 0x1020): /* Bose Companion 5 */ case USB_ID(0x074D, 0x3553): /* Outlaw RR2150 (Micronas UAC3553B) */ case USB_ID(0x1395, 0x740a): /* Sennheiser DECT */ case USB_ID(0x1901, 0x0191): /* GE B850V3 CP2114 audio interface */ From b25acd7045df332edd142efbad9594eb18d91e6d Mon Sep 17 00:00:00 2001 From: Kailang Yang Date: Fri, 3 Jan 2020 16:24:06 +0800 Subject: [PATCH 14/89] ALSA: hda/realtek - Add new codec supported for ALCS1200A commit 6d9ffcff646bbd0ede6c2a59f4cd28414ecec6e0 upstream. Add ALCS1200A supported. It was similar as ALC900. Signed-off-by: Kailang Yang Cc: Link: https://lore.kernel.org/r/a9bd3cdaa02d4fa197623448d5c51e50@realtek.com Signed-off-by: Takashi Iwai Signed-off-by: Greg Kroah-Hartman --- sound/pci/hda/patch_realtek.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/sound/pci/hda/patch_realtek.c b/sound/pci/hda/patch_realtek.c index 252888f426de..bc170f366f78 100644 --- a/sound/pci/hda/patch_realtek.c +++ b/sound/pci/hda/patch_realtek.c @@ -430,6 +430,7 @@ static void alc_fill_eapd_coef(struct hda_codec *codec) break; case 0x10ec0899: case 0x10ec0900: + case 0x10ec0b00: case 0x10ec1168: case 0x10ec1220: alc_update_coef_idx(codec, 0x7, 1<<1, 0); @@ -2526,6 +2527,7 @@ static int patch_alc882(struct hda_codec *codec) case 0x10ec0882: case 0x10ec0885: case 0x10ec0900: + case 0x10ec0b00: case 0x10ec1220: break; default: @@ -9295,6 +9297,7 @@ static const struct hda_device_id snd_hda_id_realtek[] = { HDA_CODEC_ENTRY(0x10ec0892, "ALC892", patch_alc662), HDA_CODEC_ENTRY(0x10ec0899, "ALC898", patch_alc882), HDA_CODEC_ENTRY(0x10ec0900, "ALC1150", patch_alc882), + HDA_CODEC_ENTRY(0x10ec0b00, "ALCS1200A", patch_alc882), HDA_CODEC_ENTRY(0x10ec1168, "ALC1220", patch_alc882), HDA_CODEC_ENTRY(0x10ec1220, "ALC1220", patch_alc882), {} /* terminator */ From 69ec8c0e3377f2f94ae160377fad585fbd062caf Mon Sep 17 00:00:00 2001 From: Kailang Yang Date: Tue, 7 Jan 2020 17:22:19 +0800 Subject: [PATCH 15/89] ALSA: hda/realtek - Set EAPD control to default for ALC222 commit 9194a1ebbc56d7006835e2b4cacad301201fb832 upstream. Set EAPD control to verb control. Signed-off-by: Kailang Yang Cc: Signed-off-by: Takashi Iwai Signed-off-by: Greg Kroah-Hartman --- sound/pci/hda/patch_realtek.c | 1 + 1 file changed, 1 insertion(+) diff --git a/sound/pci/hda/patch_realtek.c b/sound/pci/hda/patch_realtek.c index bc170f366f78..6062cda7f48f 100644 --- a/sound/pci/hda/patch_realtek.c +++ b/sound/pci/hda/patch_realtek.c @@ -412,6 +412,7 @@ static void alc_fill_eapd_coef(struct hda_codec *codec) case 0x10ec0672: alc_update_coef_idx(codec, 0xd, 0, 1<<14); /* EAPD Ctrl */ break; + case 0x10ec0222: case 0x10ec0623: alc_update_coef_idx(codec, 0x19, 1<<13, 0); break; From 86353aa70ed085f1b8ef9dae45491d5beb0728dd Mon Sep 17 00:00:00 2001 From: Kailang Yang Date: Wed, 8 Jan 2020 16:47:56 +0800 Subject: [PATCH 16/89] ALSA: hda/realtek - Add quirk for the bass speaker on Lenovo Yoga X1 7th gen commit 54a6a7dc107da0492a9e84fd7e9a107b3c58138d upstream. Add quirk to ALC285_FIXUP_SPEAKER2_TO_DAC1, which is the same fixup applied for X1 Carbon 7th gen in commit d2cd795c4ece ("ALSA: hda - fixup for the bass speaker on Lenovo Carbon X1 7th gen"). Signed-off-by: Kailang Yang Reviewed-by: Jaroslav Kysela Cc: Signed-off-by: Takashi Iwai Signed-off-by: Greg Kroah-Hartman --- sound/pci/hda/patch_realtek.c | 1 + 1 file changed, 1 insertion(+) diff --git a/sound/pci/hda/patch_realtek.c b/sound/pci/hda/patch_realtek.c index 6062cda7f48f..d293488dc3dd 100644 --- a/sound/pci/hda/patch_realtek.c +++ b/sound/pci/hda/patch_realtek.c @@ -7260,6 +7260,7 @@ static const struct snd_pci_quirk alc269_fixup_tbl[] = { SND_PCI_QUIRK(0x17aa, 0x224c, "Thinkpad", ALC298_FIXUP_TPT470_DOCK), SND_PCI_QUIRK(0x17aa, 0x224d, "Thinkpad", ALC298_FIXUP_TPT470_DOCK), SND_PCI_QUIRK(0x17aa, 0x225d, "Thinkpad T480", ALC269_FIXUP_LIMIT_INT_MIC_BOOST), + SND_PCI_QUIRK(0x17aa, 0x2292, "Thinkpad X1 Yoga 7th", ALC285_FIXUP_SPEAKER2_TO_DAC1), SND_PCI_QUIRK(0x17aa, 0x2293, "Thinkpad X1 Carbon 7th", ALC285_FIXUP_SPEAKER2_TO_DAC1), SND_PCI_QUIRK(0x17aa, 0x30bb, "ThinkCentre AIO", ALC233_FIXUP_LENOVO_LINE2_MIC_HOTKEY), SND_PCI_QUIRK(0x17aa, 0x30e2, "ThinkCentre AIO", ALC233_FIXUP_LENOVO_LINE2_MIC_HOTKEY), From 6de025ef5840dff4ddb6e9043ae6ca3d1b95c1cd Mon Sep 17 00:00:00 2001 From: Jarkko Sakkinen Date: Tue, 31 Dec 2019 01:19:59 +0200 Subject: [PATCH 17/89] tpm: Revert "tpm_tis: reserve chip for duration of tpm_tis_core_init" commit 9550f210492c6f88415709002f42a9d15c0e6231 upstream. Revert a commit, which was included in Linux v5.5-rc3 because it did not properly fix the issues it was supposed to fix. Fixes: 21df4a8b6018 ("tpm_tis: reserve chip for duration of tpm_tis_core_init") Link: https://bugzilla.kernel.org/show_bug.cgi?id=205935 Cc: stable@vger.kernel.org Cc: Jerry Snitselaar Cc: Dan Williams Tested-by: Dan Williams Tested-by: Xiaoping Zhou Signed-off-by: Jarkko Sakkinen Signed-off-by: Greg Kroah-Hartman --- drivers/char/tpm/tpm_tis_core.c | 35 ++++++++++++++++----------------- 1 file changed, 17 insertions(+), 18 deletions(-) diff --git a/drivers/char/tpm/tpm_tis_core.c b/drivers/char/tpm/tpm_tis_core.c index f528fc39ea6b..270f43acbb77 100644 --- a/drivers/char/tpm/tpm_tis_core.c +++ b/drivers/char/tpm/tpm_tis_core.c @@ -899,13 +899,13 @@ int tpm_tis_core_init(struct device *dev, struct tpm_tis_data *priv, int irq, if (wait_startup(chip, 0) != 0) { rc = -ENODEV; - goto err_start; + goto out_err; } /* Take control of the TPM's interrupt hardware and shut it off */ rc = tpm_tis_read32(priv, TPM_INT_ENABLE(priv->locality), &intmask); if (rc < 0) - goto err_start; + goto out_err; intmask |= TPM_INTF_CMD_READY_INT | TPM_INTF_LOCALITY_CHANGE_INT | TPM_INTF_DATA_AVAIL_INT | TPM_INTF_STS_VALID_INT; @@ -914,21 +914,21 @@ int tpm_tis_core_init(struct device *dev, struct tpm_tis_data *priv, int irq, rc = tpm_chip_start(chip); if (rc) - goto err_start; - + goto out_err; rc = tpm2_probe(chip); + tpm_chip_stop(chip); if (rc) - goto err_probe; + goto out_err; rc = tpm_tis_read32(priv, TPM_DID_VID(0), &vendor); if (rc < 0) - goto err_probe; + goto out_err; priv->manufacturer_id = vendor; rc = tpm_tis_read8(priv, TPM_RID(0), &rid); if (rc < 0) - goto err_probe; + goto out_err; dev_info(dev, "%s TPM (device-id 0x%X, rev-id %d)\n", (chip->flags & TPM_CHIP_FLAG_TPM2) ? "2.0" : "1.2", @@ -937,13 +937,13 @@ int tpm_tis_core_init(struct device *dev, struct tpm_tis_data *priv, int irq, probe = probe_itpm(chip); if (probe < 0) { rc = -ENODEV; - goto err_probe; + goto out_err; } /* Figure out the capabilities */ rc = tpm_tis_read32(priv, TPM_INTF_CAPS(priv->locality), &intfcaps); if (rc < 0) - goto err_probe; + goto out_err; dev_dbg(dev, "TPM interface capabilities (0x%x):\n", intfcaps); @@ -977,9 +977,10 @@ int tpm_tis_core_init(struct device *dev, struct tpm_tis_data *priv, int irq, if (tpm_get_timeouts(chip)) { dev_err(dev, "Could not get TPM timeouts and durations\n"); rc = -ENODEV; - goto err_probe; + goto out_err; } + tpm_chip_start(chip); chip->flags |= TPM_CHIP_FLAG_IRQ; if (irq) { tpm_tis_probe_irq_single(chip, intmask, IRQF_SHARED, @@ -990,20 +991,18 @@ int tpm_tis_core_init(struct device *dev, struct tpm_tis_data *priv, int irq, } else { tpm_tis_probe_irq(chip, intmask); } + tpm_chip_stop(chip); } - tpm_chip_stop(chip); - rc = tpm_chip_register(chip); if (rc) - goto err_start; + goto out_err; + + if (chip->ops->clk_enable != NULL) + chip->ops->clk_enable(chip, false); return 0; - -err_probe: - tpm_chip_stop(chip); - -err_start: +out_err: if ((chip->ops != NULL) && (chip->ops->clk_enable != NULL)) chip->ops->clk_enable(chip, false); From 495e9443cab9db3b8d51e837923f8170b3085c71 Mon Sep 17 00:00:00 2001 From: Stefan Berger Date: Tue, 26 Nov 2019 08:17:52 -0500 Subject: [PATCH 18/89] tpm: Revert "tpm_tis_core: Set TPM_CHIP_FLAG_IRQ before probing for interrupts" commit dda8b2af395b2ed508e2ef314ae32e122841b447 upstream. There has been a bunch of reports (one from kernel bugzilla linked) reporting that when this commit is applied it causes on some machines boot freezes. Unfortunately hardware where this commit causes a failure is not widely available (only one I'm aware is Lenovo T490), which means we cannot predict yet how long it will take to properly fix tpm_tis interrupt probing. Thus, the least worst short term action is to revert the code to the state before this commit. In long term we need fix the tpm_tis probing code to work on machines that Stefan's fix was supposed to fix. Link: https://bugzilla.kernel.org/show_bug.cgi?id=205935 Fixes: 1ea32c83c699 ("tpm_tis_core: Set TPM_CHIP_FLAG_IRQ before probing for interrupts") Cc: stable@vger.kernel.org Cc: Jerry Snitselaar Cc: Dan Williams Tested-by: Dan Williams Tested-by: Xiaoping Zhou Signed-off-by: Stefan Berger Reported-by: Jerry Snitselaar Signed-off-by: Jarkko Sakkinen Signed-off-by: Greg Kroah-Hartman --- drivers/char/tpm/tpm_tis_core.c | 1 - 1 file changed, 1 deletion(-) diff --git a/drivers/char/tpm/tpm_tis_core.c b/drivers/char/tpm/tpm_tis_core.c index 270f43acbb77..ffa9048d8f6c 100644 --- a/drivers/char/tpm/tpm_tis_core.c +++ b/drivers/char/tpm/tpm_tis_core.c @@ -981,7 +981,6 @@ int tpm_tis_core_init(struct device *dev, struct tpm_tis_data *priv, int irq, } tpm_chip_start(chip); - chip->flags |= TPM_CHIP_FLAG_IRQ; if (irq) { tpm_tis_probe_irq_single(chip, intmask, IRQF_SHARED, irq); From 2e1a0a118abe853cd9217368a551f53a0cdff15b Mon Sep 17 00:00:00 2001 From: Stefan Berger Date: Tue, 26 Nov 2019 08:17:53 -0500 Subject: [PATCH 19/89] tpm: Revert "tpm_tis_core: Turn on the TPM before probing IRQ's" commit aa4a63dd981682b1742baa01237036e48bc11923 upstream. There has been a bunch of reports (one from kernel bugzilla linked) reporting that when this commit is applied it causes on some machines boot freezes. Unfortunately hardware where this commit causes a failure is not widely available (only one I'm aware is Lenovo T490), which means we cannot predict yet how long it will take to properly fix tpm_tis interrupt probing. Thus, the least worst short term action is to revert the code to the state before this commit. In long term we need fix the tpm_tis probing code to work on machines that Stefan's fix was supposed to fix. Fixes: 21df4a8b6018 ("tpm_tis: reserve chip for duration of tpm_tis_core_init") Link: https://bugzilla.kernel.org/show_bug.cgi?id=205935 Cc: stable@vger.kernel.org Cc: Jerry Snitselaar Cc: Dan Williams Tested-by: Dan Williams Tested-by: Xiaoping Zhou Signed-off-by: Stefan Berger Reported-by: Jerry Snitselaar Signed-off-by: Jarkko Sakkinen Signed-off-by: Greg Kroah-Hartman --- drivers/char/tpm/tpm_tis_core.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/drivers/char/tpm/tpm_tis_core.c b/drivers/char/tpm/tpm_tis_core.c index ffa9048d8f6c..c3181ea9f271 100644 --- a/drivers/char/tpm/tpm_tis_core.c +++ b/drivers/char/tpm/tpm_tis_core.c @@ -980,7 +980,6 @@ int tpm_tis_core_init(struct device *dev, struct tpm_tis_data *priv, int irq, goto out_err; } - tpm_chip_start(chip); if (irq) { tpm_tis_probe_irq_single(chip, intmask, IRQF_SHARED, irq); @@ -990,7 +989,6 @@ int tpm_tis_core_init(struct device *dev, struct tpm_tis_data *priv, int irq, } else { tpm_tis_probe_irq(chip, intmask); } - tpm_chip_stop(chip); } rc = tpm_chip_register(chip); From 18c395532370dc30d4abe526d77a58c2372add2b Mon Sep 17 00:00:00 2001 From: Tadeusz Struk Date: Tue, 7 Jan 2020 14:04:48 -0800 Subject: [PATCH 20/89] tpm: Handle negative priv->response_len in tpm_common_read() commit a430e67d9a2c62a8c7b315b99e74de02018d0a96 upstream. The priv->response_length can hold the size of an response or an negative error code, and the tpm_common_read() needs to handle both cases correctly. Changed the type of response_length to signed and accounted for negative value in tpm_common_read(). Cc: stable@vger.kernel.org Fixes: d23d12484307 ("tpm: fix invalid locking in NONBLOCKING mode") Reported-by: Laura Abbott Signed-off-by: Tadeusz Struk Reviewed-by: Jarkko Sakkinen Signed-off-by: Jarkko Sakkinen Signed-off-by: Greg Kroah-Hartman --- drivers/char/tpm/tpm-dev-common.c | 2 +- drivers/char/tpm/tpm-dev.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/char/tpm/tpm-dev-common.c b/drivers/char/tpm/tpm-dev-common.c index b23b0b999232..87f449340202 100644 --- a/drivers/char/tpm/tpm-dev-common.c +++ b/drivers/char/tpm/tpm-dev-common.c @@ -130,7 +130,7 @@ ssize_t tpm_common_read(struct file *file, char __user *buf, priv->response_read = true; ret_size = min_t(ssize_t, size, priv->response_length); - if (!ret_size) { + if (ret_size <= 0) { priv->response_length = 0; goto out; } diff --git a/drivers/char/tpm/tpm-dev.h b/drivers/char/tpm/tpm-dev.h index 1089fc0bb290..f3742bcc73e3 100644 --- a/drivers/char/tpm/tpm-dev.h +++ b/drivers/char/tpm/tpm-dev.h @@ -14,7 +14,7 @@ struct file_priv { struct work_struct timeout_work; struct work_struct async_work; wait_queue_head_t async_wait; - size_t response_length; + ssize_t response_length; bool response_read; bool command_enqueued; From cfa719c2cc5989d81ffeb5994005b5a18e89a75c Mon Sep 17 00:00:00 2001 From: Chen-Yu Tsai Date: Thu, 5 Dec 2019 16:50:54 +0800 Subject: [PATCH 21/89] rtc: sun6i: Add support for RTC clocks on R40 commit 111bf02b8f544f98de53ea1f912ae01f598b161b upstream. When support for the R40 in the rtc-sun6i driver was split out for a separate compatible string, only the RTC half was covered, and not the clock half. Unfortunately this results in the whole driver not working, as the RTC half expects the clock half to have been initialized. Add support for the clock part as well. The clock part is like the H3, but does not need to export the internal oscillator, nor does it have a gateable LOSC external output. This fixes issues with WiFi and Bluetooth not working on the BPI M2U. Fixes: d6624cc75021 ("rtc: sun6i: Add R40 compatible") Cc: # 5.3.x Signed-off-by: Chen-Yu Tsai Acked-by: Maxime Ripard Link: https://lore.kernel.org/r/20191205085054.6049-1-wens@kernel.org Signed-off-by: Alexandre Belloni Signed-off-by: Greg Kroah-Hartman --- drivers/rtc/rtc-sun6i.c | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/drivers/rtc/rtc-sun6i.c b/drivers/rtc/rtc-sun6i.c index 5e2bd9f1d01e..fc32be687606 100644 --- a/drivers/rtc/rtc-sun6i.c +++ b/drivers/rtc/rtc-sun6i.c @@ -380,6 +380,22 @@ static void __init sun50i_h6_rtc_clk_init(struct device_node *node) CLK_OF_DECLARE_DRIVER(sun50i_h6_rtc_clk, "allwinner,sun50i-h6-rtc", sun50i_h6_rtc_clk_init); +/* + * The R40 user manual is self-conflicting on whether the prescaler is + * fixed or configurable. The clock diagram shows it as fixed, but there + * is also a configurable divider in the RTC block. + */ +static const struct sun6i_rtc_clk_data sun8i_r40_rtc_data = { + .rc_osc_rate = 16000000, + .fixed_prescaler = 512, +}; +static void __init sun8i_r40_rtc_clk_init(struct device_node *node) +{ + sun6i_rtc_clk_init(node, &sun8i_r40_rtc_data); +} +CLK_OF_DECLARE_DRIVER(sun8i_r40_rtc_clk, "allwinner,sun8i-r40-rtc", + sun8i_r40_rtc_clk_init); + static const struct sun6i_rtc_clk_data sun8i_v3_rtc_data = { .rc_osc_rate = 32000, .has_out_clk = 1, From 654eced27a13c5b45af9ea9ca7e4626bb050eb68 Mon Sep 17 00:00:00 2001 From: Kaitao Cheng Date: Tue, 31 Dec 2019 05:35:30 -0800 Subject: [PATCH 22/89] kernel/trace: Fix do not unregister tracepoints when register sched_migrate_task fail commit 50f9ad607ea891a9308e67b81f774c71736d1098 upstream. In the function, if register_trace_sched_migrate_task() returns error, sched_switch/sched_wakeup_new/sched_wakeup won't unregister. That is why fail_deprobe_sched_switch was added. Link: http://lkml.kernel.org/r/20191231133530.2794-1-pilgrimtao@gmail.com Cc: stable@vger.kernel.org Fixes: 478142c39c8c2 ("tracing: do not grab lock in wakeup latency function tracing") Signed-off-by: Kaitao Cheng Signed-off-by: Steven Rostedt (VMware) Signed-off-by: Greg Kroah-Hartman --- kernel/trace/trace_sched_wakeup.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/kernel/trace/trace_sched_wakeup.c b/kernel/trace/trace_sched_wakeup.c index 5e43b9664eca..617e297f46dc 100644 --- a/kernel/trace/trace_sched_wakeup.c +++ b/kernel/trace/trace_sched_wakeup.c @@ -630,7 +630,7 @@ static void start_wakeup_tracer(struct trace_array *tr) if (ret) { pr_info("wakeup trace: Couldn't activate tracepoint" " probe to kernel_sched_migrate_task\n"); - return; + goto fail_deprobe_sched_switch; } wakeup_reset(tr); @@ -648,6 +648,8 @@ static void start_wakeup_tracer(struct trace_array *tr) printk(KERN_ERR "failed to start wakeup tracer\n"); return; +fail_deprobe_sched_switch: + unregister_trace_sched_switch(probe_wakeup_sched_switch, NULL); fail_deprobe_wake_new: unregister_trace_sched_wakeup_new(probe_wakeup, NULL); fail_deprobe: From 907062a7503a64b1d48df4bf4d3cde9c4d58a267 Mon Sep 17 00:00:00 2001 From: "Steven Rostedt (VMware)" Date: Thu, 2 Jan 2020 22:02:41 -0500 Subject: [PATCH 23/89] tracing: Have stack tracer compile when MCOUNT_INSN_SIZE is not defined commit b8299d362d0837ae39e87e9019ebe6b736e0f035 upstream. On some archs with some configurations, MCOUNT_INSN_SIZE is not defined, and this makes the stack tracer fail to compile. Just define it to zero in this case. Link: https://lore.kernel.org/r/202001020219.zvE3vsty%lkp@intel.com Cc: stable@vger.kernel.org Fixes: 4df297129f622 ("tracing: Remove most or all of stack tracer stack size from stack_max_size") Reported-by: kbuild test robot Signed-off-by: Steven Rostedt (VMware) Signed-off-by: Greg Kroah-Hartman --- kernel/trace/trace_stack.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/kernel/trace/trace_stack.c b/kernel/trace/trace_stack.c index 4df9a209f7ca..c557f42a9397 100644 --- a/kernel/trace/trace_stack.c +++ b/kernel/trace/trace_stack.c @@ -283,6 +283,11 @@ static void check_stack(unsigned long ip, unsigned long *stack) local_irq_restore(flags); } +/* Some archs may not define MCOUNT_INSN_SIZE */ +#ifndef MCOUNT_INSN_SIZE +# define MCOUNT_INSN_SIZE 0 +#endif + static void stack_trace_call(unsigned long ip, unsigned long parent_ip, struct ftrace_ops *op, struct pt_regs *pt_regs) From 5270a752defa4d71e4a3ca7e2a1026e9806340e4 Mon Sep 17 00:00:00 2001 From: "Joel Fernandes (Google)" Date: Thu, 2 Jan 2020 14:46:25 -0500 Subject: [PATCH 24/89] tracing: Change offset type to s32 in preempt/irq tracepoints commit bf44f488e168368cae4139b4b33c3d0aaa11679c upstream. Discussion in the below link reported that symbols in modules can appear to be before _stext on ARM architecture, causing wrapping with the offsets of this tracepoint. Change the offset type to s32 to fix this. Link: http://lore.kernel.org/r/20191127154428.191095-1-antonio.borneo@st.com Link: http://lkml.kernel.org/r/20200102194625.226436-1-joel@joelfernandes.org Cc: Bjorn Helgaas Cc: David Sterba Cc: Ingo Molnar Cc: Mike Rapoport Cc: "Rafael J. Wysocki" Cc: Sakari Ailus Cc: Antonio Borneo Cc: stable@vger.kernel.org Fixes: d59158162e032 ("tracing: Add support for preempt and irq enable/disable events") Signed-off-by: Joel Fernandes (Google) Signed-off-by: Steven Rostedt (VMware) Signed-off-by: Greg Kroah-Hartman --- include/trace/events/preemptirq.h | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/include/trace/events/preemptirq.h b/include/trace/events/preemptirq.h index 95fba0471e5b..3f249e150c0c 100644 --- a/include/trace/events/preemptirq.h +++ b/include/trace/events/preemptirq.h @@ -18,13 +18,13 @@ DECLARE_EVENT_CLASS(preemptirq_template, TP_ARGS(ip, parent_ip), TP_STRUCT__entry( - __field(u32, caller_offs) - __field(u32, parent_offs) + __field(s32, caller_offs) + __field(s32, parent_offs) ), TP_fast_assign( - __entry->caller_offs = (u32)(ip - (unsigned long)_stext); - __entry->parent_offs = (u32)(parent_ip - (unsigned long)_stext); + __entry->caller_offs = (s32)(ip - (unsigned long)_stext); + __entry->parent_offs = (s32)(parent_ip - (unsigned long)_stext); ), TP_printk("caller=%pS parent=%pS", From 2cc7eb656286e761fec47eccccacc953db6e0cff Mon Sep 17 00:00:00 2001 From: Alan Stern Date: Tue, 10 Dec 2019 16:26:11 -0500 Subject: [PATCH 25/89] HID: Fix slab-out-of-bounds read in hid_field_extract commit 8ec321e96e056de84022c032ffea253431a83c3c upstream. The syzbot fuzzer found a slab-out-of-bounds bug in the HID report handler. The bug was caused by a report descriptor which included a field with size 12 bits and count 4899, for a total size of 7349 bytes. The usbhid driver uses at most a single-page 4-KB buffer for reports. In the test there wasn't any problem about overflowing the buffer, since only one byte was received from the device. Rather, the bug occurred when the HID core tried to extract the data from the report fields, which caused it to try reading data beyond the end of the allocated buffer. This patch fixes the problem by rejecting any report whose total length exceeds the HID_MAX_BUFFER_SIZE limit (minus one byte to allow for a possible report index). In theory a device could have a report longer than that, but if there was such a thing we wouldn't handle it correctly anyway. Reported-and-tested-by: syzbot+09ef48aa58261464b621@syzkaller.appspotmail.com Signed-off-by: Alan Stern CC: Signed-off-by: Jiri Kosina Signed-off-by: Greg Kroah-Hartman --- drivers/hid/hid-core.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/drivers/hid/hid-core.c b/drivers/hid/hid-core.c index e0b241bd3070..851fe54ea59e 100644 --- a/drivers/hid/hid-core.c +++ b/drivers/hid/hid-core.c @@ -288,6 +288,12 @@ static int hid_add_field(struct hid_parser *parser, unsigned report_type, unsign offset = report->size; report->size += parser->global.report_size * parser->global.report_count; + /* Total size check: Allow for possible report index byte */ + if (report->size > (HID_MAX_BUFFER_SIZE - 1) << 3) { + hid_err(parser->device, "report is too long\n"); + return -1; + } + if (!parser->local.usage_index) /* Ignore padding fields */ return 0; From 566dbc0db3fc58335df35336083feb44649051a3 Mon Sep 17 00:00:00 2001 From: Marcel Holtmann Date: Wed, 4 Dec 2019 03:43:55 +0100 Subject: [PATCH 26/89] HID: uhid: Fix returning EPOLLOUT from uhid_char_poll commit be54e7461ffdc5809b67d2aeefc1ddc9a91470c7 upstream. Always return EPOLLOUT from uhid_char_poll to allow polling /dev/uhid for writable state. Fixes: 1f9dec1e0164 ("HID: uhid: allow poll()'ing on uhid devices") Signed-off-by: Marcel Holtmann Cc: stable@vger.kernel.org Signed-off-by: Jiri Kosina Signed-off-by: Greg Kroah-Hartman --- drivers/hid/uhid.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/hid/uhid.c b/drivers/hid/uhid.c index fa0cc0899827..935c3d0a3b63 100644 --- a/drivers/hid/uhid.c +++ b/drivers/hid/uhid.c @@ -772,7 +772,7 @@ static __poll_t uhid_char_poll(struct file *file, poll_table *wait) if (uhid->head != uhid->tail) return EPOLLIN | EPOLLRDNORM; - return 0; + return EPOLLOUT | EPOLLWRNORM; } static const struct file_operations uhid_fops = { From e38d7bb30cbe8f9070607a424bb25f27c3cf3d35 Mon Sep 17 00:00:00 2001 From: Marcel Holtmann Date: Wed, 4 Dec 2019 03:37:13 +0100 Subject: [PATCH 27/89] HID: hidraw: Fix returning EPOLLOUT from hidraw_poll commit 9f3b61dc1dd7b81e99e7ed23776bb64a35f39e1a upstream. When polling a connected /dev/hidrawX device, it is useful to get the EPOLLOUT when writing is possible. Since writing is possible as soon as the device is connected, always return it. Right now EPOLLOUT is only returned when there are also input reports are available. This works if devices start sending reports when connected, but some HID devices might need an output report first before sending any input reports. This change will allow using EPOLLOUT here as well. Fixes: 378b80370aa1 ("hidraw: Return EPOLLOUT from hidraw_poll") Signed-off-by: Marcel Holtmann Cc: stable@vger.kernel.org Signed-off-by: Jiri Kosina Signed-off-by: Greg Kroah-Hartman --- drivers/hid/hidraw.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/hid/hidraw.c b/drivers/hid/hidraw.c index bbc6ec1aa5cb..c25e95c19cad 100644 --- a/drivers/hid/hidraw.c +++ b/drivers/hid/hidraw.c @@ -252,10 +252,10 @@ static __poll_t hidraw_poll(struct file *file, poll_table *wait) poll_wait(file, &list->hidraw->wait, wait); if (list->head != list->tail) - return EPOLLIN | EPOLLRDNORM | EPOLLOUT; + return EPOLLIN | EPOLLRDNORM; if (!list->hidraw->exist) return EPOLLERR | EPOLLHUP; - return 0; + return EPOLLOUT | EPOLLWRNORM; } static int hidraw_open(struct inode *inode, struct file *file) From 4091fbf6cc143c8ccd8275eaa642b2f2afe7c4ab Mon Sep 17 00:00:00 2001 From: Dmitry Torokhov Date: Sat, 7 Dec 2019 13:05:18 -0800 Subject: [PATCH 28/89] HID: hid-input: clear unmapped usages commit 4f3882177240a1f55e45a3d241d3121341bead78 upstream. We should not be leaving half-mapped usages with potentially invalid keycodes, as that may confuse hidinput_find_key() when the key is located by index, which may end up feeding way too large keycode into the VT keyboard handler and cause OOB write there: BUG: KASAN: global-out-of-bounds in clear_bit include/asm-generic/bitops-instrumented.h:56 [inline] BUG: KASAN: global-out-of-bounds in kbd_keycode drivers/tty/vt/keyboard.c:1411 [inline] BUG: KASAN: global-out-of-bounds in kbd_event+0xe6b/0x3790 drivers/tty/vt/keyboard.c:1495 Write of size 8 at addr ffffffff89a1b2d8 by task syz-executor108/1722 ... kbd_keycode drivers/tty/vt/keyboard.c:1411 [inline] kbd_event+0xe6b/0x3790 drivers/tty/vt/keyboard.c:1495 input_to_handler+0x3b6/0x4c0 drivers/input/input.c:118 input_pass_values.part.0+0x2e3/0x720 drivers/input/input.c:145 input_pass_values drivers/input/input.c:949 [inline] input_set_keycode+0x290/0x320 drivers/input/input.c:954 evdev_handle_set_keycode_v2+0xc4/0x120 drivers/input/evdev.c:882 evdev_do_ioctl drivers/input/evdev.c:1150 [inline] Cc: stable@vger.kernel.org Reported-by: syzbot+19340dff067c2d3835c0@syzkaller.appspotmail.com Signed-off-by: Dmitry Torokhov Tested-by: Benjamin Tissoires Signed-off-by: Jiri Kosina Signed-off-by: Greg Kroah-Hartman --- drivers/hid/hid-input.c | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/drivers/hid/hid-input.c b/drivers/hid/hid-input.c index 63855f275a38..dea9cc65bf80 100644 --- a/drivers/hid/hid-input.c +++ b/drivers/hid/hid-input.c @@ -1132,9 +1132,15 @@ static void hidinput_configure_usage(struct hid_input *hidinput, struct hid_fiel } mapped: - if (device->driver->input_mapped && device->driver->input_mapped(device, - hidinput, field, usage, &bit, &max) < 0) - goto ignore; + if (device->driver->input_mapped && + device->driver->input_mapped(device, hidinput, field, usage, + &bit, &max) < 0) { + /* + * The driver indicated that no further generic handling + * of the usage is desired. + */ + return; + } set_bit(usage->type, input->evbit); @@ -1215,9 +1221,11 @@ mapped: set_bit(MSC_SCAN, input->mscbit); } -ignore: return; +ignore: + usage->type = 0; + usage->code = 0; } static void hidinput_handle_scroll(struct hid_usage *usage, From 39f711b69799c49e0e385494b9b8c0787f51293f Mon Sep 17 00:00:00 2001 From: Dmitry Torokhov Date: Fri, 13 Dec 2019 14:56:16 -0800 Subject: [PATCH 29/89] Input: add safety guards to input_set_keycode() commit cb222aed03d798fc074be55e59d9a112338ee784 upstream. If we happen to have a garbage in input device's keycode table with values too big we'll end up doing clear_bit() with offset way outside of our bitmaps, damaging other objects within an input device or even outside of it. Let's add sanity checks to the returned old keycodes. Reported-by: syzbot+c769968809f9359b07aa@syzkaller.appspotmail.com Reported-by: syzbot+76f3a30e88d256644c78@syzkaller.appspotmail.com Link: https://lore.kernel.org/r/20191207212757.GA245964@dtor-ws Signed-off-by: Dmitry Torokhov Signed-off-by: Greg Kroah-Hartman --- drivers/input/input.c | 26 ++++++++++++++++---------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/drivers/input/input.c b/drivers/input/input.c index 55086279d044..ee6c3234df36 100644 --- a/drivers/input/input.c +++ b/drivers/input/input.c @@ -878,16 +878,18 @@ static int input_default_setkeycode(struct input_dev *dev, } } - __clear_bit(*old_keycode, dev->keybit); - __set_bit(ke->keycode, dev->keybit); - - for (i = 0; i < dev->keycodemax; i++) { - if (input_fetch_keycode(dev, i) == *old_keycode) { - __set_bit(*old_keycode, dev->keybit); - break; /* Setting the bit twice is useless, so break */ + if (*old_keycode <= KEY_MAX) { + __clear_bit(*old_keycode, dev->keybit); + for (i = 0; i < dev->keycodemax; i++) { + if (input_fetch_keycode(dev, i) == *old_keycode) { + __set_bit(*old_keycode, dev->keybit); + /* Setting the bit twice is useless, so break */ + break; + } } } + __set_bit(ke->keycode, dev->keybit); return 0; } @@ -943,9 +945,13 @@ int input_set_keycode(struct input_dev *dev, * Simulate keyup event if keycode is not present * in the keymap anymore */ - if (test_bit(EV_KEY, dev->evbit) && - !is_event_supported(old_keycode, dev->keybit, KEY_MAX) && - __test_and_clear_bit(old_keycode, dev->key)) { + if (old_keycode > KEY_MAX) { + dev_warn(dev->dev.parent ?: &dev->dev, + "%s: got too big old keycode %#x\n", + __func__, old_keycode); + } else if (test_bit(EV_KEY, dev->evbit) && + !is_event_supported(old_keycode, dev->keybit, KEY_MAX) && + __test_and_clear_bit(old_keycode, dev->key)) { struct input_value vals[] = { { EV_KEY, old_keycode, 0 }, input_value_sync From 854ac5dee5be03a280faf1a4f7907b4e9a4b3be1 Mon Sep 17 00:00:00 2001 From: Arnd Bergmann Date: Fri, 13 Dec 2019 14:06:58 -0800 Subject: [PATCH 30/89] Input: input_event - fix struct padding on sparc64 commit f729a1b0f8df7091cea3729fc0e414f5326e1163 upstream. Going through all uses of timeval, I noticed that we screwed up input_event in the previous attempts to fix it: The time fields now match between kernel and user space, but all following fields are in the wrong place. Add the required padding that is implied by the glibc timeval definition to fix the layout, and use a struct initializer to avoid leaking kernel stack data. Fixes: 141e5dcaa735 ("Input: input_event - fix the CONFIG_SPARC64 mixup") Fixes: 2e746942ebac ("Input: input_event - provide override for sparc64") Signed-off-by: Arnd Bergmann Link: https://lore.kernel.org/r/20191213204936.3643476-2-arnd@arndb.de Cc: stable@vger.kernel.org Signed-off-by: Dmitry Torokhov Signed-off-by: Greg Kroah-Hartman --- drivers/input/evdev.c | 14 +++++++------- drivers/input/misc/uinput.c | 14 +++++++++----- include/uapi/linux/input.h | 1 + 3 files changed, 17 insertions(+), 12 deletions(-) diff --git a/drivers/input/evdev.c b/drivers/input/evdev.c index d7dd6fcf2db0..f918fca9ada3 100644 --- a/drivers/input/evdev.c +++ b/drivers/input/evdev.c @@ -224,13 +224,13 @@ static void __pass_event(struct evdev_client *client, */ client->tail = (client->head - 2) & (client->bufsize - 1); - client->buffer[client->tail].input_event_sec = - event->input_event_sec; - client->buffer[client->tail].input_event_usec = - event->input_event_usec; - client->buffer[client->tail].type = EV_SYN; - client->buffer[client->tail].code = SYN_DROPPED; - client->buffer[client->tail].value = 0; + client->buffer[client->tail] = (struct input_event) { + .input_event_sec = event->input_event_sec, + .input_event_usec = event->input_event_usec, + .type = EV_SYN, + .code = SYN_DROPPED, + .value = 0, + }; client->packet_head = client->tail; } diff --git a/drivers/input/misc/uinput.c b/drivers/input/misc/uinput.c index 84051f20b18a..002654ec7040 100644 --- a/drivers/input/misc/uinput.c +++ b/drivers/input/misc/uinput.c @@ -74,12 +74,16 @@ static int uinput_dev_event(struct input_dev *dev, struct uinput_device *udev = input_get_drvdata(dev); struct timespec64 ts; - udev->buff[udev->head].type = type; - udev->buff[udev->head].code = code; - udev->buff[udev->head].value = value; ktime_get_ts64(&ts); - udev->buff[udev->head].input_event_sec = ts.tv_sec; - udev->buff[udev->head].input_event_usec = ts.tv_nsec / NSEC_PER_USEC; + + udev->buff[udev->head] = (struct input_event) { + .input_event_sec = ts.tv_sec, + .input_event_usec = ts.tv_nsec / NSEC_PER_USEC, + .type = type, + .code = code, + .value = value, + }; + udev->head = (udev->head + 1) % UINPUT_BUFFER_SIZE; wake_up_interruptible(&udev->waitq); diff --git a/include/uapi/linux/input.h b/include/uapi/linux/input.h index f056b2a00d5c..9a61c28ed3ae 100644 --- a/include/uapi/linux/input.h +++ b/include/uapi/linux/input.h @@ -34,6 +34,7 @@ struct input_event { __kernel_ulong_t __sec; #if defined(__sparc__) && defined(__arch64__) unsigned int __usec; + unsigned int __pad; #else __kernel_ulong_t __usec; #endif From 4f1fa1164ddfd339d961c9ff563e11f77e6bd514 Mon Sep 17 00:00:00 2001 From: Matt Roper Date: Mon, 23 Dec 2019 17:20:25 -0800 Subject: [PATCH 31/89] drm/i915: Add Wa_1408615072 and Wa_1407596294 to icl,ehl commit a7f3ad37f80d0d5eec9dad156964c0dac800a80e upstream. Workaround database indicates we should disable clock gating of both the vsunit and hsunit. Bspec: 33450 Bspec: 33451 Cc: stable@kernel.vger.org Cc: Lucas De Marchi Cc: Matt Atwood Cc: Radhakrishna Sripada Signed-off-by: Matt Roper Link: https://patchwork.freedesktop.org/patch/msgid/20191224012026.3157766-3-matthew.d.roper@intel.com Reviewed-by: Lucas De Marchi (cherry picked from commit b9cf9dac3dac4c1d2a47d34f30ec53c0423cecf8) Signed-off-by: Joonas Lahtinen Signed-off-by: Greg Kroah-Hartman --- drivers/gpu/drm/i915/i915_reg.h | 4 +++- drivers/gpu/drm/i915/intel_pm.c | 8 ++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/i915/i915_reg.h b/drivers/gpu/drm/i915/i915_reg.h index f8ee9aba3955..165f14d11a6d 100644 --- a/drivers/gpu/drm/i915/i915_reg.h +++ b/drivers/gpu/drm/i915/i915_reg.h @@ -4049,7 +4049,9 @@ enum { #define GWUNIT_CLKGATE_DIS (1 << 16) #define UNSLICE_UNIT_LEVEL_CLKGATE _MMIO(0x9434) -#define VFUNIT_CLKGATE_DIS (1 << 20) +#define VFUNIT_CLKGATE_DIS REG_BIT(20) +#define HSUNIT_CLKGATE_DIS REG_BIT(8) +#define VSUNIT_CLKGATE_DIS REG_BIT(3) #define INF_UNIT_LEVEL_CLKGATE _MMIO(0x9560) #define CGPSF_CLKGATE_DIS (1 << 3) diff --git a/drivers/gpu/drm/i915/intel_pm.c b/drivers/gpu/drm/i915/intel_pm.c index 2efe1d12d5a9..f6a60a0d1f94 100644 --- a/drivers/gpu/drm/i915/intel_pm.c +++ b/drivers/gpu/drm/i915/intel_pm.c @@ -9194,6 +9194,14 @@ static void icl_init_clock_gating(struct drm_i915_private *dev_priv) /* WaEnable32PlaneMode:icl */ I915_WRITE(GEN9_CSFE_CHICKEN1_RCS, _MASKED_BIT_ENABLE(GEN11_ENABLE_32_PLANE_MODE)); + + /* + * Wa_1408615072:icl,ehl (vsunit) + * Wa_1407596294:icl,ehl (hsunit) + */ + intel_uncore_rmw(&dev_priv->uncore, UNSLICE_UNIT_LEVEL_CLKGATE, + 0, VSUNIT_CLKGATE_DIS | HSUNIT_CLKGATE_DIS); + } static void cnp_init_clock_gating(struct drm_i915_private *dev_priv) From 825fb807747b5f19e30e8c8fcd08e065fb609ddd Mon Sep 17 00:00:00 2001 From: Alex Deucher Date: Mon, 6 Jan 2020 15:24:47 -0500 Subject: [PATCH 32/89] Revert "drm/amdgpu: Set no-retry as default." MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit commit 7aec9ec1cf324d5c5a8d17b9c78a34c388e5f17b upstream. This reverts commit 51bfac71cade386966791a8db87a5912781d249f. This causes stability issues on some raven boards. Revert for now until a proper fix is completed. Bug: https://gitlab.freedesktop.org/drm/amd/issues/934 Bug: https://bugzilla.kernel.org/show_bug.cgi?id=206017 Reviewed-by: Felix Kuehling Reviewed-by: Christian König Signed-off-by: Alex Deucher Cc: stable@vger.kernel.org Signed-off-by: Greg Kroah-Hartman --- drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c index b19157b19fa0..33a1099e2f33 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c @@ -145,7 +145,7 @@ int amdgpu_async_gfx_ring = 1; int amdgpu_mcbp = 0; int amdgpu_discovery = -1; int amdgpu_mes = 0; -int amdgpu_noretry = 1; +int amdgpu_noretry; struct amdgpu_mgpu_info mgpu_info = { .mutex = __MUTEX_INITIALIZER(mgpu_info.mutex), @@ -613,7 +613,7 @@ MODULE_PARM_DESC(mes, module_param_named(mes, amdgpu_mes, int, 0444); MODULE_PARM_DESC(noretry, - "Disable retry faults (0 = retry enabled, 1 = retry disabled (default))"); + "Disable retry faults (0 = retry enabled (default), 1 = retry disabled)"); module_param_named(noretry, amdgpu_noretry, int, 0644); #ifdef CONFIG_HSA_AMD From c001b900caa6ddafd5becf09fef7614141cf583e Mon Sep 17 00:00:00 2001 From: Chen-Yu Tsai Date: Tue, 7 Jan 2020 15:01:13 +0800 Subject: [PATCH 33/89] drm/sun4i: tcon: Set RGB DCLK min. divider based on hardware model commit 4396393fb96449c56423fb4b351f76e45a6bcaf6 upstream. In commit 0b8e7bbde5e7 ("drm/sun4i: tcon: Set min division of TCON0_DCLK to 1.") it was assumed that all TCON variants support a minimum divider of 1 if only DCLK was used. However, the oldest generation of hardware only supports minimum divider of 4 if only DCLK is used. If a divider of 1 was used on this old hardware, some scrolling artifact would appear. A divider of 2 seemed OK, but a divider of 3 had artifacts as well. Set the minimum divider when outputing to parallel RGB based on the hardware model, with a minimum of 4 for the oldest (A10/A10s/A13/A20) hardware, and a minimum of 1 for the rest. A value is not set for the TCON variants lacking channel 0. This fixes the scrolling artifacts seen on my A13 tablet. Fixes: 0b8e7bbde5e7 ("drm/sun4i: tcon: Set min division of TCON0_DCLK to 1.") Cc: # 5.4.x Signed-off-by: Chen-Yu Tsai Signed-off-by: Maxime Ripard Link: https://patchwork.freedesktop.org/patch/msgid/20200107070113.28951-1-wens@kernel.org Signed-off-by: Greg Kroah-Hartman --- drivers/gpu/drm/sun4i/sun4i_tcon.c | 15 ++++++++++++--- drivers/gpu/drm/sun4i/sun4i_tcon.h | 1 + 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/drivers/gpu/drm/sun4i/sun4i_tcon.c b/drivers/gpu/drm/sun4i/sun4i_tcon.c index b89439ed210d..27c80c9e2b83 100644 --- a/drivers/gpu/drm/sun4i/sun4i_tcon.c +++ b/drivers/gpu/drm/sun4i/sun4i_tcon.c @@ -488,7 +488,7 @@ static void sun4i_tcon0_mode_set_rgb(struct sun4i_tcon *tcon, WARN_ON(!tcon->quirks->has_channel_0); - tcon->dclk_min_div = 1; + tcon->dclk_min_div = tcon->quirks->dclk_min_div; tcon->dclk_max_div = 127; sun4i_tcon0_mode_set_common(tcon, mode); @@ -1425,12 +1425,14 @@ static int sun8i_r40_tcon_tv_set_mux(struct sun4i_tcon *tcon, static const struct sun4i_tcon_quirks sun4i_a10_quirks = { .has_channel_0 = true, .has_channel_1 = true, + .dclk_min_div = 4, .set_mux = sun4i_a10_tcon_set_mux, }; static const struct sun4i_tcon_quirks sun5i_a13_quirks = { .has_channel_0 = true, .has_channel_1 = true, + .dclk_min_div = 4, .set_mux = sun5i_a13_tcon_set_mux, }; @@ -1439,6 +1441,7 @@ static const struct sun4i_tcon_quirks sun6i_a31_quirks = { .has_channel_1 = true, .has_lvds_alt = true, .needs_de_be_mux = true, + .dclk_min_div = 1, .set_mux = sun6i_tcon_set_mux, }; @@ -1446,11 +1449,13 @@ static const struct sun4i_tcon_quirks sun6i_a31s_quirks = { .has_channel_0 = true, .has_channel_1 = true, .needs_de_be_mux = true, + .dclk_min_div = 1, }; static const struct sun4i_tcon_quirks sun7i_a20_quirks = { .has_channel_0 = true, .has_channel_1 = true, + .dclk_min_div = 4, /* Same display pipeline structure as A10 */ .set_mux = sun4i_a10_tcon_set_mux, }; @@ -1458,11 +1463,13 @@ static const struct sun4i_tcon_quirks sun7i_a20_quirks = { static const struct sun4i_tcon_quirks sun8i_a33_quirks = { .has_channel_0 = true, .has_lvds_alt = true, + .dclk_min_div = 1, }; static const struct sun4i_tcon_quirks sun8i_a83t_lcd_quirks = { .supports_lvds = true, .has_channel_0 = true, + .dclk_min_div = 1, }; static const struct sun4i_tcon_quirks sun8i_a83t_tv_quirks = { @@ -1476,11 +1483,13 @@ static const struct sun4i_tcon_quirks sun8i_r40_tv_quirks = { static const struct sun4i_tcon_quirks sun8i_v3s_quirks = { .has_channel_0 = true, + .dclk_min_div = 1, }; static const struct sun4i_tcon_quirks sun9i_a80_tcon_lcd_quirks = { - .has_channel_0 = true, - .needs_edp_reset = true, + .has_channel_0 = true, + .needs_edp_reset = true, + .dclk_min_div = 1, }; static const struct sun4i_tcon_quirks sun9i_a80_tcon_tv_quirks = { diff --git a/drivers/gpu/drm/sun4i/sun4i_tcon.h b/drivers/gpu/drm/sun4i/sun4i_tcon.h index f9f1fe80b206..a62ec826ae71 100644 --- a/drivers/gpu/drm/sun4i/sun4i_tcon.h +++ b/drivers/gpu/drm/sun4i/sun4i_tcon.h @@ -224,6 +224,7 @@ struct sun4i_tcon_quirks { bool needs_de_be_mux; /* sun6i needs mux to select backend */ bool needs_edp_reset; /* a80 edp reset needed for tcon0 access */ bool supports_lvds; /* Does the TCON support an LVDS output? */ + u8 dclk_min_div; /* minimum divider for TCON0 DCLK */ /* callback to handle tcon muxing options */ int (*set_mux)(struct sun4i_tcon *, const struct drm_encoder *); From 03eb90320651d91c62f69116b604d6834242fb8f Mon Sep 17 00:00:00 2001 From: Geert Uytterhoeven Date: Mon, 30 Dec 2019 14:27:34 +0100 Subject: [PATCH 34/89] drm/fb-helper: Round up bits_per_pixel if possible commit f30e27779d3031a092c2a177b7fb76adccc45241 upstream. When userspace requests a video mode parameter value that is not supported, frame buffer device drivers should round it up to a supported value, if possible, instead of just rejecting it. This allows applications to quickly scan for supported video modes. Currently this rule is not followed for the number of bits per pixel, causing e.g. "fbset -depth N" to fail, if N is smaller than the current number of bits per pixel. Fix this by returning an error only if bits per pixel is too large, and setting it to the current value otherwise. See also Documentation/fb/framebuffer.rst, Section 2 (Programmer's View of /dev/fb*"). Fixes: 865afb11949e5bf4 ("drm/fb-helper: reject any changes to the fbdev") Cc: stable@vger.kernel.org Signed-off-by: Geert Uytterhoeven Signed-off-by: Daniel Vetter Link: https://patchwork.freedesktop.org/patch/msgid/20191230132734.4538-1-geert+renesas@glider.be Signed-off-by: Greg Kroah-Hartman --- drivers/gpu/drm/drm_fb_helper.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/drm_fb_helper.c b/drivers/gpu/drm/drm_fb_helper.c index a7ba5b4902d6..8d193a58363d 100644 --- a/drivers/gpu/drm/drm_fb_helper.c +++ b/drivers/gpu/drm/drm_fb_helper.c @@ -1320,7 +1320,7 @@ int drm_fb_helper_check_var(struct fb_var_screeninfo *var, * Changes struct fb_var_screeninfo are currently not pushed back * to KMS, hence fail if different settings are requested. */ - if (var->bits_per_pixel != fb->format->cpp[0] * 8 || + if (var->bits_per_pixel > fb->format->cpp[0] * 8 || var->xres > fb->width || var->yres > fb->height || var->xres_virtual > fb->width || var->yres_virtual > fb->height) { DRM_DEBUG("fb requested width/height/bpp can't fit in current fb " @@ -1345,6 +1345,11 @@ int drm_fb_helper_check_var(struct fb_var_screeninfo *var, drm_fb_helper_fill_pixel_fmt(var, fb->format->depth); } + /* + * Likewise, bits_per_pixel should be rounded up to a supported value. + */ + var->bits_per_pixel = fb->format->cpp[0] * 8; + /* * drm fbdev emulation doesn't support changing the pixel format at all, * so reject all pixel format changing requests. From 59aa28fcf770605911f176a0715fe4c840e84f8a Mon Sep 17 00:00:00 2001 From: Wayne Lin Date: Fri, 3 Jan 2020 13:50:01 +0800 Subject: [PATCH 35/89] drm/dp_mst: correct the shifting in DP_REMOTE_I2C_READ commit c4e4fccc5d52d881afaac11d3353265ef4eccb8b upstream. [Why] According to DP spec, it should shift left 4 digits for NO_STOP_BIT in REMOTE_I2C_READ message. Not 5 digits. In current code, NO_STOP_BIT is always set to zero which means I2C master is always generating a I2C stop at the end of each I2C write transaction while handling REMOTE_I2C_READ sideband message. This issue might have the generated I2C signal not meeting the requirement. Take random read in I2C for instance, I2C master should generate a repeat start to start to read data after writing the read address. This issue will cause the I2C master to generate a stop-start rather than a re-start which is not expected in I2C random read. [How] Correct the shifting value of NO_STOP_BIT for DP_REMOTE_I2C_READ case in drm_dp_encode_sideband_req(). Changes since v1:(https://patchwork.kernel.org/patch/11312667/) * Add more descriptions in commit and cc to stable Fixes: ad7f8a1f9ced ("drm/helper: add Displayport multi-stream helper (v0.6)") Reviewed-by: Harry Wentland Signed-off-by: Wayne Lin Cc: stable@vger.kernel.org Signed-off-by: Lyude Paul Link: https://patchwork.freedesktop.org/patch/msgid/20200103055001.10287-1-Wayne.Lin@amd.com Signed-off-by: Greg Kroah-Hartman --- drivers/gpu/drm/drm_dp_mst_topology.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/gpu/drm/drm_dp_mst_topology.c b/drivers/gpu/drm/drm_dp_mst_topology.c index 86c17896b532..a48a4c21b1b3 100644 --- a/drivers/gpu/drm/drm_dp_mst_topology.c +++ b/drivers/gpu/drm/drm_dp_mst_topology.c @@ -339,7 +339,7 @@ static void drm_dp_encode_sideband_req(struct drm_dp_sideband_msg_req_body *req, memcpy(&buf[idx], req->u.i2c_read.transactions[i].bytes, req->u.i2c_read.transactions[i].num_bytes); idx += req->u.i2c_read.transactions[i].num_bytes; - buf[idx] = (req->u.i2c_read.transactions[i].no_stop_bit & 0x1) << 5; + buf[idx] = (req->u.i2c_read.transactions[i].no_stop_bit & 0x1) << 4; buf[idx] |= (req->u.i2c_read.transactions[i].i2c_transaction_delay & 0xf); idx++; } From 843f6795576ee240e05f53ffbf2353d677268d4b Mon Sep 17 00:00:00 2001 From: Matt Roper Date: Tue, 31 Dec 2019 11:07:13 -0800 Subject: [PATCH 36/89] drm/i915: Add Wa_1407352427:icl,ehl commit 25b79ad51bf04a8aa67b5bccd631fc05f963b8e0 upstream. The workaround database now indicates we need to disable psdunit clock gating as well. v3: - Rebase on top of other workarounds that have landed. - Restrict cc:stable tag to 5.2+ since that's when ICL was first officially supported. Bspec: 32354 Bspec: 33450 Bspec: 33451 Suggested-by: Lionel Landwerlin Cc: stable@vger.kernel.org # v5.2+ Cc: Lionel Landwerlin Cc: Lucas De Marchi Cc: Matt Atwood Signed-off-by: Matt Roper Acked-by: Lionel Landwerlin Link: https://patchwork.freedesktop.org/patch/msgid/20191231190713.1549533-1-matthew.d.roper@intel.com (cherry picked from commit 1cd21a7c5679015352e8a6f46813aced51d71bb8) Signed-off-by: Joonas Lahtinen Signed-off-by: Greg Kroah-Hartman --- drivers/gpu/drm/i915/i915_reg.h | 4 ++++ drivers/gpu/drm/i915/intel_pm.c | 3 +++ 2 files changed, 7 insertions(+) diff --git a/drivers/gpu/drm/i915/i915_reg.h b/drivers/gpu/drm/i915/i915_reg.h index 165f14d11a6d..7b6e68f082f8 100644 --- a/drivers/gpu/drm/i915/i915_reg.h +++ b/drivers/gpu/drm/i915/i915_reg.h @@ -4053,6 +4053,10 @@ enum { #define HSUNIT_CLKGATE_DIS REG_BIT(8) #define VSUNIT_CLKGATE_DIS REG_BIT(3) +#define UNSLICE_UNIT_LEVEL_CLKGATE2 _MMIO(0x94e4) +#define VSUNIT_CLKGATE_DIS_TGL REG_BIT(19) +#define PSDUNIT_CLKGATE_DIS REG_BIT(5) + #define INF_UNIT_LEVEL_CLKGATE _MMIO(0x9560) #define CGPSF_CLKGATE_DIS (1 << 3) diff --git a/drivers/gpu/drm/i915/intel_pm.c b/drivers/gpu/drm/i915/intel_pm.c index f6a60a0d1f94..3ccfc025fde2 100644 --- a/drivers/gpu/drm/i915/intel_pm.c +++ b/drivers/gpu/drm/i915/intel_pm.c @@ -9202,6 +9202,9 @@ static void icl_init_clock_gating(struct drm_i915_private *dev_priv) intel_uncore_rmw(&dev_priv->uncore, UNSLICE_UNIT_LEVEL_CLKGATE, 0, VSUNIT_CLKGATE_DIS | HSUNIT_CLKGATE_DIS); + /* Wa_1407352427:icl,ehl */ + intel_uncore_rmw(&dev_priv->uncore, UNSLICE_UNIT_LEVEL_CLKGATE2, + 0, PSDUNIT_CLKGATE_DIS); } static void cnp_init_clock_gating(struct drm_i915_private *dev_priv) From e1a17ea063a8bf0d735dbd477694857a1b3f4690 Mon Sep 17 00:00:00 2001 From: Chris Wilson Date: Mon, 6 Jan 2020 12:39:21 +0000 Subject: [PATCH 37/89] drm/i915/gt: Mark up virtual engine uabi_instance commit 1325008f5c8dbc84aa835d98af8447fa0569bc4d upstream. Be sure to initialise the uabi_instance on the virtual engine to the special invalid value, just in case we ever peek at it from the uAPI. Reported-by: Tvrtko Ursulin Fixes: 750e76b4f9f6 ("drm/i915/gt: Move the [class][inst] lookup for engines onto the GT") Signed-off-by: Chris Wilson Cc: Tvrtko Ursulin Cc: # v5.4+ Reviewed-by: Tvrtko Ursulin Link: https://patchwork.freedesktop.org/patch/msgid/20200106123921.2543886-1-chris@chris-wilson.co.uk (cherry picked from commit f75fc37b5e70b75f21550410f88e2379648120e2) Signed-off-by: Joonas Lahtinen Signed-off-by: Greg Kroah-Hartman --- drivers/gpu/drm/i915/gt/intel_lrc.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/drivers/gpu/drm/i915/gt/intel_lrc.c b/drivers/gpu/drm/i915/gt/intel_lrc.c index d564bfcab6a3..1ba31969c7d2 100644 --- a/drivers/gpu/drm/i915/gt/intel_lrc.c +++ b/drivers/gpu/drm/i915/gt/intel_lrc.c @@ -3716,9 +3716,11 @@ intel_execlists_create_virtual(struct i915_gem_context *ctx, ve->base.i915 = ctx->i915; ve->base.gt = siblings[0]->gt; ve->base.id = -1; + ve->base.class = OTHER_CLASS; ve->base.uabi_class = I915_ENGINE_CLASS_INVALID; ve->base.instance = I915_ENGINE_CLASS_INVALID_VIRTUAL; + ve->base.uabi_instance = I915_ENGINE_CLASS_INVALID_VIRTUAL; /* * The decision on whether to submit a request using semaphores From 578ab7d4d9477c89d494c7beaf815de003290078 Mon Sep 17 00:00:00 2001 From: Kaike Wan Date: Thu, 19 Dec 2019 18:19:20 -0500 Subject: [PATCH 38/89] IB/hfi1: Adjust flow PSN with the correct resync_psn commit b2ff0d510182eb5cc05a65d1b2371af62c4b170c upstream. When a TID RDMA ACK to RESYNC request is received, the flow PSNs for pending TID RDMA WRITE segments will be adjusted with the next flow generation number, based on the resync_psn value extracted from the flow PSN of the TID RDMA ACK packet. The resync_psn value indicates the last flow PSN for which a TID RDMA WRITE DATA packet has been received by the responder and the requester should resend TID RDMA WRITE DATA packets, starting from the next flow PSN. However, if resync_psn points to the last flow PSN for a segment and the next segment flow PSN starts with a new generation number, use of the old resync_psn to adjust the flow PSN for the next segment will lead to miscalculation, resulting in WARN_ON and sge rewinding errors: WARNING: CPU: 4 PID: 146961 at /nfs/site/home/phcvs2/gitrepo/ifs-all/components/Drivers/tmp/rpmbuild/BUILD/ifs-kernel-updates-3.10.0_957.el7.x86_64/hfi1/tid_rdma.c:4764 hfi1_rc_rcv_tid_rdma_ack+0x8f6/0xa90 [hfi1] Modules linked in: ib_ipoib(OE) hfi1(OE) rdmavt(OE) rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfsv3 nfs_acl nfs lockd grace fscache iTCO_wdt iTCO_vendor_support skx_edac intel_powerclamp coretemp intel_rapl iosf_mbi kvm irqbypass crc32_pclmul ghash_clmulni_intel ib_isert iscsi_target_mod target_core_mod aesni_intel lrw gf128mul glue_helper ablk_helper cryptd rpcrdma sunrpc opa_vnic ast ttm ib_iser libiscsi drm_kms_helper scsi_transport_iscsi ipmi_ssif syscopyarea sysfillrect sysimgblt fb_sys_fops drm joydev ipmi_si pcspkr sg drm_panel_orientation_quirks ipmi_devintf lpc_ich i2c_i801 ipmi_msghandler wmi rdma_ucm ib_ucm ib_uverbs acpi_cpufreq acpi_power_meter ib_umad rdma_cm ib_cm iw_cm ip_tables ext4 mbcache jbd2 sd_mod crc_t10dif crct10dif_generic crct10dif_pclmul i2c_algo_bit crct10dif_common crc32c_intel e1000e ib_core ahci libahci ptp libata pps_core nfit libnvdimm [last unloaded: rdmavt] CPU: 4 PID: 146961 Comm: kworker/4:0H Kdump: loaded Tainted: G W OE ------------ 3.10.0-957.el7.x86_64 #1 Hardware name: Intel Corporation S2600WFT/S2600WFT, BIOS SE5C620.86B.0X.02.0117.040420182310 04/04/2018 Workqueue: hfi0_0 _hfi1_do_tid_send [hfi1] Call Trace: [] dump_stack+0x19/0x1b [] __warn+0xd8/0x100 [] warn_slowpath_null+0x1d/0x20 [] hfi1_rc_rcv_tid_rdma_ack+0x8f6/0xa90 [hfi1] [] hfi1_kdeth_eager_rcv+0x1dc/0x210 [hfi1] [] ? hfi1_kdeth_expected_rcv+0x1ef/0x210 [hfi1] [] kdeth_process_eager+0x35/0x90 [hfi1] [] handle_receive_interrupt_nodma_rtail+0x17a/0x2b0 [hfi1] [] receive_context_interrupt+0x23/0x40 [hfi1] [] __handle_irq_event_percpu+0x44/0x1c0 [] handle_irq_event_percpu+0x32/0x80 [] handle_irq_event+0x3c/0x60 [] handle_edge_irq+0x7f/0x150 [] handle_irq+0xe4/0x1a0 [] do_IRQ+0x4d/0xf0 [] common_interrupt+0x162/0x162 [] ? swiotlb_map_page+0x49/0x150 [] hfi1_verbs_send_dma+0x291/0xb70 [hfi1] [] ? hfi1_wait_kmem+0xf0/0xf0 [hfi1] [] hfi1_verbs_send+0x126/0x2b0 [hfi1] [] _hfi1_do_tid_send+0x1d3/0x320 [hfi1] [] process_one_work+0x17f/0x440 [] worker_thread+0x126/0x3c0 [] ? manage_workers.isra.25+0x2a0/0x2a0 [] kthread+0xd1/0xe0 [] ? insert_kthread_work+0x40/0x40 [] ret_from_fork_nospec_begin+0x7/0x21 [] ? insert_kthread_work+0x40/0x40 This patch fixes the issue by adjusting the resync_psn first if the flow generation has been advanced for a pending segment. Fixes: 9e93e967f7b4 ("IB/hfi1: Add a function to receive TID RDMA ACK packet") Link: https://lore.kernel.org/r/20191219231920.51069.37147.stgit@awfm-01.aw.intel.com Cc: Reviewed-by: Mike Marciniszyn Signed-off-by: Kaike Wan Signed-off-by: Dennis Dalessandro Signed-off-by: Jason Gunthorpe Signed-off-by: Greg Kroah-Hartman --- drivers/infiniband/hw/hfi1/tid_rdma.c | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/drivers/infiniband/hw/hfi1/tid_rdma.c b/drivers/infiniband/hw/hfi1/tid_rdma.c index e53f542b60af..8a2e0d9351e9 100644 --- a/drivers/infiniband/hw/hfi1/tid_rdma.c +++ b/drivers/infiniband/hw/hfi1/tid_rdma.c @@ -4633,6 +4633,15 @@ void hfi1_rc_rcv_tid_rdma_ack(struct hfi1_packet *packet) */ fpsn = full_flow_psn(flow, flow->flow_state.spsn); req->r_ack_psn = psn; + /* + * If resync_psn points to the last flow PSN for a + * segment and the new segment (likely from a new + * request) starts with a new generation number, we + * need to adjust resync_psn accordingly. + */ + if (flow->flow_state.generation != + (resync_psn >> HFI1_KDETH_BTH_SEQ_SHIFT)) + resync_psn = mask_psn(fpsn - 1); flow->resync_npkts += delta_psn(mask_psn(resync_psn + 1), fpsn); /* From e791b56a298b3d17a880a0d875e2787d17fa0d59 Mon Sep 17 00:00:00 2001 From: Johan Hovold Date: Tue, 10 Dec 2019 12:32:30 +0100 Subject: [PATCH 39/89] can: kvaser_usb: fix interface sanity check commit 5660493c637c9d83786f1c9297f403eae44177b6 upstream. Make sure to use the current alternate setting when verifying the interface descriptors to avoid binding to an invalid interface. Failing to do so could cause the driver to misbehave or trigger a WARN() in usb_submit_urb() that kernels with panic_on_warn set would choke on. Fixes: aec5fb2268b7 ("can: kvaser_usb: Add support for Kvaser USB hydra family") Cc: stable # 4.19 Cc: Jimmy Assarsson Cc: Christer Beskow Cc: Nicklas Johansson Cc: Martin Henriksson Signed-off-by: Johan Hovold Signed-off-by: Marc Kleine-Budde Signed-off-by: Greg Kroah-Hartman --- drivers/net/can/usb/kvaser_usb/kvaser_usb_hydra.c | 2 +- drivers/net/can/usb/kvaser_usb/kvaser_usb_leaf.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/net/can/usb/kvaser_usb/kvaser_usb_hydra.c b/drivers/net/can/usb/kvaser_usb/kvaser_usb_hydra.c index 5fc0be564274..7ab87a758754 100644 --- a/drivers/net/can/usb/kvaser_usb/kvaser_usb_hydra.c +++ b/drivers/net/can/usb/kvaser_usb/kvaser_usb_hydra.c @@ -1590,7 +1590,7 @@ static int kvaser_usb_hydra_setup_endpoints(struct kvaser_usb *dev) struct usb_endpoint_descriptor *ep; int i; - iface_desc = &dev->intf->altsetting[0]; + iface_desc = dev->intf->cur_altsetting; for (i = 0; i < iface_desc->desc.bNumEndpoints; ++i) { ep = &iface_desc->endpoint[i].desc; diff --git a/drivers/net/can/usb/kvaser_usb/kvaser_usb_leaf.c b/drivers/net/can/usb/kvaser_usb/kvaser_usb_leaf.c index ae4c37e1bb75..1b9957f12459 100644 --- a/drivers/net/can/usb/kvaser_usb/kvaser_usb_leaf.c +++ b/drivers/net/can/usb/kvaser_usb/kvaser_usb_leaf.c @@ -1310,7 +1310,7 @@ static int kvaser_usb_leaf_setup_endpoints(struct kvaser_usb *dev) struct usb_endpoint_descriptor *endpoint; int i; - iface_desc = &dev->intf->altsetting[0]; + iface_desc = dev->intf->cur_altsetting; for (i = 0; i < iface_desc->desc.bNumEndpoints; ++i) { endpoint = &iface_desc->endpoint[i].desc; From a05b169456468190549af475076e6d6b630b172a Mon Sep 17 00:00:00 2001 From: Johan Hovold Date: Tue, 10 Dec 2019 12:32:31 +0100 Subject: [PATCH 40/89] can: gs_usb: gs_usb_probe(): use descriptors of current altsetting commit 2f361cd9474ab2c4ab9ac8db20faf81e66c6279b upstream. Make sure to always use the descriptors of the current alternate setting to avoid future issues when accessing fields that may differ between settings. Signed-off-by: Johan Hovold Fixes: d08e973a77d1 ("can: gs_usb: Added support for the GS_USB CAN devices") Signed-off-by: Marc Kleine-Budde Signed-off-by: Greg Kroah-Hartman --- drivers/net/can/usb/gs_usb.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/net/can/usb/gs_usb.c b/drivers/net/can/usb/gs_usb.c index 2f74f6704c12..a4b4b742c80c 100644 --- a/drivers/net/can/usb/gs_usb.c +++ b/drivers/net/can/usb/gs_usb.c @@ -918,7 +918,7 @@ static int gs_usb_probe(struct usb_interface *intf, GS_USB_BREQ_HOST_FORMAT, USB_DIR_OUT | USB_TYPE_VENDOR | USB_RECIP_INTERFACE, 1, - intf->altsetting[0].desc.bInterfaceNumber, + intf->cur_altsetting->desc.bInterfaceNumber, hconf, sizeof(*hconf), 1000); @@ -941,7 +941,7 @@ static int gs_usb_probe(struct usb_interface *intf, GS_USB_BREQ_DEVICE_CONFIG, USB_DIR_IN | USB_TYPE_VENDOR | USB_RECIP_INTERFACE, 1, - intf->altsetting[0].desc.bInterfaceNumber, + intf->cur_altsetting->desc.bInterfaceNumber, dconf, sizeof(*dconf), 1000); From 1ad3ee211d331cdc60260952c3b3c61ddda4ed64 Mon Sep 17 00:00:00 2001 From: Sean Nyekjaer Date: Wed, 11 Dec 2019 14:58:52 +0100 Subject: [PATCH 41/89] can: tcan4x5x: tcan4x5x_can_probe(): get the device out of standby before register access commit 3069ce620daed85e4ef2b0c087dca2509f809470 upstream. The m_can tries to detect if Non ISO Operation is available while in standby mode, this function results in the following error: | tcan4x5x spi2.0 (unnamed net_device) (uninitialized): Failed to init module | tcan4x5x spi2.0: m_can device registered (irq=84, version=32) | tcan4x5x spi2.0 can2: TCAN4X5X successfully initialized. When the tcan device comes out of reset it goes in standby mode. The m_can driver tries to access the control register but fails due to the device being in standby mode. So this patch will put the tcan device in normal mode before the m_can driver does the initialization. Fixes: 5443c226ba91 ("can: tcan4x5x: Add tcan4x5x driver to the kernel") Cc: stable@vger.kernel.org Signed-off-by: Sean Nyekjaer Acked-by: Dan Murphy Signed-off-by: Marc Kleine-Budde Signed-off-by: Greg Kroah-Hartman --- drivers/net/can/m_can/tcan4x5x.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/drivers/net/can/m_can/tcan4x5x.c b/drivers/net/can/m_can/tcan4x5x.c index d5d4bfa9c8fd..d797912e665a 100644 --- a/drivers/net/can/m_can/tcan4x5x.c +++ b/drivers/net/can/m_can/tcan4x5x.c @@ -445,6 +445,10 @@ static int tcan4x5x_can_probe(struct spi_device *spi) tcan4x5x_power_enable(priv->power, 1); + ret = tcan4x5x_init(mcan_class); + if (ret) + goto out_power; + ret = m_can_class_register(mcan_class); if (ret) goto out_power; From 584b8299f7250b69396138823b5a94f33016340f Mon Sep 17 00:00:00 2001 From: Florian Faber Date: Thu, 26 Dec 2019 19:51:24 +0100 Subject: [PATCH 42/89] can: mscan: mscan_rx_poll(): fix rx path lockup when returning from polling to irq mode commit 2d77bd61a2927be8f4e00d9478fe6996c47e8d45 upstream. Under load, the RX side of the mscan driver can get stuck while TX still works. Restarting the interface locks up the system. This behaviour could be reproduced reliably on a MPC5121e based system. The patch fixes the return value of the NAPI polling function (should be the number of processed packets, not constant 1) and the condition under which IRQs are enabled again after polling is finished. With this patch, no more lockups were observed over a test period of ten days. Fixes: afa17a500a36 ("net/can: add driver for mscan family & mpc52xx_mscan") Signed-off-by: Florian Faber Cc: linux-stable Signed-off-by: Marc Kleine-Budde Signed-off-by: Greg Kroah-Hartman --- drivers/net/can/mscan/mscan.c | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/drivers/net/can/mscan/mscan.c b/drivers/net/can/mscan/mscan.c index 8caf7af0dee2..99101d7027a8 100644 --- a/drivers/net/can/mscan/mscan.c +++ b/drivers/net/can/mscan/mscan.c @@ -381,13 +381,12 @@ static int mscan_rx_poll(struct napi_struct *napi, int quota) struct net_device *dev = napi->dev; struct mscan_regs __iomem *regs = priv->reg_base; struct net_device_stats *stats = &dev->stats; - int npackets = 0; - int ret = 1; + int work_done = 0; struct sk_buff *skb; struct can_frame *frame; u8 canrflg; - while (npackets < quota) { + while (work_done < quota) { canrflg = in_8(®s->canrflg); if (!(canrflg & (MSCAN_RXF | MSCAN_ERR_IF))) break; @@ -408,18 +407,18 @@ static int mscan_rx_poll(struct napi_struct *napi, int quota) stats->rx_packets++; stats->rx_bytes += frame->can_dlc; - npackets++; + work_done++; netif_receive_skb(skb); } - if (!(in_8(®s->canrflg) & (MSCAN_RXF | MSCAN_ERR_IF))) { - napi_complete(&priv->napi); - clear_bit(F_RX_PROGRESS, &priv->flags); - if (priv->can.state < CAN_STATE_BUS_OFF) - out_8(®s->canrier, priv->shadow_canrier); - ret = 0; + if (work_done < quota) { + if (likely(napi_complete_done(&priv->napi, work_done))) { + clear_bit(F_RX_PROGRESS, &priv->flags); + if (priv->can.state < CAN_STATE_BUS_OFF) + out_8(®s->canrier, priv->shadow_canrier); + } } - return ret; + return work_done; } static irqreturn_t mscan_isr(int irq, void *dev_id) From a0dbd93fb148764c5ccc099307bba0b68e7fda4d Mon Sep 17 00:00:00 2001 From: Oliver Hartkopp Date: Sat, 7 Dec 2019 19:34:18 +0100 Subject: [PATCH 43/89] can: can_dropped_invalid_skb(): ensure an initialized headroom in outgoing CAN sk_buffs commit e7153bf70c3496bac00e7e4f395bb8d8394ac0ea upstream. KMSAN sysbot detected a read access to an untinitialized value in the headroom of an outgoing CAN related sk_buff. When using CAN sockets this area is filled appropriately - but when using a packet socket this initialization is missing. The problematic read access occurs in the CAN receive path which can only be triggered when the sk_buff is sent through a (virtual) CAN interface. So we check in the sending path whether we need to perform the missing initializations. Fixes: d3b58c47d330d ("can: replace timestamp as unique skb attribute") Reported-by: syzbot+b02ff0707a97e4e79ebb@syzkaller.appspotmail.com Signed-off-by: Oliver Hartkopp Tested-by: Oliver Hartkopp Cc: linux-stable # >= v4.1 Signed-off-by: Marc Kleine-Budde Signed-off-by: Greg Kroah-Hartman --- include/linux/can/dev.h | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/include/linux/can/dev.h b/include/linux/can/dev.h index 9b3c720a31b1..5e3d45525bd3 100644 --- a/include/linux/can/dev.h +++ b/include/linux/can/dev.h @@ -18,6 +18,7 @@ #include #include #include +#include #include /* @@ -91,6 +92,36 @@ struct can_priv { #define get_can_dlc(i) (min_t(__u8, (i), CAN_MAX_DLC)) #define get_canfd_dlc(i) (min_t(__u8, (i), CANFD_MAX_DLC)) +/* Check for outgoing skbs that have not been created by the CAN subsystem */ +static inline bool can_skb_headroom_valid(struct net_device *dev, + struct sk_buff *skb) +{ + /* af_packet creates a headroom of HH_DATA_MOD bytes which is fine */ + if (WARN_ON_ONCE(skb_headroom(skb) < sizeof(struct can_skb_priv))) + return false; + + /* af_packet does not apply CAN skb specific settings */ + if (skb->ip_summed == CHECKSUM_NONE) { + /* init headroom */ + can_skb_prv(skb)->ifindex = dev->ifindex; + can_skb_prv(skb)->skbcnt = 0; + + skb->ip_summed = CHECKSUM_UNNECESSARY; + + /* preform proper loopback on capable devices */ + if (dev->flags & IFF_ECHO) + skb->pkt_type = PACKET_LOOPBACK; + else + skb->pkt_type = PACKET_HOST; + + skb_reset_mac_header(skb); + skb_reset_network_header(skb); + skb_reset_transport_header(skb); + } + + return true; +} + /* Drop a given socketbuffer if it does not contain a valid CAN frame. */ static inline bool can_dropped_invalid_skb(struct net_device *dev, struct sk_buff *skb) @@ -108,6 +139,9 @@ static inline bool can_dropped_invalid_skb(struct net_device *dev, } else goto inval_skb; + if (!can_skb_headroom_valid(dev, skb)) + goto inval_skb; + return false; inval_skb: From 5d09934c8c3a45085e14cb47328b1fa37a3b260b Mon Sep 17 00:00:00 2001 From: Hans de Goede Date: Sun, 5 Jan 2020 17:03:56 +0100 Subject: [PATCH 44/89] gpiolib: acpi: Turn dmi_system_id table into a generic quirk table commit 1ad1b54099c231aed8f6f257065c1b322583f264 upstream. Turn the existing run_edge_events_on_boot_blacklist dmi_system_id table into a generic quirk table, storing the quirks in the driver_data ptr. This is a preparation patch for adding other types of (DMI based) quirks. Cc: stable@vger.kernel.org Reviewed-by: Andy Shevchenko Acked-by: Mika Westerberg Signed-off-by: Hans de Goede Link: https://lore.kernel.org/r/20200105160357.97154-2-hdegoede@redhat.com Signed-off-by: Linus Walleij Signed-off-by: Greg Kroah-Hartman --- drivers/gpio/gpiolib-acpi.c | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/drivers/gpio/gpiolib-acpi.c b/drivers/gpio/gpiolib-acpi.c index 59ccfd24627d..2c09ba799dee 100644 --- a/drivers/gpio/gpiolib-acpi.c +++ b/drivers/gpio/gpiolib-acpi.c @@ -21,6 +21,8 @@ #include "gpiolib.h" #include "gpiolib-acpi.h" +#define QUIRK_NO_EDGE_EVENTS_ON_BOOT 0x01l + static int run_edge_events_on_boot = -1; module_param(run_edge_events_on_boot, int, 0444); MODULE_PARM_DESC(run_edge_events_on_boot, @@ -1302,7 +1304,7 @@ static int acpi_gpio_handle_deferred_request_irqs(void) /* We must use _sync so that this runs after the first deferred_probe run */ late_initcall_sync(acpi_gpio_handle_deferred_request_irqs); -static const struct dmi_system_id run_edge_events_on_boot_blacklist[] = { +static const struct dmi_system_id gpiolib_acpi_quirks[] = { { /* * The Minix Neo Z83-4 has a micro-USB-B id-pin handler for @@ -1312,7 +1314,8 @@ static const struct dmi_system_id run_edge_events_on_boot_blacklist[] = { .matches = { DMI_MATCH(DMI_SYS_VENDOR, "MINIX"), DMI_MATCH(DMI_PRODUCT_NAME, "Z83-4"), - } + }, + .driver_data = (void *)QUIRK_NO_EDGE_EVENTS_ON_BOOT, }, { /* @@ -1324,15 +1327,23 @@ static const struct dmi_system_id run_edge_events_on_boot_blacklist[] = { .matches = { DMI_MATCH(DMI_SYS_VENDOR, "Wortmann_AG"), DMI_MATCH(DMI_PRODUCT_NAME, "TERRA_PAD_1061"), - } + }, + .driver_data = (void *)QUIRK_NO_EDGE_EVENTS_ON_BOOT, }, {} /* Terminating entry */ }; static int acpi_gpio_setup_params(void) { + const struct dmi_system_id *id; + long quirks = 0; + + id = dmi_first_match(gpiolib_acpi_quirks); + if (id) + quirks = (long)id->driver_data; + if (run_edge_events_on_boot < 0) { - if (dmi_check_system(run_edge_events_on_boot_blacklist)) + if (quirks & QUIRK_NO_EDGE_EVENTS_ON_BOOT) run_edge_events_on_boot = 0; else run_edge_events_on_boot = 1; From 85b60d32210ae5c57999b94ff26c0120a47f9fe4 Mon Sep 17 00:00:00 2001 From: Hans de Goede Date: Sun, 5 Jan 2020 17:03:57 +0100 Subject: [PATCH 45/89] gpiolib: acpi: Add honor_wakeup module-option + quirk mechanism commit aa23ca3d98f756d5b1e503fb140665fb24a41a38 upstream. On some laptops enabling wakeup on the GPIO interrupts used for ACPI _AEI event handling causes spurious wakeups. This commit adds a new honor_wakeup option, defaulting to true (our current behavior), which can be used to disable wakeup on troublesome hardware to avoid these spurious wakeups. This is a workaround for an architectural problem with s2idle under Linux where we do not have any mechanism to immediately go back to sleep after wakeup events, other then for embedded-controller events using the standard ACPI EC interface, for details see: https://lore.kernel.org/linux-acpi/61450f9b-cbc6-0c09-8b3a-aff6bf9a0b3c@redhat.com/ One series of laptops which is not able to suspend without this workaround is the HP x2 10 Cherry Trail models, this commit adds a DMI based quirk which makes sets honor_wakeup to false on these models. Cc: stable@vger.kernel.org Reviewed-by: Andy Shevchenko Acked-by: Mika Westerberg Signed-off-by: Hans de Goede Link: https://lore.kernel.org/r/20200105160357.97154-3-hdegoede@redhat.com Signed-off-by: Linus Walleij Signed-off-by: Greg Kroah-Hartman --- drivers/gpio/gpiolib-acpi.c | 32 +++++++++++++++++++++++++++++++- 1 file changed, 31 insertions(+), 1 deletion(-) diff --git a/drivers/gpio/gpiolib-acpi.c b/drivers/gpio/gpiolib-acpi.c index 2c09ba799dee..1f98e988c0d3 100644 --- a/drivers/gpio/gpiolib-acpi.c +++ b/drivers/gpio/gpiolib-acpi.c @@ -22,12 +22,18 @@ #include "gpiolib-acpi.h" #define QUIRK_NO_EDGE_EVENTS_ON_BOOT 0x01l +#define QUIRK_NO_WAKEUP 0x02l static int run_edge_events_on_boot = -1; module_param(run_edge_events_on_boot, int, 0444); MODULE_PARM_DESC(run_edge_events_on_boot, "Run edge _AEI event-handlers at boot: 0=no, 1=yes, -1=auto"); +static int honor_wakeup = -1; +module_param(honor_wakeup, int, 0444); +MODULE_PARM_DESC(honor_wakeup, + "Honor the ACPI wake-capable flag: 0=no, 1=yes, -1=auto"); + /** * struct acpi_gpio_event - ACPI GPIO event handler data * @@ -276,7 +282,7 @@ static acpi_status acpi_gpiochip_alloc_event(struct acpi_resource *ares, event->handle = evt_handle; event->handler = handler; event->irq = irq; - event->irq_is_wake = agpio->wake_capable == ACPI_WAKE_CAPABLE; + event->irq_is_wake = honor_wakeup && agpio->wake_capable == ACPI_WAKE_CAPABLE; event->pin = pin; event->desc = desc; @@ -1330,6 +1336,23 @@ static const struct dmi_system_id gpiolib_acpi_quirks[] = { }, .driver_data = (void *)QUIRK_NO_EDGE_EVENTS_ON_BOOT, }, + { + /* + * Various HP X2 10 Cherry Trail models use an external + * embedded-controller connected via I2C + an ACPI GPIO + * event handler. The embedded controller generates various + * spurious wakeup events when suspended. So disable wakeup + * for its handler (it uses the only ACPI GPIO event handler). + * This breaks wakeup when opening the lid, the user needs + * to press the power-button to wakeup the system. The + * alternative is suspend simply not working, which is worse. + */ + .matches = { + DMI_MATCH(DMI_SYS_VENDOR, "HP"), + DMI_MATCH(DMI_PRODUCT_NAME, "HP x2 Detachable 10-p0XX"), + }, + .driver_data = (void *)QUIRK_NO_WAKEUP, + }, {} /* Terminating entry */ }; @@ -1349,6 +1372,13 @@ static int acpi_gpio_setup_params(void) run_edge_events_on_boot = 1; } + if (honor_wakeup < 0) { + if (quirks & QUIRK_NO_WAKEUP) + honor_wakeup = 0; + else + honor_wakeup = 1; + } + return 0; } From c617a3b777b92a0e80ceff2dffaae9350d4c3850 Mon Sep 17 00:00:00 2001 From: Kees Cook Date: Wed, 8 Jan 2020 10:06:54 -0800 Subject: [PATCH 46/89] pstore/ram: Regularize prz label allocation lifetime commit e163fdb3f7f8c62dccf194f3f37a7bcb3c333aa8 upstream. In my attempt to fix a memory leak, I introduced a double-free in the pstore error path. Instead of trying to manage the allocation lifetime between persistent_ram_new() and its callers, adjust the logic so persistent_ram_new() always takes a kstrdup() copy, and leaves the caller's allocation lifetime up to the caller. Therefore callers are _always_ responsible for freeing their label. Before, it only needed freeing when the prz itself failed to allocate, and not in any of the other prz failure cases, which callers would have no visibility into, which is the root design problem that lead to both the leak and now double-free bugs. Reported-by: Cengiz Can Link: https://lore.kernel.org/lkml/d4ec59002ede4aaf9928c7f7526da87c@kernel.wtf Fixes: 8df955a32a73 ("pstore/ram: Fix error-path memory leak in persistent_ram_new() callers") Cc: stable@vger.kernel.org Signed-off-by: Kees Cook Signed-off-by: Greg Kroah-Hartman --- fs/pstore/ram.c | 4 ++-- fs/pstore/ram_core.c | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/fs/pstore/ram.c b/fs/pstore/ram.c index 487ee39b438a..013486b5125e 100644 --- a/fs/pstore/ram.c +++ b/fs/pstore/ram.c @@ -583,12 +583,12 @@ static int ramoops_init_przs(const char *name, prz_ar[i] = persistent_ram_new(*paddr, zone_sz, sig, &cxt->ecc_info, cxt->memtype, flags, label); + kfree(label); if (IS_ERR(prz_ar[i])) { err = PTR_ERR(prz_ar[i]); dev_err(dev, "failed to request %s mem region (0x%zx@0x%llx): %d\n", name, record_size, (unsigned long long)*paddr, err); - kfree(label); while (i > 0) { i--; @@ -629,12 +629,12 @@ static int ramoops_init_prz(const char *name, label = kasprintf(GFP_KERNEL, "ramoops:%s", name); *prz = persistent_ram_new(*paddr, sz, sig, &cxt->ecc_info, cxt->memtype, PRZ_FLAG_ZAP_OLD, label); + kfree(label); if (IS_ERR(*prz)) { int err = PTR_ERR(*prz); dev_err(dev, "failed to request %s mem region (0x%zx@0x%llx): %d\n", name, sz, (unsigned long long)*paddr, err); - kfree(label); return err; } diff --git a/fs/pstore/ram_core.c b/fs/pstore/ram_core.c index 8823f65888f0..1f4d8c06f9be 100644 --- a/fs/pstore/ram_core.c +++ b/fs/pstore/ram_core.c @@ -574,7 +574,7 @@ struct persistent_ram_zone *persistent_ram_new(phys_addr_t start, size_t size, /* Initialize general buffer state. */ raw_spin_lock_init(&prz->buffer_lock); prz->flags = flags; - prz->label = label; + prz->label = kstrdup(label, GFP_KERNEL); ret = persistent_ram_buffer_map(start, size, prz, memtype); if (ret) From f2a6955c9b74cf20f471ab8a33b2aa6d257af013 Mon Sep 17 00:00:00 2001 From: Malcolm Priestley Date: Fri, 20 Dec 2019 21:15:59 +0000 Subject: [PATCH 47/89] staging: vt6656: set usb_set_intfdata on driver fail. commit c0bcf9f3f5b661d4ace2a64a79ef661edd2a4dc8 upstream. intfdata will contain stale pointer when the device is detached after failed initialization when referenced in vt6656_disconnect Provide driver access to it here and NULL it. Cc: stable Signed-off-by: Malcolm Priestley Link: https://lore.kernel.org/r/6de448d7-d833-ef2e-dd7b-3ef9992fee0e@gmail.com Signed-off-by: Greg Kroah-Hartman --- drivers/staging/vt6656/device.h | 1 + drivers/staging/vt6656/main_usb.c | 1 + drivers/staging/vt6656/wcmd.c | 1 + 3 files changed, 3 insertions(+) diff --git a/drivers/staging/vt6656/device.h b/drivers/staging/vt6656/device.h index 6074ceda78bf..50e1c8918040 100644 --- a/drivers/staging/vt6656/device.h +++ b/drivers/staging/vt6656/device.h @@ -259,6 +259,7 @@ struct vnt_private { u8 mac_hw; /* netdev */ struct usb_device *usb; + struct usb_interface *intf; u64 tsf_time; u8 rx_rate; diff --git a/drivers/staging/vt6656/main_usb.c b/drivers/staging/vt6656/main_usb.c index 856ba97aec4f..8b45b8578372 100644 --- a/drivers/staging/vt6656/main_usb.c +++ b/drivers/staging/vt6656/main_usb.c @@ -993,6 +993,7 @@ vt6656_probe(struct usb_interface *intf, const struct usb_device_id *id) priv = hw->priv; priv->hw = hw; priv->usb = udev; + priv->intf = intf; vnt_set_options(priv); diff --git a/drivers/staging/vt6656/wcmd.c b/drivers/staging/vt6656/wcmd.c index 3eb2f11a5de1..2c5250ca2801 100644 --- a/drivers/staging/vt6656/wcmd.c +++ b/drivers/staging/vt6656/wcmd.c @@ -99,6 +99,7 @@ void vnt_run_command(struct work_struct *work) if (vnt_init(priv)) { /* If fail all ends TODO retry */ dev_err(&priv->usb->dev, "failed to start\n"); + usb_set_intfdata(priv->intf, NULL); ieee80211_free_hw(priv->hw); return; } From b062fb0ca0aa5ddb195133164e6e135bb51a0c79 Mon Sep 17 00:00:00 2001 From: Malcolm Priestley Date: Fri, 20 Dec 2019 21:14:59 +0000 Subject: [PATCH 48/89] staging: vt6656: Fix non zero logical return of, usb_control_msg commit 58c3e681b04dd57c70d0dcb7b69fe52d043ff75a upstream. Starting with commit 59608cb1de1856 ("staging: vt6656: clean function's error path in usbpipe.c") the usb control functions have returned errors throughout driver with only logical variable checking. However, usb_control_msg return the amount of bytes transferred this means that normal operation causes errors. Correct the return function so only return zero when transfer is successful. Cc: stable # v5.3+ Signed-off-by: Malcolm Priestley Link: https://lore.kernel.org/r/08e88842-6f78-a2e3-a7a0-139fec960b2b@gmail.com Signed-off-by: Greg Kroah-Hartman --- drivers/staging/vt6656/usbpipe.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/drivers/staging/vt6656/usbpipe.c b/drivers/staging/vt6656/usbpipe.c index d3304df6bd53..488ebd98773d 100644 --- a/drivers/staging/vt6656/usbpipe.c +++ b/drivers/staging/vt6656/usbpipe.c @@ -59,7 +59,9 @@ int vnt_control_out(struct vnt_private *priv, u8 request, u16 value, kfree(usb_buffer); - if (ret >= 0 && ret < (int)length) + if (ret == (int)length) + ret = 0; + else ret = -EIO; end_unlock: @@ -103,7 +105,9 @@ int vnt_control_in(struct vnt_private *priv, u8 request, u16 value, kfree(usb_buffer); - if (ret >= 0 && ret < (int)length) + if (ret == (int)length) + ret = 0; + else ret = -EIO; end_unlock: From f1fcfe22933bdef1d5c25fdee81adb7abda5dd1a Mon Sep 17 00:00:00 2001 From: Peter Chen Date: Fri, 27 Dec 2019 17:10:04 +0800 Subject: [PATCH 49/89] usb: cdns3: should not use the same dev_id for shared interrupt handler commit af58e1fca9840192f14b6f03c59595d64bff9127 upstream. Both drd and gadget interrupt handler use the struct cdns3 pointer as dev_id, it causes devm_free_irq at cdns3_gadget_exit doesn't free gadget's interrupt handler, it freed drd's handler. So, when the host interrupt occurs, the gadget's interrupt hanlder is still called, and causes below oops. To fix it, we use gadget's private data priv_dev as interrupt dev_id for gadget. Unable to handle kernel NULL pointer dereference at virtual address 0000000000000380 Mem abort info: ESR = 0x96000006 EC = 0x25: DABT (current EL), IL = 32 bits SET = 0, FnV = 0 EA = 0, S1PTW = 0 Data abort info: ISV = 0, ISS = 0x00000006 CM = 0, WnR = 0 user pgtable: 4k pages, 48-bit VAs, pgdp=0000000971d79000 [0000000000000380] pgd=0000000971d6f003, pud=0000000971d6e003, pmd=0000000000000000 Internal error: Oops: 96000006 [#1] PREEMPT SMP Modules linked in: mxc_jpeg_encdec crct10dif_ce fsl_imx8_ddr_perf CPU: 0 PID: 0 Comm: swapper/0 Not tainted 5.4.0-03486-g69f4e7d9c54a-dirty #254 Hardware name: Freescale i.MX8QM MEK (DT) pstate: 00000085 (nzcv daIf -PAN -UAO) pc : cdns3_device_irq_handler+0x1c/0xb8 lr : __handle_irq_event_percpu+0x78/0x2c0 sp : ffff800010003e30 x29: ffff800010003e30 x28: ffff8000129bb000 x27: ffff8000126e9000 x26: ffff0008f61b5600 x25: ffff800011fe1018 x24: ffff8000126ea120 x23: ffff800010003f04 x22: 0000000000000000 x21: 0000000000000093 x20: ffff0008f61b5600 x19: ffff0008f5061a80 x18: 0000000000000000 x17: 0000000000000000 x16: 0000000000000000 x15: 0000000000000000 x14: 003d090000000000 x13: 00003d0900000000 x12: 0000000000000000 x11: 00003d0900000000 x10: 0000000000000040 x9 : ffff800012708cb8 x8 : ffff800012708cb0 x7 : ffff0008f7c7a9d0 x6 : 0000000000000000 x5 : ffff0008f7c7a910 x4 : ffff8008ed359000 x3 : ffff800010003f40 x2 : 0000000000000000 x1 : ffff0008f5061a80 x0 : ffff800010161a60 Call trace: cdns3_device_irq_handler+0x1c/0xb8 __handle_irq_event_percpu+0x78/0x2c0 handle_irq_event_percpu+0x40/0x98 handle_irq_event+0x4c/0xd0 handle_fasteoi_irq+0xbc/0x168 generic_handle_irq+0x34/0x50 __handle_domain_irq+0x6c/0xc0 gic_handle_irq+0xd4/0x174 el1_irq+0xb8/0x180 arch_cpu_idle+0x3c/0x230 default_idle_call+0x38/0x40 do_idle+0x20c/0x298 cpu_startup_entry+0x28/0x48 rest_init+0xdc/0xe8 arch_call_rest_init+0x14/0x1c start_kernel+0x48c/0x4b8 Code: aa0103f3 aa1e03e0 d503201f f9409662 (f941c040) ---[ end trace 091dcf4dee011b0e ]--- Kernel panic - not syncing: Fatal exception in interrupt SMP: stopping secondary CPUs Kernel Offset: disabled CPU features: 0x0002,2100600c Memory Limit: none ---[ end Kernel panic - not syncing: Fatal exception in interrupt ]--- Fixes: 7733f6c32e36 ("usb: cdns3: Add Cadence USB3 DRD Driver") Cc: #v5.4 Signed-off-by: Peter Chen Link: https://lore.kernel.org/r/1577437804-18146-1-git-send-email-peter.chen@nxp.com Signed-off-by: Greg Kroah-Hartman --- drivers/usb/cdns3/gadget.c | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/drivers/usb/cdns3/gadget.c b/drivers/usb/cdns3/gadget.c index 4c1e75509303..02f6ca2cb1ba 100644 --- a/drivers/usb/cdns3/gadget.c +++ b/drivers/usb/cdns3/gadget.c @@ -1375,13 +1375,10 @@ static void cdns3_check_usb_interrupt_proceed(struct cdns3_device *priv_dev, */ static irqreturn_t cdns3_device_irq_handler(int irq, void *data) { - struct cdns3_device *priv_dev; - struct cdns3 *cdns = data; + struct cdns3_device *priv_dev = data; irqreturn_t ret = IRQ_NONE; u32 reg; - priv_dev = cdns->gadget_dev; - /* check USB device interrupt */ reg = readl(&priv_dev->regs->usb_ists); if (reg) { @@ -1419,14 +1416,12 @@ static irqreturn_t cdns3_device_irq_handler(int irq, void *data) */ static irqreturn_t cdns3_device_thread_irq_handler(int irq, void *data) { - struct cdns3_device *priv_dev; - struct cdns3 *cdns = data; + struct cdns3_device *priv_dev = data; irqreturn_t ret = IRQ_NONE; unsigned long flags; int bit; u32 reg; - priv_dev = cdns->gadget_dev; spin_lock_irqsave(&priv_dev->lock, flags); reg = readl(&priv_dev->regs->usb_ists); @@ -2539,7 +2534,7 @@ void cdns3_gadget_exit(struct cdns3 *cdns) priv_dev = cdns->gadget_dev; - devm_free_irq(cdns->dev, cdns->dev_irq, cdns); + devm_free_irq(cdns->dev, cdns->dev_irq, priv_dev); pm_runtime_mark_last_busy(cdns->dev); pm_runtime_put_autosuspend(cdns->dev); @@ -2710,7 +2705,8 @@ static int __cdns3_gadget_init(struct cdns3 *cdns) ret = devm_request_threaded_irq(cdns->dev, cdns->dev_irq, cdns3_device_irq_handler, cdns3_device_thread_irq_handler, - IRQF_SHARED, dev_name(cdns->dev), cdns); + IRQF_SHARED, dev_name(cdns->dev), + cdns->gadget_dev); if (ret) goto err0; From 21e327468ea1adbaeb19ef6cb788e905d045c381 Mon Sep 17 00:00:00 2001 From: Colin Ian King Date: Tue, 7 Jan 2020 12:39:01 +0000 Subject: [PATCH 50/89] usb: ohci-da8xx: ensure error return on variable error is set commit ba9b40810bb43e6bf73b395012b98633c03f7f59 upstream. Currently when an error occurs when calling devm_gpiod_get_optional or calling gpiod_to_irq it causes an uninitialized error return in variable 'error' to be returned. Fix this by ensuring the error variable is set from da8xx_ohci->oc_gpio and oc_irq. Thanks to Dan Carpenter for spotting the uninitialized error in the gpiod_to_irq failure case. Addresses-Coverity: ("Uninitialized scalar variable") Fixes: d193abf1c913 ("usb: ohci-da8xx: add vbus and overcurrent gpios") Signed-off-by: Colin Ian King Cc: stable Acked-by: Alan Stern Link: https://lore.kernel.org/r/20200107123901.101190-1-colin.king@canonical.com Signed-off-by: Greg Kroah-Hartman --- drivers/usb/host/ohci-da8xx.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/drivers/usb/host/ohci-da8xx.c b/drivers/usb/host/ohci-da8xx.c index 38183ac438c6..1371b0c249ec 100644 --- a/drivers/usb/host/ohci-da8xx.c +++ b/drivers/usb/host/ohci-da8xx.c @@ -415,13 +415,17 @@ static int ohci_da8xx_probe(struct platform_device *pdev) } da8xx_ohci->oc_gpio = devm_gpiod_get_optional(dev, "oc", GPIOD_IN); - if (IS_ERR(da8xx_ohci->oc_gpio)) + if (IS_ERR(da8xx_ohci->oc_gpio)) { + error = PTR_ERR(da8xx_ohci->oc_gpio); goto err; + } if (da8xx_ohci->oc_gpio) { oc_irq = gpiod_to_irq(da8xx_ohci->oc_gpio); - if (oc_irq < 0) + if (oc_irq < 0) { + error = oc_irq; goto err; + } error = devm_request_threaded_irq(dev, oc_irq, NULL, ohci_da8xx_oc_thread, IRQF_TRIGGER_RISING | From 2989d2209f2e639caf824edb711f031491e35ccb Mon Sep 17 00:00:00 2001 From: Douglas Gilbert Date: Sun, 29 Dec 2019 22:35:44 -0500 Subject: [PATCH 51/89] USB-PD tcpm: bad warning+size, PPS adapters commit c215e48e97d232249a33849fc46fc50311043e11 upstream. Augmented Power Delivery Objects (A)PDO_s are used by USB-C PD power adapters to advertize the voltages and currents they support. There can be up to 7 PDO_s but before PPS (programmable power supply) there were seldom more than 4 or 5. Recently Samsung released an optional PPS 45 Watt power adapter (EP-TA485) that has 7 PDO_s. It is for the Galaxy 10+ tablet and charges it quicker than the adapter supplied at purchase. The EP-TA485 causes an overzealous WARN_ON to soil the log plus it miscalculates the number of bytes to read. So this bug has been there for some time but goes undetected for the majority of USB-C PD power adapters on the market today that have 6 or less PDO_s. That may soon change as more USB-C PD adapters with PPS come to market. Tested on a EP-TA485 and an older Lenovo PN: SA10M13950 USB-C 65 Watt adapter (without PPS and has 4 PDO_s) plus several other PD power adapters. Signed-off-by: Douglas Gilbert Reviewed-by: Guenter Roeck Cc: stable Link: https://lore.kernel.org/r/20191230033544.1809-1-dgilbert@interlog.com Signed-off-by: Greg Kroah-Hartman --- drivers/usb/typec/tcpm/tcpci.c | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/drivers/usb/typec/tcpm/tcpci.c b/drivers/usb/typec/tcpm/tcpci.c index c1f7073a56de..8b4ff9fff340 100644 --- a/drivers/usb/typec/tcpm/tcpci.c +++ b/drivers/usb/typec/tcpm/tcpci.c @@ -432,20 +432,30 @@ irqreturn_t tcpci_irq(struct tcpci *tcpci) if (status & TCPC_ALERT_RX_STATUS) { struct pd_message msg; - unsigned int cnt; + unsigned int cnt, payload_cnt; u16 header; regmap_read(tcpci->regmap, TCPC_RX_BYTE_CNT, &cnt); + /* + * 'cnt' corresponds to READABLE_BYTE_COUNT in section 4.4.14 + * of the TCPCI spec [Rev 2.0 Ver 1.0 October 2017] and is + * defined in table 4-36 as one greater than the number of + * bytes received. And that number includes the header. So: + */ + if (cnt > 3) + payload_cnt = cnt - (1 + sizeof(msg.header)); + else + payload_cnt = 0; tcpci_read16(tcpci, TCPC_RX_HDR, &header); msg.header = cpu_to_le16(header); - if (WARN_ON(cnt > sizeof(msg.payload))) - cnt = sizeof(msg.payload); + if (WARN_ON(payload_cnt > sizeof(msg.payload))) + payload_cnt = sizeof(msg.payload); - if (cnt > 0) + if (payload_cnt > 0) regmap_raw_read(tcpci->regmap, TCPC_RX_DATA, - &msg.payload, cnt); + &msg.payload, payload_cnt); /* Read complete, clear RX status alert bit */ tcpci_write16(tcpci, TCPC_ALERT, TCPC_ALERT_RX_STATUS); From 855e2d927f41febe8af04dc8fb11807ab72165dc Mon Sep 17 00:00:00 2001 From: Daniele Palmas Date: Thu, 19 Dec 2019 11:07:07 +0100 Subject: [PATCH 52/89] USB: serial: option: add ZLP support for 0x1bc7/0x9010 commit 2438c3a19dec5e98905fd3ffcc2f24716aceda6b upstream. Telit FN980 flashing device 0x1bc7/0x9010 requires zero packet to be sent if out data size is is equal to the endpoint max size. Signed-off-by: Daniele Palmas [ johan: switch operands in conditional ] Cc: stable Signed-off-by: Johan Hovold Signed-off-by: Greg Kroah-Hartman --- drivers/usb/serial/option.c | 8 ++++++++ drivers/usb/serial/usb-wwan.h | 1 + drivers/usb/serial/usb_wwan.c | 4 ++++ 3 files changed, 13 insertions(+) diff --git a/drivers/usb/serial/option.c b/drivers/usb/serial/option.c index fea09a3f491f..2d919d0e6e45 100644 --- a/drivers/usb/serial/option.c +++ b/drivers/usb/serial/option.c @@ -567,6 +567,9 @@ static void option_instat_callback(struct urb *urb); /* Interface must have two endpoints */ #define NUMEP2 BIT(16) +/* Device needs ZLP */ +#define ZLP BIT(17) + static const struct usb_device_id option_ids[] = { { USB_DEVICE(OPTION_VENDOR_ID, OPTION_PRODUCT_COLT) }, @@ -1198,6 +1201,8 @@ static const struct usb_device_id option_ids[] = { .driver_info = NCTRL(0) | RSVD(1) }, { USB_DEVICE_INTERFACE_CLASS(TELIT_VENDOR_ID, 0x1901, 0xff), /* Telit LN940 (MBIM) */ .driver_info = NCTRL(0) }, + { USB_DEVICE(TELIT_VENDOR_ID, 0x9010), /* Telit SBL FN980 flashing device */ + .driver_info = NCTRL(0) | ZLP }, { USB_DEVICE_AND_INTERFACE_INFO(ZTE_VENDOR_ID, ZTE_PRODUCT_MF622, 0xff, 0xff, 0xff) }, /* ZTE WCDMA products */ { USB_DEVICE_AND_INTERFACE_INFO(ZTE_VENDOR_ID, 0x0002, 0xff, 0xff, 0xff), .driver_info = RSVD(1) }, @@ -2099,6 +2104,9 @@ static int option_attach(struct usb_serial *serial) if (!(device_flags & NCTRL(iface_desc->bInterfaceNumber))) data->use_send_setup = 1; + if (device_flags & ZLP) + data->use_zlp = 1; + spin_lock_init(&data->susp_lock); usb_set_serial_data(serial, data); diff --git a/drivers/usb/serial/usb-wwan.h b/drivers/usb/serial/usb-wwan.h index 1c120eaf4091..934e9361cf6b 100644 --- a/drivers/usb/serial/usb-wwan.h +++ b/drivers/usb/serial/usb-wwan.h @@ -38,6 +38,7 @@ struct usb_wwan_intf_private { spinlock_t susp_lock; unsigned int suspended:1; unsigned int use_send_setup:1; + unsigned int use_zlp:1; int in_flight; unsigned int open_ports; void *private; diff --git a/drivers/usb/serial/usb_wwan.c b/drivers/usb/serial/usb_wwan.c index 7e855c87e4f7..13be21aad2f4 100644 --- a/drivers/usb/serial/usb_wwan.c +++ b/drivers/usb/serial/usb_wwan.c @@ -461,6 +461,7 @@ static struct urb *usb_wwan_setup_urb(struct usb_serial_port *port, void (*callback) (struct urb *)) { struct usb_serial *serial = port->serial; + struct usb_wwan_intf_private *intfdata = usb_get_serial_data(serial); struct urb *urb; urb = usb_alloc_urb(0, GFP_KERNEL); /* No ISO */ @@ -471,6 +472,9 @@ static struct urb *usb_wwan_setup_urb(struct usb_serial_port *port, usb_sndbulkpipe(serial->dev, endpoint) | dir, buf, len, callback, ctx); + if (intfdata->use_zlp && dir == USB_DIR_OUT) + urb->transfer_flags |= URB_ZERO_PACKET; + return urb; } From 32199ac31d263eaaebc7139894277ca5e88d22da Mon Sep 17 00:00:00 2001 From: Tony Lindgren Date: Tue, 7 Jan 2020 09:26:24 -0600 Subject: [PATCH 53/89] usb: musb: fix idling for suspend after disconnect interrupt commit 5fbf7a2534703fd71159d3d71504b0ad01b43394 upstream. When disconnected as USB B-device, suspend interrupt should come before diconnect interrupt, because the DP/DM pins are shorter than the VBUS/GND pins on the USB connectors. But we sometimes get a suspend interrupt after disconnect interrupt. In that case we have devctl set to 99 with VBUS still valid and musb_pm_runtime_check_session() wrongly thinks we have an active session. We have no other interrupts after disconnect coming in this case at least with the omap2430 glue. Let's fix the issue by checking the interrupt status again with delayed work for the devctl 99 case. In the suspend after disconnect case the devctl session bit has cleared by then and musb can idle. For a typical USB B-device connect case we just continue with normal interrupts. Fixes: 467d5c980709 ("usb: musb: Implement session bit based runtime PM for musb-core") Cc: Merlijn Wajer Cc: Pavel Machek Cc: Sebastian Reichel Cc: stable@vger.kernel.org Signed-off-by: Tony Lindgren Signed-off-by: Bin Liu Link: https://lore.kernel.org/r/20200107152625.857-2-b-liu@ti.com Signed-off-by: Greg Kroah-Hartman --- drivers/usb/musb/musb_core.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/drivers/usb/musb/musb_core.c b/drivers/usb/musb/musb_core.c index bd63450af76a..774e18f82f8d 100644 --- a/drivers/usb/musb/musb_core.c +++ b/drivers/usb/musb/musb_core.c @@ -1840,6 +1840,9 @@ ATTRIBUTE_GROUPS(musb); #define MUSB_QUIRK_B_INVALID_VBUS_91 (MUSB_DEVCTL_BDEVICE | \ (2 << MUSB_DEVCTL_VBUS_SHIFT) | \ MUSB_DEVCTL_SESSION) +#define MUSB_QUIRK_B_DISCONNECT_99 (MUSB_DEVCTL_BDEVICE | \ + (3 << MUSB_DEVCTL_VBUS_SHIFT) | \ + MUSB_DEVCTL_SESSION) #define MUSB_QUIRK_A_DISCONNECT_19 ((3 << MUSB_DEVCTL_VBUS_SHIFT) | \ MUSB_DEVCTL_SESSION) @@ -1862,6 +1865,11 @@ static void musb_pm_runtime_check_session(struct musb *musb) s = MUSB_DEVCTL_FSDEV | MUSB_DEVCTL_LSDEV | MUSB_DEVCTL_HR; switch (devctl & ~s) { + case MUSB_QUIRK_B_DISCONNECT_99: + musb_dbg(musb, "Poll devctl in case of suspend after disconnect\n"); + schedule_delayed_work(&musb->irq_work, + msecs_to_jiffies(1000)); + break; case MUSB_QUIRK_B_INVALID_VBUS_91: if (musb->quirk_retries && !musb->flush_irq_work) { musb_dbg(musb, From 25eecc1f14754071a61fbbd0ed32d1cf7309e0ab Mon Sep 17 00:00:00 2001 From: Paul Cercueil Date: Tue, 7 Jan 2020 09:26:25 -0600 Subject: [PATCH 54/89] usb: musb: Disable pullup at init commit 96a0c12843109e5c4d5eb1e09d915fdd0ce31d25 upstream. The pullup may be already enabled before the driver is initialized. This happens for instance on JZ4740. It has to be disabled at init time, as we cannot guarantee that a gadget driver will be bound to the UDC. Signed-off-by: Paul Cercueil Suggested-by: Bin Liu Cc: stable@vger.kernel.org Signed-off-by: Bin Liu Link: https://lore.kernel.org/r/20200107152625.857-3-b-liu@ti.com Signed-off-by: Greg Kroah-Hartman --- drivers/usb/musb/musb_core.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/drivers/usb/musb/musb_core.c b/drivers/usb/musb/musb_core.c index 774e18f82f8d..bf083c1f997f 100644 --- a/drivers/usb/musb/musb_core.c +++ b/drivers/usb/musb/musb_core.c @@ -2318,6 +2318,9 @@ musb_init_controller(struct device *dev, int nIrq, void __iomem *ctrl) musb_disable_interrupts(musb); musb_writeb(musb->mregs, MUSB_DEVCTL, 0); + /* MUSB_POWER_SOFTCONN might be already set, JZ4740 does this. */ + musb_writeb(musb->mregs, MUSB_POWER, 0); + /* Init IRQ workqueue before request_irq */ INIT_DELAYED_WORK(&musb->irq_work, musb_irq_work); INIT_DELAYED_WORK(&musb->deassert_reset_work, musb_deassert_reset); From 26219c6befa15a79250cf40026918ab9c4ae0200 Mon Sep 17 00:00:00 2001 From: Paul Cercueil Date: Mon, 16 Dec 2019 10:18:43 -0600 Subject: [PATCH 55/89] usb: musb: dma: Correct parameter passed to IRQ handler commit c80d0f4426c7fdc7efd6ae8d8b021dcfc89b4254 upstream. The IRQ handler was passed a pointer to a struct dma_controller, but the argument was then casted to a pointer to a struct musb_dma_controller. Fixes: 427c4f333474 ("usb: struct device - replace bus_id with dev_name(), dev_set_name()") Signed-off-by: Paul Cercueil Tested-by: Artur Rojek Cc: stable@vger.kernel.org Signed-off-by: Bin Liu Link: https://lore.kernel.org/r/20191216161844.772-2-b-liu@ti.com Signed-off-by: Greg Kroah-Hartman --- drivers/usb/musb/musbhsdma.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/usb/musb/musbhsdma.c b/drivers/usb/musb/musbhsdma.c index 5fc6825745f2..2d3751d885b4 100644 --- a/drivers/usb/musb/musbhsdma.c +++ b/drivers/usb/musb/musbhsdma.c @@ -425,7 +425,7 @@ struct dma_controller *musbhs_dma_controller_create(struct musb *musb, controller->controller.channel_abort = dma_channel_abort; if (request_irq(irq, dma_controller_irq, 0, - dev_name(musb->controller), &controller->controller)) { + dev_name(musb->controller), controller)) { dev_err(dev, "request_irq %d failed!\n", irq); musb_dma_controller_destroy(&controller->controller); From 23533ddf9e4f8155e8952e5e2e2635a8488db592 Mon Sep 17 00:00:00 2001 From: Ian Abbott Date: Fri, 27 Dec 2019 17:00:54 +0000 Subject: [PATCH 56/89] staging: comedi: adv_pci1710: fix AI channels 16-31 for PCI-1713 commit a9d3a9cedc1330c720e0ddde1978a8e7771da5ab upstream. The Advantech PCI-1713 has 32 analog input channels, but an incorrect bit-mask in the definition of the `PCI171X_MUX_CHANH(x)` and PCI171X_MUX_CHANL(x)` macros is causing channels 16 to 31 to be aliases of channels 0 to 15. Change the bit-mask value from 0xf to 0xff to fix it. Note that the channel numbers will have been range checked already, so the bit-mask isn't really needed. Fixes: 92c65e5553ed ("staging: comedi: adv_pci1710: define the mux control register bits") Reported-by: Dmytro Fil Cc: # v4.5+ Signed-off-by: Ian Abbott Link: https://lore.kernel.org/r/20191227170054.32051-1-abbotti@mev.co.uk Signed-off-by: Greg Kroah-Hartman Signed-off-by: Greg Kroah-Hartman --- drivers/staging/comedi/drivers/adv_pci1710.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/staging/comedi/drivers/adv_pci1710.c b/drivers/staging/comedi/drivers/adv_pci1710.c index dbff0f7e7cf5..ddc0dc93d08b 100644 --- a/drivers/staging/comedi/drivers/adv_pci1710.c +++ b/drivers/staging/comedi/drivers/adv_pci1710.c @@ -46,8 +46,8 @@ #define PCI171X_RANGE_UNI BIT(4) #define PCI171X_RANGE_GAIN(x) (((x) & 0x7) << 0) #define PCI171X_MUX_REG 0x04 /* W: A/D multiplexor control */ -#define PCI171X_MUX_CHANH(x) (((x) & 0xf) << 8) -#define PCI171X_MUX_CHANL(x) (((x) & 0xf) << 0) +#define PCI171X_MUX_CHANH(x) (((x) & 0xff) << 8) +#define PCI171X_MUX_CHANL(x) (((x) & 0xff) << 0) #define PCI171X_MUX_CHAN(x) (PCI171X_MUX_CHANH(x) | PCI171X_MUX_CHANL(x)) #define PCI171X_STATUS_REG 0x06 /* R: status register */ #define PCI171X_STATUS_IRQ BIT(11) /* 1=IRQ occurred */ From 42609009ea1f0768f4e065c4cfa76c650c6ea2f3 Mon Sep 17 00:00:00 2001 From: Malcolm Priestley Date: Fri, 20 Dec 2019 21:15:09 +0000 Subject: [PATCH 57/89] staging: vt6656: correct return of vnt_init_registers. commit 7de6155c8968a3342d1bef3f7a2084d31ae6e4be upstream. The driver standard error returns remove bool false conditions. Cc: stable # v5.3+ Signed-off-by: Malcolm Priestley Link: https://lore.kernel.org/r/072ec0b3-425f-277e-130c-1e3a116c90d6@gmail.com Signed-off-by: Greg Kroah-Hartman --- drivers/staging/vt6656/main_usb.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/staging/vt6656/main_usb.c b/drivers/staging/vt6656/main_usb.c index 8b45b8578372..c26882e2bb80 100644 --- a/drivers/staging/vt6656/main_usb.c +++ b/drivers/staging/vt6656/main_usb.c @@ -950,7 +950,7 @@ static const struct ieee80211_ops vnt_mac_ops = { int vnt_init(struct vnt_private *priv) { - if (!(vnt_init_registers(priv))) + if (vnt_init_registers(priv)) return -EAGAIN; SET_IEEE80211_PERM_ADDR(priv->hw, priv->permanent_net_addr); From de8757801ef0920b4114c430125b8c02fedbc828 Mon Sep 17 00:00:00 2001 From: Malcolm Priestley Date: Fri, 20 Dec 2019 21:15:24 +0000 Subject: [PATCH 58/89] staging: vt6656: limit reg output to block size commit 69cc1f925e1aa74b96e2ace67e3453a50d091d2f upstream. vnt_control_out appears to fail when BBREG is greater than 64 writes. Create new function that will relay an array in no larger than the indicated block size. It appears that this command has always failed but was ignored by driver until the introduction of error checking. Cc: stable # v5.3+ Signed-off-by: Malcolm Priestley Link: https://lore.kernel.org/r/a41f0601-df46-ce6e-ab7c-35e697946e2a@gmail.com Signed-off-by: Greg Kroah-Hartman --- drivers/staging/vt6656/baseband.c | 4 ++-- drivers/staging/vt6656/usbpipe.c | 17 +++++++++++++++++ drivers/staging/vt6656/usbpipe.h | 5 +++++ 3 files changed, 24 insertions(+), 2 deletions(-) diff --git a/drivers/staging/vt6656/baseband.c b/drivers/staging/vt6656/baseband.c index 8d19ae71e7cc..4e651b698617 100644 --- a/drivers/staging/vt6656/baseband.c +++ b/drivers/staging/vt6656/baseband.c @@ -449,8 +449,8 @@ int vnt_vt3184_init(struct vnt_private *priv) memcpy(array, addr, length); - ret = vnt_control_out(priv, MESSAGE_TYPE_WRITE, 0, - MESSAGE_REQUEST_BBREG, length, array); + ret = vnt_control_out_blocks(priv, VNT_REG_BLOCK_SIZE, + MESSAGE_REQUEST_BBREG, length, array); if (ret) goto end; diff --git a/drivers/staging/vt6656/usbpipe.c b/drivers/staging/vt6656/usbpipe.c index 488ebd98773d..d977d4777e4f 100644 --- a/drivers/staging/vt6656/usbpipe.c +++ b/drivers/staging/vt6656/usbpipe.c @@ -76,6 +76,23 @@ int vnt_control_out_u8(struct vnt_private *priv, u8 reg, u8 reg_off, u8 data) reg_off, reg, sizeof(u8), &data); } +int vnt_control_out_blocks(struct vnt_private *priv, + u16 block, u8 reg, u16 length, u8 *data) +{ + int ret = 0, i; + + for (i = 0; i < length; i += block) { + u16 len = min_t(int, length - i, block); + + ret = vnt_control_out(priv, MESSAGE_TYPE_WRITE, + i, reg, len, data + i); + if (ret) + goto end; + } +end: + return ret; +} + int vnt_control_in(struct vnt_private *priv, u8 request, u16 value, u16 index, u16 length, u8 *buffer) { diff --git a/drivers/staging/vt6656/usbpipe.h b/drivers/staging/vt6656/usbpipe.h index 95147ec7b96a..b65d9c01a211 100644 --- a/drivers/staging/vt6656/usbpipe.h +++ b/drivers/staging/vt6656/usbpipe.h @@ -18,6 +18,8 @@ #include "device.h" +#define VNT_REG_BLOCK_SIZE 64 + int vnt_control_out(struct vnt_private *priv, u8 request, u16 value, u16 index, u16 length, u8 *buffer); int vnt_control_in(struct vnt_private *priv, u8 request, u16 value, @@ -26,6 +28,9 @@ int vnt_control_in(struct vnt_private *priv, u8 request, u16 value, int vnt_control_out_u8(struct vnt_private *priv, u8 reg, u8 ref_off, u8 data); int vnt_control_in_u8(struct vnt_private *priv, u8 reg, u8 reg_off, u8 *data); +int vnt_control_out_blocks(struct vnt_private *priv, + u16 block, u8 reg, u16 len, u8 *data); + int vnt_start_interrupt_urb(struct vnt_private *priv); int vnt_submit_rx_urb(struct vnt_private *priv, struct vnt_rcb *rcb); int vnt_tx_context(struct vnt_private *priv, From e7fecc2112fad78e1a1912f3714412c912d406ee Mon Sep 17 00:00:00 2001 From: Michael Straube Date: Sat, 28 Dec 2019 15:37:25 +0100 Subject: [PATCH 59/89] staging: rtl8188eu: Add device code for TP-Link TL-WN727N v5.21 commit 58dcc5bf4030cab548d5c98cd4cd3632a5444d5a upstream. This device was added to the stand-alone driver on github. Add it to the staging driver as well. Link: https://github.com/lwfinger/rtl8188eu/commit/b9b537aa25a8 Signed-off-by: Michael Straube Cc: stable Link: https://lore.kernel.org/r/20191228143725.24455-1-straube.linux@gmail.com Signed-off-by: Greg Kroah-Hartman --- drivers/staging/rtl8188eu/os_dep/usb_intf.c | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/staging/rtl8188eu/os_dep/usb_intf.c b/drivers/staging/rtl8188eu/os_dep/usb_intf.c index a7cac0719b8b..b5d42f411dd8 100644 --- a/drivers/staging/rtl8188eu/os_dep/usb_intf.c +++ b/drivers/staging/rtl8188eu/os_dep/usb_intf.c @@ -37,6 +37,7 @@ static const struct usb_device_id rtw_usb_id_tbl[] = { {USB_DEVICE(0x2001, 0x3311)}, /* DLink GO-USB-N150 REV B1 */ {USB_DEVICE(0x2001, 0x331B)}, /* D-Link DWA-121 rev B1 */ {USB_DEVICE(0x2357, 0x010c)}, /* TP-Link TL-WN722N v2 */ + {USB_DEVICE(0x2357, 0x0111)}, /* TP-Link TL-WN727N v5.21 */ {USB_DEVICE(0x0df6, 0x0076)}, /* Sitecom N150 v2 */ {USB_DEVICE(USB_VENDER_ID_REALTEK, 0xffef)}, /* Rosewill RNX-N150NUB */ {} /* Terminating entry */ From d6ac466168aa4b1a6217e4b023d3fe23a8bcafe7 Mon Sep 17 00:00:00 2001 From: Punit Agrawal Date: Thu, 19 Dec 2019 19:03:45 +0900 Subject: [PATCH 60/89] serdev: Don't claim unsupported ACPI serial devices commit c5ee0b3104e0b292d353e63fd31cb8c692645d8c upstream. Serdev sub-system claims all ACPI serial devices that are not already initialised. As a result, no device node is created for serial ports on certain boards such as the Apollo Lake based UP2. This has the unintended consequence of not being able to raise the login prompt via serial connection. Introduce a blacklist to reject ACPI serial devices that should not be claimed by serdev sub-system. Add the peripheral ids for Intel HS UART to the blacklist to bring back serial port on SoCs carrying them. Cc: stable@vger.kernel.org Signed-off-by: Punit Agrawal Acked-by: Hans de Goede Acked-by: Johan Hovold Cc: Rob Herring Link: https://lore.kernel.org/r/20191219100345.911093-1-punit1.agrawal@toshiba.co.jp Signed-off-by: Greg Kroah-Hartman --- drivers/tty/serdev/core.c | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/drivers/tty/serdev/core.c b/drivers/tty/serdev/core.c index a0ac16ee6575..a9719858c950 100644 --- a/drivers/tty/serdev/core.c +++ b/drivers/tty/serdev/core.c @@ -582,6 +582,12 @@ static acpi_status acpi_serdev_register_device(struct serdev_controller *ctrl, return AE_OK; } +static const struct acpi_device_id serdev_acpi_devices_blacklist[] = { + { "INT3511", 0 }, + { "INT3512", 0 }, + { }, +}; + static acpi_status acpi_serdev_add_device(acpi_handle handle, u32 level, void *data, void **return_value) { @@ -591,6 +597,10 @@ static acpi_status acpi_serdev_add_device(acpi_handle handle, u32 level, if (acpi_bus_get_device(handle, &adev)) return AE_OK; + /* Skip if black listed */ + if (!acpi_match_device_ids(adev, serdev_acpi_devices_blacklist)) + return AE_OK; + return acpi_serdev_register_device(ctrl, adev); } From d12d10211b4cc7b253e26fb9ebd3ae3fa9fb6a00 Mon Sep 17 00:00:00 2001 From: Patrick Steinhardt Date: Fri, 27 Dec 2019 00:56:18 +0100 Subject: [PATCH 61/89] iommu/vt-d: Fix adding non-PCI devices to Intel IOMMU commit 4a350a0ee5b0a14f826fcdf60dd1a3199cafbfd6 upstream. Starting with commit fa212a97f3a3 ("iommu/vt-d: Probe DMA-capable ACPI name space devices"), we now probe DMA-capable ACPI name space devices. On Dell XPS 13 9343, which has an Intel LPSS platform device INTL9C60 enumerated via ACPI, this change leads to the following warning: ------------[ cut here ]------------ WARNING: CPU: 1 PID: 1 at pci_device_group+0x11a/0x130 CPU: 1 PID: 1 Comm: swapper/0 Tainted: G T 5.5.0-rc3+ #22 Hardware name: Dell Inc. XPS 13 9343/0310JH, BIOS A20 06/06/2019 RIP: 0010:pci_device_group+0x11a/0x130 Code: f0 ff ff 48 85 c0 49 89 c4 75 c4 48 8d 74 24 10 48 89 ef e8 48 ef ff ff 48 85 c0 49 89 c4 75 af e8 db f7 ff ff 49 89 c4 eb a5 <0f> 0b 49 c7 c4 ea ff ff ff eb 9a e8 96 1e c7 ff 66 0f 1f 44 00 00 RSP: 0000:ffffc0d6c0043cb0 EFLAGS: 00010202 RAX: 0000000000000000 RBX: ffffa3d1d43dd810 RCX: 0000000000000000 RDX: ffffa3d1d4fecf80 RSI: ffffa3d12943dcc0 RDI: ffffa3d1d43dd810 RBP: ffffa3d1d43dd810 R08: 0000000000000000 R09: ffffa3d1d4c04a80 R10: ffffa3d1d4c00880 R11: ffffa3d1d44ba000 R12: 0000000000000000 R13: ffffa3d1d4383b80 R14: ffffa3d1d4c090d0 R15: ffffa3d1d4324530 FS: 0000000000000000(0000) GS:ffffa3d1d6700000(0000) knlGS:0000000000000000 CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 CR2: 0000000000000000 CR3: 000000000460a001 CR4: 00000000003606e0 Call Trace: ? iommu_group_get_for_dev+0x81/0x1f0 ? intel_iommu_add_device+0x61/0x170 ? iommu_probe_device+0x43/0xd0 ? intel_iommu_init+0x1fa2/0x2235 ? pci_iommu_init+0x52/0xe7 ? e820__memblock_setup+0x15c/0x15c ? do_one_initcall+0xcc/0x27e ? kernel_init_freeable+0x169/0x259 ? rest_init+0x95/0x95 ? kernel_init+0x5/0xeb ? ret_from_fork+0x35/0x40 ---[ end trace 28473e7abc25b92c ]--- DMAR: ACPI name space devices didn't probe correctly The bug results from the fact that while we now enumerate ACPI devices, we aren't able to handle any non-PCI device when generating the device group. Fix the issue by implementing an Intel-specific callback that returns `pci_device_group` only if the device is a PCI device. Otherwise, it will return a generic device group. Fixes: fa212a97f3a3 ("iommu/vt-d: Probe DMA-capable ACPI name space devices") Signed-off-by: Patrick Steinhardt Cc: stable@vger.kernel.org # v5.3+ Acked-by: Lu Baolu Signed-off-by: Joerg Roedel Signed-off-by: Greg Kroah-Hartman --- drivers/iommu/intel-iommu.c | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/drivers/iommu/intel-iommu.c b/drivers/iommu/intel-iommu.c index 4ce797d4259f..67aa317de6db 100644 --- a/drivers/iommu/intel-iommu.c +++ b/drivers/iommu/intel-iommu.c @@ -5786,6 +5786,13 @@ static void intel_iommu_apply_resv_region(struct device *dev, WARN_ON_ONCE(!reserve_iova(&dmar_domain->iovad, start, end)); } +static struct iommu_group *intel_iommu_device_group(struct device *dev) +{ + if (dev_is_pci(dev)) + return pci_device_group(dev); + return generic_device_group(dev); +} + #ifdef CONFIG_INTEL_IOMMU_SVM struct intel_iommu *intel_svm_device_to_iommu(struct device *dev) { @@ -5958,7 +5965,7 @@ const struct iommu_ops intel_iommu_ops = { .get_resv_regions = intel_iommu_get_resv_regions, .put_resv_regions = intel_iommu_put_resv_regions, .apply_resv_region = intel_iommu_apply_resv_region, - .device_group = pci_device_group, + .device_group = intel_iommu_device_group, .dev_has_feat = intel_iommu_dev_has_feat, .dev_feat_enabled = intel_iommu_dev_feat_enabled, .dev_enable_feat = intel_iommu_dev_enable_feat, From 46e4035d558dfe85d2adf5159cdcd1c1155a3109 Mon Sep 17 00:00:00 2001 From: Sudip Mukherjee Date: Thu, 12 Dec 2019 13:16:02 +0000 Subject: [PATCH 62/89] tty: link tty and port before configuring it as console commit fb2b90014d782d80d7ebf663e50f96d8c507a73c upstream. There seems to be a race condition in tty drivers and I could see on many boot cycles a NULL pointer dereference as tty_init_dev() tries to do 'tty->port->itty = tty' even though tty->port is NULL. 'tty->port' will be set by the driver and if the driver has not yet done it before we open the tty device we can get to this situation. By adding some extra debug prints, I noticed that: 6.650130: uart_add_one_port 6.663849: register_console 6.664846: tty_open 6.674391: tty_init_dev 6.675456: tty_port_link_device uart_add_one_port() registers the console, as soon as it registers, the userspace tries to use it and that leads to tty_open() but uart_add_one_port() has not yet done tty_port_link_device() and so tty->port is not yet configured when control reaches tty_init_dev(). Further look into the code and tty_port_link_device() is done by uart_add_one_port(). After registering the console uart_add_one_port() will call tty_port_register_device_attr_serdev() and tty_port_link_device() is called from this. Call add tty_port_link_device() before uart_configure_port() is done and add a check in tty_port_link_device() so that it only links the port if it has not been done yet. Suggested-by: Jiri Slaby Signed-off-by: Sudip Mukherjee Cc: stable Link: https://lore.kernel.org/r/20191212131602.29504-1-sudipm.mukherjee@gmail.com Signed-off-by: Greg Kroah-Hartman --- drivers/tty/serial/serial_core.c | 1 + drivers/tty/tty_port.c | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/drivers/tty/serial/serial_core.c b/drivers/tty/serial/serial_core.c index b0a6eb106edb..7c2782785736 100644 --- a/drivers/tty/serial/serial_core.c +++ b/drivers/tty/serial/serial_core.c @@ -2834,6 +2834,7 @@ int uart_add_one_port(struct uart_driver *drv, struct uart_port *uport) if (uport->cons && uport->dev) of_console_check(uport->dev->of_node, uport->cons->name, uport->line); + tty_port_link_device(port, drv->tty_driver, uport->line); uart_configure_port(drv, state, uport); port->console = uart_console(uport); diff --git a/drivers/tty/tty_port.c b/drivers/tty/tty_port.c index 044c3cbdcfa4..5023c85ebc6e 100644 --- a/drivers/tty/tty_port.c +++ b/drivers/tty/tty_port.c @@ -89,7 +89,8 @@ void tty_port_link_device(struct tty_port *port, { if (WARN_ON(index >= driver->num)) return; - driver->ports[index] = port; + if (!driver->ports[index]) + driver->ports[index] = port; } EXPORT_SYMBOL_GPL(tty_port_link_device); From d32d6d2bee55168dc27cda2e401793d0c19dbe21 Mon Sep 17 00:00:00 2001 From: Sudip Mukherjee Date: Fri, 27 Dec 2019 17:44:34 +0000 Subject: [PATCH 63/89] tty: always relink the port commit 273f632912f1b24b642ba5b7eb5022e43a72f3b5 upstream. If the serial device is disconnected and reconnected, it re-enumerates properly but does not link it. fwiw, linking means just saving the port index, so allow it always as there is no harm in saving the same value again even if it tries to relink with the same port. Fixes: fb2b90014d78 ("tty: link tty and port before configuring it as console") Reported-by: Kenneth R. Crudup Signed-off-by: Sudip Mukherjee Cc: stable Link: https://lore.kernel.org/r/20191227174434.12057-1-sudipm.mukherjee@gmail.com Signed-off-by: Greg Kroah-Hartman --- drivers/tty/tty_port.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/drivers/tty/tty_port.c b/drivers/tty/tty_port.c index 5023c85ebc6e..044c3cbdcfa4 100644 --- a/drivers/tty/tty_port.c +++ b/drivers/tty/tty_port.c @@ -89,8 +89,7 @@ void tty_port_link_device(struct tty_port *port, { if (WARN_ON(index >= driver->num)) return; - if (!driver->ports[index]) - driver->ports[index] = port; + driver->ports[index] = port; } EXPORT_SYMBOL_GPL(tty_port_link_device); From bac641643c70d1ec4a4146bfc5665a1a51efb2e7 Mon Sep 17 00:00:00 2001 From: Amanieu d'Antras Date: Thu, 2 Jan 2020 18:24:07 +0100 Subject: [PATCH 64/89] arm64: Move __ARCH_WANT_SYS_CLONE3 definition to uapi headers commit 3e3c8ca5a351350031f0f3d5ecedf7048b1b9008 upstream. Previously this was only defined in the internal headers which resulted in __NR_clone3 not being defined in the user headers. Signed-off-by: Amanieu d'Antras Cc: linux-arm-kernel@lists.infradead.org Cc: # 5.3.x Reviewed-by: Arnd Bergmann Link: https://lore.kernel.org/r/20200102172413.654385-2-amanieu@gmail.com Signed-off-by: Christian Brauner Signed-off-by: Greg Kroah-Hartman --- arch/arm64/include/asm/unistd.h | 1 - arch/arm64/include/uapi/asm/unistd.h | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/arm64/include/asm/unistd.h b/arch/arm64/include/asm/unistd.h index 2629a68b8724..5af82587909e 100644 --- a/arch/arm64/include/asm/unistd.h +++ b/arch/arm64/include/asm/unistd.h @@ -42,7 +42,6 @@ #endif #define __ARCH_WANT_SYS_CLONE -#define __ARCH_WANT_SYS_CLONE3 #ifndef __COMPAT_SYSCALL_NR #include diff --git a/arch/arm64/include/uapi/asm/unistd.h b/arch/arm64/include/uapi/asm/unistd.h index 4703d218663a..f83a70e07df8 100644 --- a/arch/arm64/include/uapi/asm/unistd.h +++ b/arch/arm64/include/uapi/asm/unistd.h @@ -19,5 +19,6 @@ #define __ARCH_WANT_NEW_STAT #define __ARCH_WANT_SET_GET_RLIMIT #define __ARCH_WANT_TIME32_SYSCALLS +#define __ARCH_WANT_SYS_CLONE3 #include From 472f8a5821acb6c2c869fbc35dc52ed0df90ab0d Mon Sep 17 00:00:00 2001 From: Amanieu d'Antras Date: Thu, 2 Jan 2020 18:24:08 +0100 Subject: [PATCH 65/89] arm64: Implement copy_thread_tls commit a4376f2fbcc8084832f2f114577c8d68234c7903 upstream. This is required for clone3 which passes the TLS value through a struct rather than a register. Signed-off-by: Amanieu d'Antras Cc: linux-arm-kernel@lists.infradead.org Cc: # 5.3.x Acked-by: Will Deacon Link: https://lore.kernel.org/r/20200102172413.654385-3-amanieu@gmail.com Signed-off-by: Christian Brauner Signed-off-by: Greg Kroah-Hartman --- arch/arm64/Kconfig | 1 + arch/arm64/kernel/process.c | 10 +++++----- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig index 3f047afb982c..6ccd2ed30963 100644 --- a/arch/arm64/Kconfig +++ b/arch/arm64/Kconfig @@ -139,6 +139,7 @@ config ARM64 select HAVE_CMPXCHG_DOUBLE select HAVE_CMPXCHG_LOCAL select HAVE_CONTEXT_TRACKING + select HAVE_COPY_THREAD_TLS select HAVE_DEBUG_BUGVERBOSE select HAVE_DEBUG_KMEMLEAK select HAVE_DMA_CONTIGUOUS diff --git a/arch/arm64/kernel/process.c b/arch/arm64/kernel/process.c index 71f788cd2b18..d54586d5b031 100644 --- a/arch/arm64/kernel/process.c +++ b/arch/arm64/kernel/process.c @@ -360,8 +360,8 @@ int arch_dup_task_struct(struct task_struct *dst, struct task_struct *src) asmlinkage void ret_from_fork(void) asm("ret_from_fork"); -int copy_thread(unsigned long clone_flags, unsigned long stack_start, - unsigned long stk_sz, struct task_struct *p) +int copy_thread_tls(unsigned long clone_flags, unsigned long stack_start, + unsigned long stk_sz, struct task_struct *p, unsigned long tls) { struct pt_regs *childregs = task_pt_regs(p); @@ -394,11 +394,11 @@ int copy_thread(unsigned long clone_flags, unsigned long stack_start, } /* - * If a TLS pointer was passed to clone (4th argument), use it - * for the new thread. + * If a TLS pointer was passed to clone, use it for the new + * thread. */ if (clone_flags & CLONE_SETTLS) - p->thread.uw.tp_value = childregs->regs[3]; + p->thread.uw.tp_value = tls; } else { memset(childregs, 0, sizeof(struct pt_regs)); childregs->pstate = PSR_MODE_EL1h; From ca7bbad38dd16cdb645a56984730e15f9850e9eb Mon Sep 17 00:00:00 2001 From: Amanieu d'Antras Date: Thu, 2 Jan 2020 18:24:09 +0100 Subject: [PATCH 66/89] arm: Implement copy_thread_tls commit 167ee0b82429cb5df272808c7a21370b7c961ab2 upstream. This is required for clone3 which passes the TLS value through a struct rather than a register. Signed-off-by: Amanieu d'Antras Cc: linux-arm-kernel@lists.infradead.org Cc: # 5.3.x Link: https://lore.kernel.org/r/20200102172413.654385-4-amanieu@gmail.com Signed-off-by: Christian Brauner Signed-off-by: Greg Kroah-Hartman --- arch/arm/Kconfig | 1 + arch/arm/kernel/process.c | 6 +++--- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/arch/arm/Kconfig b/arch/arm/Kconfig index 8a50efb559f3..39002d769d95 100644 --- a/arch/arm/Kconfig +++ b/arch/arm/Kconfig @@ -73,6 +73,7 @@ config ARM select HAVE_ARM_SMCCC if CPU_V7 select HAVE_EBPF_JIT if !CPU_ENDIAN_BE32 select HAVE_CONTEXT_TRACKING + select HAVE_COPY_THREAD_TLS select HAVE_C_RECORDMCOUNT select HAVE_DEBUG_KMEMLEAK select HAVE_DMA_CONTIGUOUS if MMU diff --git a/arch/arm/kernel/process.c b/arch/arm/kernel/process.c index 9485acc520a4..e7fac125ea0d 100644 --- a/arch/arm/kernel/process.c +++ b/arch/arm/kernel/process.c @@ -224,8 +224,8 @@ void release_thread(struct task_struct *dead_task) asmlinkage void ret_from_fork(void) __asm__("ret_from_fork"); int -copy_thread(unsigned long clone_flags, unsigned long stack_start, - unsigned long stk_sz, struct task_struct *p) +copy_thread_tls(unsigned long clone_flags, unsigned long stack_start, + unsigned long stk_sz, struct task_struct *p, unsigned long tls) { struct thread_info *thread = task_thread_info(p); struct pt_regs *childregs = task_pt_regs(p); @@ -259,7 +259,7 @@ copy_thread(unsigned long clone_flags, unsigned long stack_start, clear_ptrace_hw_breakpoint(p); if (clone_flags & CLONE_SETTLS) - thread->tp_value[0] = childregs->ARM_r3; + thread->tp_value[0] = tls; thread->tp_value[1] = get_tpuser(); thread_notify(THREAD_NOTIFY_COPY, thread); From 0bdd4e601a9a7b0e80ab30ebcb7bea827b7c9871 Mon Sep 17 00:00:00 2001 From: Amanieu d'Antras Date: Thu, 2 Jan 2020 18:24:10 +0100 Subject: [PATCH 67/89] parisc: Implement copy_thread_tls commit d2f36c787b2181561d8b95814f8cdad64b348ad7 upstream. This is required for clone3 which passes the TLS value through a struct rather than a register. Signed-off-by: Amanieu d'Antras Cc: linux-parisc@vger.kernel.org Cc: # 5.3.x Link: https://lore.kernel.org/r/20200102172413.654385-5-amanieu@gmail.com Signed-off-by: Christian Brauner Signed-off-by: Greg Kroah-Hartman --- arch/parisc/Kconfig | 1 + arch/parisc/kernel/process.c | 8 ++++---- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/arch/parisc/Kconfig b/arch/parisc/Kconfig index b16237c95ea3..0c29d6cb2c8d 100644 --- a/arch/parisc/Kconfig +++ b/arch/parisc/Kconfig @@ -62,6 +62,7 @@ config PARISC select HAVE_FTRACE_MCOUNT_RECORD if HAVE_DYNAMIC_FTRACE select HAVE_KPROBES_ON_FTRACE select HAVE_DYNAMIC_FTRACE_WITH_REGS + select HAVE_COPY_THREAD_TLS help The PA-RISC microprocessor is designed by Hewlett-Packard and used diff --git a/arch/parisc/kernel/process.c b/arch/parisc/kernel/process.c index ecc5c2771208..230a6422b99f 100644 --- a/arch/parisc/kernel/process.c +++ b/arch/parisc/kernel/process.c @@ -208,8 +208,8 @@ arch_initcall(parisc_idle_init); * Copy architecture-specific thread state */ int -copy_thread(unsigned long clone_flags, unsigned long usp, - unsigned long kthread_arg, struct task_struct *p) +copy_thread_tls(unsigned long clone_flags, unsigned long usp, + unsigned long kthread_arg, struct task_struct *p, unsigned long tls) { struct pt_regs *cregs = &(p->thread.regs); void *stack = task_stack_page(p); @@ -254,9 +254,9 @@ copy_thread(unsigned long clone_flags, unsigned long usp, cregs->ksp = (unsigned long)stack + THREAD_SZ_ALGN + FRAME_SIZE; cregs->kpc = (unsigned long) &child_return; - /* Setup thread TLS area from the 4th parameter in clone */ + /* Setup thread TLS area */ if (clone_flags & CLONE_SETTLS) - cregs->cr27 = cregs->gr[23]; + cregs->cr27 = tls; } return 0; From 0b6a32ef88530dc2f55adf4a5e8a16453c33b0ff Mon Sep 17 00:00:00 2001 From: Amanieu d'Antras Date: Thu, 2 Jan 2020 18:24:11 +0100 Subject: [PATCH 68/89] riscv: Implement copy_thread_tls commit 20bda4ed62f507ed72e30e817b43c65fdba60be7 upstream. This is required for clone3 which passes the TLS value through a struct rather than a register. Signed-off-by: Amanieu d'Antras Cc: linux-riscv@lists.infradead.org Cc: # 5.3.x Link: https://lore.kernel.org/r/20200102172413.654385-6-amanieu@gmail.com Signed-off-by: Christian Brauner Signed-off-by: Greg Kroah-Hartman --- arch/riscv/Kconfig | 1 + arch/riscv/kernel/process.c | 6 +++--- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/arch/riscv/Kconfig b/arch/riscv/Kconfig index 8eebbc8860bb..ade9699aa0dd 100644 --- a/arch/riscv/Kconfig +++ b/arch/riscv/Kconfig @@ -61,6 +61,7 @@ config RISCV select SPARSEMEM_STATIC if 32BIT select ARCH_WANT_DEFAULT_TOPDOWN_MMAP_LAYOUT if MMU select HAVE_ARCH_MMAP_RND_BITS + select HAVE_COPY_THREAD_TLS config ARCH_MMAP_RND_BITS_MIN default 18 if 64BIT diff --git a/arch/riscv/kernel/process.c b/arch/riscv/kernel/process.c index 85e3c39bb60b..330b34706aa0 100644 --- a/arch/riscv/kernel/process.c +++ b/arch/riscv/kernel/process.c @@ -99,8 +99,8 @@ int arch_dup_task_struct(struct task_struct *dst, struct task_struct *src) return 0; } -int copy_thread(unsigned long clone_flags, unsigned long usp, - unsigned long arg, struct task_struct *p) +int copy_thread_tls(unsigned long clone_flags, unsigned long usp, + unsigned long arg, struct task_struct *p, unsigned long tls) { struct pt_regs *childregs = task_pt_regs(p); @@ -120,7 +120,7 @@ int copy_thread(unsigned long clone_flags, unsigned long usp, if (usp) /* User fork */ childregs->sp = usp; if (clone_flags & CLONE_SETTLS) - childregs->tp = childregs->a5; + childregs->tp = tls; childregs->a0 = 0; /* Return value of fork() */ p->thread.ra = (unsigned long)ret_from_fork; } From 3981d85a91824e529de5ee5128ec0b5bb63d1a42 Mon Sep 17 00:00:00 2001 From: Amanieu d'Antras Date: Thu, 2 Jan 2020 18:24:12 +0100 Subject: [PATCH 69/89] xtensa: Implement copy_thread_tls commit c346b94f8c5d1b7d637522c908209de93305a8eb upstream. This is required for clone3 which passes the TLS value through a struct rather than a register. Signed-off-by: Amanieu d'Antras Cc: linux-xtensa@linux-xtensa.org Cc: # 5.3.x Link: https://lore.kernel.org/r/20200102172413.654385-7-amanieu@gmail.com Signed-off-by: Christian Brauner Signed-off-by: Greg Kroah-Hartman --- arch/xtensa/Kconfig | 1 + arch/xtensa/kernel/process.c | 8 ++++---- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/arch/xtensa/Kconfig b/arch/xtensa/Kconfig index a8e7beb6b7b5..8352037322df 100644 --- a/arch/xtensa/Kconfig +++ b/arch/xtensa/Kconfig @@ -22,6 +22,7 @@ config XTENSA select HAVE_ARCH_JUMP_LABEL select HAVE_ARCH_KASAN if MMU select HAVE_ARCH_TRACEHOOK + select HAVE_COPY_THREAD_TLS select HAVE_DEBUG_KMEMLEAK select HAVE_DMA_CONTIGUOUS select HAVE_EXIT_THREAD diff --git a/arch/xtensa/kernel/process.c b/arch/xtensa/kernel/process.c index db278a9e80c7..7cbf8bd6d922 100644 --- a/arch/xtensa/kernel/process.c +++ b/arch/xtensa/kernel/process.c @@ -202,8 +202,9 @@ int arch_dup_task_struct(struct task_struct *dst, struct task_struct *src) * involved. Much simpler to just not copy those live frames across. */ -int copy_thread(unsigned long clone_flags, unsigned long usp_thread_fn, - unsigned long thread_fn_arg, struct task_struct *p) +int copy_thread_tls(unsigned long clone_flags, unsigned long usp_thread_fn, + unsigned long thread_fn_arg, struct task_struct *p, + unsigned long tls) { struct pt_regs *childregs = task_pt_regs(p); @@ -264,9 +265,8 @@ int copy_thread(unsigned long clone_flags, unsigned long usp_thread_fn, ®s->areg[XCHAL_NUM_AREGS - len/4], len); } - /* The thread pointer is passed in the '4th argument' (= a5) */ if (clone_flags & CLONE_SETTLS) - childregs->threadptr = childregs->areg[5]; + childregs->threadptr = tls; } else { p->thread.ra = MAKE_RA_FOR_CALL( (unsigned long)ret_from_kernel_thread, 1); From 4f43cdc72331bdbf3562d266969779aed7a834af Mon Sep 17 00:00:00 2001 From: Amanieu d'Antras Date: Thu, 2 Jan 2020 18:24:13 +0100 Subject: [PATCH 70/89] clone3: ensure copy_thread_tls is implemented commit dd499f7a7e34270208350a849ef103c0b3ae477f upstream. copy_thread implementations handle CLONE_SETTLS by reading the TLS value from the registers containing the syscall arguments for clone. This doesn't work with clone3 since the TLS value is passed in clone_args instead. Signed-off-by: Amanieu d'Antras Cc: # 5.3.x Link: https://lore.kernel.org/r/20200102172413.654385-8-amanieu@gmail.com Signed-off-by: Christian Brauner Signed-off-by: Greg Kroah-Hartman --- kernel/fork.c | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/kernel/fork.c b/kernel/fork.c index 6cabc124378c..755d8160e001 100644 --- a/kernel/fork.c +++ b/kernel/fork.c @@ -2513,6 +2513,16 @@ SYSCALL_DEFINE5(clone, unsigned long, clone_flags, unsigned long, newsp, #endif #ifdef __ARCH_WANT_SYS_CLONE3 + +/* + * copy_thread implementations handle CLONE_SETTLS by reading the TLS value from + * the registers containing the syscall arguments for clone. This doesn't work + * with clone3 since the TLS value is passed in clone_args instead. + */ +#ifndef CONFIG_HAVE_COPY_THREAD_TLS +#error clone3 requires copy_thread_tls support in arch +#endif + noinline static int copy_clone_args_from_user(struct kernel_clone_args *kargs, struct clone_args __user *uargs, size_t usize) From 8c9ff5c7ddcb6230fbb5b8224733a0a6250b6904 Mon Sep 17 00:00:00 2001 From: Amanieu d'Antras Date: Sat, 4 Jan 2020 13:39:30 +0100 Subject: [PATCH 71/89] um: Implement copy_thread_tls commit 457677c70c7672a4586b0b8abc396cc1ecdd376d upstream. This is required for clone3 which passes the TLS value through a struct rather than a register. Signed-off-by: Amanieu d'Antras Cc: linux-um@lists.infradead.org Cc: # 5.3.x Link: https://lore.kernel.org/r/20200104123928.1048822-1-amanieu@gmail.com Signed-off-by: Christian Brauner Signed-off-by: Greg Kroah-Hartman --- arch/um/Kconfig | 1 + arch/um/include/asm/ptrace-generic.h | 2 +- arch/um/kernel/process.c | 6 +++--- arch/x86/um/tls_32.c | 6 ++---- arch/x86/um/tls_64.c | 7 +++---- 5 files changed, 10 insertions(+), 12 deletions(-) diff --git a/arch/um/Kconfig b/arch/um/Kconfig index fec6b4ca2b6e..c56d3526a3bd 100644 --- a/arch/um/Kconfig +++ b/arch/um/Kconfig @@ -14,6 +14,7 @@ config UML select HAVE_FUTEX_CMPXCHG if FUTEX select HAVE_DEBUG_KMEMLEAK select HAVE_DEBUG_BUGVERBOSE + select HAVE_COPY_THREAD_TLS select GENERIC_IRQ_SHOW select GENERIC_CPU_DEVICES select GENERIC_CLOCKEVENTS diff --git a/arch/um/include/asm/ptrace-generic.h b/arch/um/include/asm/ptrace-generic.h index 81c647ef9c6c..adf91ef553ae 100644 --- a/arch/um/include/asm/ptrace-generic.h +++ b/arch/um/include/asm/ptrace-generic.h @@ -36,7 +36,7 @@ extern long subarch_ptrace(struct task_struct *child, long request, extern unsigned long getreg(struct task_struct *child, int regno); extern int putreg(struct task_struct *child, int regno, unsigned long value); -extern int arch_copy_tls(struct task_struct *new); +extern int arch_set_tls(struct task_struct *new, unsigned long tls); extern void clear_flushed_tls(struct task_struct *task); extern int syscall_trace_enter(struct pt_regs *regs); extern void syscall_trace_leave(struct pt_regs *regs); diff --git a/arch/um/kernel/process.c b/arch/um/kernel/process.c index 263a8f069133..17045e7211bf 100644 --- a/arch/um/kernel/process.c +++ b/arch/um/kernel/process.c @@ -153,8 +153,8 @@ void fork_handler(void) userspace(¤t->thread.regs.regs, current_thread_info()->aux_fp_regs); } -int copy_thread(unsigned long clone_flags, unsigned long sp, - unsigned long arg, struct task_struct * p) +int copy_thread_tls(unsigned long clone_flags, unsigned long sp, + unsigned long arg, struct task_struct * p, unsigned long tls) { void (*handler)(void); int kthread = current->flags & PF_KTHREAD; @@ -188,7 +188,7 @@ int copy_thread(unsigned long clone_flags, unsigned long sp, * Set a new TLS for the child thread? */ if (clone_flags & CLONE_SETTLS) - ret = arch_copy_tls(p); + ret = arch_set_tls(p, tls); } return ret; diff --git a/arch/x86/um/tls_32.c b/arch/x86/um/tls_32.c index 5bd949da7a4a..ac8eee093f9c 100644 --- a/arch/x86/um/tls_32.c +++ b/arch/x86/um/tls_32.c @@ -215,14 +215,12 @@ static int set_tls_entry(struct task_struct* task, struct user_desc *info, return 0; } -int arch_copy_tls(struct task_struct *new) +int arch_set_tls(struct task_struct *new, unsigned long tls) { struct user_desc info; int idx, ret = -EFAULT; - if (copy_from_user(&info, - (void __user *) UPT_SI(&new->thread.regs.regs), - sizeof(info))) + if (copy_from_user(&info, (void __user *) tls, sizeof(info))) goto out; ret = -EINVAL; diff --git a/arch/x86/um/tls_64.c b/arch/x86/um/tls_64.c index 3a621e0d3925..ebd3855d9b13 100644 --- a/arch/x86/um/tls_64.c +++ b/arch/x86/um/tls_64.c @@ -6,14 +6,13 @@ void clear_flushed_tls(struct task_struct *task) { } -int arch_copy_tls(struct task_struct *t) +int arch_set_tls(struct task_struct *t, unsigned long tls) { /* * If CLONE_SETTLS is set, we need to save the thread id - * (which is argument 5, child_tid, of clone) so it can be set - * during context switches. + * so it can be set during context switches. */ - t->thread.arch.fs = t->thread.regs.regs.gp[R8 / sizeof(long)]; + t->thread.arch.fs = tls; return 0; } From fa2bdff31cbf5a6e9beb67fba5e8cd59cc9ecfae Mon Sep 17 00:00:00 2001 From: Malcolm Priestley Date: Fri, 20 Dec 2019 21:15:33 +0000 Subject: [PATCH 72/89] staging: vt6656: remove bool from vnt_radio_power_on ret commit 07f59f180ee083c48c32a1e69ae1d0091444d212 upstream. The driver uses logical only error checking a bool true would flag error. Signed-off-by: Malcolm Priestley Link: https://lore.kernel.org/r/cc52b67c-9ef8-3e57-815a-44d10701919e@gmail.com Signed-off-by: Greg Kroah-Hartman --- drivers/staging/vt6656/card.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/staging/vt6656/card.c b/drivers/staging/vt6656/card.c index 56cd77fd9ea0..7958fc165462 100644 --- a/drivers/staging/vt6656/card.c +++ b/drivers/staging/vt6656/card.c @@ -719,7 +719,7 @@ end: */ int vnt_radio_power_on(struct vnt_private *priv) { - int ret = true; + int ret = 0; vnt_exit_deep_sleep(priv); From cbd6a85021a38ce3071fc50f2e11b709b0add8c7 Mon Sep 17 00:00:00 2001 From: Ganapathi Bhat Date: Thu, 21 Nov 2019 21:34:38 +0530 Subject: [PATCH 73/89] mwifiex: fix possible heap overflow in mwifiex_process_country_ie() commit 3d94a4a8373bf5f45cf5f939e88b8354dbf2311b upstream. mwifiex_process_country_ie() function parse elements of bss descriptor in beacon packet. When processing WLAN_EID_COUNTRY element, there is no upper limit check for country_ie_len before calling memcpy. The destination buffer domain_info->triplet is an array of length MWIFIEX_MAX_TRIPLET_802_11D(83). The remote attacker can build a fake AP with the same ssid as real AP, and send malicous beacon packet with long WLAN_EID_COUNTRY elemen (country_ie_len > 83). Attacker can force STA connect to fake AP on a different channel. When the victim STA connects to fake AP, will trigger the heap buffer overflow. Fix this by checking for length and if found invalid, don not connect to the AP. This fix addresses CVE-2019-14895. Reported-by: huangwen Signed-off-by: Ganapathi Bhat Signed-off-by: Kalle Valo Cc: Ben Hutchings Signed-off-by: Greg Kroah-Hartman --- drivers/net/wireless/marvell/mwifiex/sta_ioctl.c | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/drivers/net/wireless/marvell/mwifiex/sta_ioctl.c b/drivers/net/wireless/marvell/mwifiex/sta_ioctl.c index 74e50566db1f..6dd835f1efc2 100644 --- a/drivers/net/wireless/marvell/mwifiex/sta_ioctl.c +++ b/drivers/net/wireless/marvell/mwifiex/sta_ioctl.c @@ -229,6 +229,14 @@ static int mwifiex_process_country_ie(struct mwifiex_private *priv, "11D: skip setting domain info in FW\n"); return 0; } + + if (country_ie_len > + (IEEE80211_COUNTRY_STRING_LEN + MWIFIEX_MAX_TRIPLET_802_11D)) { + mwifiex_dbg(priv->adapter, ERROR, + "11D: country_ie_len overflow!, deauth AP\n"); + return -EINVAL; + } + memcpy(priv->adapter->country_code, &country_ie[2], 2); domain_info->country_code[0] = country_ie[2]; @@ -272,8 +280,9 @@ int mwifiex_bss_start(struct mwifiex_private *priv, struct cfg80211_bss *bss, priv->scan_block = false; if (bss) { - if (adapter->region_code == 0x00) - mwifiex_process_country_ie(priv, bss); + if (adapter->region_code == 0x00 && + mwifiex_process_country_ie(priv, bss)) + return -EINVAL; /* Allocate and fill new bss descriptor */ bss_desc = kzalloc(sizeof(struct mwifiex_bssdescriptor), From 3fe1ced40e189e31c21f6723fbe4bdf8d2731922 Mon Sep 17 00:00:00 2001 From: Navid Emamdoost Date: Fri, 4 Oct 2019 15:08:52 -0500 Subject: [PATCH 74/89] mwifiex: pcie: Fix memory leak in mwifiex_pcie_alloc_cmdrsp_buf commit db8fd2cde93227e566a412cf53173ffa227998bc upstream. In mwifiex_pcie_alloc_cmdrsp_buf, a new skb is allocated which should be released if mwifiex_map_pci_memory() fails. The release is added. Fixes: fc3314609047 ("mwifiex: use pci_alloc/free_consistent APIs for PCIe") Signed-off-by: Navid Emamdoost Acked-by: Ganapathi Bhat Signed-off-by: Kalle Valo Cc: Ben Hutchings Signed-off-by: Greg Kroah-Hartman --- drivers/net/wireless/marvell/mwifiex/pcie.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/drivers/net/wireless/marvell/mwifiex/pcie.c b/drivers/net/wireless/marvell/mwifiex/pcie.c index 096334e941a1..fc1706d0647d 100644 --- a/drivers/net/wireless/marvell/mwifiex/pcie.c +++ b/drivers/net/wireless/marvell/mwifiex/pcie.c @@ -1032,8 +1032,10 @@ static int mwifiex_pcie_alloc_cmdrsp_buf(struct mwifiex_adapter *adapter) } skb_put(skb, MWIFIEX_UPLD_SIZE); if (mwifiex_map_pci_memory(adapter, skb, MWIFIEX_UPLD_SIZE, - PCI_DMA_FROMDEVICE)) + PCI_DMA_FROMDEVICE)) { + kfree_skb(skb); return -1; + } card->cmdrsp_buf = skb; From 5bbe72cf486c3b983f739b3e1d98b61c8a205795 Mon Sep 17 00:00:00 2001 From: Navid Emamdoost Date: Mon, 16 Sep 2019 22:31:23 -0500 Subject: [PATCH 75/89] rpmsg: char: release allocated memory commit bbe692e349e2a1edf3fe0a29a0e05899c9c94d51 upstream. In rpmsg_eptdev_write_iter, if copy_from_iter_full fails the allocated buffer needs to be released. Signed-off-by: Navid Emamdoost Signed-off-by: Bjorn Andersson Cc: Ben Hutchings Signed-off-by: Greg Kroah-Hartman --- drivers/rpmsg/rpmsg_char.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/drivers/rpmsg/rpmsg_char.c b/drivers/rpmsg/rpmsg_char.c index eea5ebbb5119..c655074c07c2 100644 --- a/drivers/rpmsg/rpmsg_char.c +++ b/drivers/rpmsg/rpmsg_char.c @@ -227,8 +227,10 @@ static ssize_t rpmsg_eptdev_write_iter(struct kiocb *iocb, if (!kbuf) return -ENOMEM; - if (!copy_from_iter_full(kbuf, len, from)) - return -EFAULT; + if (!copy_from_iter_full(kbuf, len, from)) { + ret = -EFAULT; + goto free_kbuf; + } if (mutex_lock_interruptible(&eptdev->ept_lock)) { ret = -ERESTARTSYS; From 448fe0b67c68d36cb45c09444c6b8298130d4c5e Mon Sep 17 00:00:00 2001 From: Navid Emamdoost Date: Tue, 10 Sep 2019 18:44:15 -0500 Subject: [PATCH 76/89] scsi: bfa: release allocated memory in case of error commit 0e62395da2bd5166d7c9e14cbc7503b256a34cb0 upstream. In bfad_im_get_stats if bfa_port_get_stats fails, allocated memory needs to be released. Link: https://lore.kernel.org/r/20190910234417.22151-1-navid.emamdoost@gmail.com Signed-off-by: Navid Emamdoost Signed-off-by: Martin K. Petersen Cc: Ben Hutchings Signed-off-by: Greg Kroah-Hartman --- drivers/scsi/bfa/bfad_attr.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/drivers/scsi/bfa/bfad_attr.c b/drivers/scsi/bfa/bfad_attr.c index 29ab81df75c0..fbfce02e5b93 100644 --- a/drivers/scsi/bfa/bfad_attr.c +++ b/drivers/scsi/bfa/bfad_attr.c @@ -275,8 +275,10 @@ bfad_im_get_stats(struct Scsi_Host *shost) rc = bfa_port_get_stats(BFA_FCPORT(&bfad->bfa), fcstats, bfad_hcb_comp, &fcomp); spin_unlock_irqrestore(&bfad->bfad_lock, flags); - if (rc != BFA_STATUS_OK) + if (rc != BFA_STATUS_OK) { + kfree(fcstats); return NULL; + } wait_for_completion(&fcomp.comp); From e380d974731502d24e0353df36a883fe232c866b Mon Sep 17 00:00:00 2001 From: Navid Emamdoost Date: Thu, 19 Sep 2019 22:00:41 -0500 Subject: [PATCH 77/89] rtl8xxxu: prevent leaking urb commit a2cdd07488e666aa93a49a3fc9c9b1299e27ef3c upstream. In rtl8xxxu_submit_int_urb if usb_submit_urb fails the allocated urb should be released. Signed-off-by: Navid Emamdoost Reviewed-by: Chris Chiu Signed-off-by: Kalle Valo Cc: Ben Hutchings Signed-off-by: Greg Kroah-Hartman --- drivers/net/wireless/realtek/rtl8xxxu/rtl8xxxu_core.c | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/net/wireless/realtek/rtl8xxxu/rtl8xxxu_core.c b/drivers/net/wireless/realtek/rtl8xxxu/rtl8xxxu_core.c index 361248e97568..e7d96ac673b7 100644 --- a/drivers/net/wireless/realtek/rtl8xxxu/rtl8xxxu_core.c +++ b/drivers/net/wireless/realtek/rtl8xxxu/rtl8xxxu_core.c @@ -5447,6 +5447,7 @@ static int rtl8xxxu_submit_int_urb(struct ieee80211_hw *hw) ret = usb_submit_urb(urb, GFP_KERNEL); if (ret) { usb_unanchor_urb(urb); + usb_free_urb(urb); goto error; } From ad1e0d1976b9061bf2aca99249b0187c9bbd3334 Mon Sep 17 00:00:00 2001 From: Navid Emamdoost Date: Thu, 19 Sep 2019 20:36:26 -0500 Subject: [PATCH 78/89] ath10k: fix memory leak commit b8d17e7d93d2beb89e4f34c59996376b8b544792 upstream. In ath10k_usb_hif_tx_sg the allocated urb should be released if usb_submit_urb fails. Signed-off-by: Navid Emamdoost Signed-off-by: Kalle Valo Cc: Ben Hutchings Signed-off-by: Greg Kroah-Hartman --- drivers/net/wireless/ath/ath10k/usb.c | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/net/wireless/ath/ath10k/usb.c b/drivers/net/wireless/ath/ath10k/usb.c index 9ebe74ee4aef..1e0343081be9 100644 --- a/drivers/net/wireless/ath/ath10k/usb.c +++ b/drivers/net/wireless/ath/ath10k/usb.c @@ -443,6 +443,7 @@ static int ath10k_usb_hif_tx_sg(struct ath10k *ar, u8 pipe_id, ath10k_dbg(ar, ATH10K_DBG_USB_BULK, "usb bulk transmit failed: %d\n", ret); usb_unanchor_urb(urb); + usb_free_urb(urb); ret = -EINVAL; goto err_free_urb_to_pipe; } From 1ea36fba56ef33475b0b95276062ea242ffd53d4 Mon Sep 17 00:00:00 2001 From: Dmitry Torokhov Date: Tue, 17 Dec 2019 14:50:21 -0800 Subject: [PATCH 79/89] HID: hiddev: fix mess in hiddev_open() commit 18a1b06e5b91d47dc86c0a66a762646ea7c5d141 upstream. The open method of hiddev handler fails to bring the device out of autosuspend state as was promised in 0361a28d3f9a, as it actually has 2 blocks that try to start the transport (call hid_hw_open()) with both being guarded by the "open" counter, so the 2nd block is never executed as the first block increments the counter so it is never at 0 when we check it for the second block. Additionally hiddev_open() was leaving counter incremented on errors, causing the device to never be reopened properly if there was ever an error. Let's fix all of this by factoring out code that creates client structure and powers up the device into a separate function that is being called from usbhid_open() with the "existancelock" being held. Fixes: 0361a28d3f9a ("HID: autosuspend support for USB HID") Signed-off-by: Dmitry Torokhov Signed-off-by: Benjamin Tissoires Signed-off-by: Greg Kroah-Hartman --- drivers/hid/usbhid/hiddev.c | 97 ++++++++++++++++--------------------- 1 file changed, 42 insertions(+), 55 deletions(-) diff --git a/drivers/hid/usbhid/hiddev.c b/drivers/hid/usbhid/hiddev.c index 1f9bc4483465..c879b214a479 100644 --- a/drivers/hid/usbhid/hiddev.c +++ b/drivers/hid/usbhid/hiddev.c @@ -241,12 +241,51 @@ static int hiddev_release(struct inode * inode, struct file * file) return 0; } +static int __hiddev_open(struct hiddev *hiddev, struct file *file) +{ + struct hiddev_list *list; + int error; + + lockdep_assert_held(&hiddev->existancelock); + + list = vzalloc(sizeof(*list)); + if (!list) + return -ENOMEM; + + mutex_init(&list->thread_lock); + list->hiddev = hiddev; + + if (!hiddev->open++) { + error = hid_hw_power(hiddev->hid, PM_HINT_FULLON); + if (error < 0) + goto err_drop_count; + + error = hid_hw_open(hiddev->hid); + if (error < 0) + goto err_normal_power; + } + + spin_lock_irq(&hiddev->list_lock); + list_add_tail(&list->node, &hiddev->list); + spin_unlock_irq(&hiddev->list_lock); + + file->private_data = list; + + return 0; + +err_normal_power: + hid_hw_power(hiddev->hid, PM_HINT_NORMAL); +err_drop_count: + hiddev->open--; + vfree(list); + return error; +} + /* * open file op */ static int hiddev_open(struct inode *inode, struct file *file) { - struct hiddev_list *list; struct usb_interface *intf; struct hid_device *hid; struct hiddev *hiddev; @@ -255,66 +294,14 @@ static int hiddev_open(struct inode *inode, struct file *file) intf = usbhid_find_interface(iminor(inode)); if (!intf) return -ENODEV; + hid = usb_get_intfdata(intf); hiddev = hid->hiddev; - if (!(list = vzalloc(sizeof(struct hiddev_list)))) - return -ENOMEM; - mutex_init(&list->thread_lock); - list->hiddev = hiddev; - file->private_data = list; - - /* - * no need for locking because the USB major number - * is shared which usbcore guards against disconnect - */ - if (list->hiddev->exist) { - if (!list->hiddev->open++) { - res = hid_hw_open(hiddev->hid); - if (res < 0) - goto bail; - } - } else { - res = -ENODEV; - goto bail; - } - - spin_lock_irq(&list->hiddev->list_lock); - list_add_tail(&list->node, &hiddev->list); - spin_unlock_irq(&list->hiddev->list_lock); - mutex_lock(&hiddev->existancelock); - /* - * recheck exist with existance lock held to - * avoid opening a disconnected device - */ - if (!list->hiddev->exist) { - res = -ENODEV; - goto bail_unlock; - } - if (!list->hiddev->open++) - if (list->hiddev->exist) { - struct hid_device *hid = hiddev->hid; - res = hid_hw_power(hid, PM_HINT_FULLON); - if (res < 0) - goto bail_unlock; - res = hid_hw_open(hid); - if (res < 0) - goto bail_normal_power; - } - mutex_unlock(&hiddev->existancelock); - return 0; -bail_normal_power: - hid_hw_power(hid, PM_HINT_NORMAL); -bail_unlock: + res = hiddev->exist ? __hiddev_open(hiddev, file) : -ENODEV; mutex_unlock(&hiddev->existancelock); - spin_lock_irq(&list->hiddev->list_lock); - list_del(&list->node); - spin_unlock_irq(&list->hiddev->list_lock); -bail: - file->private_data = NULL; - vfree(list); return res; } From 3fcaac70d298fc69d4c63f5bc5494bab30ec4ee1 Mon Sep 17 00:00:00 2001 From: Alan Stern Date: Mon, 6 Jan 2020 10:43:42 -0500 Subject: [PATCH 80/89] USB: Fix: Don't skip endpoint descriptors with maxpacket=0 commit 2548288b4fb059b2da9ceada172ef763077e8a59 upstream. It turns out that even though endpoints with a maxpacket length of 0 aren't useful for data transfer, the descriptors do serve other purposes. In particular, skipping them will also skip over other class-specific descriptors for classes such as UVC. This unexpected side effect has caused some UVC cameras to stop working. In addition, the USB spec requires that when isochronous endpoint descriptors are present in an interface's altsetting 0 (which is true on some devices), the maxpacket size _must_ be set to 0. Warning about such things seems like a bad idea. This patch updates an earlier commit which would log a warning and skip these endpoint descriptors. Now we only log a warning, and we don't even do that for isochronous endpoints in altsetting 0. We don't need to worry about preventing endpoints with maxpacket = 0 from ever being used for data transfers; usb_submit_urb() already checks for this. Reported-and-tested-by: Roger Whittaker Fixes: d482c7bb0541 ("USB: Skip endpoints with 0 maxpacket length") Signed-off-by: Alan Stern CC: Laurent Pinchart Link: https://marc.info/?l=linux-usb&m=157790377329882&w=2 Link: https://lore.kernel.org/r/Pine.LNX.4.44L0.2001061040270.1514-100000@iolanthe.rowland.org Signed-off-by: Greg Kroah-Hartman --- drivers/usb/core/config.c | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/drivers/usb/core/config.c b/drivers/usb/core/config.c index 5e92c6abe306..3e94259406d7 100644 --- a/drivers/usb/core/config.c +++ b/drivers/usb/core/config.c @@ -392,12 +392,16 @@ static int usb_parse_endpoint(struct device *ddev, int cfgno, endpoint->desc.wMaxPacketSize = cpu_to_le16(8); } - /* Validate the wMaxPacketSize field */ + /* + * Validate the wMaxPacketSize field. + * Some devices have isochronous endpoints in altsetting 0; + * the USB-2 spec requires such endpoints to have wMaxPacketSize = 0 + * (see the end of section 5.6.3), so don't warn about them. + */ maxp = usb_endpoint_maxp(&endpoint->desc); - if (maxp == 0) { - dev_warn(ddev, "config %d interface %d altsetting %d endpoint 0x%X has wMaxPacketSize 0, skipping\n", + if (maxp == 0 && !(usb_endpoint_xfer_isoc(d) && asnum == 0)) { + dev_warn(ddev, "config %d interface %d altsetting %d endpoint 0x%X has invalid wMaxPacketSize 0\n", cfgno, inum, asnum, d->bEndpointAddress); - goto skip_to_next_endpoint_or_interface_descriptor; } /* Find the highest legal maxpacket size for this endpoint */ From 73681018f00c980fc512b46074e8fbee02471bee Mon Sep 17 00:00:00 2001 From: Tony Lindgren Date: Fri, 20 Dec 2019 16:21:40 +0530 Subject: [PATCH 81/89] phy: cpcap-usb: Fix error path when no host driver is loaded commit 4acb0200ab2b07843e3ef5599add3454c7440f03 upstream. If musb_mailbox() returns an error, we must still continue to finish configuring the phy. Otherwise the phy state may end up only half initialized, and this can cause the debug serial console to stop working. And this will happen if the usb driver musb controller is not loaded. Let's fix the issue by adding helper for cpcap_usb_try_musb_mailbox(). Fixes: 6d6ce40f63af ("phy: cpcap-usb: Add CPCAP PMIC USB support") Cc: Merlijn Wajer Cc: Pavel Machek Cc: Sebastian Reichel Signed-off-by: Tony Lindgren Signed-off-by: Kishon Vijay Abraham I Signed-off-by: Greg Kroah-Hartman --- drivers/phy/motorola/phy-cpcap-usb.c | 33 +++++++++++++++------------- 1 file changed, 18 insertions(+), 15 deletions(-) diff --git a/drivers/phy/motorola/phy-cpcap-usb.c b/drivers/phy/motorola/phy-cpcap-usb.c index ead06c6c2601..eeb47fddf64b 100644 --- a/drivers/phy/motorola/phy-cpcap-usb.c +++ b/drivers/phy/motorola/phy-cpcap-usb.c @@ -207,6 +207,19 @@ static int cpcap_phy_get_ints_state(struct cpcap_phy_ddata *ddata, static int cpcap_usb_set_uart_mode(struct cpcap_phy_ddata *ddata); static int cpcap_usb_set_usb_mode(struct cpcap_phy_ddata *ddata); +static void cpcap_usb_try_musb_mailbox(struct cpcap_phy_ddata *ddata, + enum musb_vbus_id_status status) +{ + int error; + + error = musb_mailbox(status); + if (!error) + return; + + dev_dbg(ddata->dev, "%s: musb_mailbox failed: %i\n", + __func__, error); +} + static void cpcap_usb_detect(struct work_struct *work) { struct cpcap_phy_ddata *ddata; @@ -226,9 +239,7 @@ static void cpcap_usb_detect(struct work_struct *work) if (error) goto out_err; - error = musb_mailbox(MUSB_ID_GROUND); - if (error) - goto out_err; + cpcap_usb_try_musb_mailbox(ddata, MUSB_ID_GROUND); error = regmap_update_bits(ddata->reg, CPCAP_REG_USBC3, CPCAP_BIT_VBUSSTBY_EN | @@ -257,9 +268,7 @@ static void cpcap_usb_detect(struct work_struct *work) error = cpcap_usb_set_usb_mode(ddata); if (error) goto out_err; - error = musb_mailbox(MUSB_ID_GROUND); - if (error) - goto out_err; + cpcap_usb_try_musb_mailbox(ddata, MUSB_ID_GROUND); return; } @@ -269,9 +278,7 @@ static void cpcap_usb_detect(struct work_struct *work) error = cpcap_usb_set_usb_mode(ddata); if (error) goto out_err; - error = musb_mailbox(MUSB_VBUS_VALID); - if (error) - goto out_err; + cpcap_usb_try_musb_mailbox(ddata, MUSB_VBUS_VALID); return; } @@ -281,9 +288,7 @@ static void cpcap_usb_detect(struct work_struct *work) if (error) goto out_err; - error = musb_mailbox(MUSB_VBUS_OFF); - if (error) - goto out_err; + cpcap_usb_try_musb_mailbox(ddata, MUSB_VBUS_OFF); dev_dbg(ddata->dev, "set UART mode\n"); @@ -649,9 +654,7 @@ static int cpcap_usb_phy_remove(struct platform_device *pdev) if (error) dev_err(ddata->dev, "could not set UART mode\n"); - error = musb_mailbox(MUSB_VBUS_OFF); - if (error) - dev_err(ddata->dev, "could not set mailbox\n"); + cpcap_usb_try_musb_mailbox(ddata, MUSB_VBUS_OFF); usb_remove_phy(&ddata->phy); cancel_delayed_work_sync(&ddata->detect_work); From 2deeba3623429a84103b26297a402634b7221973 Mon Sep 17 00:00:00 2001 From: Tony Lindgren Date: Sun, 22 Dec 2019 10:00:19 -0800 Subject: [PATCH 82/89] phy: cpcap-usb: Fix flakey host idling and enumerating of devices commit 049226b9fd7442149dcbcf55f15408f5973cceda upstream. We must let the USB host idle things properly before we switch to debug UART mode. Otherwise the USB host may never idle after disconnecting devices, and that causes the next enumeration to be flakey. Cc: Jacopo Mondi Cc: Marcel Partap Cc: Merlijn Wajer Cc: Michael Scott Cc: NeKit Cc: Pavel Machek Cc: Sebastian Reichel Acked-by: Pavel Machek Fixes: 6d6ce40f63af ("phy: cpcap-usb: Add CPCAP PMIC USB support") Signed-off-by: Tony Lindgren Signed-off-by: Kishon Vijay Abraham I Signed-off-by: Greg Kroah-Hartman --- drivers/phy/motorola/phy-cpcap-usb.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/phy/motorola/phy-cpcap-usb.c b/drivers/phy/motorola/phy-cpcap-usb.c index eeb47fddf64b..9a38741d3546 100644 --- a/drivers/phy/motorola/phy-cpcap-usb.c +++ b/drivers/phy/motorola/phy-cpcap-usb.c @@ -283,13 +283,13 @@ static void cpcap_usb_detect(struct work_struct *work) return; } + cpcap_usb_try_musb_mailbox(ddata, MUSB_VBUS_OFF); + /* Default to debug UART mode */ error = cpcap_usb_set_uart_mode(ddata); if (error) goto out_err; - cpcap_usb_try_musb_mailbox(ddata, MUSB_VBUS_OFF); - dev_dbg(ddata->dev, "set UART mode\n"); return; From 503ff0dcfba7519b86a04e82b6954dc73df80aac Mon Sep 17 00:00:00 2001 From: Florian Westphal Date: Fri, 27 Dec 2019 01:33:10 +0100 Subject: [PATCH 83/89] netfilter: arp_tables: init netns pointer in xt_tgchk_param struct commit 1b789577f655060d98d20ed0c6f9fbd469d6ba63 upstream. We get crash when the targets checkentry function tries to make use of the network namespace pointer for arptables. When the net pointer got added back in 2010, only ip/ip6/ebtables were changed to initialize it, so arptables has this set to NULL. This isn't a problem for normal arptables because no existing arptables target has a checkentry function that makes use of par->net. However, direct users of the setsockopt interface can provide any target they want as long as its registered for ARP or UNPSEC protocols. syzkaller managed to send a semi-valid arptables rule for RATEEST target which is enough to trigger NULL deref: kasan: GPF could be caused by NULL-ptr deref or user memory access general protection fault: 0000 [#1] PREEMPT SMP KASAN RIP: xt_rateest_tg_checkentry+0x11d/0xb40 net/netfilter/xt_RATEEST.c:109 [..] xt_check_target+0x283/0x690 net/netfilter/x_tables.c:1019 check_target net/ipv4/netfilter/arp_tables.c:399 [inline] find_check_entry net/ipv4/netfilter/arp_tables.c:422 [inline] translate_table+0x1005/0x1d70 net/ipv4/netfilter/arp_tables.c:572 do_replace net/ipv4/netfilter/arp_tables.c:977 [inline] do_arpt_set_ctl+0x310/0x640 net/ipv4/netfilter/arp_tables.c:1456 Fixes: add67461240c1d ("netfilter: add struct net * to target parameters") Reported-by: syzbot+d7358a458d8a81aee898@syzkaller.appspotmail.com Signed-off-by: Florian Westphal Acked-by: Cong Wang Signed-off-by: Pablo Neira Ayuso Signed-off-by: Greg Kroah-Hartman --- net/ipv4/netfilter/arp_tables.c | 27 ++++++++++++++++----------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/net/ipv4/netfilter/arp_tables.c b/net/ipv4/netfilter/arp_tables.c index 214154b47d56..069f72edb264 100644 --- a/net/ipv4/netfilter/arp_tables.c +++ b/net/ipv4/netfilter/arp_tables.c @@ -384,10 +384,11 @@ next: ; return 1; } -static inline int check_target(struct arpt_entry *e, const char *name) +static int check_target(struct arpt_entry *e, struct net *net, const char *name) { struct xt_entry_target *t = arpt_get_target(e); struct xt_tgchk_param par = { + .net = net, .table = name, .entryinfo = e, .target = t->u.kernel.target, @@ -399,8 +400,9 @@ static inline int check_target(struct arpt_entry *e, const char *name) return xt_check_target(&par, t->u.target_size - sizeof(*t), 0, false); } -static inline int -find_check_entry(struct arpt_entry *e, const char *name, unsigned int size, +static int +find_check_entry(struct arpt_entry *e, struct net *net, const char *name, + unsigned int size, struct xt_percpu_counter_alloc_state *alloc_state) { struct xt_entry_target *t; @@ -419,7 +421,7 @@ find_check_entry(struct arpt_entry *e, const char *name, unsigned int size, } t->u.kernel.target = target; - ret = check_target(e, name); + ret = check_target(e, net, name); if (ret) goto err; return 0; @@ -512,7 +514,9 @@ static inline void cleanup_entry(struct arpt_entry *e) /* Checks and translates the user-supplied table segment (held in * newinfo). */ -static int translate_table(struct xt_table_info *newinfo, void *entry0, +static int translate_table(struct net *net, + struct xt_table_info *newinfo, + void *entry0, const struct arpt_replace *repl) { struct xt_percpu_counter_alloc_state alloc_state = { 0 }; @@ -569,7 +573,7 @@ static int translate_table(struct xt_table_info *newinfo, void *entry0, /* Finally, each sanity check must pass */ i = 0; xt_entry_foreach(iter, entry0, newinfo->size) { - ret = find_check_entry(iter, repl->name, repl->size, + ret = find_check_entry(iter, net, repl->name, repl->size, &alloc_state); if (ret != 0) break; @@ -974,7 +978,7 @@ static int do_replace(struct net *net, const void __user *user, goto free_newinfo; } - ret = translate_table(newinfo, loc_cpu_entry, &tmp); + ret = translate_table(net, newinfo, loc_cpu_entry, &tmp); if (ret != 0) goto free_newinfo; @@ -1149,7 +1153,8 @@ compat_copy_entry_from_user(struct compat_arpt_entry *e, void **dstptr, } } -static int translate_compat_table(struct xt_table_info **pinfo, +static int translate_compat_table(struct net *net, + struct xt_table_info **pinfo, void **pentry0, const struct compat_arpt_replace *compatr) { @@ -1217,7 +1222,7 @@ static int translate_compat_table(struct xt_table_info **pinfo, repl.num_counters = 0; repl.counters = NULL; repl.size = newinfo->size; - ret = translate_table(newinfo, entry1, &repl); + ret = translate_table(net, newinfo, entry1, &repl); if (ret) goto free_newinfo; @@ -1270,7 +1275,7 @@ static int compat_do_replace(struct net *net, void __user *user, goto free_newinfo; } - ret = translate_compat_table(&newinfo, &loc_cpu_entry, &tmp); + ret = translate_compat_table(net, &newinfo, &loc_cpu_entry, &tmp); if (ret != 0) goto free_newinfo; @@ -1546,7 +1551,7 @@ int arpt_register_table(struct net *net, loc_cpu_entry = newinfo->entries; memcpy(loc_cpu_entry, repl->entries, repl->size); - ret = translate_table(newinfo, loc_cpu_entry, repl); + ret = translate_table(net, newinfo, loc_cpu_entry, repl); if (ret != 0) goto out_free; From 99a55d8a7fa4c788afc74d6e5ded8903d1712d1a Mon Sep 17 00:00:00 2001 From: Florian Westphal Date: Mon, 6 Jan 2020 23:34:17 +0100 Subject: [PATCH 84/89] netfilter: conntrack: dccp, sctp: handle null timeout argument commit 1d9a7acd3d1e74c2d150d8934f7f55bed6d70858 upstream. The timeout pointer can be NULL which means we should modify the per-nets timeout instead. All do this, except sctp and dccp which instead give: general protection fault: 0000 [#1] PREEMPT SMP KASAN net/netfilter/nf_conntrack_proto_dccp.c:682 ctnl_timeout_parse_policy+0x150/0x1d0 net/netfilter/nfnetlink_cttimeout.c:67 cttimeout_default_set+0x150/0x1c0 net/netfilter/nfnetlink_cttimeout.c:368 nfnetlink_rcv_msg+0xcf2/0xfb0 net/netfilter/nfnetlink.c:229 netlink_rcv_skb+0x177/0x450 net/netlink/af_netlink.c:2477 Reported-by: syzbot+46a4ad33f345d1dd346e@syzkaller.appspotmail.com Fixes: c779e849608a8 ("netfilter: conntrack: remove get_timeout() indirection") Signed-off-by: Florian Westphal Signed-off-by: Pablo Neira Ayuso Signed-off-by: Greg Kroah-Hartman --- net/netfilter/nf_conntrack_proto_dccp.c | 3 +++ net/netfilter/nf_conntrack_proto_sctp.c | 3 +++ 2 files changed, 6 insertions(+) diff --git a/net/netfilter/nf_conntrack_proto_dccp.c b/net/netfilter/nf_conntrack_proto_dccp.c index b6b14db3955b..b3f4a334f9d7 100644 --- a/net/netfilter/nf_conntrack_proto_dccp.c +++ b/net/netfilter/nf_conntrack_proto_dccp.c @@ -677,6 +677,9 @@ static int dccp_timeout_nlattr_to_obj(struct nlattr *tb[], unsigned int *timeouts = data; int i; + if (!timeouts) + timeouts = dn->dccp_timeout; + /* set default DCCP timeouts. */ for (i=0; idccp_timeout[i]; diff --git a/net/netfilter/nf_conntrack_proto_sctp.c b/net/netfilter/nf_conntrack_proto_sctp.c index fce3d93f1541..0399ae8f1188 100644 --- a/net/netfilter/nf_conntrack_proto_sctp.c +++ b/net/netfilter/nf_conntrack_proto_sctp.c @@ -594,6 +594,9 @@ static int sctp_timeout_nlattr_to_obj(struct nlattr *tb[], struct nf_sctp_net *sn = nf_sctp_pernet(net); int i; + if (!timeouts) + timeouts = sn->timeouts; + /* set default SCTP timeouts. */ for (i=0; itimeouts[i]; From f58642c1bc7946c315d8617d61916f754d90583f Mon Sep 17 00:00:00 2001 From: Florian Westphal Date: Wed, 8 Jan 2020 10:59:38 +0100 Subject: [PATCH 85/89] netfilter: ipset: avoid null deref when IPSET_ATTR_LINENO is present commit 22dad713b8a5ff488e07b821195270672f486eb2 upstream. The set uadt functions assume lineno is never NULL, but it is in case of ip_set_utest(). syzkaller managed to generate a netlink message that calls this with LINENO attr present: general protection fault: 0000 [#1] PREEMPT SMP KASAN RIP: 0010:hash_mac4_uadt+0x1bc/0x470 net/netfilter/ipset/ip_set_hash_mac.c:104 Call Trace: ip_set_utest+0x55b/0x890 net/netfilter/ipset/ip_set_core.c:1867 nfnetlink_rcv_msg+0xcf2/0xfb0 net/netfilter/nfnetlink.c:229 netlink_rcv_skb+0x177/0x450 net/netlink/af_netlink.c:2477 nfnetlink_rcv+0x1ba/0x460 net/netfilter/nfnetlink.c:563 pass a dummy lineno storage, its easier than patching all set implementations. This seems to be a day-0 bug. Cc: Jozsef Kadlecsik Reported-by: syzbot+34bd2369d38707f3f4a7@syzkaller.appspotmail.com Fixes: a7b4f989a6294 ("netfilter: ipset: IP set core support") Signed-off-by: Florian Westphal Acked-by: Jozsef Kadlecsik Signed-off-by: Pablo Neira Ayuso Signed-off-by: Greg Kroah-Hartman --- net/netfilter/ipset/ip_set_core.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/net/netfilter/ipset/ip_set_core.c b/net/netfilter/ipset/ip_set_core.c index d73d1828216a..d8143a8c034d 100644 --- a/net/netfilter/ipset/ip_set_core.c +++ b/net/netfilter/ipset/ip_set_core.c @@ -1658,6 +1658,7 @@ static int ip_set_utest(struct net *net, struct sock *ctnl, struct sk_buff *skb, struct ip_set *set; struct nlattr *tb[IPSET_ATTR_ADT_MAX + 1] = {}; int ret = 0; + u32 lineno; if (unlikely(protocol_min_failed(attr) || !attr[IPSET_ATTR_SETNAME] || @@ -1674,7 +1675,7 @@ static int ip_set_utest(struct net *net, struct sock *ctnl, struct sk_buff *skb, return -IPSET_ERR_PROTOCOL; rcu_read_lock_bh(); - ret = set->variant->uadt(set, tb, IPSET_TEST, NULL, 0, 0); + ret = set->variant->uadt(set, tb, IPSET_TEST, &lineno, 0, 0); rcu_read_unlock_bh(); /* Userspace can't trigger element to be re-added */ if (ret == -EAGAIN) From 53b9bd37af59d1def99b20707536105857eb9bd0 Mon Sep 17 00:00:00 2001 From: Akeem G Abodunrin Date: Wed, 8 Jan 2020 09:34:16 -0800 Subject: [PATCH 86/89] drm/i915/gen9: Clear residual context state on context switch commit bc8a76a152c5f9ef3b48104154a65a68a8b76946 upstream. Intel ID: PSIRT-TA-201910-001 CVEID: CVE-2019-14615 Intel GPU Hardware prior to Gen11 does not clear EU state during a context switch. This can result in information leakage between contexts. For Gen8 and Gen9, hardware provides a mechanism for fast cleardown of the EU state, by issuing a PIPE_CONTROL with bit 27 set. We can use this in a context batch buffer to explicitly cleardown the state on every context switch. As this workaround is already in place for gen8, we can borrow the code verbatim for Gen9. Signed-off-by: Mika Kuoppala Signed-off-by: Akeem G Abodunrin Cc: Kumar Valsan Prathap Cc: Chris Wilson Cc: Balestrieri Francesco Cc: Bloomfield Jon Cc: Dutt Sudeep Signed-off-by: Greg Kroah-Hartman --- drivers/gpu/drm/i915/gt/intel_lrc.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/drivers/gpu/drm/i915/gt/intel_lrc.c b/drivers/gpu/drm/i915/gt/intel_lrc.c index 1ba31969c7d2..4949b5ad860f 100644 --- a/drivers/gpu/drm/i915/gt/intel_lrc.c +++ b/drivers/gpu/drm/i915/gt/intel_lrc.c @@ -2132,6 +2132,14 @@ static u32 *gen9_init_indirectctx_bb(struct intel_engine_cs *engine, u32 *batch) /* WaFlushCoherentL3CacheLinesAtContextSwitch:skl,bxt,glk */ batch = gen8_emit_flush_coherentl3_wa(engine, batch); + /* WaClearSlmSpaceAtContextSwitch:skl,bxt,kbl,glk,cfl */ + batch = gen8_emit_pipe_control(batch, + PIPE_CONTROL_FLUSH_L3 | + PIPE_CONTROL_GLOBAL_GTT_IVB | + PIPE_CONTROL_CS_STALL | + PIPE_CONTROL_QW_WRITE, + slm_offset(engine)); + batch = emit_lri(batch, lri, ARRAY_SIZE(lri)); /* WaMediaPoolStateCmdInWABB:bxt,glk */ From adc0acf587768b7db6ca1d7c395a9116865c9e07 Mon Sep 17 00:00:00 2001 From: Greg Kroah-Hartman Date: Tue, 14 Jan 2020 20:08:40 +0100 Subject: [PATCH 87/89] Linux 5.4.12 --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index e8963f623568..45c6264f1108 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,7 @@ # SPDX-License-Identifier: GPL-2.0 VERSION = 5 PATCHLEVEL = 4 -SUBLEVEL = 11 +SUBLEVEL = 12 EXTRAVERSION = NAME = Kleptomaniac Octopus From 4eb8e72c3538dbbac6939472ddb8fb7d377176bd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maciej=20=C5=BBenczykowski?= Date: Mon, 13 Jan 2020 23:52:14 -0800 Subject: [PATCH 88/89] GKI: enable CONFIG_NETFILTER_XT_MATCH_QUOTA2_LOG=y MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Quota logging is needed to enable data metering. Bug: 147203196 Signed-off-by: Maciej Å»enczykowski Change-Id: I7219a3abd922b31249a3bccb94bc0dcff197788e --- arch/arm64/configs/gki_defconfig | 1 + arch/x86/configs/gki_defconfig | 1 + 2 files changed, 2 insertions(+) diff --git a/arch/arm64/configs/gki_defconfig b/arch/arm64/configs/gki_defconfig index 1e7fccfc07de..d808a88c9e47 100644 --- a/arch/arm64/configs/gki_defconfig +++ b/arch/arm64/configs/gki_defconfig @@ -155,6 +155,7 @@ CONFIG_NETFILTER_XT_MATCH_POLICY=y CONFIG_NETFILTER_XT_MATCH_PKTTYPE=y CONFIG_NETFILTER_XT_MATCH_QUOTA=y CONFIG_NETFILTER_XT_MATCH_QUOTA2=y +CONFIG_NETFILTER_XT_MATCH_QUOTA2_LOG=y CONFIG_NETFILTER_XT_MATCH_SOCKET=y CONFIG_NETFILTER_XT_MATCH_STATE=y CONFIG_NETFILTER_XT_MATCH_STATISTIC=y diff --git a/arch/x86/configs/gki_defconfig b/arch/x86/configs/gki_defconfig index d456c9857d6c..d9961fc7df69 100644 --- a/arch/x86/configs/gki_defconfig +++ b/arch/x86/configs/gki_defconfig @@ -128,6 +128,7 @@ CONFIG_NETFILTER_XT_MATCH_POLICY=y CONFIG_NETFILTER_XT_MATCH_PKTTYPE=y CONFIG_NETFILTER_XT_MATCH_QUOTA=y CONFIG_NETFILTER_XT_MATCH_QUOTA2=y +CONFIG_NETFILTER_XT_MATCH_QUOTA2_LOG=y CONFIG_NETFILTER_XT_MATCH_SOCKET=y CONFIG_NETFILTER_XT_MATCH_STATE=y CONFIG_NETFILTER_XT_MATCH_STATISTIC=y From 49107cf033d2e3424f8532940ef80337e77135d1 Mon Sep 17 00:00:00 2001 From: Blagovest Kolenichev Date: Sat, 7 Mar 2020 12:54:11 -0800 Subject: [PATCH 89/89] Revert "BACKPORT: FROMLIST: Update Inline Encryption from v5 to v6 of patch series" This reverts commit b7b3af961465d23a733bb44d4a49bb78d4d5b08f. This is a preparation change for merging android-5.4.12 into msm-5.4 branch. The reverted change is committed already via: c83b552741 Merge fbe changes from upstream 5.4 Change-Id: I5c9aad30ee41c04e2b77372fc309568aab88d324 Signed-off-by: Blagovest Kolenichev --- Documentation/block/inline-encryption.rst | 8 +- block/Kconfig | 15 +- block/Makefile | 1 - block/bio-crypt-ctx.c | 117 ++-- block/bio.c | 8 +- block/blk-core.c | 12 +- block/blk-crypto-fallback.c | 648 ------------------- block/blk-crypto-internal.h | 58 -- block/blk-crypto.c | 748 +++++++++++++++++++--- block/blk-merge.c | 38 +- block/bounce.c | 5 +- block/keyslot-manager.c | 256 +++----- drivers/md/dm.c | 14 +- drivers/scsi/ufs/ufshcd-crypto.c | 235 ++++--- drivers/scsi/ufs/ufshcd-crypto.h | 18 +- drivers/scsi/ufs/ufshcd.c | 10 +- drivers/scsi/ufs/ufshcd.h | 4 +- fs/buffer.c | 2 +- fs/crypto/bio.c | 7 +- fs/crypto/crypto.c | 2 +- fs/crypto/fname.c | 4 +- fs/crypto/fscrypt_private.h | 130 ++-- fs/crypto/inline_crypt.c | 305 +++++---- fs/crypto/keyring.c | 6 +- fs/crypto/keysetup.c | 108 ++-- fs/crypto/keysetup_v1.c | 16 +- fs/ext4/page-io.c | 7 +- fs/ext4/readpage.c | 8 +- fs/f2fs/data.c | 28 +- fs/f2fs/super.c | 21 - include/linux/bio-crypt-ctx.h | 299 +++++---- include/linux/blk-crypto.h | 39 +- include/linux/fscrypt.h | 34 +- include/linux/keyslot-manager.h | 94 ++- 34 files changed, 1613 insertions(+), 1692 deletions(-) delete mode 100644 block/blk-crypto-fallback.c delete mode 100644 block/blk-crypto-internal.h diff --git a/Documentation/block/inline-encryption.rst b/Documentation/block/inline-encryption.rst index 330106b23c09..202826cee95e 100644 --- a/Documentation/block/inline-encryption.rst +++ b/Documentation/block/inline-encryption.rst @@ -97,7 +97,7 @@ Blk-crypto ensures that: - The bio's encryption context is programmed into a keyslot in the KSM of the request queue that the bio is being submitted to (or the crypto API fallback - KSM if the request queue doesn't have a KSM), and that the ``bc_ksm`` + KSM if the request queue doesn't have a KSM), and that the ``processing_ksm`` in the ``bi_crypt_context`` is set to this KSM - That the bio has its own individual reference to the keyslot in this KSM. @@ -107,7 +107,7 @@ Blk-crypto ensures that: ensuring that the bio has a valid reference to the keyslot when, for e.g., the crypto API fallback KSM in blk-crypto performs crypto on the device's behalf. The individual references are ensured by increasing the refcount for the - keyslot in the ``bc_ksm`` when a bio with a programmed encryption + keyslot in the ``processing_ksm`` when a bio with a programmed encryption context is cloned. @@ -120,7 +120,7 @@ been programmed into any keyslot in any KSM (for e.g. a bio from the FS). request queue the bio is being submitted to (and if this KSM does not exist, then it will program it into blk-crypto's internal KSM for crypto API fallback). The KSM that this encryption context was programmed into is stored - as the ``bc_ksm`` in the bio's ``bi_crypt_context``. + as the ``processing_ksm`` in the bio's ``bi_crypt_context``. **Case 2:** blk-crypto is given a bio whose encryption context has already been programmed into a keyslot in the *crypto API fallback* KSM. @@ -138,7 +138,7 @@ KSM). This way, when a device driver is processing a bio, it can be sure that the bio's encryption context has been programmed into some KSM (either the device driver's request queue's KSM, or blk-crypto's crypto API fallback KSM). -It then simply needs to check if the bio's ``bc_ksm`` is the device's +It then simply needs to check if the bio's processing_ksm is the device's request queue's KSM. If so, then it should proceed with IE. If not, it should simply do nothing with respect to crypto, because some other KSM (perhaps the blk-crypto crypto API fallback KSM) is handling the en/decryption. diff --git a/block/Kconfig b/block/Kconfig index e9ce5e9bc9c6..606a67e47e68 100644 --- a/block/Kconfig +++ b/block/Kconfig @@ -179,20 +179,13 @@ config BLK_SED_OPAL config BLK_INLINE_ENCRYPTION bool "Enable inline encryption support in block layer" - help - Build the blk-crypto subsystem. Enabling this lets the - block layer handle encryption, so users can take - advantage of inline encryption hardware if present. - -config BLK_INLINE_ENCRYPTION_FALLBACK - bool "Enable crypto API fallback for blk-crypto" - depends on BLK_INLINE_ENCRYPTION select CRYPTO select CRYPTO_BLKCIPHER help - Enabling this lets the block layer handle inline encryption - by falling back to the kernel crypto API when inline - encryption hardware is not present. + Build the blk-crypto subsystem. + Enabling this lets the block layer handle encryption, + so users can take advantage of inline encryption + hardware if present. menu "Partition Types" diff --git a/block/Makefile b/block/Makefile index 9bfd4c9f9e72..8932c7e4fd07 100644 --- a/block/Makefile +++ b/block/Makefile @@ -38,4 +38,3 @@ obj-$(CONFIG_BLK_SED_OPAL) += sed-opal.o obj-$(CONFIG_BLK_PM) += blk-pm.o obj-$(CONFIG_BLK_INLINE_ENCRYPTION) += keyslot-manager.o bio-crypt-ctx.o \ blk-crypto.o -obj-$(CONFIG_BLK_INLINE_ENCRYPTION_FALLBACK) += blk-crypto-fallback.o \ No newline at end of file diff --git a/block/bio-crypt-ctx.c b/block/bio-crypt-ctx.c index 75982dabc7a3..0f7641b875e9 100644 --- a/block/bio-crypt-ctx.c +++ b/block/bio-crypt-ctx.c @@ -5,43 +5,26 @@ #include #include -#include -#include #include - -#include "blk-crypto-internal.h" +#include static int num_prealloc_crypt_ctxs = 128; - -module_param(num_prealloc_crypt_ctxs, int, 0444); -MODULE_PARM_DESC(num_prealloc_crypt_ctxs, - "Number of bio crypto contexts to preallocate"); - static struct kmem_cache *bio_crypt_ctx_cache; static mempool_t *bio_crypt_ctx_pool; -int __init bio_crypt_ctx_init(void) +int bio_crypt_ctx_init(void) { - size_t i; - bio_crypt_ctx_cache = KMEM_CACHE(bio_crypt_ctx, 0); if (!bio_crypt_ctx_cache) return -ENOMEM; - bio_crypt_ctx_pool = mempool_create_slab_pool(num_prealloc_crypt_ctxs, - bio_crypt_ctx_cache); + bio_crypt_ctx_pool = mempool_create_slab_pool( + num_prealloc_crypt_ctxs, + bio_crypt_ctx_cache); + if (!bio_crypt_ctx_pool) return -ENOMEM; - /* This is assumed in various places. */ - BUILD_BUG_ON(BLK_ENCRYPTION_MODE_INVALID != 0); - - /* Sanity check that no algorithm exceeds the defined limits. */ - for (i = 0; i < BLK_ENCRYPTION_MODE_MAX; i++) { - BUG_ON(blk_crypto_modes[i].keysize > BLK_CRYPTO_MAX_KEY_SIZE); - BUG_ON(blk_crypto_modes[i].ivsize > BLK_CRYPTO_MAX_IV_SIZE); - } - return 0; } @@ -49,43 +32,51 @@ struct bio_crypt_ctx *bio_crypt_alloc_ctx(gfp_t gfp_mask) { return mempool_alloc(bio_crypt_ctx_pool, gfp_mask); } +EXPORT_SYMBOL(bio_crypt_alloc_ctx); void bio_crypt_free_ctx(struct bio *bio) { mempool_free(bio->bi_crypt_context, bio_crypt_ctx_pool); bio->bi_crypt_context = NULL; } +EXPORT_SYMBOL(bio_crypt_free_ctx); -void bio_crypt_clone(struct bio *dst, struct bio *src, gfp_t gfp_mask) +int bio_crypt_clone(struct bio *dst, struct bio *src, gfp_t gfp_mask) { - const struct bio_crypt_ctx *src_bc = src->bi_crypt_context; - /* - * If a bio is fallback_crypted, then it will be decrypted when - * bio_endio is called. As we only want the data to be decrypted once, - * copies of the bio must not have have a crypt context. + * If a bio is swhandled, then it will be decrypted when bio_endio + * is called. As we only want the data to be decrypted once, copies + * of the bio must not have have a crypt context. */ - if (!src_bc || bio_crypt_fallback_crypted(src_bc)) - return; + if (!bio_has_crypt_ctx(src) || bio_crypt_swhandled(src)) + return 0; dst->bi_crypt_context = bio_crypt_alloc_ctx(gfp_mask); - *dst->bi_crypt_context = *src_bc; + if (!dst->bi_crypt_context) + return -ENOMEM; - if (src_bc->bc_keyslot >= 0) - keyslot_manager_get_slot(src_bc->bc_ksm, src_bc->bc_keyslot); + *dst->bi_crypt_context = *src->bi_crypt_context; + + if (bio_crypt_has_keyslot(src)) + keyslot_manager_get_slot(src->bi_crypt_context->processing_ksm, + src->bi_crypt_context->keyslot); + + return 0; } -EXPORT_SYMBOL_GPL(bio_crypt_clone); +EXPORT_SYMBOL(bio_crypt_clone); -bool bio_crypt_should_process(struct request *rq) +bool bio_crypt_should_process(struct bio *bio, struct request_queue *q) { - struct bio *bio = rq->bio; - - if (!bio || !bio->bi_crypt_context) + if (!bio_has_crypt_ctx(bio)) return false; - return rq->q->ksm == bio->bi_crypt_context->bc_ksm; + if (q->ksm != bio->bi_crypt_context->processing_ksm) + return false; + + WARN_ON(!bio_crypt_has_keyslot(bio)); + return true; } -EXPORT_SYMBOL_GPL(bio_crypt_should_process); +EXPORT_SYMBOL(bio_crypt_should_process); /* * Checks that two bio crypt contexts are compatible - i.e. that @@ -96,19 +87,23 @@ bool bio_crypt_ctx_compatible(struct bio *b_1, struct bio *b_2) struct bio_crypt_ctx *bc1 = b_1->bi_crypt_context; struct bio_crypt_ctx *bc2 = b_2->bi_crypt_context; - if (bc1 != bc2) + if (bio_has_crypt_ctx(b_1) != bio_has_crypt_ctx(b_2)) return false; - return !bc1 || bc1->bc_key == bc2->bc_key; + if (!bio_has_crypt_ctx(b_1)) + return true; + + return bc1->keyslot == bc2->keyslot && + bc1->data_unit_size_bits == bc2->data_unit_size_bits; } /* * Checks that two bio crypt contexts are compatible, and also * that their data_unit_nums are continuous (and can hence be merged) - * in the order b_1 followed by b_2. */ -bool bio_crypt_ctx_mergeable(struct bio *b_1, unsigned int b1_bytes, - struct bio *b_2) +bool bio_crypt_ctx_back_mergeable(struct bio *b_1, + unsigned int b1_sectors, + struct bio *b_2) { struct bio_crypt_ctx *bc1 = b_1->bi_crypt_context; struct bio_crypt_ctx *bc2 = b_2->bi_crypt_context; @@ -116,25 +111,35 @@ bool bio_crypt_ctx_mergeable(struct bio *b_1, unsigned int b1_bytes, if (!bio_crypt_ctx_compatible(b_1, b_2)) return false; - return !bc1 || bio_crypt_dun_is_contiguous(bc1, b1_bytes, bc2->bc_dun); + return !bio_has_crypt_ctx(b_1) || + (bc1->data_unit_num + + (b1_sectors >> (bc1->data_unit_size_bits - 9)) == + bc2->data_unit_num); } -void bio_crypt_ctx_release_keyslot(struct bio_crypt_ctx *bc) +void bio_crypt_ctx_release_keyslot(struct bio *bio) { - keyslot_manager_put_slot(bc->bc_ksm, bc->bc_keyslot); - bc->bc_ksm = NULL; - bc->bc_keyslot = -1; + struct bio_crypt_ctx *crypt_ctx = bio->bi_crypt_context; + + keyslot_manager_put_slot(crypt_ctx->processing_ksm, crypt_ctx->keyslot); + bio->bi_crypt_context->processing_ksm = NULL; + bio->bi_crypt_context->keyslot = -1; } -int bio_crypt_ctx_acquire_keyslot(struct bio_crypt_ctx *bc, - struct keyslot_manager *ksm) +int bio_crypt_ctx_acquire_keyslot(struct bio *bio, struct keyslot_manager *ksm) { - int slot = keyslot_manager_get_slot_for_key(ksm, bc->bc_key); + int slot; + enum blk_crypto_mode_num crypto_mode = bio_crypto_mode(bio); + if (!ksm) + return -ENOMEM; + + slot = keyslot_manager_get_slot_for_key(ksm, + bio_crypt_raw_key(bio), crypto_mode, + 1 << bio->bi_crypt_context->data_unit_size_bits); if (slot < 0) return slot; - bc->bc_keyslot = slot; - bc->bc_ksm = ksm; + bio_crypt_set_keyslot(bio, slot, ksm); return 0; } diff --git a/block/bio.c b/block/bio.c index d2f552ae0753..2d6a840bb67f 100644 --- a/block/bio.c +++ b/block/bio.c @@ -234,8 +234,6 @@ fallback: void bio_uninit(struct bio *bio) { bio_disassociate_blkg(bio); - - bio_crypt_free_ctx(bio); } EXPORT_SYMBOL(bio_uninit); @@ -244,6 +242,7 @@ static void bio_free(struct bio *bio) struct bio_set *bs = bio->bi_pool; void *p; + bio_crypt_free_ctx(bio); bio_uninit(bio); if (bs) { @@ -654,7 +653,10 @@ struct bio *bio_clone_fast(struct bio *bio, gfp_t gfp_mask, struct bio_set *bs) __bio_clone_fast(b, bio); - bio_crypt_clone(b, bio, gfp_mask); + if (bio_crypt_clone(b, bio, gfp_mask) < 0) { + bio_put(b); + return NULL; + } if (bio_integrity(bio) && bio_integrity_clone(b, bio, gfp_mask) < 0) { diff --git a/block/blk-core.c b/block/blk-core.c index 9e10ad79df34..808adf0bb44b 100644 --- a/block/blk-core.c +++ b/block/blk-core.c @@ -1115,11 +1115,14 @@ blk_qc_t direct_make_request(struct bio *bio) { struct request_queue *q = bio->bi_disk->queue; bool nowait = bio->bi_opf & REQ_NOWAIT; - blk_qc_t ret = BLK_QC_T_NONE; + blk_qc_t ret; if (!generic_make_request_checks(bio)) return BLK_QC_T_NONE; + if (blk_crypto_submit_bio(&bio)) + return BLK_QC_T_NONE; + if (unlikely(blk_queue_enter(q, nowait ? BLK_MQ_REQ_NOWAIT : 0))) { if (nowait && !blk_queue_dying(q)) bio->bi_status = BLK_STS_AGAIN; @@ -1129,8 +1132,7 @@ blk_qc_t direct_make_request(struct bio *bio) return BLK_QC_T_NONE; } - if (!blk_crypto_submit_bio(&bio)) - ret = q->make_request_fn(q, bio); + ret = q->make_request_fn(q, bio); blk_queue_exit(q); return ret; } @@ -1817,8 +1819,8 @@ int __init blk_dev_init(void) if (bio_crypt_ctx_init() < 0) panic("Failed to allocate mem for bio crypt ctxs\n"); - if (blk_crypto_fallback_init() < 0) - panic("Failed to init blk-crypto-fallback\n"); + if (blk_crypto_init() < 0) + panic("Failed to init blk-crypto\n"); return 0; } diff --git a/block/blk-crypto-fallback.c b/block/blk-crypto-fallback.c deleted file mode 100644 index 9773bd6a5186..000000000000 --- a/block/blk-crypto-fallback.c +++ /dev/null @@ -1,648 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -/* - * Copyright 2019 Google LLC - */ - -/* - * Refer to Documentation/block/inline-encryption.rst for detailed explanation. - */ - -#define pr_fmt(fmt) "blk-crypto-fallback: " fmt - -#include -#include -#include -#include -#include -#include -#include -#include - -#include "blk-crypto-internal.h" - -static unsigned int num_prealloc_bounce_pg = 32; -module_param(num_prealloc_bounce_pg, uint, 0); -MODULE_PARM_DESC(num_prealloc_bounce_pg, - "Number of preallocated bounce pages for the blk-crypto crypto API fallback"); - -static unsigned int blk_crypto_num_keyslots = 100; -module_param_named(num_keyslots, blk_crypto_num_keyslots, uint, 0); -MODULE_PARM_DESC(num_keyslots, - "Number of keyslots for the blk-crypto crypto API fallback"); - -static unsigned int num_prealloc_fallback_crypt_ctxs = 128; -module_param(num_prealloc_fallback_crypt_ctxs, uint, 0); -MODULE_PARM_DESC(num_prealloc_crypt_fallback_ctxs, - "Number of preallocated bio fallback crypto contexts for blk-crypto to use during crypto API fallback"); - -struct bio_fallback_crypt_ctx { - struct bio_crypt_ctx crypt_ctx; - /* - * Copy of the bvec_iter when this bio was submitted. - * We only want to en/decrypt the part of the bio as described by the - * bvec_iter upon submission because bio might be split before being - * resubmitted - */ - struct bvec_iter crypt_iter; - u64 fallback_dun[BLK_CRYPTO_DUN_ARRAY_SIZE]; -}; - -/* The following few vars are only used during the crypto API fallback */ -static struct kmem_cache *bio_fallback_crypt_ctx_cache; -static mempool_t *bio_fallback_crypt_ctx_pool; - -/* - * Allocating a crypto tfm during I/O can deadlock, so we have to preallocate - * all of a mode's tfms when that mode starts being used. Since each mode may - * need all the keyslots at some point, each mode needs its own tfm for each - * keyslot; thus, a keyslot may contain tfms for multiple modes. However, to - * match the behavior of real inline encryption hardware (which only supports a - * single encryption context per keyslot), we only allow one tfm per keyslot to - * be used at a time - the rest of the unused tfms have their keys cleared. - */ -static DEFINE_MUTEX(tfms_init_lock); -static bool tfms_inited[BLK_ENCRYPTION_MODE_MAX]; - -struct blk_crypto_decrypt_work { - struct work_struct work; - struct bio *bio; -}; - -static struct blk_crypto_keyslot { - struct crypto_skcipher *tfm; - enum blk_crypto_mode_num crypto_mode; - struct crypto_skcipher *tfms[BLK_ENCRYPTION_MODE_MAX]; -} *blk_crypto_keyslots; - -/* The following few vars are only used during the crypto API fallback */ -static struct keyslot_manager *blk_crypto_ksm; -static struct workqueue_struct *blk_crypto_wq; -static mempool_t *blk_crypto_bounce_page_pool; -static struct kmem_cache *blk_crypto_decrypt_work_cache; - -bool bio_crypt_fallback_crypted(const struct bio_crypt_ctx *bc) -{ - return bc && bc->bc_ksm == blk_crypto_ksm; -} - -/* - * This is the key we set when evicting a keyslot. This *should* be the all 0's - * key, but AES-XTS rejects that key, so we use some random bytes instead. - */ -static u8 blank_key[BLK_CRYPTO_MAX_KEY_SIZE]; - -static void blk_crypto_evict_keyslot(unsigned int slot) -{ - struct blk_crypto_keyslot *slotp = &blk_crypto_keyslots[slot]; - enum blk_crypto_mode_num crypto_mode = slotp->crypto_mode; - int err; - - WARN_ON(slotp->crypto_mode == BLK_ENCRYPTION_MODE_INVALID); - - /* Clear the key in the skcipher */ - err = crypto_skcipher_setkey(slotp->tfms[crypto_mode], blank_key, - blk_crypto_modes[crypto_mode].keysize); - WARN_ON(err); - slotp->crypto_mode = BLK_ENCRYPTION_MODE_INVALID; -} - -static int blk_crypto_keyslot_program(struct keyslot_manager *ksm, - const struct blk_crypto_key *key, - unsigned int slot) -{ - struct blk_crypto_keyslot *slotp = &blk_crypto_keyslots[slot]; - const enum blk_crypto_mode_num crypto_mode = key->crypto_mode; - int err; - - if (crypto_mode != slotp->crypto_mode && - slotp->crypto_mode != BLK_ENCRYPTION_MODE_INVALID) { - blk_crypto_evict_keyslot(slot); - } - - if (!slotp->tfms[crypto_mode]) - return -ENOMEM; - slotp->crypto_mode = crypto_mode; - err = crypto_skcipher_setkey(slotp->tfms[crypto_mode], key->raw, - key->size); - if (err) { - blk_crypto_evict_keyslot(slot); - return err; - } - return 0; -} - -static int blk_crypto_keyslot_evict(struct keyslot_manager *ksm, - const struct blk_crypto_key *key, - unsigned int slot) -{ - blk_crypto_evict_keyslot(slot); - return 0; -} - -/* - * The crypto API fallback KSM ops - only used for a bio when it specifies a - * blk_crypto_mode for which we failed to get a keyslot in the device's inline - * encryption hardware (which probably means the device doesn't have inline - * encryption hardware that supports that crypto mode). - */ -static const struct keyslot_mgmt_ll_ops blk_crypto_ksm_ll_ops = { - .keyslot_program = blk_crypto_keyslot_program, - .keyslot_evict = blk_crypto_keyslot_evict, -}; - -static void blk_crypto_encrypt_endio(struct bio *enc_bio) -{ - struct bio *src_bio = enc_bio->bi_private; - int i; - - for (i = 0; i < enc_bio->bi_vcnt; i++) - mempool_free(enc_bio->bi_io_vec[i].bv_page, - blk_crypto_bounce_page_pool); - - src_bio->bi_status = enc_bio->bi_status; - - bio_put(enc_bio); - bio_endio(src_bio); -} - -static struct bio *blk_crypto_clone_bio(struct bio *bio_src) -{ - struct bvec_iter iter; - struct bio_vec bv; - struct bio *bio; - - bio = bio_alloc_bioset(GFP_NOIO, bio_segments(bio_src), NULL); - if (!bio) - return NULL; - bio->bi_disk = bio_src->bi_disk; - bio->bi_opf = bio_src->bi_opf; - bio->bi_ioprio = bio_src->bi_ioprio; - bio->bi_write_hint = bio_src->bi_write_hint; - bio->bi_iter.bi_sector = bio_src->bi_iter.bi_sector; - bio->bi_iter.bi_size = bio_src->bi_iter.bi_size; - - bio_for_each_segment(bv, bio_src, iter) - bio->bi_io_vec[bio->bi_vcnt++] = bv; - - if (bio_integrity(bio_src) && - bio_integrity_clone(bio, bio_src, GFP_NOIO) < 0) { - bio_put(bio); - return NULL; - } - - bio_clone_blkg_association(bio, bio_src); - blkcg_bio_issue_init(bio); - - return bio; -} - -static int blk_crypto_alloc_cipher_req(struct bio *src_bio, - struct skcipher_request **ciph_req_ret, - struct crypto_wait *wait) -{ - struct skcipher_request *ciph_req; - const struct blk_crypto_keyslot *slotp; - - slotp = &blk_crypto_keyslots[src_bio->bi_crypt_context->bc_keyslot]; - ciph_req = skcipher_request_alloc(slotp->tfms[slotp->crypto_mode], - GFP_NOIO); - if (!ciph_req) { - src_bio->bi_status = BLK_STS_RESOURCE; - return -ENOMEM; - } - - skcipher_request_set_callback(ciph_req, - CRYPTO_TFM_REQ_MAY_BACKLOG | - CRYPTO_TFM_REQ_MAY_SLEEP, - crypto_req_done, wait); - *ciph_req_ret = ciph_req; - return 0; -} - -static int blk_crypto_split_bio_if_needed(struct bio **bio_ptr) -{ - struct bio *bio = *bio_ptr; - unsigned int i = 0; - unsigned int num_sectors = 0; - struct bio_vec bv; - struct bvec_iter iter; - - bio_for_each_segment(bv, bio, iter) { - num_sectors += bv.bv_len >> SECTOR_SHIFT; - if (++i == BIO_MAX_PAGES) - break; - } - if (num_sectors < bio_sectors(bio)) { - struct bio *split_bio; - - split_bio = bio_split(bio, num_sectors, GFP_NOIO, NULL); - if (!split_bio) { - bio->bi_status = BLK_STS_RESOURCE; - return -ENOMEM; - } - bio_chain(split_bio, bio); - generic_make_request(bio); - *bio_ptr = split_bio; - } - return 0; -} - -union blk_crypto_iv { - __le64 dun[BLK_CRYPTO_DUN_ARRAY_SIZE]; - u8 bytes[BLK_CRYPTO_MAX_IV_SIZE]; -}; - -static void blk_crypto_dun_to_iv(const u64 dun[BLK_CRYPTO_DUN_ARRAY_SIZE], - union blk_crypto_iv *iv) -{ - int i; - - for (i = 0; i < BLK_CRYPTO_DUN_ARRAY_SIZE; i++) - iv->dun[i] = cpu_to_le64(dun[i]); -} - -/* - * The crypto API fallback's encryption routine. - * Allocate a bounce bio for encryption, encrypt the input bio using crypto API, - * and replace *bio_ptr with the bounce bio. May split input bio if it's too - * large. - */ -static int blk_crypto_encrypt_bio(struct bio **bio_ptr) -{ - struct bio *src_bio; - struct skcipher_request *ciph_req = NULL; - DECLARE_CRYPTO_WAIT(wait); - u64 curr_dun[BLK_CRYPTO_DUN_ARRAY_SIZE]; - union blk_crypto_iv iv; - struct scatterlist src, dst; - struct bio *enc_bio; - unsigned int i, j; - int data_unit_size; - struct bio_crypt_ctx *bc; - int err = 0; - - /* Split the bio if it's too big for single page bvec */ - err = blk_crypto_split_bio_if_needed(bio_ptr); - if (err) - return err; - - src_bio = *bio_ptr; - bc = src_bio->bi_crypt_context; - data_unit_size = bc->bc_key->data_unit_size; - - /* Allocate bounce bio for encryption */ - enc_bio = blk_crypto_clone_bio(src_bio); - if (!enc_bio) { - src_bio->bi_status = BLK_STS_RESOURCE; - return -ENOMEM; - } - - /* - * Use the crypto API fallback keyslot manager to get a crypto_skcipher - * for the algorithm and key specified for this bio. - */ - err = bio_crypt_ctx_acquire_keyslot(bc, blk_crypto_ksm); - if (err) { - src_bio->bi_status = BLK_STS_IOERR; - goto out_put_enc_bio; - } - - /* and then allocate an skcipher_request for it */ - err = blk_crypto_alloc_cipher_req(src_bio, &ciph_req, &wait); - if (err) - goto out_release_keyslot; - - memcpy(curr_dun, bc->bc_dun, sizeof(curr_dun)); - sg_init_table(&src, 1); - sg_init_table(&dst, 1); - - skcipher_request_set_crypt(ciph_req, &src, &dst, data_unit_size, - iv.bytes); - - /* Encrypt each page in the bounce bio */ - for (i = 0; i < enc_bio->bi_vcnt; i++) { - struct bio_vec *enc_bvec = &enc_bio->bi_io_vec[i]; - struct page *plaintext_page = enc_bvec->bv_page; - struct page *ciphertext_page = - mempool_alloc(blk_crypto_bounce_page_pool, GFP_NOIO); - - enc_bvec->bv_page = ciphertext_page; - - if (!ciphertext_page) { - src_bio->bi_status = BLK_STS_RESOURCE; - err = -ENOMEM; - goto out_free_bounce_pages; - } - - sg_set_page(&src, plaintext_page, data_unit_size, - enc_bvec->bv_offset); - sg_set_page(&dst, ciphertext_page, data_unit_size, - enc_bvec->bv_offset); - - /* Encrypt each data unit in this page */ - for (j = 0; j < enc_bvec->bv_len; j += data_unit_size) { - blk_crypto_dun_to_iv(curr_dun, &iv); - err = crypto_wait_req(crypto_skcipher_encrypt(ciph_req), - &wait); - if (err) { - i++; - src_bio->bi_status = BLK_STS_RESOURCE; - goto out_free_bounce_pages; - } - bio_crypt_dun_increment(curr_dun, 1); - src.offset += data_unit_size; - dst.offset += data_unit_size; - } - } - - enc_bio->bi_private = src_bio; - enc_bio->bi_end_io = blk_crypto_encrypt_endio; - *bio_ptr = enc_bio; - - enc_bio = NULL; - err = 0; - goto out_free_ciph_req; - -out_free_bounce_pages: - while (i > 0) - mempool_free(enc_bio->bi_io_vec[--i].bv_page, - blk_crypto_bounce_page_pool); -out_free_ciph_req: - skcipher_request_free(ciph_req); -out_release_keyslot: - bio_crypt_ctx_release_keyslot(bc); -out_put_enc_bio: - if (enc_bio) - bio_put(enc_bio); - - return err; -} - -static void blk_crypto_free_fallback_crypt_ctx(struct bio *bio) -{ - mempool_free(container_of(bio->bi_crypt_context, - struct bio_fallback_crypt_ctx, - crypt_ctx), - bio_fallback_crypt_ctx_pool); - bio->bi_crypt_context = NULL; -} - -/* - * The crypto API fallback's main decryption routine. - * Decrypts input bio in place. - */ -static void blk_crypto_decrypt_bio(struct work_struct *work) -{ - struct blk_crypto_decrypt_work *decrypt_work = - container_of(work, struct blk_crypto_decrypt_work, work); - struct bio *bio = decrypt_work->bio; - struct skcipher_request *ciph_req = NULL; - DECLARE_CRYPTO_WAIT(wait); - struct bio_vec bv; - struct bvec_iter iter; - u64 curr_dun[BLK_CRYPTO_DUN_ARRAY_SIZE]; - union blk_crypto_iv iv; - struct scatterlist sg; - struct bio_crypt_ctx *bc = bio->bi_crypt_context; - struct bio_fallback_crypt_ctx *f_ctx = - container_of(bc, struct bio_fallback_crypt_ctx, crypt_ctx); - const int data_unit_size = bc->bc_key->data_unit_size; - unsigned int i; - int err; - - /* - * Use the crypto API fallback keyslot manager to get a crypto_skcipher - * for the algorithm and key specified for this bio. - */ - if (bio_crypt_ctx_acquire_keyslot(bc, blk_crypto_ksm)) { - bio->bi_status = BLK_STS_RESOURCE; - goto out_no_keyslot; - } - - /* and then allocate an skcipher_request for it */ - err = blk_crypto_alloc_cipher_req(bio, &ciph_req, &wait); - if (err) - goto out; - - memcpy(curr_dun, f_ctx->fallback_dun, sizeof(curr_dun)); - sg_init_table(&sg, 1); - skcipher_request_set_crypt(ciph_req, &sg, &sg, data_unit_size, - iv.bytes); - - /* Decrypt each segment in the bio */ - __bio_for_each_segment(bv, bio, iter, f_ctx->crypt_iter) { - struct page *page = bv.bv_page; - - sg_set_page(&sg, page, data_unit_size, bv.bv_offset); - - /* Decrypt each data unit in the segment */ - for (i = 0; i < bv.bv_len; i += data_unit_size) { - blk_crypto_dun_to_iv(curr_dun, &iv); - if (crypto_wait_req(crypto_skcipher_decrypt(ciph_req), - &wait)) { - bio->bi_status = BLK_STS_IOERR; - goto out; - } - bio_crypt_dun_increment(curr_dun, 1); - sg.offset += data_unit_size; - } - } - -out: - skcipher_request_free(ciph_req); - bio_crypt_ctx_release_keyslot(bc); -out_no_keyslot: - kmem_cache_free(blk_crypto_decrypt_work_cache, decrypt_work); - blk_crypto_free_fallback_crypt_ctx(bio); - bio_endio(bio); -} - -/* - * Queue bio for decryption. - * Returns true iff bio was queued for decryption. - */ -bool blk_crypto_queue_decrypt_bio(struct bio *bio) -{ - struct blk_crypto_decrypt_work *decrypt_work; - - /* If there was an IO error, don't queue for decrypt. */ - if (bio->bi_status) - goto out; - - decrypt_work = kmem_cache_zalloc(blk_crypto_decrypt_work_cache, - GFP_ATOMIC); - if (!decrypt_work) { - bio->bi_status = BLK_STS_RESOURCE; - goto out; - } - - INIT_WORK(&decrypt_work->work, blk_crypto_decrypt_bio); - decrypt_work->bio = bio; - queue_work(blk_crypto_wq, &decrypt_work->work); - - return true; -out: - blk_crypto_free_fallback_crypt_ctx(bio); - return false; -} - -/** - * blk_crypto_start_using_mode() - Start using a crypto algorithm on a device - * @mode_num: the blk_crypto_mode we want to allocate ciphers for. - * @data_unit_size: the data unit size that will be used - * @q: the request queue for the device - * - * Upper layers must call this function to ensure that a the crypto API fallback - * has transforms for this algorithm, if they become necessary. - * - * Return: 0 on success and -err on error. - */ -int blk_crypto_start_using_mode(enum blk_crypto_mode_num mode_num, - unsigned int data_unit_size, - struct request_queue *q) -{ - struct blk_crypto_keyslot *slotp; - unsigned int i; - int err = 0; - - /* - * Fast path - * Ensure that updates to blk_crypto_keyslots[i].tfms[mode_num] - * for each i are visible before we try to access them. - */ - if (likely(smp_load_acquire(&tfms_inited[mode_num]))) - return 0; - - /* - * If the keyslot manager of the request queue supports this - * crypto mode, then we don't need to allocate this mode. - */ - if (keyslot_manager_crypto_mode_supported(q->ksm, mode_num, - data_unit_size)) - return 0; - - mutex_lock(&tfms_init_lock); - if (likely(tfms_inited[mode_num])) - goto out; - - for (i = 0; i < blk_crypto_num_keyslots; i++) { - slotp = &blk_crypto_keyslots[i]; - slotp->tfms[mode_num] = crypto_alloc_skcipher( - blk_crypto_modes[mode_num].cipher_str, - 0, 0); - if (IS_ERR(slotp->tfms[mode_num])) { - err = PTR_ERR(slotp->tfms[mode_num]); - slotp->tfms[mode_num] = NULL; - goto out_free_tfms; - } - - crypto_skcipher_set_flags(slotp->tfms[mode_num], - CRYPTO_TFM_REQ_FORBID_WEAK_KEYS); - } - - /* - * Ensure that updates to blk_crypto_keyslots[i].tfms[mode_num] - * for each i are visible before we set tfms_inited[mode_num]. - */ - smp_store_release(&tfms_inited[mode_num], true); - goto out; - -out_free_tfms: - for (i = 0; i < blk_crypto_num_keyslots; i++) { - slotp = &blk_crypto_keyslots[i]; - crypto_free_skcipher(slotp->tfms[mode_num]); - slotp->tfms[mode_num] = NULL; - } -out: - mutex_unlock(&tfms_init_lock); - return err; -} - -int blk_crypto_fallback_evict_key(const struct blk_crypto_key *key) -{ - return keyslot_manager_evict_key(blk_crypto_ksm, key); -} - -int blk_crypto_fallback_submit_bio(struct bio **bio_ptr) -{ - struct bio *bio = *bio_ptr; - struct bio_crypt_ctx *bc = bio->bi_crypt_context; - struct bio_fallback_crypt_ctx *f_ctx; - - if (WARN_ON_ONCE(!tfms_inited[bc->bc_key->crypto_mode])) { - bio->bi_status = BLK_STS_IOERR; - return -EIO; - } - - if (bio_data_dir(bio) == WRITE) - return blk_crypto_encrypt_bio(bio_ptr); - - /* - * Mark bio as fallback crypted and replace the bio_crypt_ctx with - * another one contained in a bio_fallback_crypt_ctx, so that the - * fallback has space to store the info it needs for decryption. - */ - bc->bc_ksm = blk_crypto_ksm; - f_ctx = mempool_alloc(bio_fallback_crypt_ctx_pool, GFP_NOIO); - f_ctx->crypt_ctx = *bc; - memcpy(f_ctx->fallback_dun, bc->bc_dun, sizeof(f_ctx->fallback_dun)); - f_ctx->crypt_iter = bio->bi_iter; - - bio_crypt_free_ctx(bio); - bio->bi_crypt_context = &f_ctx->crypt_ctx; - - return 0; -} - -int __init blk_crypto_fallback_init(void) -{ - int i; - unsigned int crypto_mode_supported[BLK_ENCRYPTION_MODE_MAX]; - - prandom_bytes(blank_key, BLK_CRYPTO_MAX_KEY_SIZE); - - /* All blk-crypto modes have a crypto API fallback. */ - for (i = 0; i < BLK_ENCRYPTION_MODE_MAX; i++) - crypto_mode_supported[i] = 0xFFFFFFFF; - crypto_mode_supported[BLK_ENCRYPTION_MODE_INVALID] = 0; - - blk_crypto_ksm = keyslot_manager_create(blk_crypto_num_keyslots, - &blk_crypto_ksm_ll_ops, - crypto_mode_supported, NULL); - if (!blk_crypto_ksm) - return -ENOMEM; - - blk_crypto_wq = alloc_workqueue("blk_crypto_wq", - WQ_UNBOUND | WQ_HIGHPRI | - WQ_MEM_RECLAIM, num_online_cpus()); - if (!blk_crypto_wq) - return -ENOMEM; - - blk_crypto_keyslots = kcalloc(blk_crypto_num_keyslots, - sizeof(blk_crypto_keyslots[0]), - GFP_KERNEL); - if (!blk_crypto_keyslots) - return -ENOMEM; - - blk_crypto_bounce_page_pool = - mempool_create_page_pool(num_prealloc_bounce_pg, 0); - if (!blk_crypto_bounce_page_pool) - return -ENOMEM; - - blk_crypto_decrypt_work_cache = KMEM_CACHE(blk_crypto_decrypt_work, - SLAB_RECLAIM_ACCOUNT); - if (!blk_crypto_decrypt_work_cache) - return -ENOMEM; - - bio_fallback_crypt_ctx_cache = KMEM_CACHE(bio_fallback_crypt_ctx, 0); - if (!bio_fallback_crypt_ctx_cache) - return -ENOMEM; - - bio_fallback_crypt_ctx_pool = - mempool_create_slab_pool(num_prealloc_fallback_crypt_ctxs, - bio_fallback_crypt_ctx_cache); - if (!bio_fallback_crypt_ctx_pool) - return -ENOMEM; - - return 0; -} diff --git a/block/blk-crypto-internal.h b/block/blk-crypto-internal.h deleted file mode 100644 index 43351eecc97a..000000000000 --- a/block/blk-crypto-internal.h +++ /dev/null @@ -1,58 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -/* - * Copyright 2019 Google LLC - */ - -#ifndef __LINUX_BLK_CRYPTO_INTERNAL_H -#define __LINUX_BLK_CRYPTO_INTERNAL_H - -#include - -/* Represents a crypto mode supported by blk-crypto */ -struct blk_crypto_mode { - const char *cipher_str; /* crypto API name (for fallback case) */ - unsigned int keysize; /* key size in bytes */ - unsigned int ivsize; /* iv size in bytes */ -}; - -extern const struct blk_crypto_mode blk_crypto_modes[]; - -#ifdef CONFIG_BLK_INLINE_ENCRYPTION_FALLBACK - -int blk_crypto_fallback_submit_bio(struct bio **bio_ptr); - -bool blk_crypto_queue_decrypt_bio(struct bio *bio); - -int blk_crypto_fallback_evict_key(const struct blk_crypto_key *key); - -bool bio_crypt_fallback_crypted(const struct bio_crypt_ctx *bc); - -#else /* CONFIG_BLK_INLINE_ENCRYPTION_FALLBACK */ - -static inline bool bio_crypt_fallback_crypted(const struct bio_crypt_ctx *bc) -{ - return false; -} - -static inline int blk_crypto_fallback_submit_bio(struct bio **bio_ptr) -{ - pr_warn_once("blk-crypto crypto API fallback disabled; failing request"); - (*bio_ptr)->bi_status = BLK_STS_NOTSUPP; - return -EIO; -} - -static inline bool blk_crypto_queue_decrypt_bio(struct bio *bio) -{ - WARN_ON(1); - return false; -} - -static inline int -blk_crypto_fallback_evict_key(const struct blk_crypto_key *key) -{ - return 0; -} - -#endif /* CONFIG_BLK_INLINE_ENCRYPTION_FALLBACK */ - -#endif /* __LINUX_BLK_CRYPTO_INTERNAL_H */ diff --git a/block/blk-crypto.c b/block/blk-crypto.c index 2c47e8eec865..89649655bf4b 100644 --- a/block/blk-crypto.c +++ b/block/blk-crypto.c @@ -10,36 +10,219 @@ #define pr_fmt(fmt) "blk-crypto: " fmt #include -#include #include -#include -#include +#include +#include +#include +#include +#include +#include +#include -#include "blk-crypto-internal.h" +/* Represents a crypto mode supported by blk-crypto */ +struct blk_crypto_mode { + const char *cipher_str; /* crypto API name (for fallback case) */ + size_t keysize; /* key size in bytes */ +}; -const struct blk_crypto_mode blk_crypto_modes[] = { +static const struct blk_crypto_mode blk_crypto_modes[] = { [BLK_ENCRYPTION_MODE_AES_256_XTS] = { .cipher_str = "xts(aes)", .keysize = 64, - .ivsize = 16, - }, - [BLK_ENCRYPTION_MODE_AES_128_CBC_ESSIV] = { - .cipher_str = "essiv(cbc(aes),sha256)", - .keysize = 16, - .ivsize = 16, - }, - [BLK_ENCRYPTION_MODE_ADIANTUM] = { - .cipher_str = "adiantum(xchacha12,aes)", - .keysize = 32, - .ivsize = 32, }, }; +static unsigned int num_prealloc_bounce_pg = 32; +module_param(num_prealloc_bounce_pg, uint, 0); +MODULE_PARM_DESC(num_prealloc_bounce_pg, + "Number of preallocated bounce pages for blk-crypto to use during crypto API fallback encryption"); + +#define BLK_CRYPTO_MAX_KEY_SIZE 64 +static int blk_crypto_num_keyslots = 100; +module_param_named(num_keyslots, blk_crypto_num_keyslots, int, 0); +MODULE_PARM_DESC(num_keyslots, + "Number of keyslots for crypto API fallback in blk-crypto."); + +static struct blk_crypto_keyslot { + struct crypto_skcipher *tfm; + enum blk_crypto_mode_num crypto_mode; + u8 key[BLK_CRYPTO_MAX_KEY_SIZE]; + struct crypto_skcipher *tfms[ARRAY_SIZE(blk_crypto_modes)]; +} *blk_crypto_keyslots; + +/* + * Allocating a crypto tfm during I/O can deadlock, so we have to preallocate + * all of a mode's tfms when that mode starts being used. Since each mode may + * need all the keyslots at some point, each mode needs its own tfm for each + * keyslot; thus, a keyslot may contain tfms for multiple modes. However, to + * match the behavior of real inline encryption hardware (which only supports a + * single encryption context per keyslot), we only allow one tfm per keyslot to + * be used at a time - the rest of the unused tfms have their keys cleared. + */ +static struct mutex tfms_lock[ARRAY_SIZE(blk_crypto_modes)]; +static bool tfms_inited[ARRAY_SIZE(blk_crypto_modes)]; + +struct work_mem { + struct work_struct crypto_work; + struct bio *bio; +}; + +/* The following few vars are only used during the crypto API fallback */ +static struct keyslot_manager *blk_crypto_ksm; +static struct workqueue_struct *blk_crypto_wq; +static mempool_t *blk_crypto_page_pool; +static struct kmem_cache *blk_crypto_work_mem_cache; + +bool bio_crypt_swhandled(struct bio *bio) +{ + return bio_has_crypt_ctx(bio) && + bio->bi_crypt_context->processing_ksm == blk_crypto_ksm; +} + +static u8 blank_key[BLK_CRYPTO_MAX_KEY_SIZE]; +static void evict_keyslot(unsigned int slot) +{ + struct blk_crypto_keyslot *slotp = &blk_crypto_keyslots[slot]; + enum blk_crypto_mode_num crypto_mode = slotp->crypto_mode; + int err; + + WARN_ON(slotp->crypto_mode == BLK_ENCRYPTION_MODE_INVALID); + + /* Clear the key in the skcipher */ + err = crypto_skcipher_setkey(slotp->tfms[crypto_mode], blank_key, + blk_crypto_modes[crypto_mode].keysize); + WARN_ON(err); + memzero_explicit(slotp->key, BLK_CRYPTO_MAX_KEY_SIZE); + slotp->crypto_mode = BLK_ENCRYPTION_MODE_INVALID; +} + +static int blk_crypto_keyslot_program(void *priv, const u8 *key, + enum blk_crypto_mode_num crypto_mode, + unsigned int data_unit_size, + unsigned int slot) +{ + struct blk_crypto_keyslot *slotp = &blk_crypto_keyslots[slot]; + const struct blk_crypto_mode *mode = &blk_crypto_modes[crypto_mode]; + size_t keysize = mode->keysize; + int err; + + if (crypto_mode != slotp->crypto_mode && + slotp->crypto_mode != BLK_ENCRYPTION_MODE_INVALID) { + evict_keyslot(slot); + } + + if (!slotp->tfms[crypto_mode]) + return -ENOMEM; + slotp->crypto_mode = crypto_mode; + err = crypto_skcipher_setkey(slotp->tfms[crypto_mode], key, keysize); + + if (err) { + evict_keyslot(slot); + return err; + } + + memcpy(slotp->key, key, keysize); + + return 0; +} + +static int blk_crypto_keyslot_evict(void *priv, const u8 *key, + enum blk_crypto_mode_num crypto_mode, + unsigned int data_unit_size, + unsigned int slot) +{ + evict_keyslot(slot); + return 0; +} + +static int blk_crypto_keyslot_find(void *priv, + const u8 *key, + enum blk_crypto_mode_num crypto_mode, + unsigned int data_unit_size_bytes) +{ + int slot; + const size_t keysize = blk_crypto_modes[crypto_mode].keysize; + + for (slot = 0; slot < blk_crypto_num_keyslots; slot++) { + if (blk_crypto_keyslots[slot].crypto_mode == crypto_mode && + !crypto_memneq(blk_crypto_keyslots[slot].key, key, keysize)) + return slot; + } + + return -ENOKEY; +} + +static bool blk_crypto_mode_supported(void *priv, + enum blk_crypto_mode_num crypt_mode, + unsigned int data_unit_size) +{ + /* All blk_crypto_modes are required to have a crypto API fallback. */ + return true; +} + +/* + * The crypto API fallback KSM ops - only used for a bio when it specifies a + * blk_crypto_mode for which we failed to get a keyslot in the device's inline + * encryption hardware (which probably means the device doesn't have inline + * encryption hardware that supports that crypto mode). + */ +static const struct keyslot_mgmt_ll_ops blk_crypto_ksm_ll_ops = { + .keyslot_program = blk_crypto_keyslot_program, + .keyslot_evict = blk_crypto_keyslot_evict, + .keyslot_find = blk_crypto_keyslot_find, + .crypto_mode_supported = blk_crypto_mode_supported, +}; + +static void blk_crypto_encrypt_endio(struct bio *enc_bio) +{ + struct bio *src_bio = enc_bio->bi_private; + int i; + + for (i = 0; i < enc_bio->bi_vcnt; i++) + mempool_free(enc_bio->bi_io_vec[i].bv_page, + blk_crypto_page_pool); + + src_bio->bi_status = enc_bio->bi_status; + + bio_put(enc_bio); + bio_endio(src_bio); +} + +static struct bio *blk_crypto_clone_bio(struct bio *bio_src) +{ + struct bvec_iter iter; + struct bio_vec bv; + struct bio *bio; + + bio = bio_alloc_bioset(GFP_NOIO, bio_segments(bio_src), NULL); + if (!bio) + return NULL; + bio->bi_disk = bio_src->bi_disk; + bio->bi_opf = bio_src->bi_opf; + bio->bi_ioprio = bio_src->bi_ioprio; + bio->bi_write_hint = bio_src->bi_write_hint; + bio->bi_iter.bi_sector = bio_src->bi_iter.bi_sector; + bio->bi_iter.bi_size = bio_src->bi_iter.bi_size; + + bio_for_each_segment(bv, bio_src, iter) + bio->bi_io_vec[bio->bi_vcnt++] = bv; + + if (bio_integrity(bio_src) && + bio_integrity_clone(bio, bio_src, GFP_NOIO) < 0) { + bio_put(bio); + return NULL; + } + + bio_clone_blkg_association(bio, bio_src); + blkcg_bio_issue_init(bio); + + return bio; +} + /* Check that all I/O segments are data unit aligned */ static int bio_crypt_check_alignment(struct bio *bio) { - const unsigned int data_unit_size = - bio->bi_crypt_context->bc_key->data_unit_size; + int data_unit_size = 1 << bio->bi_crypt_context->data_unit_size_bits; struct bvec_iter iter; struct bio_vec bv; @@ -50,6 +233,268 @@ static int bio_crypt_check_alignment(struct bio *bio) return 0; } +static int blk_crypto_alloc_cipher_req(struct bio *src_bio, + struct skcipher_request **ciph_req_ptr, + struct crypto_wait *wait) +{ + int slot; + struct skcipher_request *ciph_req; + struct blk_crypto_keyslot *slotp; + + slot = bio_crypt_get_keyslot(src_bio); + slotp = &blk_crypto_keyslots[slot]; + ciph_req = skcipher_request_alloc(slotp->tfms[slotp->crypto_mode], + GFP_NOIO); + if (!ciph_req) { + src_bio->bi_status = BLK_STS_RESOURCE; + return -ENOMEM; + } + + skcipher_request_set_callback(ciph_req, + CRYPTO_TFM_REQ_MAY_BACKLOG | + CRYPTO_TFM_REQ_MAY_SLEEP, + crypto_req_done, wait); + *ciph_req_ptr = ciph_req; + return 0; +} + +static int blk_crypto_split_bio_if_needed(struct bio **bio_ptr) +{ + struct bio *bio = *bio_ptr; + unsigned int i = 0; + unsigned int num_sectors = 0; + struct bio_vec bv; + struct bvec_iter iter; + + bio_for_each_segment(bv, bio, iter) { + num_sectors += bv.bv_len >> SECTOR_SHIFT; + if (++i == BIO_MAX_PAGES) + break; + } + if (num_sectors < bio_sectors(bio)) { + struct bio *split_bio; + + split_bio = bio_split(bio, num_sectors, GFP_NOIO, NULL); + if (!split_bio) { + bio->bi_status = BLK_STS_RESOURCE; + return -ENOMEM; + } + bio_chain(split_bio, bio); + generic_make_request(bio); + *bio_ptr = split_bio; + } + return 0; +} + +/* + * The crypto API fallback's encryption routine. + * Allocate a bounce bio for encryption, encrypt the input bio using + * crypto API, and replace *bio_ptr with the bounce bio. May split input + * bio if it's too large. + */ +static int blk_crypto_encrypt_bio(struct bio **bio_ptr) +{ + struct bio *src_bio; + struct skcipher_request *ciph_req = NULL; + DECLARE_CRYPTO_WAIT(wait); + int err = 0; + u64 curr_dun; + union { + __le64 dun; + u8 bytes[16]; + } iv; + struct scatterlist src, dst; + struct bio *enc_bio; + struct bio_vec *enc_bvec; + int i, j; + int data_unit_size; + + /* Split the bio if it's too big for single page bvec */ + err = blk_crypto_split_bio_if_needed(bio_ptr); + if (err) + return err; + + src_bio = *bio_ptr; + data_unit_size = 1 << src_bio->bi_crypt_context->data_unit_size_bits; + + /* Allocate bounce bio for encryption */ + enc_bio = blk_crypto_clone_bio(src_bio); + if (!enc_bio) { + src_bio->bi_status = BLK_STS_RESOURCE; + return -ENOMEM; + } + + /* + * Use the crypto API fallback keyslot manager to get a crypto_skcipher + * for the algorithm and key specified for this bio. + */ + err = bio_crypt_ctx_acquire_keyslot(src_bio, blk_crypto_ksm); + if (err) { + src_bio->bi_status = BLK_STS_IOERR; + goto out_put_enc_bio; + } + + /* and then allocate an skcipher_request for it */ + err = blk_crypto_alloc_cipher_req(src_bio, &ciph_req, &wait); + if (err) + goto out_release_keyslot; + + curr_dun = bio_crypt_data_unit_num(src_bio); + sg_init_table(&src, 1); + sg_init_table(&dst, 1); + + skcipher_request_set_crypt(ciph_req, &src, &dst, + data_unit_size, iv.bytes); + + /* Encrypt each page in the bounce bio */ + for (i = 0, enc_bvec = enc_bio->bi_io_vec; i < enc_bio->bi_vcnt; + enc_bvec++, i++) { + struct page *plaintext_page = enc_bvec->bv_page; + struct page *ciphertext_page = + mempool_alloc(blk_crypto_page_pool, GFP_NOIO); + + enc_bvec->bv_page = ciphertext_page; + + if (!ciphertext_page) { + src_bio->bi_status = BLK_STS_RESOURCE; + err = -ENOMEM; + goto out_free_bounce_pages; + } + + sg_set_page(&src, plaintext_page, data_unit_size, + enc_bvec->bv_offset); + sg_set_page(&dst, ciphertext_page, data_unit_size, + enc_bvec->bv_offset); + + /* Encrypt each data unit in this page */ + for (j = 0; j < enc_bvec->bv_len; j += data_unit_size) { + memset(&iv, 0, sizeof(iv)); + iv.dun = cpu_to_le64(curr_dun); + + err = crypto_wait_req(crypto_skcipher_encrypt(ciph_req), + &wait); + if (err) { + i++; + src_bio->bi_status = BLK_STS_RESOURCE; + goto out_free_bounce_pages; + } + curr_dun++; + src.offset += data_unit_size; + dst.offset += data_unit_size; + } + } + + enc_bio->bi_private = src_bio; + enc_bio->bi_end_io = blk_crypto_encrypt_endio; + *bio_ptr = enc_bio; + + enc_bio = NULL; + err = 0; + goto out_free_ciph_req; + +out_free_bounce_pages: + while (i > 0) + mempool_free(enc_bio->bi_io_vec[--i].bv_page, + blk_crypto_page_pool); +out_free_ciph_req: + skcipher_request_free(ciph_req); +out_release_keyslot: + bio_crypt_ctx_release_keyslot(src_bio); +out_put_enc_bio: + if (enc_bio) + bio_put(enc_bio); + + return err; +} + +/* + * The crypto API fallback's main decryption routine. + * Decrypts input bio in place. + */ +static void blk_crypto_decrypt_bio(struct work_struct *w) +{ + struct work_mem *work_mem = + container_of(w, struct work_mem, crypto_work); + struct bio *bio = work_mem->bio; + struct skcipher_request *ciph_req = NULL; + DECLARE_CRYPTO_WAIT(wait); + struct bio_vec bv; + struct bvec_iter iter; + u64 curr_dun; + union { + __le64 dun; + u8 bytes[16]; + } iv; + struct scatterlist sg; + int data_unit_size = 1 << bio->bi_crypt_context->data_unit_size_bits; + int i; + int err; + + /* + * Use the crypto API fallback keyslot manager to get a crypto_skcipher + * for the algorithm and key specified for this bio. + */ + if (bio_crypt_ctx_acquire_keyslot(bio, blk_crypto_ksm)) { + bio->bi_status = BLK_STS_RESOURCE; + goto out_no_keyslot; + } + + /* and then allocate an skcipher_request for it */ + err = blk_crypto_alloc_cipher_req(bio, &ciph_req, &wait); + if (err) + goto out; + + curr_dun = bio_crypt_sw_data_unit_num(bio); + sg_init_table(&sg, 1); + skcipher_request_set_crypt(ciph_req, &sg, &sg, data_unit_size, + iv.bytes); + + /* Decrypt each segment in the bio */ + __bio_for_each_segment(bv, bio, iter, + bio->bi_crypt_context->crypt_iter) { + struct page *page = bv.bv_page; + + sg_set_page(&sg, page, data_unit_size, bv.bv_offset); + + /* Decrypt each data unit in the segment */ + for (i = 0; i < bv.bv_len; i += data_unit_size) { + memset(&iv, 0, sizeof(iv)); + iv.dun = cpu_to_le64(curr_dun); + if (crypto_wait_req(crypto_skcipher_decrypt(ciph_req), + &wait)) { + bio->bi_status = BLK_STS_IOERR; + goto out; + } + curr_dun++; + sg.offset += data_unit_size; + } + } + +out: + skcipher_request_free(ciph_req); + bio_crypt_ctx_release_keyslot(bio); +out_no_keyslot: + kmem_cache_free(blk_crypto_work_mem_cache, work_mem); + bio_endio(bio); +} + +/* Queue bio for decryption */ +static void blk_crypto_queue_decrypt_bio(struct bio *bio) +{ + struct work_mem *work_mem = + kmem_cache_zalloc(blk_crypto_work_mem_cache, GFP_ATOMIC); + + if (!work_mem) { + bio->bi_status = BLK_STS_RESOURCE; + bio_endio(bio); + return; + } + + INIT_WORK(&work_mem->crypto_work, blk_crypto_decrypt_bio); + work_mem->bio = bio; + queue_work(blk_crypto_wq, &work_mem->crypto_work); +} + /** * blk_crypto_submit_bio - handle submitting bio for inline encryption * @@ -73,20 +518,20 @@ int blk_crypto_submit_bio(struct bio **bio_ptr) { struct bio *bio = *bio_ptr; struct request_queue *q; - struct bio_crypt_ctx *bc = bio->bi_crypt_context; int err; + struct bio_crypt_ctx *crypt_ctx; - if (!bc || !bio_has_data(bio)) + if (!bio_has_crypt_ctx(bio) || !bio_has_data(bio)) return 0; /* - * When a read bio is marked for fallback decryption, its bi_iter is - * saved so that when we decrypt the bio later, we know what part of it - * was marked for fallback decryption (when the bio is passed down after + * When a read bio is marked for sw decryption, its bi_iter is saved + * so that when we decrypt the bio later, we know what part of it was + * marked for sw decryption (when the bio is passed down after * blk_crypto_submit bio, it may be split or advanced so we cannot rely * on the bi_iter while decrypting in blk_crypto_endio) */ - if (bio_crypt_fallback_crypted(bc)) + if (bio_crypt_swhandled(bio)) return 0; err = bio_crypt_check_alignment(bio); @@ -95,22 +540,21 @@ int blk_crypto_submit_bio(struct bio **bio_ptr) goto out; } + crypt_ctx = bio->bi_crypt_context; q = bio->bi_disk->queue; - if (bc->bc_ksm) { + if (bio_crypt_has_keyslot(bio)) { /* Key already programmed into device? */ - if (q->ksm == bc->bc_ksm) + if (q->ksm == crypt_ctx->processing_ksm) return 0; /* Nope, release the existing keyslot. */ - bio_crypt_ctx_release_keyslot(bc); + bio_crypt_ctx_release_keyslot(bio); } /* Get device keyslot if supported */ - if (keyslot_manager_crypto_mode_supported(q->ksm, - bc->bc_key->crypto_mode, - bc->bc_key->data_unit_size)) { - err = bio_crypt_ctx_acquire_keyslot(bc, q->ksm); + if (q->ksm) { + err = bio_crypt_ctx_acquire_keyslot(bio, q->ksm); if (!err) return 0; @@ -119,10 +563,24 @@ int blk_crypto_submit_bio(struct bio **bio_ptr) } /* Fallback to crypto API */ - err = blk_crypto_fallback_submit_bio(bio_ptr); - if (err) + if (!READ_ONCE(tfms_inited[bio->bi_crypt_context->crypto_mode])) { + err = -EIO; + bio->bi_status = BLK_STS_IOERR; goto out; + } + if (bio_data_dir(bio) == WRITE) { + /* Encrypt the data now */ + err = blk_crypto_encrypt_bio(bio_ptr); + if (err) + goto out; + } else { + /* Mark bio as swhandled */ + bio->bi_crypt_context->processing_ksm = blk_crypto_ksm; + bio->bi_crypt_context->crypt_iter = bio->bi_iter; + bio->bi_crypt_context->sw_data_unit_num = + bio->bi_crypt_context->data_unit_num; + } return 0; out: bio_endio(*bio_ptr); @@ -132,10 +590,10 @@ out: /** * blk_crypto_endio - clean up bio w.r.t inline encryption during bio_endio * - * @bio: the bio to clean up + * @bio - the bio to clean up * - * If blk_crypto_submit_bio decided to fallback to crypto API for this bio, - * we queue the bio for decryption into a workqueue and return false, + * If blk_crypto_submit_bio decided to fallback to crypto API for this + * bio, we queue the bio for decryption into a workqueue and return false, * and call bio_endio(bio) at a later time (after the bio has been decrypted). * * If the bio is not to be decrypted by the crypto API, this function releases @@ -146,97 +604,195 @@ out: */ bool blk_crypto_endio(struct bio *bio) { - struct bio_crypt_ctx *bc = bio->bi_crypt_context; - - if (!bc) + if (!bio_has_crypt_ctx(bio)) return true; - if (bio_crypt_fallback_crypted(bc)) { + if (bio_crypt_swhandled(bio)) { /* - * The only bios who's crypto is handled by the blk-crypto - * fallback when they reach here are those with - * bio_data_dir(bio) == READ, since WRITE bios that are - * encrypted by the crypto API fallback are handled by - * blk_crypto_encrypt_endio(). + * The only bios that are swhandled when they reach here + * are those with bio_data_dir(bio) == READ, since WRITE + * bios that are encrypted by the crypto API fallback are + * handled by blk_crypto_encrypt_endio. */ - return !blk_crypto_queue_decrypt_bio(bio); + + /* If there was an IO error, don't decrypt. */ + if (bio->bi_status) + return true; + + blk_crypto_queue_decrypt_bio(bio); + return false; } - if (bc->bc_keyslot >= 0) - bio_crypt_ctx_release_keyslot(bc); + if (bio_crypt_has_keyslot(bio)) + bio_crypt_ctx_release_keyslot(bio); return true; } /** - * blk_crypto_init_key() - Prepare a key for use with blk-crypto - * @blk_key: Pointer to the blk_crypto_key to initialize. - * @raw_key: Pointer to the raw key. Must be the correct length for the chosen - * @crypto_mode; see blk_crypto_modes[]. - * @crypto_mode: identifier for the encryption algorithm to use - * @data_unit_size: the data unit size to use for en/decryption + * blk_crypto_start_using_mode() - Allocate skciphers for a + * mode_num for all keyslots + * @mode_num - the blk_crypto_mode we want to allocate ciphers for. * - * Return: The blk_crypto_key that was prepared, or an ERR_PTR() on error. When - * done using the key, it must be freed with blk_crypto_free_key(). + * Upper layers (filesystems) should call this function to ensure that a + * the crypto API fallback has transforms for this algorithm, if they become + * necessary. + * + * Return: 0 on success and -err on error. */ -int blk_crypto_init_key(struct blk_crypto_key *blk_key, const u8 *raw_key, - enum blk_crypto_mode_num crypto_mode, - unsigned int data_unit_size) +int blk_crypto_start_using_mode(enum blk_crypto_mode_num mode_num, + unsigned int data_unit_size, + struct request_queue *q) { - const struct blk_crypto_mode *mode; - static siphash_key_t hash_key; - - memset(blk_key, 0, sizeof(*blk_key)); - - if (crypto_mode >= ARRAY_SIZE(blk_crypto_modes)) - return -EINVAL; - - mode = &blk_crypto_modes[crypto_mode]; - if (mode->keysize == 0) - return -EINVAL; - - if (!is_power_of_2(data_unit_size)) - return -EINVAL; - - blk_key->crypto_mode = crypto_mode; - blk_key->data_unit_size = data_unit_size; - blk_key->data_unit_size_bits = ilog2(data_unit_size); - blk_key->size = mode->keysize; - memcpy(blk_key->raw, raw_key, mode->keysize); + struct blk_crypto_keyslot *slotp; + int err = 0; + int i; /* - * The keyslot manager uses the SipHash of the key to implement O(1) key - * lookups while avoiding leaking information about the keys. It's - * precomputed here so that it only needs to be computed once per key. + * Fast path + * Ensure that updates to blk_crypto_keyslots[i].tfms[mode_num] + * for each i are visible before we try to access them. */ - get_random_once(&hash_key, sizeof(hash_key)); - blk_key->hash = siphash(raw_key, mode->keysize, &hash_key); + if (likely(smp_load_acquire(&tfms_inited[mode_num]))) + return 0; - return 0; + /* + * If the keyslot manager of the request queue supports this + * crypto mode, then we don't need to allocate this mode. + */ + if (keyslot_manager_crypto_mode_supported(q->ksm, mode_num, + data_unit_size)) { + return 0; + } + + mutex_lock(&tfms_lock[mode_num]); + if (likely(tfms_inited[mode_num])) + goto out; + + for (i = 0; i < blk_crypto_num_keyslots; i++) { + slotp = &blk_crypto_keyslots[i]; + slotp->tfms[mode_num] = crypto_alloc_skcipher( + blk_crypto_modes[mode_num].cipher_str, + 0, 0); + if (IS_ERR(slotp->tfms[mode_num])) { + err = PTR_ERR(slotp->tfms[mode_num]); + slotp->tfms[mode_num] = NULL; + goto out_free_tfms; + } + + crypto_skcipher_set_flags(slotp->tfms[mode_num], + CRYPTO_TFM_REQ_FORBID_WEAK_KEYS); + } + + /* + * Ensure that updates to blk_crypto_keyslots[i].tfms[mode_num] + * for each i are visible before we set tfms_inited[mode_num]. + */ + smp_store_release(&tfms_inited[mode_num], true); + goto out; + +out_free_tfms: + for (i = 0; i < blk_crypto_num_keyslots; i++) { + slotp = &blk_crypto_keyslots[i]; + crypto_free_skcipher(slotp->tfms[mode_num]); + slotp->tfms[mode_num] = NULL; + } +out: + mutex_unlock(&tfms_lock[mode_num]); + return err; } +EXPORT_SYMBOL(blk_crypto_start_using_mode); /** * blk_crypto_evict_key() - Evict a key from any inline encryption hardware * it may have been programmed into - * @q: The request queue who's keyslot manager this key might have been - * programmed into - * @key: The key to evict + * @q - The request queue who's keyslot manager this key might have been + * programmed into + * @key - The key to evict + * @mode - The blk_crypto_mode_num used with this key + * @data_unit_size - The data unit size used with this key * * Upper layers (filesystems) should call this function to ensure that a key * is evicted from hardware that it might have been programmed into. This * will call keyslot_manager_evict_key on the queue's keyslot manager, if one * exists, and supports the crypto algorithm with the specified data unit size. - * Otherwise, it will evict the key from the blk-crypto-fallback's ksm. + * Otherwise, it will evict the key from the blk_crypto_ksm. * * Return: 0 on success, -err on error. */ -int blk_crypto_evict_key(struct request_queue *q, - const struct blk_crypto_key *key) +int blk_crypto_evict_key(struct request_queue *q, const u8 *key, + enum blk_crypto_mode_num mode, + unsigned int data_unit_size) { - if (q->ksm && - keyslot_manager_crypto_mode_supported(q->ksm, key->crypto_mode, - key->data_unit_size)) - return keyslot_manager_evict_key(q->ksm, key); + struct keyslot_manager *ksm = blk_crypto_ksm; - return blk_crypto_fallback_evict_key(key); + if (q && q->ksm && keyslot_manager_crypto_mode_supported(q->ksm, mode, + data_unit_size)) { + ksm = q->ksm; + } + + return keyslot_manager_evict_key(ksm, key, mode, data_unit_size); +} +EXPORT_SYMBOL(blk_crypto_evict_key); + +int __init blk_crypto_init(void) +{ + int i; + int err = -ENOMEM; + + prandom_bytes(blank_key, BLK_CRYPTO_MAX_KEY_SIZE); + + blk_crypto_ksm = keyslot_manager_create(blk_crypto_num_keyslots, + &blk_crypto_ksm_ll_ops, + NULL); + if (!blk_crypto_ksm) + goto out; + + blk_crypto_wq = alloc_workqueue("blk_crypto_wq", + WQ_UNBOUND | WQ_HIGHPRI | + WQ_MEM_RECLAIM, + num_online_cpus()); + if (!blk_crypto_wq) + goto out_free_ksm; + + blk_crypto_keyslots = kcalloc(blk_crypto_num_keyslots, + sizeof(*blk_crypto_keyslots), + GFP_KERNEL); + if (!blk_crypto_keyslots) + goto out_free_workqueue; + + for (i = 0; i < blk_crypto_num_keyslots; i++) { + blk_crypto_keyslots[i].crypto_mode = + BLK_ENCRYPTION_MODE_INVALID; + } + + for (i = 0; i < ARRAY_SIZE(blk_crypto_modes); i++) + mutex_init(&tfms_lock[i]); + + blk_crypto_page_pool = + mempool_create_page_pool(num_prealloc_bounce_pg, 0); + if (!blk_crypto_page_pool) + goto out_free_keyslots; + + blk_crypto_work_mem_cache = KMEM_CACHE(work_mem, SLAB_RECLAIM_ACCOUNT); + if (!blk_crypto_work_mem_cache) + goto out_free_page_pool; + + return 0; + +out_free_page_pool: + mempool_destroy(blk_crypto_page_pool); + blk_crypto_page_pool = NULL; +out_free_keyslots: + kzfree(blk_crypto_keyslots); + blk_crypto_keyslots = NULL; +out_free_workqueue: + destroy_workqueue(blk_crypto_wq); + blk_crypto_wq = NULL; +out_free_ksm: + keyslot_manager_destroy(blk_crypto_ksm); + blk_crypto_ksm = NULL; +out: + pr_warn("No memory for blk-crypto crypto API fallback."); + return err; } diff --git a/block/blk-merge.c b/block/blk-merge.c index 50dce8cf4570..c97c02a20c6a 100644 --- a/block/blk-merge.c +++ b/block/blk-merge.c @@ -557,6 +557,9 @@ static inline int ll_new_hw_segment(struct request *req, struct bio *bio, if (blk_integrity_merge_bio(req->q, req, bio) == false) goto no_merge; + if (WARN_ON_ONCE(!bio_crypt_ctx_compatible(bio, req->bio))) + goto no_merge; + /* * This will form the start of a new hw segment. Bump both * counters. @@ -581,8 +584,6 @@ int ll_back_merge_fn(struct request *req, struct bio *bio, unsigned int nr_segs) req_set_nomerge(req->q, req); return 0; } - if (!bio_crypt_ctx_mergeable(req->bio, blk_rq_bytes(req), bio)) - return 0; return ll_new_hw_segment(req, bio, nr_segs); } @@ -599,8 +600,6 @@ int ll_front_merge_fn(struct request *req, struct bio *bio, unsigned int nr_segs req_set_nomerge(req->q, req); return 0; } - if (!bio_crypt_ctx_mergeable(bio, bio->bi_iter.bi_size, req->bio)) - return 0; return ll_new_hw_segment(req, bio, nr_segs); } @@ -645,9 +644,6 @@ static int ll_merge_requests_fn(struct request_queue *q, struct request *req, if (blk_integrity_merge_rq(q, req, next) == false) return 0; - if (!bio_crypt_ctx_mergeable(req->bio, blk_rq_bytes(req), next->bio)) - return 0; - /* Merge is OK... */ req->nr_phys_segments = total_phys_segments; return 1; @@ -718,8 +714,14 @@ static enum elv_merge blk_try_req_merge(struct request *req, { if (blk_discard_mergable(req)) return ELEVATOR_DISCARD_MERGE; - else if (blk_rq_pos(req) + blk_rq_sectors(req) == blk_rq_pos(next)) + else if (blk_rq_pos(req) + blk_rq_sectors(req) == blk_rq_pos(next)) { + if (!bio_crypt_ctx_back_mergeable(req->bio, + blk_rq_sectors(req), + next->bio)) { + return ELEVATOR_NO_MERGE; + } return ELEVATOR_BACK_MERGE; + } return ELEVATOR_NO_MERGE; } @@ -755,6 +757,9 @@ static struct request *attempt_merge(struct request_queue *q, if (req->ioprio != next->ioprio) return NULL; + if (!bio_crypt_ctx_compatible(req->bio, next->bio)) + return NULL; + /* * If we are allowed to merge, then append bio list * from next to rq and release next. merge_requests_fn @@ -896,11 +901,22 @@ bool blk_rq_merge_ok(struct request *rq, struct bio *bio) enum elv_merge blk_try_merge(struct request *rq, struct bio *bio) { - if (blk_discard_mergable(rq)) + if (blk_discard_mergable(rq)) { return ELEVATOR_DISCARD_MERGE; - else if (blk_rq_pos(rq) + blk_rq_sectors(rq) == bio->bi_iter.bi_sector) + } else if (blk_rq_pos(rq) + blk_rq_sectors(rq) == + bio->bi_iter.bi_sector) { + if (!bio_crypt_ctx_back_mergeable(rq->bio, + blk_rq_sectors(rq), bio)) { + return ELEVATOR_NO_MERGE; + } return ELEVATOR_BACK_MERGE; - else if (blk_rq_pos(rq) - bio_sectors(bio) == bio->bi_iter.bi_sector) + } else if (blk_rq_pos(rq) - bio_sectors(bio) == + bio->bi_iter.bi_sector) { + if (!bio_crypt_ctx_back_mergeable(bio, + bio_sectors(bio), rq->bio)) { + return ELEVATOR_NO_MERGE; + } return ELEVATOR_FRONT_MERGE; + } return ELEVATOR_NO_MERGE; } diff --git a/block/bounce.c b/block/bounce.c index aa57ccc6ced3..6f9a2359b22a 100644 --- a/block/bounce.c +++ b/block/bounce.c @@ -267,7 +267,10 @@ static struct bio *bounce_clone_bio(struct bio *bio_src, gfp_t gfp_mask, break; } - bio_crypt_clone(bio, bio_src, gfp_mask); + if (bio_crypt_clone(bio, bio_src, gfp_mask) < 0) { + bio_put(bio); + return NULL; + } if (bio_integrity(bio_src) && bio_integrity_clone(bio, bio_src, gfp_mask) < 0) { diff --git a/block/keyslot-manager.c b/block/keyslot-manager.c index cdf0607019d4..020931fc9f7d 100644 --- a/block/keyslot-manager.c +++ b/block/keyslot-manager.c @@ -1,5 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 /* + * keyslot-manager.c + * * Copyright 2019 Google LLC */ @@ -25,7 +27,6 @@ * Upper layers will call keyslot_manager_get_slot_for_key() to program a * key into some slot in the inline encryption hardware. */ -#include #include #include #include @@ -35,14 +36,12 @@ struct keyslot { atomic_t slot_refs; struct list_head idle_slot_node; - struct hlist_node hash_node; - struct blk_crypto_key key; }; struct keyslot_manager { unsigned int num_slots; + atomic_t num_idle_slots; struct keyslot_mgmt_ll_ops ksm_ll_ops; - unsigned int crypto_mode_supported[BLK_ENCRYPTION_MODE_MAX]; void *ll_priv_data; /* Protects programming and evicting keys from the device */ @@ -53,15 +52,6 @@ struct keyslot_manager { struct list_head idle_slots; spinlock_t idle_slots_lock; - /* - * Hash table which maps key hashes to keyslots, so that we can find a - * key's keyslot in O(1) time rather than O(num_slots). Protected by - * 'lock'. A cryptographic hash function is used so that timing attacks - * can't leak information about the raw keys. - */ - struct hlist_head *slot_hashtable; - unsigned int slot_hashtable_size; - /* Per-keyslot data */ struct keyslot slots[]; }; @@ -72,13 +62,6 @@ struct keyslot_manager { * @ksm_ll_ops: The struct keyslot_mgmt_ll_ops for the device that this keyslot * manager will use to perform operations like programming and * evicting keys. - * @crypto_mode_supported: Array of size BLK_ENCRYPTION_MODE_MAX of - * bitmasks that represents whether a crypto mode - * and data unit size are supported. The i'th bit - * of crypto_mode_supported[crypto_mode] is set iff - * a data unit size of (1 << i) is supported. We - * only support data unit sizes that are powers of - * 2. * @ll_priv_data: Private data passed as is to the functions in ksm_ll_ops. * * Allocate memory for and initialize a keyslot manager. Called by e.g. @@ -88,20 +71,20 @@ struct keyslot_manager { * Return: Pointer to constructed keyslot manager or NULL on error. */ struct keyslot_manager *keyslot_manager_create(unsigned int num_slots, - const struct keyslot_mgmt_ll_ops *ksm_ll_ops, - const unsigned int crypto_mode_supported[BLK_ENCRYPTION_MODE_MAX], - void *ll_priv_data) + const struct keyslot_mgmt_ll_ops *ksm_ll_ops, + void *ll_priv_data) { struct keyslot_manager *ksm; - unsigned int slot; - unsigned int i; + int slot; if (num_slots == 0) return NULL; /* Check that all ops are specified */ if (ksm_ll_ops->keyslot_program == NULL || - ksm_ll_ops->keyslot_evict == NULL) + ksm_ll_ops->keyslot_evict == NULL || + ksm_ll_ops->crypto_mode_supported == NULL || + ksm_ll_ops->keyslot_find == NULL) return NULL; ksm = kvzalloc(struct_size(ksm, slots, num_slots), GFP_KERNEL); @@ -109,9 +92,8 @@ struct keyslot_manager *keyslot_manager_create(unsigned int num_slots, return NULL; ksm->num_slots = num_slots; + atomic_set(&ksm->num_idle_slots, num_slots); ksm->ksm_ll_ops = *ksm_ll_ops; - memcpy(ksm->crypto_mode_supported, crypto_mode_supported, - sizeof(ksm->crypto_mode_supported)); ksm->ll_priv_data = ll_priv_data; init_rwsem(&ksm->lock); @@ -126,29 +108,9 @@ struct keyslot_manager *keyslot_manager_create(unsigned int num_slots, spin_lock_init(&ksm->idle_slots_lock); - ksm->slot_hashtable_size = roundup_pow_of_two(num_slots); - ksm->slot_hashtable = kvmalloc_array(ksm->slot_hashtable_size, - sizeof(ksm->slot_hashtable[0]), - GFP_KERNEL); - if (!ksm->slot_hashtable) - goto err_free_ksm; - for (i = 0; i < ksm->slot_hashtable_size; i++) - INIT_HLIST_HEAD(&ksm->slot_hashtable[i]); - return ksm; - -err_free_ksm: - keyslot_manager_destroy(ksm); - return NULL; -} -EXPORT_SYMBOL_GPL(keyslot_manager_create); - -static inline struct hlist_head * -hash_bucket_for_key(struct keyslot_manager *ksm, - const struct blk_crypto_key *key) -{ - return &ksm->slot_hashtable[key->hash & (ksm->slot_hashtable_size - 1)]; } +EXPORT_SYMBOL(keyslot_manager_create); static void remove_slot_from_lru_list(struct keyslot_manager *ksm, int slot) { @@ -157,32 +119,22 @@ static void remove_slot_from_lru_list(struct keyslot_manager *ksm, int slot) spin_lock_irqsave(&ksm->idle_slots_lock, flags); list_del(&ksm->slots[slot].idle_slot_node); spin_unlock_irqrestore(&ksm->idle_slots_lock, flags); + + atomic_dec(&ksm->num_idle_slots); } -static int find_keyslot(struct keyslot_manager *ksm, - const struct blk_crypto_key *key) -{ - const struct hlist_head *head = hash_bucket_for_key(ksm, key); - const struct keyslot *slotp; - - hlist_for_each_entry(slotp, head, hash_node) { - if (slotp->key.hash == key->hash && - slotp->key.crypto_mode == key->crypto_mode && - slotp->key.data_unit_size == key->data_unit_size && - !crypto_memneq(slotp->key.raw, key->raw, key->size)) - return slotp - ksm->slots; - } - return -ENOKEY; -} - -static int find_and_grab_keyslot(struct keyslot_manager *ksm, - const struct blk_crypto_key *key) +static int find_and_grab_keyslot(struct keyslot_manager *ksm, const u8 *key, + enum blk_crypto_mode_num crypto_mode, + unsigned int data_unit_size) { int slot; - slot = find_keyslot(ksm, key); + slot = ksm->ksm_ll_ops.keyslot_find(ksm->ll_priv_data, key, + crypto_mode, data_unit_size); if (slot < 0) return slot; + if (WARN_ON(slot >= ksm->num_slots)) + return -EINVAL; if (atomic_inc_return(&ksm->slots[slot].slot_refs) == 1) { /* Took first reference to this slot; remove it from LRU list */ remove_slot_from_lru_list(ksm, slot); @@ -193,32 +145,37 @@ static int find_and_grab_keyslot(struct keyslot_manager *ksm, /** * keyslot_manager_get_slot_for_key() - Program a key into a keyslot. * @ksm: The keyslot manager to program the key into. - * @key: Pointer to the key object to program, including the raw key, crypto - * mode, and data unit size. + * @key: Pointer to the bytes of the key to program. Must be the correct length + * for the chosen @crypto_mode; see blk_crypto_modes in blk-crypto.c. + * @crypto_mode: Identifier for the encryption algorithm to use. + * @data_unit_size: The data unit size to use for en/decryption. * - * Get a keyslot that's been programmed with the specified key. If one already - * exists, return it with incremented refcount. Otherwise, wait for a keyslot - * to become idle and program it. + * Get a keyslot that's been programmed with the specified key, crypto_mode, and + * data_unit_size. If one already exists, return it with incremented refcount. + * Otherwise, wait for a keyslot to become idle and program it. * * Context: Process context. Takes and releases ksm->lock. * Return: The keyslot on success, else a -errno value. */ int keyslot_manager_get_slot_for_key(struct keyslot_manager *ksm, - const struct blk_crypto_key *key) + const u8 *key, + enum blk_crypto_mode_num crypto_mode, + unsigned int data_unit_size) { int slot; int err; struct keyslot *idle_slot; down_read(&ksm->lock); - slot = find_and_grab_keyslot(ksm, key); + slot = find_and_grab_keyslot(ksm, key, crypto_mode, data_unit_size); up_read(&ksm->lock); if (slot != -ENOKEY) return slot; for (;;) { down_write(&ksm->lock); - slot = find_and_grab_keyslot(ksm, key); + slot = find_and_grab_keyslot(ksm, key, crypto_mode, + data_unit_size); if (slot != -ENOKEY) { up_write(&ksm->lock); return slot; @@ -228,43 +185,42 @@ int keyslot_manager_get_slot_for_key(struct keyslot_manager *ksm, * If we're here, that means there wasn't a slot that was * already programmed with the key. So try to program it. */ - if (!list_empty(&ksm->idle_slots)) + if (atomic_read(&ksm->num_idle_slots) > 0) break; up_write(&ksm->lock); wait_event(ksm->idle_slots_wait_queue, - !list_empty(&ksm->idle_slots)); + (atomic_read(&ksm->num_idle_slots) > 0)); } idle_slot = list_first_entry(&ksm->idle_slots, struct keyslot, idle_slot_node); slot = idle_slot - ksm->slots; - err = ksm->ksm_ll_ops.keyslot_program(ksm, key, slot); + err = ksm->ksm_ll_ops.keyslot_program(ksm->ll_priv_data, key, + crypto_mode, + data_unit_size, + slot); + if (err) { wake_up(&ksm->idle_slots_wait_queue); up_write(&ksm->lock); return err; } - /* Move this slot to the hash list for the new key. */ - if (idle_slot->key.crypto_mode != BLK_ENCRYPTION_MODE_INVALID) - hlist_del(&idle_slot->hash_node); - hlist_add_head(&idle_slot->hash_node, hash_bucket_for_key(ksm, key)); - - atomic_set(&idle_slot->slot_refs, 1); - idle_slot->key = *key; - + atomic_set(&ksm->slots[slot].slot_refs, 1); remove_slot_from_lru_list(ksm, slot); up_write(&ksm->lock); return slot; + } +EXPORT_SYMBOL(keyslot_manager_get_slot_for_key); /** * keyslot_manager_get_slot() - Increment the refcount on the specified slot. - * @ksm: The keyslot manager that we want to modify. - * @slot: The slot to increment the refcount of. + * @ksm - The keyslot manager that we want to modify. + * @slot - The slot to increment the refcount of. * * This function assumes that there is already an active reference to that slot * and simply increments the refcount. This is useful when cloning a bio that @@ -280,6 +236,7 @@ void keyslot_manager_get_slot(struct keyslot_manager *ksm, unsigned int slot) WARN_ON(atomic_inc_return(&ksm->slots[slot].slot_refs) < 2); } +EXPORT_SYMBOL(keyslot_manager_get_slot); /** * keyslot_manager_put_slot() - Release a reference to a slot @@ -300,17 +257,19 @@ void keyslot_manager_put_slot(struct keyslot_manager *ksm, unsigned int slot) list_add_tail(&ksm->slots[slot].idle_slot_node, &ksm->idle_slots); spin_unlock_irqrestore(&ksm->idle_slots_lock, flags); + atomic_inc(&ksm->num_idle_slots); wake_up(&ksm->idle_slots_wait_queue); } } +EXPORT_SYMBOL(keyslot_manager_put_slot); /** * keyslot_manager_crypto_mode_supported() - Find out if a crypto_mode/data * unit size combination is supported * by a ksm. - * @ksm: The keyslot manager to check - * @crypto_mode: The crypto mode to check for. - * @data_unit_size: The data_unit_size for the mode. + * @ksm - The keyslot manager to check + * @crypto_mode - The crypto mode to check for. + * @data_unit_size - The data_unit_size for the mode. * * Calls and returns the result of the crypto_mode_supported function specified * by the ksm. @@ -325,102 +284,69 @@ bool keyslot_manager_crypto_mode_supported(struct keyslot_manager *ksm, { if (!ksm) return false; - if (WARN_ON(crypto_mode >= BLK_ENCRYPTION_MODE_MAX)) - return false; - if (WARN_ON(!is_power_of_2(data_unit_size))) - return false; - return ksm->crypto_mode_supported[crypto_mode] & data_unit_size; + return ksm->ksm_ll_ops.crypto_mode_supported(ksm->ll_priv_data, + crypto_mode, + data_unit_size); } +EXPORT_SYMBOL(keyslot_manager_crypto_mode_supported); + +bool keyslot_manager_rq_crypto_mode_supported(struct request_queue *q, + enum blk_crypto_mode_num crypto_mode, + unsigned int data_unit_size) +{ + return keyslot_manager_crypto_mode_supported(q->ksm, crypto_mode, + data_unit_size); +} +EXPORT_SYMBOL(keyslot_manager_rq_crypto_mode_supported); /** * keyslot_manager_evict_key() - Evict a key from the lower layer device. - * @ksm: The keyslot manager to evict from - * @key: The key to evict + * @ksm - The keyslot manager to evict from + * @key - The key to evict + * @crypto_mode - The crypto algorithm the key was programmed with. + * @data_unit_size - The data_unit_size the key was programmed with. * - * Find the keyslot that the specified key was programmed into, and evict that - * slot from the lower layer device if that slot is not currently in use. + * Finds the slot that the specified key, crypto_mode, data_unit_size combo + * was programmed into, and evicts that slot from the lower layer device if + * the refcount on the slot is 0. Returns -EBUSY if the refcount is not 0, and + * -errno on error. * * Context: Process context. Takes and releases ksm->lock. - * Return: 0 on success, -EBUSY if the key is still in use, or another - * -errno value on other error. */ int keyslot_manager_evict_key(struct keyslot_manager *ksm, - const struct blk_crypto_key *key) + const u8 *key, + enum blk_crypto_mode_num crypto_mode, + unsigned int data_unit_size) { int slot; - int err; - struct keyslot *slotp; + int err = 0; down_write(&ksm->lock); - slot = find_keyslot(ksm, key); + slot = ksm->ksm_ll_ops.keyslot_find(ksm->ll_priv_data, key, + crypto_mode, + data_unit_size); + if (slot < 0) { - err = slot; - goto out_unlock; + up_write(&ksm->lock); + return slot; } - slotp = &ksm->slots[slot]; - if (atomic_read(&slotp->slot_refs) != 0) { + if (atomic_read(&ksm->slots[slot].slot_refs) == 0) { + err = ksm->ksm_ll_ops.keyslot_evict(ksm->ll_priv_data, key, + crypto_mode, + data_unit_size, + slot); + } else { err = -EBUSY; - goto out_unlock; } - err = ksm->ksm_ll_ops.keyslot_evict(ksm, key, slot); - if (err) - goto out_unlock; - hlist_del(&slotp->hash_node); - memzero_explicit(&slotp->key, sizeof(slotp->key)); - err = 0; -out_unlock: up_write(&ksm->lock); return err; } - -/** - * keyslot_manager_reprogram_all_keys() - Re-program all keyslots. - * @ksm: The keyslot manager - * - * Re-program all keyslots that are supposed to have a key programmed. This is - * intended only for use by drivers for hardware that loses its keys on reset. - * - * Context: Process context. Takes and releases ksm->lock. - */ -void keyslot_manager_reprogram_all_keys(struct keyslot_manager *ksm) -{ - unsigned int slot; - - down_write(&ksm->lock); - for (slot = 0; slot < ksm->num_slots; slot++) { - const struct keyslot *slotp = &ksm->slots[slot]; - int err; - - if (slotp->key.crypto_mode == BLK_ENCRYPTION_MODE_INVALID) - continue; - - err = ksm->ksm_ll_ops.keyslot_program(ksm, &slotp->key, slot); - WARN_ON(err); - } - up_write(&ksm->lock); -} -EXPORT_SYMBOL_GPL(keyslot_manager_reprogram_all_keys); - -/** - * keyslot_manager_private() - return the private data stored with ksm - * @ksm: The keyslot manager - * - * Returns the private data passed to the ksm when it was created. - */ -void *keyslot_manager_private(struct keyslot_manager *ksm) -{ - return ksm->ll_priv_data; -} -EXPORT_SYMBOL_GPL(keyslot_manager_private); +EXPORT_SYMBOL(keyslot_manager_evict_key); void keyslot_manager_destroy(struct keyslot_manager *ksm) { - if (ksm) { - kvfree(ksm->slot_hashtable); - memzero_explicit(ksm, struct_size(ksm, slots, ksm->num_slots)); - kvfree(ksm); - } + kvfree(ksm); } -EXPORT_SYMBOL_GPL(keyslot_manager_destroy); +EXPORT_SYMBOL(keyslot_manager_destroy); diff --git a/drivers/md/dm.c b/drivers/md/dm.c index 59fe829aac9c..67c24294d7c8 100644 --- a/drivers/md/dm.c +++ b/drivers/md/dm.c @@ -1322,13 +1322,15 @@ static int clone_bio(struct dm_target_io *tio, struct bio *bio, sector_t sector, unsigned len) { struct bio *clone = &tio->clone; + int ret; __bio_clone_fast(clone, bio); - bio_crypt_clone(clone, bio, GFP_NOIO); + ret = bio_crypt_clone(clone, bio, GFP_NOIO); + if (ret < 0) + return ret; if (bio_integrity(bio)) { - int r; if (unlikely(!dm_target_has_integrity(tio->ti->type) && !dm_target_passes_integrity(tio->ti->type))) { DMWARN("%s: the target %s doesn't support integrity data.", @@ -1337,9 +1339,11 @@ static int clone_bio(struct dm_target_io *tio, struct bio *bio, return -EIO; } - r = bio_integrity_clone(clone, bio, GFP_NOIO); - if (r < 0) - return r; + ret = bio_integrity_clone(clone, bio, GFP_NOIO); + if (ret < 0) { + bio_crypt_free_ctx(clone); + return ret; + } } bio_advance(clone, to_bytes(sector - clone->bi_iter.bi_sector)); diff --git a/drivers/scsi/ufs/ufshcd-crypto.c b/drivers/scsi/ufs/ufshcd-crypto.c index 1cfd9a327049..75d834180f8b 100644 --- a/drivers/scsi/ufs/ufshcd-crypto.c +++ b/drivers/scsi/ufs/ufshcd-crypto.c @@ -3,7 +3,8 @@ * Copyright 2019 Google LLC */ -#include +#include + #include "ufshcd.h" #include "ufshcd-crypto.h" @@ -24,23 +25,19 @@ static u8 get_data_unit_size_mask(unsigned int data_unit_size) static size_t get_keysize_bytes(enum ufs_crypto_key_size size) { switch (size) { - case UFS_CRYPTO_KEY_SIZE_128: - return 16; - case UFS_CRYPTO_KEY_SIZE_192: - return 24; - case UFS_CRYPTO_KEY_SIZE_256: - return 32; - case UFS_CRYPTO_KEY_SIZE_512: - return 64; - default: - return 0; + case UFS_CRYPTO_KEY_SIZE_128: return 16; + case UFS_CRYPTO_KEY_SIZE_192: return 24; + case UFS_CRYPTO_KEY_SIZE_256: return 32; + case UFS_CRYPTO_KEY_SIZE_512: return 64; + default: return 0; } } -static int ufshcd_crypto_cap_find(struct ufs_hba *hba, +static int ufshcd_crypto_cap_find(void *hba_p, enum blk_crypto_mode_num crypto_mode, unsigned int data_unit_size) { + struct ufs_hba *hba = hba_p; enum ufs_crypto_alg ufs_alg; u8 data_unit_mask; int cap_idx; @@ -55,8 +52,7 @@ static int ufshcd_crypto_cap_find(struct ufs_hba *hba, ufs_alg = UFS_CRYPTO_ALG_AES_XTS; ufs_key_size = UFS_CRYPTO_KEY_SIZE_256; break; - default: - return -EINVAL; + default: return -EINVAL; } data_unit_mask = get_data_unit_size_mask(data_unit_size); @@ -105,10 +101,8 @@ static int ufshcd_crypto_cfg_entry_write_key(union ufs_crypto_cfg_entry *cfg, memcpy(cfg->crypto_key + UFS_CRYPTO_KEY_MAX_SIZE/2, key + key_size_bytes/2, key_size_bytes/2); return 0; - case UFS_CRYPTO_ALG_BITLOCKER_AES_CBC: - /* fall through */ - case UFS_CRYPTO_ALG_AES_ECB: - /* fall through */ + case UFS_CRYPTO_ALG_BITLOCKER_AES_CBC: // fallthrough + case UFS_CRYPTO_ALG_AES_ECB: // fallthrough case UFS_CRYPTO_ALG_ESSIV_AES_CBC: memcpy(cfg->crypto_key, key, key_size_bytes); return 0; @@ -117,15 +111,13 @@ static int ufshcd_crypto_cfg_entry_write_key(union ufs_crypto_cfg_entry *cfg, return -EINVAL; } -static void ufshcd_program_key(struct ufs_hba *hba, - const union ufs_crypto_cfg_entry *cfg, - int slot) +static void program_key(struct ufs_hba *hba, + const union ufs_crypto_cfg_entry *cfg, + int slot) { int i; u32 slot_offset = hba->crypto_cfg_register + slot * sizeof(*cfg); - pm_runtime_get_sync(hba->dev); - ufshcd_hold(hba, false); /* Clear the dword 16 */ ufshcd_writel(hba, 0, slot_offset + 16 * sizeof(cfg->reg_val[0])); /* Ensure that CFGE is cleared before programming the key */ @@ -145,45 +137,29 @@ static void ufshcd_program_key(struct ufs_hba *hba, ufshcd_writel(hba, le32_to_cpu(cfg->reg_val[16]), slot_offset + 16 * sizeof(cfg->reg_val[0])); wmb(); - ufshcd_release(hba); - pm_runtime_put_sync(hba->dev); } -static void ufshcd_clear_keyslot(struct ufs_hba *hba, int slot) -{ - union ufs_crypto_cfg_entry cfg = { 0 }; - - ufshcd_program_key(hba, &cfg, slot); -} - -/* Clear all keyslots at driver init time */ -static void ufshcd_clear_all_keyslots(struct ufs_hba *hba) -{ - int slot; - - for (slot = 0; slot < ufshcd_num_keyslots(hba); slot++) - ufshcd_clear_keyslot(hba, slot); -} - -static int ufshcd_crypto_keyslot_program(struct keyslot_manager *ksm, - const struct blk_crypto_key *key, +static int ufshcd_crypto_keyslot_program(void *hba_p, const u8 *key, + enum blk_crypto_mode_num crypto_mode, + unsigned int data_unit_size, unsigned int slot) { - struct ufs_hba *hba = keyslot_manager_private(ksm); + struct ufs_hba *hba = hba_p; int err = 0; u8 data_unit_mask; union ufs_crypto_cfg_entry cfg; + union ufs_crypto_cfg_entry *cfg_arr = hba->crypto_cfgs; int cap_idx; - cap_idx = ufshcd_crypto_cap_find(hba, key->crypto_mode, - key->data_unit_size); + cap_idx = ufshcd_crypto_cap_find(hba_p, crypto_mode, + data_unit_size); if (!ufshcd_is_crypto_enabled(hba) || !ufshcd_keyslot_valid(hba, slot) || !ufshcd_cap_idx_valid(hba, cap_idx)) return -EINVAL; - data_unit_mask = get_data_unit_size_mask(key->data_unit_size); + data_unit_mask = get_data_unit_size_mask(data_unit_size); if (!(data_unit_mask & hba->crypto_cap_array[cap_idx].sdus_mask)) return -EINVAL; @@ -193,74 +169,134 @@ static int ufshcd_crypto_keyslot_program(struct keyslot_manager *ksm, cfg.crypto_cap_idx = cap_idx; cfg.config_enable |= UFS_CRYPTO_CONFIGURATION_ENABLE; - err = ufshcd_crypto_cfg_entry_write_key(&cfg, key->raw, - hba->crypto_cap_array[cap_idx]); + err = ufshcd_crypto_cfg_entry_write_key(&cfg, key, + hba->crypto_cap_array[cap_idx]); if (err) return err; - ufshcd_program_key(hba, &cfg, slot); + program_key(hba, &cfg, slot); + memcpy(&cfg_arr[slot], &cfg, sizeof(cfg)); memzero_explicit(&cfg, sizeof(cfg)); + return 0; } -static int ufshcd_crypto_keyslot_evict(struct keyslot_manager *ksm, - const struct blk_crypto_key *key, +static int ufshcd_crypto_keyslot_find(void *hba_p, + const u8 *key, + enum blk_crypto_mode_num crypto_mode, + unsigned int data_unit_size) +{ + struct ufs_hba *hba = hba_p; + int err = 0; + int slot; + u8 data_unit_mask; + union ufs_crypto_cfg_entry cfg; + union ufs_crypto_cfg_entry *cfg_arr = hba->crypto_cfgs; + int cap_idx; + + cap_idx = ufshcd_crypto_cap_find(hba_p, crypto_mode, + data_unit_size); + + if (!ufshcd_is_crypto_enabled(hba) || + !ufshcd_cap_idx_valid(hba, cap_idx)) + return -EINVAL; + + data_unit_mask = get_data_unit_size_mask(data_unit_size); + + if (!(data_unit_mask & hba->crypto_cap_array[cap_idx].sdus_mask)) + return -EINVAL; + + memset(&cfg, 0, sizeof(cfg)); + err = ufshcd_crypto_cfg_entry_write_key(&cfg, key, + hba->crypto_cap_array[cap_idx]); + + if (err) + return -EINVAL; + + for (slot = 0; slot < NUM_KEYSLOTS(hba); slot++) { + if ((cfg_arr[slot].config_enable & + UFS_CRYPTO_CONFIGURATION_ENABLE) && + data_unit_mask == cfg_arr[slot].data_unit_size && + cap_idx == cfg_arr[slot].crypto_cap_idx && + !crypto_memneq(&cfg.crypto_key, cfg_arr[slot].crypto_key, + UFS_CRYPTO_KEY_MAX_SIZE)) { + memzero_explicit(&cfg, sizeof(cfg)); + return slot; + } + } + + memzero_explicit(&cfg, sizeof(cfg)); + return -ENOKEY; +} + +static int ufshcd_crypto_keyslot_evict(void *hba_p, const u8 *key, + enum blk_crypto_mode_num crypto_mode, + unsigned int data_unit_size, unsigned int slot) { - struct ufs_hba *hba = keyslot_manager_private(ksm); + struct ufs_hba *hba = hba_p; + int i = 0; + u32 reg_base; + union ufs_crypto_cfg_entry *cfg_arr = hba->crypto_cfgs; if (!ufshcd_is_crypto_enabled(hba) || !ufshcd_keyslot_valid(hba, slot)) return -EINVAL; + memset(&cfg_arr[slot], 0, sizeof(cfg_arr[slot])); + reg_base = hba->crypto_cfg_register + slot * sizeof(cfg_arr[0]); + /* * Clear the crypto cfg on the device. Clearing CFGE * might not be sufficient, so just clear the entire cfg. */ - ufshcd_clear_keyslot(hba, slot); + for (i = 0; i < sizeof(cfg_arr[0]); i += sizeof(__le32)) + ufshcd_writel(hba, 0, reg_base + i); + wmb(); return 0; } +static bool ufshcd_crypto_mode_supported(void *hba_p, + enum blk_crypto_mode_num crypto_mode, + unsigned int data_unit_size) +{ + return ufshcd_crypto_cap_find(hba_p, crypto_mode, data_unit_size) >= 0; +} + /* Functions implementing UFSHCI v2.1 specification behaviour */ void ufshcd_crypto_enable_spec(struct ufs_hba *hba) { + union ufs_crypto_cfg_entry *cfg_arr = hba->crypto_cfgs; + int slot; + if (!ufshcd_hba_is_crypto_supported(hba)) return; hba->caps |= UFSHCD_CAP_CRYPTO; - - /* Reset might clear all keys, so reprogram all the keys. */ - keyslot_manager_reprogram_all_keys(hba->ksm); + /* + * Reset might clear all keys, so reprogram all the keys. + * Also serves to clear keys on driver init. + */ + for (slot = 0; slot < NUM_KEYSLOTS(hba); slot++) + program_key(hba, &cfg_arr[slot], slot); } -EXPORT_SYMBOL_GPL(ufshcd_crypto_enable_spec); +EXPORT_SYMBOL(ufshcd_crypto_enable_spec); void ufshcd_crypto_disable_spec(struct ufs_hba *hba) { hba->caps &= ~UFSHCD_CAP_CRYPTO; } -EXPORT_SYMBOL_GPL(ufshcd_crypto_disable_spec); +EXPORT_SYMBOL(ufshcd_crypto_disable_spec); static const struct keyslot_mgmt_ll_ops ufshcd_ksm_ops = { .keyslot_program = ufshcd_crypto_keyslot_program, .keyslot_evict = ufshcd_crypto_keyslot_evict, + .keyslot_find = ufshcd_crypto_keyslot_find, + .crypto_mode_supported = ufshcd_crypto_mode_supported, }; -enum blk_crypto_mode_num ufshcd_blk_crypto_mode_num_for_alg_dusize( - enum ufs_crypto_alg ufs_crypto_alg, - enum ufs_crypto_key_size key_size) -{ - /* - * This is currently the only mode that UFS and blk-crypto both support. - */ - if (ufs_crypto_alg == UFS_CRYPTO_ALG_AES_XTS && - key_size == UFS_CRYPTO_KEY_SIZE_256) - return BLK_ENCRYPTION_MODE_AES_256_XTS; - - return BLK_ENCRYPTION_MODE_INVALID; -} - /** * ufshcd_hba_init_crypto - Read crypto capabilities, init crypto fields in hba * @hba: Per adapter instance @@ -272,8 +308,6 @@ int ufshcd_hba_init_crypto_spec(struct ufs_hba *hba, { int cap_idx = 0; int err = 0; - unsigned int crypto_modes_supported[BLK_ENCRYPTION_MODE_MAX]; - enum blk_crypto_mode_num blk_mode_num; /* Default to disabling crypto */ hba->caps &= ~UFSHCD_CAP_CRYPTO; @@ -302,7 +336,16 @@ int ufshcd_hba_init_crypto_spec(struct ufs_hba *hba, goto out; } - memset(crypto_modes_supported, 0, sizeof(crypto_modes_supported)); + hba->crypto_cfgs = + devm_kcalloc(hba->dev, + NUM_KEYSLOTS(hba), + sizeof(hba->crypto_cfgs[0]), + GFP_KERNEL); + if (!hba->crypto_cfgs) { + err = -ENOMEM; + goto out_free_cfg_mem; + } + /* * Store all the capabilities now so that we don't need to repeatedly * access the device each time we want to know its capabilities @@ -313,35 +356,26 @@ int ufshcd_hba_init_crypto_spec(struct ufs_hba *hba, cpu_to_le32(ufshcd_readl(hba, REG_UFS_CRYPTOCAP + cap_idx * sizeof(__le32))); - blk_mode_num = ufshcd_blk_crypto_mode_num_for_alg_dusize( - hba->crypto_cap_array[cap_idx].algorithm_id, - hba->crypto_cap_array[cap_idx].key_size); - if (blk_mode_num == BLK_ENCRYPTION_MODE_INVALID) - continue; - crypto_modes_supported[blk_mode_num] |= - hba->crypto_cap_array[cap_idx].sdus_mask * 512; } - ufshcd_clear_all_keyslots(hba); - - hba->ksm = keyslot_manager_create(ufshcd_num_keyslots(hba), ksm_ops, - crypto_modes_supported, hba); + hba->ksm = keyslot_manager_create(NUM_KEYSLOTS(hba), ksm_ops, hba); if (!hba->ksm) { err = -ENOMEM; - goto out_free_caps; + goto out_free_crypto_cfgs; } return 0; - -out_free_caps: +out_free_crypto_cfgs: + devm_kfree(hba->dev, hba->crypto_cfgs); +out_free_cfg_mem: devm_kfree(hba->dev, hba->crypto_cap_array); out: /* Indicate that init failed by setting crypto_capabilities to 0 */ hba->crypto_capabilities.reg_val = 0; return err; } -EXPORT_SYMBOL_GPL(ufshcd_hba_init_crypto_spec); +EXPORT_SYMBOL(ufshcd_hba_init_crypto_spec); void ufshcd_crypto_setup_rq_keyslot_manager_spec(struct ufs_hba *hba, struct request_queue *q) @@ -351,26 +385,26 @@ void ufshcd_crypto_setup_rq_keyslot_manager_spec(struct ufs_hba *hba, q->ksm = hba->ksm; } -EXPORT_SYMBOL_GPL(ufshcd_crypto_setup_rq_keyslot_manager_spec); +EXPORT_SYMBOL(ufshcd_crypto_setup_rq_keyslot_manager_spec); void ufshcd_crypto_destroy_rq_keyslot_manager_spec(struct ufs_hba *hba, struct request_queue *q) { keyslot_manager_destroy(hba->ksm); } -EXPORT_SYMBOL_GPL(ufshcd_crypto_destroy_rq_keyslot_manager_spec); +EXPORT_SYMBOL(ufshcd_crypto_destroy_rq_keyslot_manager_spec); int ufshcd_prepare_lrbp_crypto_spec(struct ufs_hba *hba, struct scsi_cmnd *cmd, struct ufshcd_lrb *lrbp) { - struct bio_crypt_ctx *bc; + int key_slot; - if (!bio_crypt_should_process(cmd->request)) { + if (!cmd->request->bio || + !bio_crypt_should_process(cmd->request->bio, cmd->request->q)) { lrbp->crypto_enable = false; return 0; } - bc = cmd->request->bio->bi_crypt_context; if (WARN_ON(!ufshcd_is_crypto_enabled(hba))) { /* @@ -379,16 +413,17 @@ int ufshcd_prepare_lrbp_crypto_spec(struct ufs_hba *hba, */ return -EINVAL; } - if (!ufshcd_keyslot_valid(hba, bc->bc_keyslot)) + key_slot = bio_crypt_get_keyslot(cmd->request->bio); + if (!ufshcd_keyslot_valid(hba, key_slot)) return -EINVAL; lrbp->crypto_enable = true; - lrbp->crypto_key_slot = bc->bc_keyslot; - lrbp->data_unit_num = bc->bc_dun[0]; + lrbp->crypto_key_slot = key_slot; + lrbp->data_unit_num = bio_crypt_data_unit_num(cmd->request->bio); return 0; } -EXPORT_SYMBOL_GPL(ufshcd_prepare_lrbp_crypto_spec); +EXPORT_SYMBOL(ufshcd_prepare_lrbp_crypto_spec); /* Crypto Variant Ops Support */ diff --git a/drivers/scsi/ufs/ufshcd-crypto.h b/drivers/scsi/ufs/ufshcd-crypto.h index 24a587c7a94e..3c03d0e23e87 100644 --- a/drivers/scsi/ufs/ufshcd-crypto.h +++ b/drivers/scsi/ufs/ufshcd-crypto.h @@ -11,10 +11,7 @@ #include "ufshcd.h" #include "ufshci.h" -static inline int ufshcd_num_keyslots(struct ufs_hba *hba) -{ - return hba->crypto_capabilities.config_count + 1; -} +#define NUM_KEYSLOTS(hba) (hba->crypto_capabilities.config_count + 1) static inline bool ufshcd_keyslot_valid(struct ufs_hba *hba, unsigned int slot) { @@ -22,7 +19,7 @@ static inline bool ufshcd_keyslot_valid(struct ufs_hba *hba, unsigned int slot) * The actual number of configurations supported is (CFGC+1), so slot * numbers range from 0 to config_count inclusive. */ - return slot < ufshcd_num_keyslots(hba); + return slot < NUM_KEYSLOTS(hba); } static inline bool ufshcd_hba_is_crypto_supported(struct ufs_hba *hba) @@ -54,11 +51,6 @@ void ufshcd_crypto_setup_rq_keyslot_manager_spec(struct ufs_hba *hba, void ufshcd_crypto_destroy_rq_keyslot_manager_spec(struct ufs_hba *hba, struct request_queue *q); -static inline bool ufshcd_lrbp_crypto_enabled(struct ufshcd_lrb *lrbp) -{ - return lrbp->crypto_enable; -} - /* Crypto Variant Ops Support */ void ufshcd_crypto_enable(struct ufs_hba *hba); @@ -126,14 +118,10 @@ static inline int ufshcd_prepare_lrbp_crypto(struct ufs_hba *hba, struct scsi_cmnd *cmd, struct ufshcd_lrb *lrbp) { + lrbp->crypto_enable = false; return 0; } -static inline bool ufshcd_lrbp_crypto_enabled(struct ufshcd_lrb *lrbp) -{ - return false; -} - static inline int ufshcd_complete_lrbp_crypto(struct ufs_hba *hba, struct scsi_cmnd *cmd, struct ufshcd_lrb *lrbp) diff --git a/drivers/scsi/ufs/ufshcd.c b/drivers/scsi/ufs/ufshcd.c index 308be8685e86..9216198da200 100644 --- a/drivers/scsi/ufs/ufshcd.c +++ b/drivers/scsi/ufs/ufshcd.c @@ -2246,15 +2246,13 @@ static void ufshcd_prepare_req_desc_hdr(struct ufshcd_lrb *lrbp, dword_0 |= UTP_REQ_DESC_INT_CMD; /* Transfer request descriptor header fields */ - if (ufshcd_lrbp_crypto_enabled(lrbp)) { -#if IS_ENABLED(CONFIG_SCSI_UFS_CRYPTO) + if (lrbp->crypto_enable) { dword_0 |= UTP_REQ_DESC_CRYPTO_ENABLE_CMD; dword_0 |= lrbp->crypto_key_slot; req_desc->header.dword_1 = - cpu_to_le32(lower_32_bits(lrbp->data_unit_num)); + cpu_to_le32((u32)lrbp->data_unit_num); req_desc->header.dword_3 = - cpu_to_le32(upper_32_bits(lrbp->data_unit_num)); -#endif /* CONFIG_SCSI_UFS_CRYPTO */ + cpu_to_le32((u32)(lrbp->data_unit_num >> 32)); } else { /* dword_1 and dword_3 are reserved, hence they are set to 0 */ req_desc->header.dword_1 = 0; @@ -2557,9 +2555,7 @@ static int ufshcd_compose_dev_cmd(struct ufs_hba *hba, lrbp->task_tag = tag; lrbp->lun = 0; /* device management cmd is not specific to any LUN */ lrbp->intr_cmd = true; /* No interrupt aggregation */ -#if IS_ENABLED(CONFIG_SCSI_UFS_CRYPTO) lrbp->crypto_enable = false; /* No crypto operations */ -#endif hba->dev_cmd.type = cmd_type; return ufshcd_comp_devman_upiu(hba, lrbp); diff --git a/drivers/scsi/ufs/ufshcd.h b/drivers/scsi/ufs/ufshcd.h index 9f1452299803..6fe75302df59 100644 --- a/drivers/scsi/ufs/ufshcd.h +++ b/drivers/scsi/ufs/ufshcd.h @@ -194,11 +194,9 @@ struct ufshcd_lrb { bool intr_cmd; ktime_t issue_time_stamp; ktime_t compl_time_stamp; -#if IS_ENABLED(CONFIG_SCSI_UFS_CRYPTO) bool crypto_enable; u8 crypto_key_slot; u64 data_unit_num; -#endif /* CONFIG_SCSI_UFS_CRYPTO */ bool req_abort_skip; }; @@ -548,6 +546,7 @@ struct ufs_stats { * @crypto_capabilities: Content of crypto capabilities register (0x100) * @crypto_cap_array: Array of crypto capabilities * @crypto_cfg_register: Start of the crypto cfg array + * @crypto_cfgs: Array of crypto configurations (i.e. config for each slot) * @ksm: the keyslot manager tied to this hba */ struct ufs_hba { @@ -778,6 +777,7 @@ struct ufs_hba { union ufs_crypto_capabilities crypto_capabilities; union ufs_crypto_cap_entry *crypto_cap_array; u32 crypto_cfg_register; + union ufs_crypto_cfg_entry *crypto_cfgs; struct keyslot_manager *ksm; #endif /* CONFIG_SCSI_UFS_CRYPTO */ }; diff --git a/fs/buffer.c b/fs/buffer.c index 32141c922fed..2e64fa95a1ac 100644 --- a/fs/buffer.c +++ b/fs/buffer.c @@ -3046,7 +3046,7 @@ static int submit_bh_wbc(int op, int op_flags, struct buffer_head *bh, */ bio = bio_alloc(GFP_NOIO, 1); - fscrypt_set_bio_crypt_ctx_bh(bio, bh, GFP_NOIO); + fscrypt_set_bio_crypt_ctx_bh(bio, bh, GFP_NOIO | __GFP_NOFAIL); bio->bi_iter.bi_sector = bh->b_blocknr * (bh->b_size >> 9); bio_set_dev(bio, bh->b_bdev); diff --git a/fs/crypto/bio.c b/fs/crypto/bio.c index d28d8e803554..956798debf71 100644 --- a/fs/crypto/bio.c +++ b/fs/crypto/bio.c @@ -73,8 +73,11 @@ int fscrypt_zeroout_range(const struct inode *inode, pgoff_t lblk, err = -ENOMEM; goto errout; } - fscrypt_set_bio_crypt_ctx(bio, inode, lblk, GFP_NOIO); - + err = fscrypt_set_bio_crypt_ctx(bio, inode, lblk, GFP_NOIO); + if (err) { + bio_put(bio); + goto errout; + } bio_set_dev(bio, inode->i_sb->s_bdev); bio->bi_iter.bi_sector = pblk << (blockbits - 9); bio_set_op_attrs(bio, REQ_OP_WRITE, 0); diff --git a/fs/crypto/crypto.c b/fs/crypto/crypto.c index 7549d76a0aba..3719efa546c6 100644 --- a/fs/crypto/crypto.c +++ b/fs/crypto/crypto.c @@ -97,7 +97,7 @@ int fscrypt_crypt_block(const struct inode *inode, fscrypt_direction_t rw, DECLARE_CRYPTO_WAIT(wait); struct scatterlist dst, src; struct fscrypt_info *ci = inode->i_crypt_info; - struct crypto_skcipher *tfm = ci->ci_key.tfm; + struct crypto_skcipher *tfm = ci->ci_ctfm; int res = 0; if (WARN_ON_ONCE(len <= 0)) diff --git a/fs/crypto/fname.c b/fs/crypto/fname.c index 3aafddaab703..3da3707c10e3 100644 --- a/fs/crypto/fname.c +++ b/fs/crypto/fname.c @@ -40,7 +40,7 @@ int fname_encrypt(struct inode *inode, const struct qstr *iname, struct skcipher_request *req = NULL; DECLARE_CRYPTO_WAIT(wait); struct fscrypt_info *ci = inode->i_crypt_info; - struct crypto_skcipher *tfm = ci->ci_key.tfm; + struct crypto_skcipher *tfm = ci->ci_ctfm; union fscrypt_iv iv; struct scatterlist sg; int res; @@ -93,7 +93,7 @@ static int fname_decrypt(struct inode *inode, DECLARE_CRYPTO_WAIT(wait); struct scatterlist src_sg, dst_sg; struct fscrypt_info *ci = inode->i_crypt_info; - struct crypto_skcipher *tfm = ci->ci_key.tfm; + struct crypto_skcipher *tfm = ci->ci_ctfm; union fscrypt_iv iv; int res; diff --git a/fs/crypto/fscrypt_private.h b/fs/crypto/fscrypt_private.h index 19b7a8ea5954..c731bd4245c5 100644 --- a/fs/crypto/fscrypt_private.h +++ b/fs/crypto/fscrypt_private.h @@ -15,6 +15,8 @@ #include #include +struct fscrypt_master_key; + #define CONST_STRLEN(str) (sizeof(str) - 1) #define FS_KEY_DERIVATION_NONCE_SIZE 16 @@ -152,20 +154,6 @@ struct fscrypt_symlink_data { char encrypted_path[1]; } __packed; -/** - * struct fscrypt_prepared_key - a key prepared for actual encryption/decryption - * @tfm: crypto API transform object - * @blk_key: key for blk-crypto - * - * Normally only one of the fields will be non-NULL. - */ -struct fscrypt_prepared_key { - struct crypto_skcipher *tfm; -#ifdef CONFIG_FS_ENCRYPTION_INLINE_CRYPT - struct fscrypt_blk_crypto_key *blk_key; -#endif -}; - /* * fscrypt_info - the "encryption key" for an inode * @@ -175,20 +163,20 @@ struct fscrypt_prepared_key { */ struct fscrypt_info { - /* The key in a form prepared for actual encryption/decryption */ - struct fscrypt_prepared_key ci_key; - - /* True if the key should be freed when this fscrypt_info is freed */ - bool ci_owns_key; + /* The actual crypto transform used for encryption and decryption */ + struct crypto_skcipher *ci_ctfm; #ifdef CONFIG_FS_ENCRYPTION_INLINE_CRYPT /* - * True if this inode will use inline encryption (blk-crypto) instead of - * the traditional filesystem-layer encryption. + * The raw key for inline encryption, if this file is using inline + * encryption rather than the traditional filesystem layer encryption. */ - bool ci_inlinecrypt; + const u8 *ci_inline_crypt_key; #endif + /* True if the key should be freed when this fscrypt_info is freed */ + bool ci_owns_key; + /* * Encryption mode used for this inode. It corresponds to either the * contents or filenames encryption mode, depending on the inode type. @@ -213,7 +201,7 @@ struct fscrypt_info { /* * If non-NULL, then encryption is done using the master key directly - * and ci_key will equal ci_direct_key->dk_key. + * and ci_ctfm will equal ci_direct_key->dk_ctfm. */ struct fscrypt_direct_key *ci_direct_key; @@ -277,7 +265,6 @@ union fscrypt_iv { u8 nonce[FS_KEY_DERIVATION_NONCE_SIZE]; }; u8 raw[FSCRYPT_MAX_IV_SIZE]; - __le64 dun[FSCRYPT_MAX_IV_SIZE / sizeof(__le64)]; }; void fscrypt_generate_iv(union fscrypt_iv *iv, u64 lblk_num, @@ -319,71 +306,49 @@ extern void fscrypt_destroy_hkdf(struct fscrypt_hkdf *hkdf); /* inline_crypt.c */ #ifdef CONFIG_FS_ENCRYPTION_INLINE_CRYPT -extern void fscrypt_select_encryption_impl(struct fscrypt_info *ci); +extern bool fscrypt_should_use_inline_encryption(const struct fscrypt_info *ci); -static inline bool -fscrypt_using_inline_encryption(const struct fscrypt_info *ci) -{ - return ci->ci_inlinecrypt; -} +extern int fscrypt_set_inline_crypt_key(struct fscrypt_info *ci, + const u8 *derived_key); -extern int fscrypt_prepare_inline_crypt_key( - struct fscrypt_prepared_key *prep_key, - const u8 *raw_key, - const struct fscrypt_info *ci); +extern void fscrypt_free_inline_crypt_key(struct fscrypt_info *ci); -extern void fscrypt_destroy_inline_crypt_key( - struct fscrypt_prepared_key *prep_key); +extern int fscrypt_setup_per_mode_inline_crypt_key( + struct fscrypt_info *ci, + struct fscrypt_master_key *mk); -/* - * Check whether the crypto transform or blk-crypto key has been allocated in - * @prep_key, depending on which encryption implementation the file will use. - */ -static inline bool -fscrypt_is_key_prepared(struct fscrypt_prepared_key *prep_key, - const struct fscrypt_info *ci) -{ - /* - * The READ_ONCE() here pairs with the smp_store_release() in - * fscrypt_prepare_key(). (This only matters for the per-mode keys, - * which are shared by multiple inodes.) - */ - if (fscrypt_using_inline_encryption(ci)) - return READ_ONCE(prep_key->blk_key) != NULL; - return READ_ONCE(prep_key->tfm) != NULL; -} +extern void fscrypt_evict_inline_crypt_keys(struct fscrypt_master_key *mk); #else /* CONFIG_FS_ENCRYPTION_INLINE_CRYPT */ -static inline void fscrypt_select_encryption_impl(struct fscrypt_info *ci) -{ -} - -static inline bool fscrypt_using_inline_encryption( +static inline bool fscrypt_should_use_inline_encryption( const struct fscrypt_info *ci) { return false; } -static inline int -fscrypt_prepare_inline_crypt_key(struct fscrypt_prepared_key *prep_key, - const u8 *raw_key, - const struct fscrypt_info *ci) +static inline int fscrypt_set_inline_crypt_key(struct fscrypt_info *ci, + const u8 *derived_key) { WARN_ON(1); return -EOPNOTSUPP; } -static inline void -fscrypt_destroy_inline_crypt_key(struct fscrypt_prepared_key *prep_key) +static inline void fscrypt_free_inline_crypt_key(struct fscrypt_info *ci) { } -static inline bool -fscrypt_is_key_prepared(struct fscrypt_prepared_key *prep_key, - const struct fscrypt_info *ci) +static inline int fscrypt_setup_per_mode_inline_crypt_key( + struct fscrypt_info *ci, + struct fscrypt_master_key *mk) +{ + WARN_ON(1); + return -EOPNOTSUPP; +} + +static inline void fscrypt_evict_inline_crypt_keys( + struct fscrypt_master_key *mk) { - return READ_ONCE(prep_key->tfm) != NULL; } #endif /* !CONFIG_FS_ENCRYPTION_INLINE_CRYPT */ @@ -476,12 +441,25 @@ struct fscrypt_master_key { struct list_head mk_decrypted_inodes; spinlock_t mk_decrypted_inodes_lock; - /* Per-mode keys for DIRECT_KEY policies, allocated on-demand */ - struct fscrypt_prepared_key mk_direct_keys[__FSCRYPT_MODE_MAX + 1]; + /* Crypto API transforms for DIRECT_KEY policies, allocated on-demand */ + struct crypto_skcipher *mk_direct_tfms[__FSCRYPT_MODE_MAX + 1]; - /* Per-mode keys for IV_INO_LBLK_64 policies, allocated on-demand */ - struct fscrypt_prepared_key mk_iv_ino_lblk_64_keys[__FSCRYPT_MODE_MAX + 1]; + /* + * Crypto API transforms for filesystem-layer implementation of + * IV_INO_LBLK_64 policies, allocated on-demand. + */ + struct crypto_skcipher *mk_iv_ino_lblk_64_tfms[__FSCRYPT_MODE_MAX + 1]; +#ifdef CONFIG_FS_ENCRYPTION_INLINE_CRYPT + /* Raw keys for IV_INO_LBLK_64 policies, allocated on-demand */ + u8 *mk_iv_ino_lblk_64_raw_keys[__FSCRYPT_MODE_MAX + 1]; + + /* The data unit size being used for inline encryption */ + unsigned int mk_data_unit_size; + + /* The filesystem's block device */ + struct block_device *mk_bdev; +#endif } __randomize_layout; static inline bool @@ -548,11 +526,9 @@ fscrypt_mode_supports_direct_key(const struct fscrypt_mode *mode) return mode->ivsize >= offsetofend(union fscrypt_iv, nonce); } -extern int fscrypt_prepare_key(struct fscrypt_prepared_key *prep_key, - const u8 *raw_key, - const struct fscrypt_info *ci); - -extern void fscrypt_destroy_prepared_key(struct fscrypt_prepared_key *prep_key); +extern struct crypto_skcipher * +fscrypt_allocate_skcipher(struct fscrypt_mode *mode, const u8 *raw_key, + const struct inode *inode); extern int fscrypt_set_derived_key(struct fscrypt_info *ci, const u8 *derived_key); diff --git a/fs/crypto/inline_crypt.c b/fs/crypto/inline_crypt.c index aa04e8a7d674..e41c6d66ff0d 100644 --- a/fs/crypto/inline_crypt.c +++ b/fs/crypto/inline_crypt.c @@ -15,126 +15,187 @@ #include #include #include +#include #include "fscrypt_private.h" -struct fscrypt_blk_crypto_key { - struct blk_crypto_key base; - int num_devs; - struct request_queue *devs[]; -}; - -/* Enable inline encryption for this file if supported. */ -void fscrypt_select_encryption_impl(struct fscrypt_info *ci) +/* Return true iff inline encryption should be used for this file */ +bool fscrypt_should_use_inline_encryption(const struct fscrypt_info *ci) { const struct inode *inode = ci->ci_inode; struct super_block *sb = inode->i_sb; /* The file must need contents encryption, not filenames encryption */ if (!S_ISREG(inode->i_mode)) - return; + return false; /* blk-crypto must implement the needed encryption algorithm */ if (ci->ci_mode->blk_crypto_mode == BLK_ENCRYPTION_MODE_INVALID) - return; + return false; + + /* DIRECT_KEY needs a 24+ byte IV, so it can't work with 8-byte DUNs */ + if (fscrypt_is_direct_key_policy(&ci->ci_policy)) + return false; /* The filesystem must be mounted with -o inlinecrypt */ if (!sb->s_cop->inline_crypt_enabled || !sb->s_cop->inline_crypt_enabled(sb)) - return; + return false; - ci->ci_inlinecrypt = true; + return true; } -int fscrypt_prepare_inline_crypt_key(struct fscrypt_prepared_key *prep_key, - const u8 *raw_key, - const struct fscrypt_info *ci) +/* Set a per-file inline encryption key (for passing to blk-crypto) */ +int fscrypt_set_inline_crypt_key(struct fscrypt_info *ci, const u8 *derived_key) { - const struct inode *inode = ci->ci_inode; - struct super_block *sb = inode->i_sb; - enum blk_crypto_mode_num crypto_mode = ci->ci_mode->blk_crypto_mode; - int num_devs = 1; - int queue_refs = 0; - struct fscrypt_blk_crypto_key *blk_key; - int err; - int i; + const struct fscrypt_mode *mode = ci->ci_mode; + const struct super_block *sb = ci->ci_inode->i_sb; - if (sb->s_cop->get_num_devices) - num_devs = sb->s_cop->get_num_devices(sb); - if (WARN_ON(num_devs < 1)) + ci->ci_inline_crypt_key = kmemdup(derived_key, mode->keysize, GFP_NOFS); + if (!ci->ci_inline_crypt_key) + return -ENOMEM; + ci->ci_owns_key = true; + + return blk_crypto_start_using_mode(mode->blk_crypto_mode, + sb->s_blocksize, + sb->s_bdev->bd_queue); +} + +/* Free a per-file inline encryption key and evict it from blk-crypto */ +void fscrypt_free_inline_crypt_key(struct fscrypt_info *ci) +{ + if (ci->ci_inline_crypt_key != NULL) { + const struct fscrypt_mode *mode = ci->ci_mode; + const struct super_block *sb = ci->ci_inode->i_sb; + + blk_crypto_evict_key(sb->s_bdev->bd_queue, + ci->ci_inline_crypt_key, + mode->blk_crypto_mode, sb->s_blocksize); + kzfree(ci->ci_inline_crypt_key); + } +} + +/* + * Set up ->inline_crypt_key (for passing to blk-crypto) for inodes which use an + * IV_INO_LBLK_64 encryption policy. + * + * Return: 0 on success, -errno on failure + */ +int fscrypt_setup_per_mode_inline_crypt_key(struct fscrypt_info *ci, + struct fscrypt_master_key *mk) +{ + static DEFINE_MUTEX(inline_crypt_setup_mutex); + const struct super_block *sb = ci->ci_inode->i_sb; + struct block_device *bdev = sb->s_bdev; + const struct fscrypt_mode *mode = ci->ci_mode; + const u8 mode_num = mode - fscrypt_modes; + u8 *raw_key; + u8 hkdf_info[sizeof(mode_num) + sizeof(sb->s_uuid)]; + int err; + + if (WARN_ON(mode_num > __FSCRYPT_MODE_MAX)) return -EINVAL; - blk_key = kzalloc(struct_size(blk_key, devs, num_devs), GFP_NOFS); - if (!blk_key) - return -ENOMEM; - - blk_key->num_devs = num_devs; - if (num_devs == 1) - blk_key->devs[0] = bdev_get_queue(sb->s_bdev); - else - sb->s_cop->get_devices(sb, blk_key->devs); - - err = blk_crypto_init_key(&blk_key->base, raw_key, crypto_mode, - sb->s_blocksize); - if (err) { - fscrypt_err(inode, "error %d initializing blk-crypto key", err); - goto fail; + /* pairs with smp_store_release() below */ + raw_key = smp_load_acquire(&mk->mk_iv_ino_lblk_64_raw_keys[mode_num]); + if (raw_key) { + err = 0; + goto out; } - /* - * We have to start using blk-crypto on all the filesystem's devices. - * We also have to save all the request_queue's for later so that the - * key can be evicted from them. This is needed because some keys - * aren't destroyed until after the filesystem was already unmounted - * (namely, the per-mode keys in struct fscrypt_master_key). - */ - for (i = 0; i < num_devs; i++) { - if (!blk_get_queue(blk_key->devs[i])) { - fscrypt_err(inode, "couldn't get request_queue"); - err = -EAGAIN; - goto fail; - } - queue_refs++; + mutex_lock(&inline_crypt_setup_mutex); - err = blk_crypto_start_using_mode(crypto_mode, sb->s_blocksize, - blk_key->devs[i]); - if (err) { - fscrypt_err(inode, - "error %d starting to use blk-crypto", err); - goto fail; - } + raw_key = mk->mk_iv_ino_lblk_64_raw_keys[mode_num]; + if (raw_key) { + err = 0; + goto out_unlock; } - /* - * Pairs with READ_ONCE() in fscrypt_is_key_prepared(). (Only matters - * for the per-mode keys, which are shared by multiple inodes.) - */ - smp_store_release(&prep_key->blk_key, blk_key); - return 0; -fail: - for (i = 0; i < queue_refs; i++) - blk_put_queue(blk_key->devs[i]); - kzfree(blk_key); + raw_key = kmalloc(mode->keysize, GFP_NOFS); + if (!raw_key) { + err = -ENOMEM; + goto out_unlock; + } + + BUILD_BUG_ON(sizeof(mode_num) != 1); + BUILD_BUG_ON(sizeof(sb->s_uuid) != 16); + BUILD_BUG_ON(sizeof(hkdf_info) != 17); + hkdf_info[0] = mode_num; + memcpy(&hkdf_info[1], &sb->s_uuid, sizeof(sb->s_uuid)); + + err = fscrypt_hkdf_expand(&mk->mk_secret.hkdf, + HKDF_CONTEXT_IV_INO_LBLK_64_KEY, + hkdf_info, sizeof(hkdf_info), + raw_key, mode->keysize); + if (err) + goto out_unlock; + + err = blk_crypto_start_using_mode(mode->blk_crypto_mode, + sb->s_blocksize, bdev->bd_queue); + if (err) + goto out_unlock; + + /* + * When a master key's first inline encryption key is set up, save a + * reference to the filesystem's block device so that the inline + * encryption keys can be evicted when the master key is destroyed. + */ + if (!mk->mk_bdev) { + mk->mk_bdev = bdgrab(bdev); + mk->mk_data_unit_size = sb->s_blocksize; + } + + /* pairs with smp_load_acquire() above */ + smp_store_release(&mk->mk_iv_ino_lblk_64_raw_keys[mode_num], raw_key); + err = 0; +out_unlock: + mutex_unlock(&inline_crypt_setup_mutex); +out: + if (err == 0) { + ci->ci_inline_crypt_key = raw_key; + /* + * Since each struct fscrypt_master_key belongs to a particular + * filesystem (a struct super_block), there should be only one + * block device, and only one data unit size as it should equal + * the filesystem's blocksize (i.e. s_blocksize). + */ + if (WARN_ON(mk->mk_bdev != bdev)) + err = -EINVAL; + if (WARN_ON(mk->mk_data_unit_size != sb->s_blocksize)) + err = -EINVAL; + } else { + kzfree(raw_key); + } return err; } -void fscrypt_destroy_inline_crypt_key(struct fscrypt_prepared_key *prep_key) +/* + * Evict per-mode inline encryption keys from blk-crypto when a master key is + * destroyed. + */ +void fscrypt_evict_inline_crypt_keys(struct fscrypt_master_key *mk) { - struct fscrypt_blk_crypto_key *blk_key = prep_key->blk_key; - int i; + struct block_device *bdev = mk->mk_bdev; + size_t i; - if (blk_key) { - for (i = 0; i < blk_key->num_devs; i++) { - blk_crypto_evict_key(blk_key->devs[i], &blk_key->base); - blk_put_queue(blk_key->devs[i]); + if (!bdev) /* No inline encryption keys? */ + return; + + for (i = 0; i < ARRAY_SIZE(mk->mk_iv_ino_lblk_64_raw_keys); i++) { + u8 *raw_key = mk->mk_iv_ino_lblk_64_raw_keys[i]; + + if (raw_key != NULL) { + blk_crypto_evict_key(bdev->bd_queue, raw_key, + fscrypt_modes[i].blk_crypto_mode, + mk->mk_data_unit_size); + kzfree(raw_key); } - kzfree(blk_key); } + bdput(bdev); } /** - * fscrypt_inode_uses_inline_crypto - test whether an inode uses inline - * encryption + * fscrypt_inode_uses_inline_crypto - test whether an inode uses inline encryption * @inode: an inode * * Return: true if the inode requires file contents encryption and if the @@ -144,13 +205,12 @@ void fscrypt_destroy_inline_crypt_key(struct fscrypt_prepared_key *prep_key) bool fscrypt_inode_uses_inline_crypto(const struct inode *inode) { return IS_ENCRYPTED(inode) && S_ISREG(inode->i_mode) && - inode->i_crypt_info->ci_inlinecrypt; + inode->i_crypt_info->ci_inline_crypt_key != NULL; } EXPORT_SYMBOL_GPL(fscrypt_inode_uses_inline_crypto); /** - * fscrypt_inode_uses_fs_layer_crypto - test whether an inode uses fs-layer - * encryption + * fscrypt_inode_uses_fs_layer_crypto - test whether an inode uses fs-layer encryption * @inode: an inode * * Return: true if the inode requires file contents encryption and if the @@ -160,22 +220,22 @@ EXPORT_SYMBOL_GPL(fscrypt_inode_uses_inline_crypto); bool fscrypt_inode_uses_fs_layer_crypto(const struct inode *inode) { return IS_ENCRYPTED(inode) && S_ISREG(inode->i_mode) && - !inode->i_crypt_info->ci_inlinecrypt; + inode->i_crypt_info->ci_inline_crypt_key == NULL; } EXPORT_SYMBOL_GPL(fscrypt_inode_uses_fs_layer_crypto); -static void fscrypt_generate_dun(const struct fscrypt_info *ci, u64 lblk_num, - u64 dun[BLK_CRYPTO_DUN_ARRAY_SIZE]) +static inline u64 fscrypt_generate_dun(const struct fscrypt_info *ci, + u64 lblk_num) { union fscrypt_iv iv; - int i; fscrypt_generate_iv(&iv, lblk_num, ci); - - BUILD_BUG_ON(FSCRYPT_MAX_IV_SIZE > BLK_CRYPTO_MAX_IV_SIZE); - memset(dun, 0, BLK_CRYPTO_MAX_IV_SIZE); - for (i = 0; i < ci->ci_mode->ivsize/sizeof(dun[0]); i++) - dun[i] = le64_to_cpu(iv.dun[i]); + /* + * fscrypt_should_use_inline_encryption() ensures we never get here if + * more than the first 8 bytes of the IV are nonzero. + */ + BUG_ON(memchr_inv(&iv.raw[8], 0, ci->ci_mode->ivsize - 8)); + return le64_to_cpu(iv.lblk_num); } /** @@ -183,8 +243,7 @@ static void fscrypt_generate_dun(const struct fscrypt_info *ci, u64 lblk_num, * @bio: a bio which will eventually be submitted to the file * @inode: the file's inode * @first_lblk: the first file logical block number in the I/O - * @gfp_mask: memory allocation flags - these must be a waiting mask so that - * bio_crypt_set_ctx can't fail. + * @gfp_mask: memory allocation flags * * If the contents of the file should be encrypted (or decrypted) with inline * encryption, then assign the appropriate encryption context to the bio. @@ -193,18 +252,24 @@ static void fscrypt_generate_dun(const struct fscrypt_info *ci, u64 lblk_num, * otherwise fscrypt_mergeable_bio() won't work as intended. * * The encryption context will be freed automatically when the bio is freed. + * + * Return: 0 on success, -errno on failure. If __GFP_NOFAIL is specified, this + * is guaranteed to succeed. */ -void fscrypt_set_bio_crypt_ctx(struct bio *bio, const struct inode *inode, - u64 first_lblk, gfp_t gfp_mask) +int fscrypt_set_bio_crypt_ctx(struct bio *bio, const struct inode *inode, + u64 first_lblk, gfp_t gfp_mask) { const struct fscrypt_info *ci = inode->i_crypt_info; - u64 dun[BLK_CRYPTO_DUN_ARRAY_SIZE]; + u64 dun; if (!fscrypt_inode_uses_inline_crypto(inode)) - return; + return 0; - fscrypt_generate_dun(ci, first_lblk, dun); - bio_crypt_set_ctx(bio, &ci->ci_key.blk_key->base, dun, gfp_mask); + dun = fscrypt_generate_dun(ci, first_lblk); + + return bio_crypt_set_ctx(bio, ci->ci_inline_crypt_key, + ci->ci_mode->blk_crypto_mode, + dun, inode->i_blkbits, gfp_mask); } EXPORT_SYMBOL_GPL(fscrypt_set_bio_crypt_ctx); @@ -233,24 +298,27 @@ static bool bh_get_inode_and_lblk_num(const struct buffer_head *bh, } /** - * fscrypt_set_bio_crypt_ctx_bh - prepare a file contents bio for inline - * encryption + * fscrypt_set_bio_crypt_ctx_bh - prepare a file contents bio for inline encryption * @bio: a bio which will eventually be submitted to the file * @first_bh: the first buffer_head for which I/O will be submitted * @gfp_mask: memory allocation flags * * Same as fscrypt_set_bio_crypt_ctx(), except this takes a buffer_head instead * of an inode and block number directly. + * + * Return: 0 on success, -errno on failure */ -void fscrypt_set_bio_crypt_ctx_bh(struct bio *bio, +int fscrypt_set_bio_crypt_ctx_bh(struct bio *bio, const struct buffer_head *first_bh, gfp_t gfp_mask) { const struct inode *inode; u64 first_lblk; - if (bh_get_inode_and_lblk_num(first_bh, &inode, &first_lblk)) - fscrypt_set_bio_crypt_ctx(bio, inode, first_lblk, gfp_mask); + if (!bh_get_inode_and_lblk_num(first_bh, &inode, &first_lblk)) + return 0; + + return fscrypt_set_bio_crypt_ctx(bio, inode, first_lblk, gfp_mask); } EXPORT_SYMBOL_GPL(fscrypt_set_bio_crypt_ctx_bh); @@ -274,24 +342,27 @@ EXPORT_SYMBOL_GPL(fscrypt_set_bio_crypt_ctx_bh); bool fscrypt_mergeable_bio(struct bio *bio, const struct inode *inode, u64 next_lblk) { - const struct bio_crypt_ctx *bc = bio->bi_crypt_context; - u64 next_dun[BLK_CRYPTO_DUN_ARRAY_SIZE]; + const struct bio_crypt_ctx *bc; + const u8 *next_key; + u64 next_dun; - if (!!bc != fscrypt_inode_uses_inline_crypto(inode)) + if (bio_has_crypt_ctx(bio) != fscrypt_inode_uses_inline_crypto(inode)) return false; - if (!bc) + if (!bio_has_crypt_ctx(bio)) return true; + bc = bio->bi_crypt_context; + next_key = inode->i_crypt_info->ci_inline_crypt_key; + next_dun = fscrypt_generate_dun(inode->i_crypt_info, next_lblk); /* * Comparing the key pointers is good enough, as all I/O for each key * uses the same pointer. I.e., there's currently no need to support * merging requests where the keys are the same but the pointers differ. */ - if (bc->bc_key != &inode->i_crypt_info->ci_key.blk_key->base) - return false; - - fscrypt_generate_dun(inode->i_crypt_info, next_lblk, next_dun); - return bio_crypt_dun_is_contiguous(bc, bio->bi_iter.bi_size, next_dun); + return next_key == bc->raw_key && + next_dun == bc->data_unit_num + + (bio_sectors(bio) >> + (bc->data_unit_size_bits - SECTOR_SHIFT)); } EXPORT_SYMBOL_GPL(fscrypt_mergeable_bio); @@ -312,7 +383,7 @@ bool fscrypt_mergeable_bio_bh(struct bio *bio, u64 next_lblk; if (!bh_get_inode_and_lblk_num(next_bh, &inode, &next_lblk)) - return !bio->bi_crypt_context; + return !bio_has_crypt_ctx(bio); return fscrypt_mergeable_bio(bio, inode, next_lblk); } diff --git a/fs/crypto/keyring.c b/fs/crypto/keyring.c index e6b0f0da1eaa..05cc274e5c13 100644 --- a/fs/crypto/keyring.c +++ b/fs/crypto/keyring.c @@ -44,10 +44,12 @@ static void free_master_key(struct fscrypt_master_key *mk) wipe_master_key_secret(&mk->mk_secret); for (i = 0; i <= __FSCRYPT_MODE_MAX; i++) { - fscrypt_destroy_prepared_key(&mk->mk_direct_keys[i]); - fscrypt_destroy_prepared_key(&mk->mk_iv_ino_lblk_64_keys[i]); + crypto_free_skcipher(mk->mk_direct_tfms[i]); + crypto_free_skcipher(mk->mk_iv_ino_lblk_64_tfms[i]); } + fscrypt_evict_inline_crypt_keys(mk); + key_put(mk->mk_users); kzfree(mk); } diff --git a/fs/crypto/keysetup.c b/fs/crypto/keysetup.c index f7e909d690b2..8070dad9a541 100644 --- a/fs/crypto/keysetup.c +++ b/fs/crypto/keysetup.c @@ -32,7 +32,6 @@ struct fscrypt_mode fscrypt_modes[] = { .cipher_str = "essiv(cbc(aes),sha256)", .keysize = 16, .ivsize = 16, - .blk_crypto_mode = BLK_ENCRYPTION_MODE_AES_128_CBC_ESSIV, }, [FSCRYPT_MODE_AES_128_CTS] = { .friendly_name = "AES-128-CTS-CBC", @@ -45,7 +44,6 @@ struct fscrypt_mode fscrypt_modes[] = { .cipher_str = "adiantum(xchacha12,aes)", .keysize = 32, .ivsize = 32, - .blk_crypto_mode = BLK_ENCRYPTION_MODE_ADIANTUM, }, }; @@ -65,9 +63,9 @@ select_encryption_mode(const union fscrypt_policy *policy, } /* Create a symmetric cipher object for the given encryption mode and key */ -static struct crypto_skcipher * -fscrypt_allocate_skcipher(struct fscrypt_mode *mode, const u8 *raw_key, - const struct inode *inode) +struct crypto_skcipher *fscrypt_allocate_skcipher(struct fscrypt_mode *mode, + const u8 *raw_key, + const struct inode *inode) { struct crypto_skcipher *tfm; int err; @@ -109,55 +107,33 @@ err_free_tfm: return ERR_PTR(err); } -/* - * Prepare the crypto transform object or blk-crypto key in @prep_key, given the - * raw key, encryption mode, and flag indicating which encryption implementation - * (fs-layer or blk-crypto) will be used. - */ -int fscrypt_prepare_key(struct fscrypt_prepared_key *prep_key, - const u8 *raw_key, const struct fscrypt_info *ci) -{ - struct crypto_skcipher *tfm; - - if (fscrypt_using_inline_encryption(ci)) - return fscrypt_prepare_inline_crypt_key(prep_key, raw_key, ci); - - tfm = fscrypt_allocate_skcipher(ci->ci_mode, raw_key, ci->ci_inode); - if (IS_ERR(tfm)) - return PTR_ERR(tfm); - /* - * Pairs with READ_ONCE() in fscrypt_is_key_prepared(). (Only matters - * for the per-mode keys, which are shared by multiple inodes.) - */ - smp_store_release(&prep_key->tfm, tfm); - return 0; -} - -/* Destroy a crypto transform object and/or blk-crypto key. */ -void fscrypt_destroy_prepared_key(struct fscrypt_prepared_key *prep_key) -{ - crypto_free_skcipher(prep_key->tfm); - fscrypt_destroy_inline_crypt_key(prep_key); -} - /* Given the per-file key, set up the file's crypto transform object */ int fscrypt_set_derived_key(struct fscrypt_info *ci, const u8 *derived_key) { + struct crypto_skcipher *tfm; + + if (fscrypt_should_use_inline_encryption(ci)) + return fscrypt_set_inline_crypt_key(ci, derived_key); + + tfm = fscrypt_allocate_skcipher(ci->ci_mode, derived_key, ci->ci_inode); + if (IS_ERR(tfm)) + return PTR_ERR(tfm); + + ci->ci_ctfm = tfm; ci->ci_owns_key = true; - return fscrypt_prepare_key(&ci->ci_key, derived_key, ci); + return 0; } static int setup_per_mode_key(struct fscrypt_info *ci, struct fscrypt_master_key *mk, - struct fscrypt_prepared_key *keys, + struct crypto_skcipher **tfms, u8 hkdf_context, bool include_fs_uuid) { - static DEFINE_MUTEX(mode_key_setup_mutex); const struct inode *inode = ci->ci_inode; const struct super_block *sb = inode->i_sb; struct fscrypt_mode *mode = ci->ci_mode; const u8 mode_num = mode - fscrypt_modes; - struct fscrypt_prepared_key *prep_key; + struct crypto_skcipher *tfm, *prev_tfm; u8 mode_key[FSCRYPT_MAX_KEY_SIZE]; u8 hkdf_info[sizeof(mode_num) + sizeof(sb->s_uuid)]; unsigned int hkdf_infolen = 0; @@ -166,16 +142,10 @@ static int setup_per_mode_key(struct fscrypt_info *ci, if (WARN_ON(mode_num > __FSCRYPT_MODE_MAX)) return -EINVAL; - prep_key = &keys[mode_num]; - if (fscrypt_is_key_prepared(prep_key, ci)) { - ci->ci_key = *prep_key; - return 0; - } - - mutex_lock(&mode_key_setup_mutex); - - if (fscrypt_is_key_prepared(prep_key, ci)) - goto done_unlock; + /* pairs with cmpxchg() below */ + tfm = READ_ONCE(tfms[mode_num]); + if (likely(tfm != NULL)) + goto done; BUILD_BUG_ON(sizeof(mode_num) != 1); BUILD_BUG_ON(sizeof(sb->s_uuid) != 16); @@ -190,17 +160,21 @@ static int setup_per_mode_key(struct fscrypt_info *ci, hkdf_context, hkdf_info, hkdf_infolen, mode_key, mode->keysize); if (err) - goto out_unlock; - err = fscrypt_prepare_key(prep_key, mode_key, ci); + return err; + tfm = fscrypt_allocate_skcipher(mode, mode_key, inode); memzero_explicit(mode_key, mode->keysize); - if (err) - goto out_unlock; -done_unlock: - ci->ci_key = *prep_key; - err = 0; -out_unlock: - mutex_unlock(&mode_key_setup_mutex); - return err; + if (IS_ERR(tfm)) + return PTR_ERR(tfm); + + /* pairs with READ_ONCE() above */ + prev_tfm = cmpxchg(&tfms[mode_num], NULL, tfm); + if (prev_tfm != NULL) { + crypto_free_skcipher(tfm); + tfm = prev_tfm; + } +done: + ci->ci_ctfm = tfm; + return 0; } static int fscrypt_setup_v2_file_key(struct fscrypt_info *ci, @@ -224,7 +198,7 @@ static int fscrypt_setup_v2_file_key(struct fscrypt_info *ci, ci->ci_mode->friendly_name); return -EINVAL; } - return setup_per_mode_key(ci, mk, mk->mk_direct_keys, + return setup_per_mode_key(ci, mk, mk->mk_direct_tfms, HKDF_CONTEXT_DIRECT_KEY, false); } else if (ci->ci_policy.v2.flags & FSCRYPT_POLICY_FLAG_IV_INO_LBLK_64) { @@ -234,7 +208,9 @@ static int fscrypt_setup_v2_file_key(struct fscrypt_info *ci, * the IVs. This format is optimized for use with inline * encryption hardware compliant with the UFS or eMMC standards. */ - return setup_per_mode_key(ci, mk, mk->mk_iv_ino_lblk_64_keys, + if (fscrypt_should_use_inline_encryption(ci)) + return fscrypt_setup_per_mode_inline_crypt_key(ci, mk); + return setup_per_mode_key(ci, mk, mk->mk_iv_ino_lblk_64_tfms, HKDF_CONTEXT_IV_INO_LBLK_64_KEY, true); } @@ -269,8 +245,6 @@ static int setup_file_encryption_key(struct fscrypt_info *ci, struct fscrypt_key_specifier mk_spec; int err; - fscrypt_select_encryption_impl(ci); - switch (ci->ci_policy.version) { case FSCRYPT_POLICY_V1: mk_spec.type = FSCRYPT_KEY_SPEC_TYPE_DESCRIPTOR; @@ -362,8 +336,10 @@ static void put_crypt_info(struct fscrypt_info *ci) if (ci->ci_direct_key) fscrypt_put_direct_key(ci->ci_direct_key); - else if (ci->ci_owns_key) - fscrypt_destroy_prepared_key(&ci->ci_key); + else if (ci->ci_owns_key) { + crypto_free_skcipher(ci->ci_ctfm); + fscrypt_free_inline_crypt_key(ci); + } key = ci->ci_master_key; if (key) { diff --git a/fs/crypto/keysetup_v1.c b/fs/crypto/keysetup_v1.c index 423ee9b64108..5298ef22aa85 100644 --- a/fs/crypto/keysetup_v1.c +++ b/fs/crypto/keysetup_v1.c @@ -146,7 +146,7 @@ struct fscrypt_direct_key { struct hlist_node dk_node; refcount_t dk_refcount; const struct fscrypt_mode *dk_mode; - struct fscrypt_prepared_key dk_key; + struct crypto_skcipher *dk_ctfm; u8 dk_descriptor[FSCRYPT_KEY_DESCRIPTOR_SIZE]; u8 dk_raw[FSCRYPT_MAX_KEY_SIZE]; }; @@ -154,7 +154,7 @@ struct fscrypt_direct_key { static void free_direct_key(struct fscrypt_direct_key *dk) { if (dk) { - fscrypt_destroy_prepared_key(&dk->dk_key); + crypto_free_skcipher(dk->dk_ctfm); kzfree(dk); } } @@ -199,8 +199,6 @@ find_or_insert_direct_key(struct fscrypt_direct_key *to_insert, continue; if (ci->ci_mode != dk->dk_mode) continue; - if (!fscrypt_is_key_prepared(&dk->dk_key, ci)) - continue; if (crypto_memneq(raw_key, dk->dk_raw, ci->ci_mode->keysize)) continue; /* using existing tfm with same (descriptor, mode, raw_key) */ @@ -233,9 +231,13 @@ fscrypt_get_direct_key(const struct fscrypt_info *ci, const u8 *raw_key) return ERR_PTR(-ENOMEM); refcount_set(&dk->dk_refcount, 1); dk->dk_mode = ci->ci_mode; - err = fscrypt_prepare_key(&dk->dk_key, raw_key, ci); - if (err) + dk->dk_ctfm = fscrypt_allocate_skcipher(ci->ci_mode, raw_key, + ci->ci_inode); + if (IS_ERR(dk->dk_ctfm)) { + err = PTR_ERR(dk->dk_ctfm); + dk->dk_ctfm = NULL; goto err_free_dk; + } memcpy(dk->dk_descriptor, ci->ci_policy.v1.master_key_descriptor, FSCRYPT_KEY_DESCRIPTOR_SIZE); memcpy(dk->dk_raw, raw_key, ci->ci_mode->keysize); @@ -272,7 +274,7 @@ static int setup_v1_file_key_direct(struct fscrypt_info *ci, if (IS_ERR(dk)) return PTR_ERR(dk); ci->ci_direct_key = dk; - ci->ci_key = dk->dk_key; + ci->ci_ctfm = dk->dk_ctfm; return 0; } diff --git a/fs/ext4/page-io.c b/fs/ext4/page-io.c index b55038236845..46a4aeef8275 100644 --- a/fs/ext4/page-io.c +++ b/fs/ext4/page-io.c @@ -362,11 +362,16 @@ static int io_submit_init_bio(struct ext4_io_submit *io, struct buffer_head *bh) { struct bio *bio; + int err; bio = bio_alloc(GFP_NOIO, BIO_MAX_PAGES); if (!bio) return -ENOMEM; - fscrypt_set_bio_crypt_ctx_bh(bio, bh, GFP_NOIO); + err = fscrypt_set_bio_crypt_ctx_bh(bio, bh, GFP_NOIO); + if (err) { + bio_put(bio); + return err; + } bio->bi_iter.bi_sector = bh->b_blocknr * (bh->b_size >> 9); bio_set_dev(bio, bh->b_bdev); bio->bi_end_io = ext4_end_bio; diff --git a/fs/ext4/readpage.c b/fs/ext4/readpage.c index 522ccc42b7e5..e21216e7ae24 100644 --- a/fs/ext4/readpage.c +++ b/fs/ext4/readpage.c @@ -406,8 +406,12 @@ int ext4_mpage_readpages(struct address_space *mapping, min_t(int, nr_pages, BIO_MAX_PAGES)); if (!bio) goto set_error_page; - fscrypt_set_bio_crypt_ctx(bio, inode, next_block, - GFP_KERNEL); + if (fscrypt_set_bio_crypt_ctx(bio, inode, next_block, + GFP_KERNEL) != 0) { + bio_put(bio); + bio = NULL; + goto set_error_page; + } ctx = get_bio_post_read_ctx(inode, bio, page->index); if (IS_ERR(ctx)) { bio_put(bio); diff --git a/fs/f2fs/data.c b/fs/f2fs/data.c index b494242e66b1..ce44cda13bfe 100644 --- a/fs/f2fs/data.c +++ b/fs/f2fs/data.c @@ -307,7 +307,7 @@ static struct bio *__bio_alloc(struct f2fs_io_info *fio, int npages) return bio; } -static void f2fs_set_bio_crypt_ctx(struct bio *bio, const struct inode *inode, +static int f2fs_set_bio_crypt_ctx(struct bio *bio, const struct inode *inode, pgoff_t first_idx, const struct f2fs_io_info *fio, gfp_t gfp_mask) @@ -316,8 +316,10 @@ static void f2fs_set_bio_crypt_ctx(struct bio *bio, const struct inode *inode, * The f2fs garbage collector sets ->encrypted_page when it wants to * read/write raw data without encryption. */ - if (!fio || !fio->encrypted_page) - fscrypt_set_bio_crypt_ctx(bio, inode, first_idx, gfp_mask); + if (fio && fio->encrypted_page) + return 0; + + return fscrypt_set_bio_crypt_ctx(bio, inode, first_idx, gfp_mask); } static bool f2fs_crypt_mergeable_bio(struct bio *bio, const struct inode *inode, @@ -505,6 +507,7 @@ int f2fs_submit_page_bio(struct f2fs_io_info *fio) struct bio *bio; struct page *page = fio->encrypted_page ? fio->encrypted_page : fio->page; + int err; if (!f2fs_is_valid_blkaddr(fio->sbi, fio->new_blkaddr, fio->is_por ? META_POR : (__is_meta_io(fio) ? @@ -517,8 +520,12 @@ int f2fs_submit_page_bio(struct f2fs_io_info *fio) /* Allocate a new bio */ bio = __bio_alloc(fio, 1); - f2fs_set_bio_crypt_ctx(bio, fio->page->mapping->host, - fio->page->index, fio, GFP_NOIO); + err = f2fs_set_bio_crypt_ctx(bio, fio->page->mapping->host, + fio->page->index, fio, GFP_NOIO); + if (err) { + bio_put(bio); + return err; + } if (bio_add_page(bio, page, PAGE_SIZE, 0) < PAGE_SIZE) { bio_put(bio); @@ -599,7 +606,7 @@ alloc_new: bio = __bio_alloc(fio, BIO_MAX_PAGES); f2fs_set_bio_crypt_ctx(bio, fio->page->mapping->host, fio->page->index, fio, - GFP_NOIO); + GFP_NOIO | __GFP_NOFAIL); bio_set_op_attrs(bio, fio->op, fio->op_flags); } @@ -683,7 +690,7 @@ alloc_new: io->bio = __bio_alloc(fio, BIO_MAX_PAGES); f2fs_set_bio_crypt_ctx(io->bio, fio->page->mapping->host, fio->page->index, fio, - GFP_NOIO); + GFP_NOIO | __GFP_NOFAIL); io->fio = *fio; } @@ -723,12 +730,17 @@ static struct bio *f2fs_grab_read_bio(struct inode *inode, block_t blkaddr, struct bio *bio; struct bio_post_read_ctx *ctx; unsigned int post_read_steps = 0; + int err; bio = f2fs_bio_alloc(sbi, min_t(int, nr_pages, BIO_MAX_PAGES), false); if (!bio) return ERR_PTR(-ENOMEM); - f2fs_set_bio_crypt_ctx(bio, inode, first_idx, NULL, GFP_NOFS); + err = f2fs_set_bio_crypt_ctx(bio, inode, first_idx, NULL, GFP_NOFS); + if (err) { + bio_put(bio); + return ERR_PTR(err); + } f2fs_target_device(sbi, blkaddr, bio); bio->bi_end_io = f2fs_read_end_io; diff --git a/fs/f2fs/super.c b/fs/f2fs/super.c index eecfe09c52e7..850a2a2394d8 100644 --- a/fs/f2fs/super.c +++ b/fs/f2fs/super.c @@ -2339,25 +2339,6 @@ static bool f2fs_inline_crypt_enabled(struct super_block *sb) return F2FS_OPTION(F2FS_SB(sb)).inlinecrypt; } -static int f2fs_get_num_devices(struct super_block *sb) -{ - struct f2fs_sb_info *sbi = F2FS_SB(sb); - - if (f2fs_is_multi_device(sbi)) - return sbi->s_ndevs; - return 1; -} - -static void f2fs_get_devices(struct super_block *sb, - struct request_queue **devs) -{ - struct f2fs_sb_info *sbi = F2FS_SB(sb); - int i; - - for (i = 0; i < sbi->s_ndevs; i++) - devs[i] = bdev_get_queue(FDEV(i).bdev); -} - static const struct fscrypt_operations f2fs_cryptops = { .key_prefix = "f2fs:", .get_context = f2fs_get_context, @@ -2368,8 +2349,6 @@ static const struct fscrypt_operations f2fs_cryptops = { .has_stable_inodes = f2fs_has_stable_inodes, .get_ino_and_lblk_bits = f2fs_get_ino_and_lblk_bits, .inline_crypt_enabled = f2fs_inline_crypt_enabled, - .get_num_devices = f2fs_get_num_devices, - .get_devices = f2fs_get_devices, }; #endif diff --git a/include/linux/bio-crypt-ctx.h b/include/linux/bio-crypt-ctx.h index 4535df0a6349..7c389f310bab 100644 --- a/include/linux/bio-crypt-ctx.h +++ b/include/linux/bio-crypt-ctx.h @@ -6,188 +6,221 @@ #define __LINUX_BIO_CRYPT_CTX_H enum blk_crypto_mode_num { - BLK_ENCRYPTION_MODE_INVALID, - BLK_ENCRYPTION_MODE_AES_256_XTS, - BLK_ENCRYPTION_MODE_AES_128_CBC_ESSIV, - BLK_ENCRYPTION_MODE_ADIANTUM, - BLK_ENCRYPTION_MODE_MAX, + BLK_ENCRYPTION_MODE_INVALID = 0, + BLK_ENCRYPTION_MODE_AES_256_XTS = 1, }; #ifdef CONFIG_BLOCK #include #ifdef CONFIG_BLK_INLINE_ENCRYPTION - -#define BLK_CRYPTO_MAX_KEY_SIZE 64 - -/** - * struct blk_crypto_key - an inline encryption key - * @crypto_mode: encryption algorithm this key is for - * @data_unit_size: the data unit size for all encryption/decryptions with this - * key. This is the size in bytes of each individual plaintext and - * ciphertext. This is always a power of 2. It might be e.g. the - * filesystem block size or the disk sector size. - * @data_unit_size_bits: log2 of data_unit_size - * @size: size of this key in bytes (determined by @crypto_mode) - * @hash: hash of this key, for keyslot manager use only - * @raw: the raw bytes of this key. Only the first @size bytes are used. - * - * A blk_crypto_key is immutable once created, and many bios can reference it at - * the same time. It must not be freed until all bios using it have completed. - */ -struct blk_crypto_key { - enum blk_crypto_mode_num crypto_mode; - unsigned int data_unit_size; - unsigned int data_unit_size_bits; - unsigned int size; - unsigned int hash; - u8 raw[BLK_CRYPTO_MAX_KEY_SIZE]; -}; - -#define BLK_CRYPTO_MAX_IV_SIZE 32 -#define BLK_CRYPTO_DUN_ARRAY_SIZE (BLK_CRYPTO_MAX_IV_SIZE/sizeof(u64)) - -/** - * struct bio_crypt_ctx - an inline encryption context - * @bc_key: the key, algorithm, and data unit size to use - * @bc_keyslot: the keyslot that has been assigned for this key in @bc_ksm, - * or -1 if no keyslot has been assigned yet. - * @bc_dun: the data unit number (starting IV) to use - * @bc_ksm: the keyslot manager into which the key has been programmed with - * @bc_keyslot, or NULL if this key hasn't yet been programmed. - * - * A bio_crypt_ctx specifies that the contents of the bio will be encrypted (for - * write requests) or decrypted (for read requests) inline by the storage device - * or controller, or by the crypto API fallback. - */ struct bio_crypt_ctx { - const struct blk_crypto_key *bc_key; - int bc_keyslot; - - /* Data unit number */ - u64 bc_dun[BLK_CRYPTO_DUN_ARRAY_SIZE]; + int keyslot; + const u8 *raw_key; + enum blk_crypto_mode_num crypto_mode; + u64 data_unit_num; + unsigned int data_unit_size_bits; /* * The keyslot manager where the key has been programmed * with keyslot. */ - struct keyslot_manager *bc_ksm; + struct keyslot_manager *processing_ksm; + + /* + * Copy of the bvec_iter when this bio was submitted. + * We only want to en/decrypt the part of the bio + * as described by the bvec_iter upon submission because + * bio might be split before being resubmitted + */ + struct bvec_iter crypt_iter; + u64 sw_data_unit_num; }; -int bio_crypt_ctx_init(void); - -struct bio_crypt_ctx *bio_crypt_alloc_ctx(gfp_t gfp_mask); - -void bio_crypt_free_ctx(struct bio *bio); +extern int bio_crypt_clone(struct bio *dst, struct bio *src, + gfp_t gfp_mask); static inline bool bio_has_crypt_ctx(struct bio *bio) { return bio->bi_crypt_context; } -void bio_crypt_clone(struct bio *dst, struct bio *src, gfp_t gfp_mask); - -static inline void bio_crypt_set_ctx(struct bio *bio, - const struct blk_crypto_key *key, - u64 dun[BLK_CRYPTO_DUN_ARRAY_SIZE], - gfp_t gfp_mask) -{ - struct bio_crypt_ctx *bc = bio_crypt_alloc_ctx(gfp_mask); - - bc->bc_key = key; - memcpy(bc->bc_dun, dun, sizeof(bc->bc_dun)); - bc->bc_ksm = NULL; - bc->bc_keyslot = -1; - - bio->bi_crypt_context = bc; -} - -void bio_crypt_ctx_release_keyslot(struct bio_crypt_ctx *bc); - -int bio_crypt_ctx_acquire_keyslot(struct bio_crypt_ctx *bc, - struct keyslot_manager *ksm); - -struct request; -bool bio_crypt_should_process(struct request *rq); - -static inline bool bio_crypt_dun_is_contiguous(const struct bio_crypt_ctx *bc, - unsigned int bytes, - u64 next_dun[BLK_CRYPTO_DUN_ARRAY_SIZE]) -{ - int i = 0; - unsigned int inc = bytes >> bc->bc_key->data_unit_size_bits; - - while (inc && i < BLK_CRYPTO_DUN_ARRAY_SIZE) { - if (bc->bc_dun[i] + inc != next_dun[i]) - return false; - inc = ((bc->bc_dun[i] + inc) < inc); - i++; - } - - return true; -} - - -static inline void bio_crypt_dun_increment(u64 dun[BLK_CRYPTO_DUN_ARRAY_SIZE], - unsigned int inc) -{ - int i = 0; - - while (inc && i < BLK_CRYPTO_DUN_ARRAY_SIZE) { - dun[i] += inc; - inc = (dun[i] < inc); - i++; - } -} - static inline void bio_crypt_advance(struct bio *bio, unsigned int bytes) { - struct bio_crypt_ctx *bc = bio->bi_crypt_context; - - if (!bc) - return; - - bio_crypt_dun_increment(bc->bc_dun, - bytes >> bc->bc_key->data_unit_size_bits); + if (bio_has_crypt_ctx(bio)) { + bio->bi_crypt_context->data_unit_num += + bytes >> bio->bi_crypt_context->data_unit_size_bits; + } } -bool bio_crypt_ctx_compatible(struct bio *b_1, struct bio *b_2); +extern bool bio_crypt_swhandled(struct bio *bio); -bool bio_crypt_ctx_mergeable(struct bio *b_1, unsigned int b1_bytes, - struct bio *b_2); +static inline bool bio_crypt_has_keyslot(struct bio *bio) +{ + return bio->bi_crypt_context->keyslot >= 0; +} + +extern int bio_crypt_ctx_init(void); + +extern struct bio_crypt_ctx *bio_crypt_alloc_ctx(gfp_t gfp_mask); + +extern void bio_crypt_free_ctx(struct bio *bio); + +static inline int bio_crypt_set_ctx(struct bio *bio, + const u8 *raw_key, + enum blk_crypto_mode_num crypto_mode, + u64 dun, + unsigned int dun_bits, + gfp_t gfp_mask) +{ + struct bio_crypt_ctx *crypt_ctx; + + crypt_ctx = bio_crypt_alloc_ctx(gfp_mask); + if (!crypt_ctx) + return -ENOMEM; + + crypt_ctx->raw_key = raw_key; + crypt_ctx->data_unit_num = dun; + crypt_ctx->data_unit_size_bits = dun_bits; + crypt_ctx->crypto_mode = crypto_mode; + crypt_ctx->processing_ksm = NULL; + crypt_ctx->keyslot = -1; + bio->bi_crypt_context = crypt_ctx; + + return 0; +} + +static inline void bio_set_data_unit_num(struct bio *bio, u64 dun) +{ + bio->bi_crypt_context->data_unit_num = dun; +} + +static inline int bio_crypt_get_keyslot(struct bio *bio) +{ + return bio->bi_crypt_context->keyslot; +} + +static inline void bio_crypt_set_keyslot(struct bio *bio, + unsigned int keyslot, + struct keyslot_manager *ksm) +{ + bio->bi_crypt_context->keyslot = keyslot; + bio->bi_crypt_context->processing_ksm = ksm; +} + +extern void bio_crypt_ctx_release_keyslot(struct bio *bio); + +extern int bio_crypt_ctx_acquire_keyslot(struct bio *bio, + struct keyslot_manager *ksm); + +static inline const u8 *bio_crypt_raw_key(struct bio *bio) +{ + return bio->bi_crypt_context->raw_key; +} + +static inline enum blk_crypto_mode_num bio_crypto_mode(struct bio *bio) +{ + return bio->bi_crypt_context->crypto_mode; +} + +static inline u64 bio_crypt_data_unit_num(struct bio *bio) +{ + return bio->bi_crypt_context->data_unit_num; +} + +static inline u64 bio_crypt_sw_data_unit_num(struct bio *bio) +{ + return bio->bi_crypt_context->sw_data_unit_num; +} + +extern bool bio_crypt_should_process(struct bio *bio, struct request_queue *q); + +extern bool bio_crypt_ctx_compatible(struct bio *b_1, struct bio *b_2); + +extern bool bio_crypt_ctx_back_mergeable(struct bio *b_1, + unsigned int b1_sectors, + struct bio *b_2); #else /* CONFIG_BLK_INLINE_ENCRYPTION */ +struct keyslot_manager; + static inline int bio_crypt_ctx_init(void) { return 0; } +static inline int bio_crypt_clone(struct bio *dst, struct bio *src, + gfp_t gfp_mask) +{ + return 0; +} + +static inline void bio_crypt_advance(struct bio *bio, + unsigned int bytes) { } + static inline bool bio_has_crypt_ctx(struct bio *bio) { return false; } -static inline void bio_crypt_clone(struct bio *dst, struct bio *src, - gfp_t gfp_mask) { } - static inline void bio_crypt_free_ctx(struct bio *bio) { } -static inline void bio_crypt_advance(struct bio *bio, unsigned int bytes) { } +static inline void bio_crypt_set_ctx(struct bio *bio, + u8 *raw_key, + enum blk_crypto_mode_num crypto_mode, + u64 dun, + unsigned int dun_bits, + gfp_t gfp_mask) { } + +static inline bool bio_crypt_swhandled(struct bio *bio) +{ + return false; +} + +static inline void bio_set_data_unit_num(struct bio *bio, u64 dun) { } + +static inline bool bio_crypt_has_keyslot(struct bio *bio) +{ + return false; +} + +static inline void bio_crypt_set_keyslot(struct bio *bio, + unsigned int keyslot, + struct keyslot_manager *ksm) { } + +static inline int bio_crypt_get_keyslot(struct bio *bio) +{ + return -1; +} + +static inline u8 *bio_crypt_raw_key(struct bio *bio) +{ + return NULL; +} + +static inline u64 bio_crypt_data_unit_num(struct bio *bio) +{ + return 0; +} + +static inline bool bio_crypt_should_process(struct bio *bio, + struct request_queue *q) +{ + return false; +} static inline bool bio_crypt_ctx_compatible(struct bio *b_1, struct bio *b_2) { return true; } -static inline bool bio_crypt_ctx_mergeable(struct bio *b_1, - unsigned int b1_bytes, - struct bio *b_2) +static inline bool bio_crypt_ctx_back_mergeable(struct bio *b_1, + unsigned int b1_sectors, + struct bio *b_2) { return true; } #endif /* CONFIG_BLK_INLINE_ENCRYPTION */ - #endif /* CONFIG_BLOCK */ - #endif /* __LINUX_BIO_CRYPT_CTX_H */ diff --git a/include/linux/blk-crypto.h b/include/linux/blk-crypto.h index 07d32e33f0bb..2a07401244a6 100644 --- a/include/linux/blk-crypto.h +++ b/include/linux/blk-crypto.h @@ -6,23 +6,32 @@ #ifndef __LINUX_BLK_CRYPTO_H #define __LINUX_BLK_CRYPTO_H +#include #include #ifdef CONFIG_BLK_INLINE_ENCRYPTION +int blk_crypto_init(void); + int blk_crypto_submit_bio(struct bio **bio_ptr); bool blk_crypto_endio(struct bio *bio); -int blk_crypto_init_key(struct blk_crypto_key *blk_key, const u8 *raw_key, - enum blk_crypto_mode_num crypto_mode, - unsigned int data_unit_size); +int blk_crypto_start_using_mode(enum blk_crypto_mode_num mode_num, + unsigned int data_unit_size, + struct request_queue *q); -int blk_crypto_evict_key(struct request_queue *q, - const struct blk_crypto_key *key); +int blk_crypto_evict_key(struct request_queue *q, const u8 *key, + enum blk_crypto_mode_num mode, + unsigned int data_unit_size); #else /* CONFIG_BLK_INLINE_ENCRYPTION */ +static inline int blk_crypto_init(void) +{ + return 0; +} + static inline int blk_crypto_submit_bio(struct bio **bio_ptr) { return 0; @@ -33,31 +42,21 @@ static inline bool blk_crypto_endio(struct bio *bio) return true; } -#endif /* CONFIG_BLK_INLINE_ENCRYPTION */ - -#ifdef CONFIG_BLK_INLINE_ENCRYPTION_FALLBACK - -int blk_crypto_start_using_mode(enum blk_crypto_mode_num mode_num, - unsigned int data_unit_size, - struct request_queue *q); - -int blk_crypto_fallback_init(void); - -#else /* CONFIG_BLK_INLINE_ENCRYPTION_FALLBACK */ - static inline int blk_crypto_start_using_mode(enum blk_crypto_mode_num mode_num, unsigned int data_unit_size, struct request_queue *q) { - return 0; + return -EOPNOTSUPP; } -static inline int blk_crypto_fallback_init(void) +static inline int blk_crypto_evict_key(struct request_queue *q, const u8 *key, + enum blk_crypto_mode_num mode, + unsigned int data_unit_size) { return 0; } -#endif /* CONFIG_BLK_INLINE_ENCRYPTION_FALLBACK */ +#endif /* CONFIG_BLK_INLINE_ENCRYPTION */ #endif /* __LINUX_BLK_CRYPTO_H */ diff --git a/include/linux/fscrypt.h b/include/linux/fscrypt.h index 0b15a0a6a354..607076207f73 100644 --- a/include/linux/fscrypt.h +++ b/include/linux/fscrypt.h @@ -65,9 +65,6 @@ struct fscrypt_operations { void (*get_ino_and_lblk_bits)(struct super_block *sb, int *ino_bits_ret, int *lblk_bits_ret); bool (*inline_crypt_enabled)(struct super_block *sb); - int (*get_num_devices)(struct super_block *sb); - void (*get_devices)(struct super_block *sb, - struct request_queue **devs); }; static inline bool fscrypt_has_encryption_key(const struct inode *inode) @@ -553,13 +550,12 @@ extern bool fscrypt_inode_uses_inline_crypto(const struct inode *inode); extern bool fscrypt_inode_uses_fs_layer_crypto(const struct inode *inode); -extern void fscrypt_set_bio_crypt_ctx(struct bio *bio, - const struct inode *inode, - u64 first_lblk, gfp_t gfp_mask); +extern int fscrypt_set_bio_crypt_ctx(struct bio *bio, const struct inode *inode, + u64 first_lblk, gfp_t gfp_mask); -extern void fscrypt_set_bio_crypt_ctx_bh(struct bio *bio, - const struct buffer_head *first_bh, - gfp_t gfp_mask); +extern int fscrypt_set_bio_crypt_ctx_bh(struct bio *bio, + const struct buffer_head *first_bh, + gfp_t gfp_mask); extern bool fscrypt_mergeable_bio(struct bio *bio, const struct inode *inode, u64 next_lblk); @@ -578,14 +574,20 @@ static inline bool fscrypt_inode_uses_fs_layer_crypto(const struct inode *inode) return IS_ENCRYPTED(inode) && S_ISREG(inode->i_mode); } -static inline void fscrypt_set_bio_crypt_ctx(struct bio *bio, - const struct inode *inode, - u64 first_lblk, gfp_t gfp_mask) { } +static inline int fscrypt_set_bio_crypt_ctx(struct bio *bio, + const struct inode *inode, + u64 first_lblk, gfp_t gfp_mask) +{ + return 0; +} -static inline void fscrypt_set_bio_crypt_ctx_bh( - struct bio *bio, - const struct buffer_head *first_bh, - gfp_t gfp_mask) { } +static inline int fscrypt_set_bio_crypt_ctx_bh( + struct bio *bio, + const struct buffer_head *first_bh, + gfp_t gfp_mask) +{ + return 0; +} static inline bool fscrypt_mergeable_bio(struct bio *bio, const struct inode *inode, diff --git a/include/linux/keyslot-manager.h b/include/linux/keyslot-manager.h index fbc423fe5cd5..0777ade7907c 100644 --- a/include/linux/keyslot-manager.h +++ b/include/linux/keyslot-manager.h @@ -3,58 +3,96 @@ * Copyright 2019 Google LLC */ +#include + +#ifdef CONFIG_BLOCK + #ifndef __LINUX_KEYSLOT_MANAGER_H #define __LINUX_KEYSLOT_MANAGER_H -#include - -struct keyslot_manager; - /** * struct keyslot_mgmt_ll_ops - functions to manage keyslots in hardware - * @keyslot_program: Program the specified key into the specified slot in the - * inline encryption hardware. + * @keyslot_program: Program the specified key and algorithm into the + * specified slot in the inline encryption hardware. * @keyslot_evict: Evict key from the specified keyslot in the hardware. - * The key is provided so that e.g. dm layers can evict - * keys from the devices that they map over. + * The key, crypto_mode and data_unit_size are also passed + * down so that e.g. dm layers can evict keys from + * the devices that they map over. * Returns 0 on success, -errno otherwise. + * @crypto_mode_supported: Check whether a crypto_mode and data_unit_size + * combo is supported. + * @keyslot_find: Returns the slot number that matches the key, + * or -ENOKEY if no match found, or -errno on + * error. * * This structure should be provided by storage device drivers when they set up * a keyslot manager - this structure holds the function ptrs that the keyslot * manager will use to manipulate keyslots in the hardware. */ struct keyslot_mgmt_ll_ops { - int (*keyslot_program)(struct keyslot_manager *ksm, - const struct blk_crypto_key *key, + int (*keyslot_program)(void *ll_priv_data, const u8 *key, + enum blk_crypto_mode_num crypto_mode, + unsigned int data_unit_size, unsigned int slot); - int (*keyslot_evict)(struct keyslot_manager *ksm, - const struct blk_crypto_key *key, + int (*keyslot_evict)(void *ll_priv_data, const u8 *key, + enum blk_crypto_mode_num crypto_mode, + unsigned int data_unit_size, unsigned int slot); + bool (*crypto_mode_supported)(void *ll_priv_data, + enum blk_crypto_mode_num crypto_mode, + unsigned int data_unit_size); + int (*keyslot_find)(void *ll_priv_data, const u8 *key, + enum blk_crypto_mode_num crypto_mode, + unsigned int data_unit_size); }; -struct keyslot_manager *keyslot_manager_create(unsigned int num_slots, - const struct keyslot_mgmt_ll_ops *ksm_ops, - const unsigned int crypto_mode_supported[BLK_ENCRYPTION_MODE_MAX], - void *ll_priv_data); +#ifdef CONFIG_BLK_INLINE_ENCRYPTION +struct keyslot_manager; -int keyslot_manager_get_slot_for_key(struct keyslot_manager *ksm, - const struct blk_crypto_key *key); +extern struct keyslot_manager *keyslot_manager_create(unsigned int num_slots, + const struct keyslot_mgmt_ll_ops *ksm_ops, + void *ll_priv_data); -void keyslot_manager_get_slot(struct keyslot_manager *ksm, unsigned int slot); +extern int +keyslot_manager_get_slot_for_key(struct keyslot_manager *ksm, + const u8 *key, + enum blk_crypto_mode_num crypto_mode, + unsigned int data_unit_size); -void keyslot_manager_put_slot(struct keyslot_manager *ksm, unsigned int slot); +extern void keyslot_manager_get_slot(struct keyslot_manager *ksm, + unsigned int slot); -bool keyslot_manager_crypto_mode_supported(struct keyslot_manager *ksm, - enum blk_crypto_mode_num crypto_mode, - unsigned int data_unit_size); +extern void keyslot_manager_put_slot(struct keyslot_manager *ksm, + unsigned int slot); -int keyslot_manager_evict_key(struct keyslot_manager *ksm, - const struct blk_crypto_key *key); +extern bool +keyslot_manager_crypto_mode_supported(struct keyslot_manager *ksm, + enum blk_crypto_mode_num crypto_mode, + unsigned int data_unit_size); -void keyslot_manager_reprogram_all_keys(struct keyslot_manager *ksm); +extern bool +keyslot_manager_rq_crypto_mode_supported(struct request_queue *q, + enum blk_crypto_mode_num crypto_mode, + unsigned int data_unit_size); -void *keyslot_manager_private(struct keyslot_manager *ksm); +extern int keyslot_manager_evict_key(struct keyslot_manager *ksm, + const u8 *key, + enum blk_crypto_mode_num crypto_mode, + unsigned int data_unit_size); -void keyslot_manager_destroy(struct keyslot_manager *ksm); +extern void keyslot_manager_destroy(struct keyslot_manager *ksm); + +#else /* CONFIG_BLK_INLINE_ENCRYPTION */ + +static inline bool +keyslot_manager_rq_crypto_mode_supported(struct request_queue *q, + enum blk_crypto_mode_num crypto_mode, + unsigned int data_unit_size) +{ + return false; +} +#endif /* CONFIG_BLK_INLINE_ENCRYPTION */ #endif /* __LINUX_KEYSLOT_MANAGER_H */ + +#endif /* CONFIG_BLOCK */