diff --git a/Documentation/block/inline-encryption.rst b/Documentation/block/inline-encryption.rst index 202826cee95e..330106b23c09 100644 --- a/Documentation/block/inline-encryption.rst +++ b/Documentation/block/inline-encryption.rst @@ -97,7 +97,7 @@ Blk-crypto ensures that: - The bio's encryption context is programmed into a keyslot in the KSM of the request queue that the bio is being submitted to (or the crypto API fallback - KSM if the request queue doesn't have a KSM), and that the ``processing_ksm`` + KSM if the request queue doesn't have a KSM), and that the ``bc_ksm`` in the ``bi_crypt_context`` is set to this KSM - That the bio has its own individual reference to the keyslot in this KSM. @@ -107,7 +107,7 @@ Blk-crypto ensures that: ensuring that the bio has a valid reference to the keyslot when, for e.g., the crypto API fallback KSM in blk-crypto performs crypto on the device's behalf. The individual references are ensured by increasing the refcount for the - keyslot in the ``processing_ksm`` when a bio with a programmed encryption + keyslot in the ``bc_ksm`` when a bio with a programmed encryption context is cloned. @@ -120,7 +120,7 @@ been programmed into any keyslot in any KSM (for e.g. a bio from the FS). request queue the bio is being submitted to (and if this KSM does not exist, then it will program it into blk-crypto's internal KSM for crypto API fallback). The KSM that this encryption context was programmed into is stored - as the ``processing_ksm`` in the bio's ``bi_crypt_context``. + as the ``bc_ksm`` in the bio's ``bi_crypt_context``. **Case 2:** blk-crypto is given a bio whose encryption context has already been programmed into a keyslot in the *crypto API fallback* KSM. @@ -138,7 +138,7 @@ KSM). This way, when a device driver is processing a bio, it can be sure that the bio's encryption context has been programmed into some KSM (either the device driver's request queue's KSM, or blk-crypto's crypto API fallback KSM). -It then simply needs to check if the bio's processing_ksm is the device's +It then simply needs to check if the bio's ``bc_ksm`` is the device's request queue's KSM. If so, then it should proceed with IE. If not, it should simply do nothing with respect to crypto, because some other KSM (perhaps the blk-crypto crypto API fallback KSM) is handling the en/decryption. diff --git a/abi_gki_aarch64.xml b/abi_gki_aarch64.xml index 286187d01e5a..2ce464781f62 100644 --- a/abi_gki_aarch64.xml +++ b/abi_gki_aarch64.xml @@ -2170,7 +2170,7 @@ - + @@ -2196,7 +2196,7 @@ - + @@ -20952,7 +20952,45 @@ - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -20969,13 +21007,14 @@ - - - - - - - + + + + + + + + @@ -20994,12 +21033,12 @@ - + - + @@ -21027,7 +21066,7 @@ - + @@ -21075,13 +21114,13 @@ - + - - + + @@ -21108,8 +21147,8 @@ - - + + @@ -21141,223 +21180,227 @@ - - + + - + - + - + - - + + - + - + - + - + - + - + - + - - + + - - + + - + - - + + - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + - + - + - - + + - - + + - + - - + + - + - + - + - + - - + + - + - + - + - + - + - + - + - + - + - + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -21386,7 +21429,7 @@ - + @@ -21398,7 +21441,7 @@ - + @@ -21446,10 +21489,10 @@ - + - + @@ -21473,7 +21516,7 @@ - + @@ -21512,7 +21555,7 @@ - + @@ -21539,21 +21582,21 @@ - + - + - + - + - + @@ -21561,7 +21604,7 @@ - + @@ -21578,15 +21621,15 @@ - + - + - + @@ -21594,12 +21637,12 @@ - + - + @@ -21616,8 +21659,8 @@ - - + + @@ -21625,7 +21668,7 @@ - + @@ -21649,7 +21692,7 @@ - + @@ -21657,7 +21700,7 @@ - + @@ -21815,7 +21858,7 @@ - + @@ -21883,12 +21926,12 @@ - + - + - + @@ -21909,7 +21952,7 @@ - + @@ -21938,13 +21981,13 @@ - + - + - + @@ -21965,16 +22008,16 @@ - + - + - + - + @@ -21985,7 +22028,7 @@ - + @@ -21996,10 +22039,10 @@ - + - + @@ -22010,15 +22053,15 @@ - + - + - + @@ -22026,7 +22069,7 @@ - + @@ -22037,20 +22080,20 @@ - + - + - + - + - + @@ -22059,7 +22102,7 @@ - + @@ -22070,15 +22113,15 @@ - + - + - + - + @@ -22186,10 +22229,10 @@ - + - + @@ -22198,9 +22241,9 @@ - + - + @@ -22209,7 +22252,7 @@ - + @@ -22218,7 +22261,7 @@ - + @@ -22244,7 +22287,7 @@ - + @@ -22252,7 +22295,7 @@ - + @@ -22263,28 +22306,28 @@ - + - + - + - - - + + + - + @@ -22293,13 +22336,13 @@ - - + + - + @@ -22322,16 +22365,16 @@ - + - + - + - + @@ -22360,22 +22403,22 @@ - + - + - + - + - + @@ -22398,7 +22441,7 @@ - + @@ -22412,7 +22455,7 @@ - + @@ -22421,50 +22464,50 @@ - + - + - + - - + + - + - + - + - + - + - + - + - + - + @@ -22479,34 +22522,34 @@ - + - + - + - + - + - + - + @@ -22518,16 +22561,16 @@ - + - + - + - + @@ -22541,7 +22584,7 @@ - + @@ -22550,16 +22593,16 @@ - + - + - + - + @@ -22574,25 +22617,25 @@ - + - + - + - + @@ -22600,13 +22643,13 @@ - + - + - + @@ -22635,13 +22678,13 @@ - + - + @@ -22650,9 +22693,9 @@ - + - + @@ -22664,20 +22707,20 @@ - + - + - + - + @@ -22685,9 +22728,9 @@ - + - + @@ -22696,8 +22739,8 @@ - - + + @@ -22705,32 +22748,32 @@ - + - + - + - + - + - + - + @@ -22738,26 +22781,26 @@ - + - + - + - + - + - + - + - + @@ -22766,74 +22809,74 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -22845,7 +22888,7 @@ - + @@ -22878,10 +22921,10 @@ - + - + @@ -22896,7 +22939,7 @@ - + @@ -22905,25 +22948,25 @@ - + - + - + - + - + - + - + @@ -22938,7 +22981,7 @@ - + @@ -22950,10 +22993,10 @@ - + - + @@ -22962,10 +23005,10 @@ - + - + @@ -22973,7 +23016,7 @@ - + @@ -22987,7 +23030,7 @@ - + @@ -23014,7 +23057,7 @@ - + @@ -23040,15 +23083,15 @@ - + - + - + - + @@ -23056,48 +23099,48 @@ - + - + - + - + - + - + - + - + - + - + - + @@ -23112,7 +23155,7 @@ - + @@ -23123,7 +23166,7 @@ - + @@ -23134,10 +23177,10 @@ - + - + @@ -23145,27 +23188,27 @@ - + - + - + - + - + - + - + - + @@ -23183,12 +23226,12 @@ - + - + - + @@ -23197,10 +23240,10 @@ - + - + @@ -23208,12 +23251,12 @@ - + - + - + @@ -23246,7 +23289,7 @@ - + @@ -23255,7 +23298,7 @@ - + @@ -23267,30 +23310,30 @@ - + - + - + - + - + - + - + @@ -23299,13 +23342,13 @@ - + - + - + @@ -23332,10 +23375,10 @@ - + - + @@ -23344,7 +23387,7 @@ - + @@ -23377,27 +23420,27 @@ - + - + - + - + - + - + - + @@ -23408,15 +23451,15 @@ - + - + - + @@ -23424,40 +23467,40 @@ - + - + - + - + - + - + - + - + - + - + @@ -23465,7 +23508,7 @@ - + @@ -23473,7 +23516,7 @@ - + @@ -23496,33 +23539,33 @@ - + - + - + - + - + - + - + - + - + - + @@ -23531,13 +23574,13 @@ - + - + @@ -23588,7 +23631,7 @@ - + @@ -23597,10 +23640,10 @@ - + - + @@ -23615,13 +23658,13 @@ - + - + @@ -23636,7 +23679,7 @@ - + @@ -23663,7 +23706,7 @@ - + @@ -23678,13 +23721,13 @@ - + - + @@ -23708,10 +23751,10 @@ - + - + @@ -23728,19 +23771,19 @@ - + - + - + - + - + @@ -23748,19 +23791,19 @@ - + - + - + - + @@ -23775,22 +23818,22 @@ - + - + - + - + - + @@ -23814,7 +23857,7 @@ - + @@ -23826,22 +23869,22 @@ - + - - + + - + - + @@ -23849,7 +23892,7 @@ - + @@ -23857,20 +23900,20 @@ - + - - - + + + - + - + @@ -23885,15 +23928,15 @@ - + - + - + @@ -23902,7 +23945,7 @@ - + @@ -23917,108 +23960,108 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - - - - + + + + @@ -24035,7 +24078,7 @@ - + @@ -24062,7 +24105,7 @@ - + @@ -24095,10 +24138,10 @@ - - + + - + @@ -24122,10 +24165,10 @@ - + - + @@ -24149,13 +24192,13 @@ - + - + @@ -24176,7 +24219,7 @@ - + @@ -24191,10 +24234,10 @@ - + - + @@ -24202,7 +24245,7 @@ - + @@ -24211,7 +24254,7 @@ - + @@ -24222,7 +24265,7 @@ - + @@ -24235,7 +24278,7 @@ - + @@ -24252,9 +24295,9 @@ - + - + @@ -24263,13 +24306,13 @@ - + - + - + @@ -24278,13 +24321,13 @@ - + - + - + @@ -24314,7 +24357,7 @@ - + @@ -24325,13 +24368,13 @@ - + - + @@ -24343,31 +24386,31 @@ - + - + - + - + - + - + - + - + @@ -24444,16 +24487,16 @@ - + - + - + @@ -24461,27 +24504,27 @@ - + - + - + - + - + - + - + - + @@ -24582,7 +24625,7 @@ - + @@ -24590,10 +24633,10 @@ - + - + @@ -24608,8 +24651,8 @@ - - + + @@ -24626,18 +24669,18 @@ - + - + - - - + + + @@ -24651,45 +24694,57 @@ - - + + - + - + - + - - - - - - - - - - + - - - - + - + + + + + + + + + + + + + + + + + + + + + + + + - - + + + @@ -24697,174 +24752,176 @@ - + - - + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - + + + + + + + @@ -24887,7 +24944,7 @@ - + @@ -24896,315 +24953,315 @@ - + - - + + - - + + - + - + - + - + - + - - - - - - - - - - - - - - - - - - - - - - - - - - - + - + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + - + + - - - - + + + - - - + + + - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - + + + - - - - + + + + - - + + - + - - - - + + + + - - + + - - - + + + - - - - + + + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + - - + - - - + - - - + + - + + + + + - - + + + + + + + + + + + + - - - + + + + + + + + + - - + + + + + + + - - - + + + - + + + + + + + + + + + - + @@ -25212,7 +25269,7 @@ - + @@ -25220,10 +25277,9 @@ - - + @@ -25241,9 +25297,9 @@ - - - + + + @@ -25254,11 +25310,11 @@ - + - + @@ -25268,7 +25324,7 @@ - + @@ -25290,56 +25346,56 @@ - - + + - + - - + + - - - - - - - + + + + + + + - - + + - + - - + + - + - + - - + + - + - + @@ -25350,9 +25406,9 @@ - + - + @@ -25361,7 +25417,7 @@ - + @@ -25370,7 +25426,7 @@ - + @@ -25388,7 +25444,7 @@ - + @@ -25397,7 +25453,7 @@ - + @@ -25405,9 +25461,9 @@ - + - + @@ -25422,7 +25478,7 @@ - + @@ -25446,7 +25502,7 @@ - + @@ -25484,10 +25540,10 @@ - + - + @@ -25496,10 +25552,10 @@ - + - + @@ -25514,12 +25570,12 @@ - + - + @@ -25530,8 +25586,8 @@ - - + + @@ -25539,10 +25595,10 @@ - + - + @@ -25559,7 +25615,7 @@ - + @@ -25588,7 +25644,7 @@ - + @@ -25603,87 +25659,87 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -25710,10 +25766,10 @@ - + - + @@ -25724,22 +25780,22 @@ - + - + - + - + - + @@ -25750,7 +25806,7 @@ - + @@ -25761,24 +25817,24 @@ - + - + - + - + - + - + - + @@ -25786,15 +25842,15 @@ - + - + - + @@ -25806,7 +25862,7 @@ - + @@ -25824,7 +25880,7 @@ - + @@ -25850,16 +25906,16 @@ - + - + - + @@ -25873,7 +25929,7 @@ - + @@ -25887,7 +25943,7 @@ - + @@ -25922,22 +25978,22 @@ - + - + - + - + @@ -25955,7 +26011,7 @@ - + @@ -25967,7 +26023,7 @@ - + @@ -25988,7 +26044,7 @@ - + @@ -26036,25 +26092,25 @@ - + - + - + - + - + @@ -26066,19 +26122,19 @@ - + - + - + - + @@ -26093,7 +26149,7 @@ - + @@ -26102,22 +26158,22 @@ - - + + - + - - + + - + - + @@ -26128,7 +26184,7 @@ - + @@ -26145,7 +26201,7 @@ - + @@ -26170,7 +26226,7 @@ - + @@ -26302,7 +26358,7 @@ - + @@ -26310,9 +26366,9 @@ - + - + @@ -26327,7 +26383,7 @@ - + @@ -26336,7 +26392,7 @@ - + @@ -26351,23 +26407,23 @@ - + - + - + - + @@ -26377,9 +26433,9 @@ - + - + @@ -26387,7 +26443,7 @@ - + @@ -26406,10 +26462,10 @@ - + - + @@ -26435,13 +26491,13 @@ - + - + - + @@ -26455,7 +26511,7 @@ - + @@ -26463,10 +26519,10 @@ - + - + @@ -26483,7 +26539,7 @@ - + @@ -26491,10 +26547,10 @@ - + - + @@ -26505,7 +26561,7 @@ - + @@ -26513,16 +26569,16 @@ - - + + - + - + @@ -26534,7 +26590,7 @@ - + @@ -26543,7 +26599,7 @@ - + @@ -26555,19 +26611,19 @@ - + - + - + - + - + @@ -26576,20 +26632,20 @@ - + - + - + - + - + @@ -26598,24 +26654,24 @@ - + - + - - + + - + - + @@ -26644,15 +26700,15 @@ - + - + - + @@ -26681,7 +26737,7 @@ - + @@ -26689,7 +26745,7 @@ - + @@ -26697,7 +26753,7 @@ - + @@ -26705,7 +26761,7 @@ - + @@ -26713,9 +26769,9 @@ - + - + @@ -26736,17 +26792,17 @@ - + - + - + - + - + @@ -26758,9 +26814,9 @@ - + - + @@ -26772,27 +26828,13 @@ - - - - - - - - - - - - - - - - + + - + @@ -26801,19 +26843,19 @@ - + - + - + - + - + @@ -26825,8 +26867,8 @@ - - + + @@ -26837,13 +26879,13 @@ - + - + @@ -26878,7 +26920,7 @@ - + @@ -26886,7 +26928,7 @@ - + @@ -26894,66 +26936,64 @@ - - - - - - - - - - - + + + + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -26974,7 +27014,7 @@ - + @@ -26991,120 +27031,120 @@ - + - + - + - + - + - - - - - - - - - - - - - - - + - + + - - - + + + + + + - + + + + + + + + + + + - + - + - - - - - - - - - - - - - - - + - - - + + + + + + + + + + + + + + + + + - - + + - - + + - - + + - + @@ -27123,7 +27163,7 @@ - + @@ -27158,18 +27198,18 @@ - - - - + + + + - + - + @@ -27186,7 +27226,7 @@ - + @@ -27194,11 +27234,11 @@ - + - + - + @@ -27211,7 +27251,7 @@ - + @@ -27222,7 +27262,7 @@ - + @@ -27233,7 +27273,7 @@ - + @@ -27241,7 +27281,7 @@ - + @@ -27251,18 +27291,18 @@ - + - + - + @@ -27271,15 +27311,15 @@ - + - + - + @@ -27290,7 +27330,7 @@ - + @@ -27298,55 +27338,55 @@ - - - - - - + + + + + + - + - + - + - + - + - + - + - + - - + + - - + + - + @@ -27360,8 +27400,8 @@ - - + + @@ -27372,17 +27412,19 @@ - + - + + + - + @@ -27408,7 +27450,7 @@ - + @@ -27422,68 +27464,68 @@ - - + + - + - + - - + + - - + + - + - - + + - + - + - - + + - - + + - + - + @@ -27513,22 +27555,22 @@ - + - + - - - + + + - + @@ -27539,7 +27581,7 @@ - + @@ -27558,25 +27600,25 @@ - - + + - + - + - - + + @@ -27590,15 +27632,15 @@ - + - + - + @@ -27623,17 +27665,17 @@ - - - + + + - + - + @@ -27641,7 +27683,7 @@ - + @@ -27656,16 +27698,16 @@ - + - + - + - + @@ -27680,42 +27722,42 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -27728,7 +27770,7 @@ - + @@ -27736,26 +27778,26 @@ - + - - - - - - - - - + + + + - - - - - - - + + + + + + + + + + + + @@ -27790,65 +27832,65 @@ - - - + + + - - - - + + + + - - - + + + - - - - + + + + - - - + + + - - + + - + - - + + - - - + + + - - + + - - + + @@ -27933,7 +27975,7 @@ - + @@ -27941,9 +27983,9 @@ - - - + + + @@ -27960,7 +28002,7 @@ - + @@ -27968,7 +28010,7 @@ - + @@ -27982,7 +28024,7 @@ - + @@ -28017,9 +28059,9 @@ - + - + @@ -28028,7 +28070,7 @@ - + @@ -28036,12 +28078,12 @@ - + - + - + @@ -28072,7 +28114,7 @@ - + @@ -28080,26 +28122,26 @@ - - + + - + - + - + - + - + @@ -28112,7 +28154,7 @@ - + @@ -28123,7 +28165,7 @@ - + @@ -28134,7 +28176,7 @@ - + @@ -28142,13 +28184,13 @@ - + - + @@ -28162,22 +28204,22 @@ - - + + - + - + - - + + @@ -28197,7 +28239,7 @@ - + @@ -28208,19 +28250,19 @@ - + - - - - + + + + - + @@ -28262,22 +28304,22 @@ - + - + - + - + @@ -28291,7 +28333,7 @@ - + @@ -28345,7 +28387,7 @@ - + @@ -28354,18 +28396,18 @@ - + - + - - + + @@ -28378,15 +28420,15 @@ - + - - - - + + + + @@ -28400,8 +28442,8 @@ - - + + @@ -28415,14 +28457,14 @@ - + - + - + @@ -28512,16 +28554,16 @@ - + - + - + @@ -28560,8 +28602,8 @@ - - + + @@ -28571,7 +28613,7 @@ - + @@ -28588,21 +28630,29 @@ - + - - - + + + + + + + + + + + - - - + + + @@ -28612,13 +28662,13 @@ - + - - + + @@ -28664,9 +28714,9 @@ - + - + @@ -28681,10 +28731,10 @@ - + - + @@ -28702,36 +28752,36 @@ - + - + - + - + - + - + - + @@ -28746,8 +28796,8 @@ - - + + @@ -28766,44 +28816,44 @@ - + - + - + - + - - - + + + - + - + - + - + - + @@ -28812,15 +28862,15 @@ - + - + - + @@ -28829,7 +28879,7 @@ - + @@ -28839,7 +28889,7 @@ - + @@ -28847,13 +28897,13 @@ - + - + @@ -28864,13 +28914,13 @@ - + - + - + @@ -28884,19 +28934,19 @@ - + - + - + - + @@ -28906,23 +28956,23 @@ - + - + - + - + - + - + @@ -28936,7 +28986,7 @@ - + @@ -28956,33 +29006,33 @@ - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + - - + + + + + - - + + @@ -29013,85 +29063,85 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -29102,7 +29152,7 @@ - + @@ -29117,7 +29167,7 @@ - + @@ -29128,25 +29178,25 @@ - - - + + + - - + + - + - + - + @@ -29155,183 +29205,39 @@ - + - + - + - + - + - + - + - + - - - - - - - - - - - - - - - - - - - - - - - - - - - - + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + @@ -29342,39 +29248,7 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + @@ -29385,10 +29259,10 @@ - + - + @@ -29402,23 +29276,7 @@ - - - - - - - - - - - - - - - - - + @@ -29429,24 +29287,24 @@ - + - + - + - + - + - + @@ -29458,7 +29316,7 @@ - + @@ -29470,7 +29328,7 @@ - + @@ -29478,73 +29336,79 @@ - + - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - - + + + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + @@ -29555,190 +29419,188 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - + - + - + - + - + - + - + - - + + - + - - - - + + + + - - + + - + - - + + - - + + - - + + - - - + + + - - + + - + - - - - + + + + - - + + - + - - - + + + - - - - - - + + - - + - - + + + - - + + + + + + - + - - - - - - - - + + + + + + + + - + @@ -29761,7 +29623,7 @@ - + @@ -29775,9 +29637,9 @@ - + - + @@ -29787,10 +29649,10 @@ - + - + @@ -29808,7 +29670,7 @@ - + @@ -29823,19 +29685,19 @@ - + - + - + - + - + @@ -29913,7 +29775,7 @@ - + @@ -29946,7 +29808,7 @@ - + @@ -30003,7 +29865,7 @@ - + @@ -30012,65 +29874,65 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -30081,21 +29943,21 @@ - + - + - + - + @@ -30106,7 +29968,7 @@ - + @@ -30119,7 +29981,7 @@ - + @@ -30132,13 +29994,13 @@ - + - + @@ -30156,7 +30018,7 @@ - + @@ -30186,8 +30048,8 @@ - - + + @@ -30195,8 +30057,8 @@ - - + + @@ -30204,108 +30066,108 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -30341,16 +30203,16 @@ - + - + - + - + @@ -30362,10 +30224,10 @@ - + - + @@ -30379,27 +30241,27 @@ - + - + - + - + - + - + - + - + @@ -30420,10 +30282,10 @@ - + - + @@ -30435,7 +30297,7 @@ - + @@ -30446,13 +30308,13 @@ - + - + - + @@ -30460,8 +30322,8 @@ - - + + @@ -30472,7 +30334,7 @@ - + @@ -30486,7 +30348,7 @@ - + @@ -30497,7 +30359,7 @@ - + @@ -30511,7 +30373,7 @@ - + @@ -30525,10 +30387,10 @@ - + - + @@ -30555,7 +30417,7 @@ - + @@ -30570,29 +30432,29 @@ - + - + - + - + - + - + - + - + @@ -30615,31 +30477,31 @@ - + - + - + - + - + - + - + - + - + @@ -30660,7 +30522,7 @@ - + @@ -30698,10 +30560,10 @@ - + - + @@ -30725,20 +30587,20 @@ - + - + - + - - + + @@ -30746,7 +30608,7 @@ - + @@ -30754,26 +30616,26 @@ - + - + - + - + - + - + - + - + @@ -30802,20 +30664,20 @@ - + - + - + - + - + @@ -30824,18 +30686,18 @@ - + - + - + - + - + @@ -30844,10 +30706,10 @@ - + - + @@ -30855,7 +30717,7 @@ - + @@ -30863,22 +30725,22 @@ - + - + - + - + - + - + @@ -30890,27 +30752,27 @@ - + - + - + - + - + - + - + @@ -30921,7 +30783,7 @@ - + @@ -30930,7 +30792,7 @@ - + @@ -30959,20 +30821,6 @@ - - - - - - - - - - - - - - @@ -31015,13 +30863,13 @@ - + - + @@ -31104,7 +30952,7 @@ - + @@ -31113,13 +30961,13 @@ - + - + - + @@ -31295,7 +31143,7 @@ - + @@ -31304,12 +31152,12 @@ - + - + @@ -31342,51 +31190,51 @@ - + - + - - - + + + - - + + - - - - - - - - - - - - + + + + + + + + + + + + - - - - - - - - - + + + + + + + + + - - - - - + + + + + @@ -31399,45 +31247,45 @@ - - - - - - + + + + + + - - - - - - - - + + + + + + + + - - - + + + - - - - + + + + - + - - - - - - - - - - - + + + + + + + + + + + @@ -31445,7 +31293,7 @@ - + @@ -31459,40 +31307,40 @@ - + - + - + - + - + - + - + @@ -31501,7 +31349,7 @@ - + @@ -31512,7 +31360,7 @@ - + @@ -31529,16 +31377,16 @@ - + - + - + @@ -31546,7 +31394,7 @@ - + @@ -31554,19 +31402,19 @@ - + - + - + @@ -31575,7 +31423,7 @@ - + @@ -31583,50 +31431,50 @@ - + - + - - + + - + - + - + - + - + - + @@ -31634,7 +31482,7 @@ - + @@ -31642,66 +31490,66 @@ - + - + - + - + - - + + - - + + - + - + - + - + - + @@ -31715,16 +31563,16 @@ - + - + - + @@ -31732,7 +31580,7 @@ - + @@ -31755,7 +31603,7 @@ - + @@ -31763,32 +31611,32 @@ - + - + - + - + - + - + - + - + @@ -31802,7 +31650,7 @@ - + @@ -31931,7 +31779,7 @@ - + @@ -32571,158 +32419,147 @@ - + - + - + - + - - + + - - + + - + - + - + - + - + - - + + - + - - - - - - - - - - - - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - - + + @@ -32731,17 +32568,17 @@ - - - + + + - - + + @@ -32750,7 +32587,7 @@ - + @@ -32764,35 +32601,35 @@ - + - + - + - + - + - - + + - + @@ -32812,13 +32649,13 @@ - + - + - + @@ -32830,7 +32667,7 @@ - + @@ -32838,59 +32675,59 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -32902,16 +32739,16 @@ - - + + - + - + - + @@ -32931,190 +32768,190 @@ - + - + - + - + - + - + - + - + - + - + - + - - - - + + + + - + - + - + - + - + - + - + - + - + - + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - - - - + + - - + + - - + + - + - - + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - - - - - - - + - - + + - - + + - - + + - - + + - - + + - - - + + + - - + + - - + + @@ -33138,80 +32975,80 @@ - + - + - + - + - + - + - + - + - + - + - + - - + + - - - - - + + + + + - - + + - + - + - + @@ -33220,7 +33057,7 @@ - + @@ -33271,7 +33108,7 @@ - + @@ -33280,7 +33117,7 @@ - + @@ -33340,7 +33177,7 @@ - + @@ -33349,7 +33186,7 @@ - + @@ -33361,7 +33198,7 @@ - + @@ -33475,13 +33312,13 @@ - + - + @@ -33493,16 +33330,16 @@ - + - + - + @@ -33517,12 +33354,12 @@ - + - + @@ -33531,13 +33368,13 @@ - + - + @@ -33546,10 +33383,10 @@ - + - + @@ -33579,7 +33416,7 @@ - + @@ -33597,33 +33434,33 @@ - + - + - + - + - + - + - - + + - + - + @@ -33632,7 +33469,7 @@ - + @@ -33640,34 +33477,34 @@ - + - + - + - + - + - + - + - + - + - + @@ -33676,10 +33513,10 @@ - + - + @@ -33699,37 +33536,37 @@ - + - + - + - + - + - + - + - + - - + + - + @@ -33737,99 +33574,99 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + - + - - + + - - + + - - + + - - + + - - - + + + - - + + - - + + + + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - - + @@ -33844,102 +33681,102 @@ - + - + - + - + - + - + - + - + - + - + - - - - - - - + + + + + + + - + - + - + - + - + - + - + - + - - + + - - + + - - + + - - + + - - + + @@ -33954,81 +33791,81 @@ - - + + - + - + - + - + - + - - - + + + - + - + - + - + - + - + - + - + - + - + - + - + - + - - + + @@ -34052,12 +33889,12 @@ - - + + - + @@ -34083,22 +33920,22 @@ - + - + - + - + @@ -34106,20 +33943,20 @@ - + - + - + - + - + @@ -34140,13 +33977,13 @@ - + - + - + @@ -34161,16 +33998,16 @@ - + - + - + @@ -34182,7 +34019,7 @@ - + @@ -34194,7 +34031,7 @@ - + @@ -34208,7 +34045,7 @@ - + @@ -34223,13 +34060,13 @@ - + - + @@ -34253,19 +34090,19 @@ - + - + - + - + @@ -34358,50 +34195,50 @@ - + - + - + - + - + - + - + - + - + - + - + @@ -34442,16 +34279,16 @@ - + - + - + - + @@ -34496,7 +34333,7 @@ - + @@ -34516,8 +34353,8 @@ - - + + @@ -34525,113 +34362,113 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -34645,12 +34482,12 @@ - + - + - + @@ -34659,26 +34496,26 @@ - + - + - + - + - + - + @@ -34687,193 +34524,193 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - + - + - + - + - + - + - + - + + + + + + + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - - - - - - + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - + - + - + - + - - + + - - + + - + - - - - - + + + + + - + - + - + - + @@ -34885,7 +34722,7 @@ - + @@ -34893,7 +34730,7 @@ - + @@ -34902,10 +34739,10 @@ - + - + @@ -34926,7 +34763,7 @@ - + @@ -34934,7 +34771,7 @@ - + @@ -34945,13 +34782,13 @@ - + - + - + @@ -34987,24 +34824,24 @@ - + - + - + - + - + - + @@ -35016,7 +34853,7 @@ - + @@ -35024,109 +34861,109 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -35134,7 +34971,7 @@ - + @@ -35143,14 +34980,14 @@ - + - + @@ -35173,21 +35010,21 @@ - + - + - + - + - + @@ -35223,16 +35060,16 @@ - + - + - + - + @@ -35277,10 +35114,10 @@ - - + + - + @@ -35289,16 +35126,16 @@ - + - + - + @@ -35319,7 +35156,7 @@ - + @@ -35330,18 +35167,18 @@ - + - + - + - + @@ -35350,43 +35187,43 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -35401,63 +35238,63 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -35475,10 +35312,10 @@ - + - + @@ -35487,278 +35324,278 @@ - + - + - + - + - + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - + - + - + - + - + - + - + - + - - + + - - - + + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - - + + + - - - + + + - - - + + + - - + + - - + + + + + + + + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - - - - - - + - - + + - - + + - - + + + + + + + + + + + + + + + + + + + + - + + - - - - - - - - - - - - - - - - - - - - + - - + + - - + + - + - + - + - + @@ -35771,21 +35608,21 @@ - - + + - - + + - + - + @@ -35793,37 +35630,37 @@ - + - + - - - - + + + + - + - + - + - + @@ -35832,13 +35669,13 @@ - + - + - + @@ -35856,10 +35693,10 @@ - + - + @@ -35898,7 +35735,7 @@ - + @@ -35922,112 +35759,112 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -36036,9 +35873,9 @@ - + - + @@ -36050,7 +35887,7 @@ - + @@ -36064,10 +35901,10 @@ - + - + @@ -36075,12 +35912,12 @@ - + - + - + @@ -36092,110 +35929,110 @@ - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + - - - - - - - - - + + + + + - + - + - + - + - + - - + + - - - + + + - - - + + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + @@ -36204,13 +36041,13 @@ - + - + - + @@ -36237,9 +36074,9 @@ - + - + @@ -36254,44 +36091,44 @@ - + - + - + - + - + - + - + - - - - - - + + + + + + - + - + @@ -36301,65 +36138,65 @@ - + - - - + + + - - + + - + - - + + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -36383,7 +36220,7 @@ - + @@ -36392,7 +36229,7 @@ - + @@ -36404,21 +36241,21 @@ - + - + - + - + - + - + @@ -36472,9 +36309,9 @@ - + - + @@ -36486,10 +36323,10 @@ - + - + @@ -36498,12 +36335,12 @@ - + - + - + @@ -36524,15 +36361,15 @@ - + - + - + @@ -36543,19 +36380,19 @@ - + - + - + - + @@ -36579,13 +36416,13 @@ - + - + - + @@ -36603,7 +36440,7 @@ - + @@ -36633,7 +36470,7 @@ - + @@ -36672,10 +36509,10 @@ - + - + @@ -36683,10 +36520,10 @@ - + - + @@ -36713,7 +36550,7 @@ - + @@ -36722,161 +36559,161 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - - - - + + + + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -36890,7 +36727,7 @@ - + @@ -36898,13 +36735,13 @@ - + - + @@ -36942,7 +36779,7 @@ - + @@ -36974,7 +36811,7 @@ - + @@ -36998,7 +36835,7 @@ - + @@ -37015,15 +36852,15 @@ - + - + - + - + @@ -37037,410 +36874,410 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + - + - + - - + + - + - + - + - - + + - - + + - - + + - - - + + + - - + + - - + + - - - + + + - - + + - - + + - - + + - - + + - - + + - - + + - - - + + + - - - + + + - - - + + + - - + + - - + + - - + + - - - + + + - + - + - + - - + + - - + + - + - - + + - - + + - - + + - - + + + + + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - - - + - - + + - - + + - - + + - - - + + + - - + + - - + + - + - - + + - + - + - + - + @@ -37494,7 +37331,7 @@ - + @@ -37503,19 +37340,19 @@ - + - + - + - + - + @@ -37524,7 +37361,7 @@ - + @@ -37535,55 +37372,55 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -37616,13 +37453,13 @@ - + - + - + @@ -37631,10 +37468,10 @@ - + - + @@ -37649,7 +37486,7 @@ - + @@ -37676,28 +37513,28 @@ - + - + - + - + - + @@ -37708,10 +37545,10 @@ - + - + @@ -37728,18 +37565,18 @@ - + - + - + - + - + @@ -37748,16 +37585,16 @@ - + - + - + @@ -37768,7 +37605,7 @@ - + @@ -37803,27 +37640,27 @@ - + - + - + - + - + - + @@ -37838,91 +37675,91 @@ - + - - - + + + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -37931,16 +37768,16 @@ - + - + - + - + @@ -37955,10 +37792,10 @@ - + - + @@ -37979,27 +37816,27 @@ - + - + - + - + - + - + @@ -38008,7 +37845,7 @@ - + @@ -38017,7 +37854,7 @@ - + @@ -38031,10 +37868,10 @@ - + - + @@ -38043,16 +37880,16 @@ - + - + - + - + @@ -38076,85 +37913,85 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - - + + @@ -38165,13 +38002,13 @@ - - + + - + @@ -38181,46 +38018,46 @@ - + - + - + - + - + - + - + - + - + - + - - + + @@ -38231,24 +38068,24 @@ - - - + + + - + - + - + @@ -38271,15 +38108,15 @@ - + - + - + @@ -38287,7 +38124,7 @@ - + @@ -38295,7 +38132,7 @@ - + @@ -38316,10 +38153,10 @@ - + - + @@ -38333,9 +38170,9 @@ - + - + @@ -38344,10 +38181,10 @@ - + - + @@ -38368,7 +38205,7 @@ - + @@ -38380,27 +38217,27 @@ - + - + - + - + - + - + @@ -38409,10 +38246,10 @@ - + - + @@ -38420,23 +38257,23 @@ - + - + - + - + - + - + @@ -38447,89 +38284,89 @@ - + - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - - + - + @@ -38540,297 +38377,297 @@ - + - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + - + - + - - + + - + - - + + - + - - + + - + - - + + - + - - + + - + - + - - + + - + + + + + + - - - + + + + + + + + + + + + + + + + + + + + + + - - - - + + - - + - - - - + + - - - + - + - + + - - + + - + + - - - - - - - - - - - - - - - - - - - - + - - + + - + - - + + - - - + + + - - + + - - + + - - + + - - + + - + + + + + + + + + + + + + + + + + + - + + - + + - + + - + + - - + + + - - + + - - - + + + - - + + + + - - - - - - - - - - - - - - - - - - - - - - - - - + - + - - + + @@ -38853,28 +38690,28 @@ - + - + - + - + - + - + - - + + @@ -38891,7 +38728,7 @@ - + @@ -38906,10 +38743,10 @@ - + - + @@ -38920,13 +38757,13 @@ - + - + - + @@ -38935,32 +38772,32 @@ - + - + - + - + - + - + - + @@ -38977,121 +38814,121 @@ - + - + - + - + - + - + - + - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + - + - + - + - + - + - + - + - + - + - - + + - + - + - - + + - - + + - - + + - + @@ -39103,7 +38940,7 @@ - + @@ -39132,35 +38969,35 @@ - + - + - + - + - + - + - + - + - + @@ -39178,159 +39015,159 @@ - - - - - - + + + + + + - + - + - - + + - - + + - - + + - - + + - - + + - - + + - + - + - + - + - + - + - - + + - + - + - + - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + - + - - + + - + - + - + - + - + - + - + - + @@ -39360,73 +39197,73 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -39435,7 +39272,7 @@ - + @@ -39498,49 +39335,49 @@ - + - + - + - + - + - + - + - + - + - + - + - + @@ -39554,22 +39391,22 @@ - + - + - + - + @@ -39617,7 +39454,7 @@ - + @@ -39635,16 +39472,16 @@ - + - + - + @@ -39653,27 +39490,27 @@ - + - + - + - + - + - + - + @@ -39700,13 +39537,13 @@ - + - + @@ -39748,7 +39585,7 @@ - + @@ -39772,169 +39609,169 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -39961,7 +39798,7 @@ - + @@ -39970,26 +39807,26 @@ - + - + - + - + - + - + - + @@ -40001,59 +39838,59 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -40077,10 +39914,10 @@ - + - + @@ -40088,32 +39925,32 @@ - + - + - + - + - + - + - + @@ -40125,16 +39962,16 @@ - + - + - + @@ -40145,7 +39982,7 @@ - + @@ -40166,7 +40003,7 @@ - + @@ -40174,18 +40011,18 @@ - + - + - + - + - + @@ -40199,7 +40036,7 @@ - + @@ -40216,19 +40053,19 @@ - + - + - + - + - + @@ -40271,7 +40108,7 @@ - + @@ -40279,7 +40116,7 @@ - + @@ -40381,13 +40218,13 @@ - + - + @@ -40396,42 +40233,42 @@ - + - + - + - + - + - + - + - + - + - + - + - + @@ -40458,9 +40295,9 @@ - + - + @@ -40469,41 +40306,41 @@ - + - + - + - + - + - + - + - + - + - + - + - + @@ -40512,18 +40349,18 @@ - + - + - + - + @@ -40532,16 +40369,16 @@ - + - + - + - + @@ -40556,10 +40393,10 @@ - + - + @@ -40568,22 +40405,22 @@ - + - + - + - + - + @@ -40595,7 +40432,7 @@ - + @@ -40610,12 +40447,12 @@ - + - + - + @@ -40624,16 +40461,16 @@ - + - + - + @@ -40642,110 +40479,110 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -40775,7 +40612,7 @@ - + @@ -40787,50 +40624,50 @@ - + - + - + - + - + - + - + - - + + - + - + - + - + @@ -40845,7 +40682,7 @@ - + @@ -40859,12 +40696,12 @@ - + - + - + @@ -40873,17 +40710,17 @@ - + - + - - - + + + @@ -40900,109 +40737,109 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -41041,22 +40878,22 @@ - + - + - + - + - + - + @@ -41071,68 +40908,68 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -41146,7 +40983,7 @@ - + @@ -41161,7 +40998,7 @@ - + @@ -41169,12 +41006,12 @@ - + - + - + @@ -41183,21 +41020,21 @@ - + - + - + - + - + @@ -41217,7 +41054,7 @@ - + @@ -41237,34 +41074,34 @@ - + - + - + - + - + - + - + - + @@ -41276,7 +41113,7 @@ - + @@ -41294,13 +41131,13 @@ - + - + @@ -41326,16 +41163,16 @@ - + - + - + - + @@ -41386,10 +41223,10 @@ - + - + @@ -41400,7 +41237,7 @@ - + @@ -41444,27 +41281,27 @@ - + - + - + - + - + - + - + - + @@ -41530,7 +41367,7 @@ - + @@ -41556,7 +41393,7 @@ - + @@ -41600,7 +41437,7 @@ - + @@ -41629,13 +41466,13 @@ - + - + - + @@ -41643,90 +41480,90 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -41746,7 +41583,7 @@ - + @@ -41766,7 +41603,7 @@ - + @@ -41792,10 +41629,10 @@ - + - + @@ -41809,13 +41646,13 @@ - + - + - + @@ -41823,7 +41660,7 @@ - + @@ -41837,7 +41674,7 @@ - + @@ -41845,15 +41682,15 @@ - + - + - + @@ -41861,19 +41698,19 @@ - + - + - + @@ -41884,7 +41721,7 @@ - + @@ -41920,13 +41757,13 @@ - + - + @@ -41935,9 +41772,9 @@ - + - + @@ -41952,7 +41789,7 @@ - + @@ -41960,10 +41797,10 @@ - + - + @@ -41971,77 +41808,77 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -42059,7 +41896,7 @@ - + @@ -42068,18 +41905,18 @@ - + - + - + - + @@ -42097,45 +41934,45 @@ - + - + - + - + - + - + - + - + - + - + - + - + @@ -42149,116 +41986,116 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -42282,7 +42119,7 @@ - + @@ -42293,7 +42130,7 @@ - + @@ -42307,37 +42144,37 @@ - - + + - + - + - + - + - + - + - + - + - + - + @@ -42375,43 +42212,43 @@ - + - + - + - + - + - + - + - + - + @@ -42441,33 +42278,33 @@ - + - + - + - + - + - + - + - + - + - + @@ -42481,7 +42318,7 @@ - + @@ -42489,21 +42326,21 @@ - + - + - + - + - + - + @@ -42514,39 +42351,39 @@ - + - + - + - + - + - + - + - + - + - + - + - + @@ -42566,12 +42403,12 @@ - + - + @@ -42580,20 +42417,20 @@ - + - + - + - + - + @@ -42610,1000 +42447,929 @@ - + - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + - - - - - + + + - + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - + + + + + + + + + + + - + - + - + - - + + - - - + + + + + + + + + + + + + + + - - + + + + + + - - - - - - + + + - - + + + + - - - - - - - + + - - - - + + + - - + + + + + + + + + + + + + - - - + + + - - - + + + - - - + + + - - - - + + + - - - - + + + - - - - + + + - - - - + + + + + + - + + - + + + + - - - - - - - - - - - - - - - - - - - - - - - - - + + - - - + + + - - + + - - + + - - + + - - + + - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - + + + + + - - + + + + + + + - - - - + + + + + - - - - - - + + + + + + - - - - - + + + + + - - - - - - - + + + - - - - - + + + - - - - - - + + + + - - - - - + + + + + + - - - + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + - - + + - - - - + + + + - - - + + + - + + - - + + - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - + + - + + - - - - + + + + - - - - - - - - - - - - + + + - - + + - - + + - - - - + - - - + + - - - - - - - - - - - - - - - - - - - - + - - + + - - + + - - - + + + - - + + - - - + + + - - + + - + + + + + + + + + + + + + + + + + + + + + + + + + + - - - + + + + - - - + + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - + - - + + - - + + - - + + + + + + + + + + - - + + + + + + + + + + + + + - + - + - - - + + - - + - + + - + + - - + - + - - + + - - + + + - + + + - - - - - - - - - - - - - - - - - - - - - - + - - + + - - + + - - - + + + - - + + - - + + + + + + + + + + + + + + + + + + + + + + - - + - - + + - - + + - - - - - - - - - - + - - - + + + + - - - - - - - - - - - - - + + - - + + - - + + - - + + - - + + - - - + + + - + - - + + - + - + - + - - + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + - + @@ -43618,50 +43384,99 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -43670,24 +43485,24 @@ - + - + - + - + - + - + @@ -43696,7 +43511,7 @@ - + @@ -43704,36 +43519,36 @@ - + - + - + - + - + - + - + - + - + - + @@ -43744,7 +43559,7 @@ - + @@ -43767,19 +43582,19 @@ - + - + - + @@ -43824,11 +43639,34 @@ - + - - + + + + + + + + + + + + + + + + + + + + + + + + + @@ -43839,9 +43677,9 @@ - + - + @@ -43859,9 +43697,9 @@ - + - + @@ -43873,238 +43711,128 @@ - + - + - + - - - - - - - - - - - - - - - - - - - - - - - - - - - + - - - - - - - - - - - - - - - - - - - - - - - - + - + - + - + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + - + - + - + - + - + - + - + - + - + - + - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + - - + + - - + + - - + + - - + + - - + + - + - - + + - - + + - - - + + + @@ -44113,102 +43841,102 @@ - + - - + + - - + + - + - - - + + + - + - - + + - - - + + + - - + + - - + + - + - - + + - - + + - - - + + + - + - + - + - + - - - + + + - + - + - + - + @@ -44223,7 +43951,7 @@ - + @@ -44234,7 +43962,7 @@ - + @@ -44264,7 +43992,7 @@ - + @@ -44272,13 +44000,13 @@ - + - + @@ -44287,27 +44015,27 @@ - + - + - + - + - + - + - + @@ -44316,7 +44044,7 @@ - + @@ -44327,7 +44055,7 @@ - + @@ -44345,7 +44073,7 @@ - + @@ -44354,71 +44082,71 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -44439,1374 +44167,7 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + @@ -45964,12 +44325,12 @@ - + - + - + @@ -45982,7 +44343,7 @@ - + @@ -45993,7 +44354,7 @@ - + @@ -46004,7 +44365,7 @@ - + @@ -46012,13 +44373,13 @@ - + - + @@ -46032,15 +44393,15 @@ - + - + - + @@ -46051,7 +44412,7 @@ - + @@ -46059,23 +44420,1396 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - + + - + - - + + - + @@ -46087,29 +45821,29 @@ - - - + + + - - - + + + - + - + @@ -46117,7 +45851,7 @@ - + @@ -46125,7 +45859,7 @@ - + @@ -46136,36 +45870,37 @@ - - - - + + + + + - - + + - - + + - - + + - + - - + + - + @@ -46183,20 +45918,20 @@ - + - + - + - + - - + + @@ -46221,7 +45956,7 @@ - + @@ -46231,7 +45966,7 @@ - + @@ -46240,11 +45975,11 @@ - - - + + + - + @@ -46262,8 +45997,8 @@ - - + + @@ -46285,7 +46020,7 @@ - + @@ -46378,15 +46113,15 @@ - - + + - + @@ -46394,76 +46129,76 @@ - + - + - + - + - + - + - + - + - - + + - - + + - - + + - + - + - - + + @@ -46480,22 +46215,22 @@ - + - + - - - + + + - - + + - + @@ -46515,11 +46250,11 @@ - + - + @@ -46533,13 +46268,13 @@ - + - + @@ -46563,7 +46298,7 @@ - + @@ -46571,7 +46306,7 @@ - + @@ -46579,15 +46314,15 @@ - + - - - - + + + + @@ -46607,7 +46342,7 @@ - + @@ -46662,15 +46397,15 @@ - + - + - + @@ -46683,7 +46418,7 @@ - + @@ -46692,8 +46427,8 @@ - - + + @@ -46704,33 +46439,33 @@ - + - - - + + + - + - + - + - + - + @@ -46741,10 +46476,10 @@ - + - + @@ -46753,7 +46488,7 @@ - + @@ -46762,37 +46497,37 @@ - + - + - + - + - + - + - + - + - + - + @@ -46885,22 +46620,22 @@ - + - + - + - + - + @@ -46909,7 +46644,7 @@ - + @@ -46921,7 +46656,7 @@ - + @@ -46931,9 +46666,9 @@ - + - + @@ -46954,7 +46689,7 @@ - + @@ -46995,13 +46730,13 @@ - + - + - + @@ -47016,20 +46751,20 @@ - + - + - + - + - + @@ -47038,7 +46773,7 @@ - + @@ -47053,7 +46788,7 @@ - + @@ -47067,7 +46802,7 @@ - + @@ -47079,7 +46814,7 @@ - + @@ -47093,10 +46828,10 @@ - + - + @@ -47104,13 +46839,13 @@ - + - + @@ -47118,7 +46853,7 @@ - + @@ -47133,13 +46868,13 @@ - + - + - + @@ -47154,27 +46889,27 @@ - + - + - + - + - + - + - + - + @@ -47182,13 +46917,13 @@ - + - + @@ -47202,7 +46937,7 @@ - + @@ -47216,7 +46951,7 @@ - + @@ -47225,10 +46960,10 @@ - + - + @@ -47245,16 +46980,16 @@ - + - + - + - + @@ -47271,7 +47006,7 @@ - + @@ -47282,21 +47017,21 @@ - + - + - + - + - + @@ -47305,7 +47040,7 @@ - + @@ -47313,7 +47048,7 @@ - + @@ -47331,7 +47066,7 @@ - + @@ -47357,24 +47092,24 @@ - + - + - + - + - + @@ -47410,9 +47145,9 @@ - + - + @@ -47421,13 +47156,13 @@ - + - + @@ -47456,14 +47191,14 @@ - + - + @@ -47471,16 +47206,16 @@ - + - + - + @@ -47488,7 +47223,7 @@ - + @@ -47502,54 +47237,54 @@ - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + - - + + - + - - - - - - + + + + + + - + - + - + @@ -47561,13 +47296,13 @@ - + - + @@ -47576,19 +47311,19 @@ - + - + - + - + @@ -47642,37 +47377,37 @@ - + - + - + - + - + - + - + - + - + @@ -47695,13 +47430,13 @@ - + - + - + @@ -47758,13 +47493,13 @@ - + - + - + @@ -47781,8 +47516,8 @@ - - + + @@ -47796,7 +47531,7 @@ - + @@ -47807,140 +47542,140 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -47951,52 +47686,52 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -48008,40 +47743,40 @@ - + - + - + - + - + - + - + - + - + - + - + @@ -48049,42 +47784,42 @@ - + - + - + - + - + - + - + - + - + - + - + - + @@ -48092,7 +47827,7 @@ - + @@ -48104,48 +47839,48 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -48166,12 +47901,12 @@ - + - + - + @@ -48248,15 +47983,15 @@ - + - + - + @@ -48273,15 +48008,15 @@ - + - + - + - + @@ -48289,13 +48024,13 @@ - + - + @@ -48321,7 +48056,7 @@ - + @@ -48329,13 +48064,13 @@ - + - + @@ -48346,15 +48081,15 @@ - + - + - + - + @@ -48363,13 +48098,13 @@ - + - + @@ -48380,7 +48115,7 @@ - + @@ -48389,372 +48124,372 @@ - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - + - - + + - - + + - - + + - - + + - + - + - + - - + + - + - + - + - + - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - + + + + + + + - + + - - + + - - - - + + + - - + + - + + + - - + + + - - + + + - - - + + + + + - - + + + + + + - - - - - - + + + + - - + - - - + + - - - + + - - + + - - - + - - - + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + - - + + - - + + - - + + - - + + - - - + + + - + - + - - + + - - + + - + - - + + - - + + - - + + - - - + + + - - - + + + - - - + + + - - + + - - + + @@ -48762,13 +48497,13 @@ - + - + @@ -48795,48 +48530,48 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -48871,7 +48606,7 @@ - + @@ -48889,7 +48624,7 @@ - + @@ -48909,12 +48644,12 @@ - + - + - + @@ -48926,33 +48661,33 @@ - + - + - + - + - + - + - + - + - + @@ -48967,7 +48702,7 @@ - + @@ -48975,59 +48710,59 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -49036,191 +48771,191 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - + - - + + - - + + - + - + - - + + - - - + + + - - + + - - - + + + - - - + + + - - + + - - - + + + - - + + - - + + - - + + - - - + + + - - - - - + + + + + - - + + - + - - + + - - - + + + - - + + - - - + + + - - - + + + - - + + - + - + - + - + - + - + - + - + @@ -49231,7 +48966,7 @@ - + @@ -49276,16 +49011,16 @@ - + - + - + - + @@ -49297,10 +49032,10 @@ - + - + @@ -49315,19 +49050,19 @@ - + - + - + - + - + @@ -49357,16 +49092,16 @@ - + - + - + @@ -49380,7 +49115,7 @@ - + @@ -49397,24 +49132,24 @@ - + - + - + - + - + - + - + @@ -49429,13 +49164,13 @@ - + - + - + @@ -49446,16 +49181,16 @@ - + - + - + - + @@ -49463,7 +49198,7 @@ - + @@ -49471,32 +49206,32 @@ - + - + - + - + - + - + - + - + - + @@ -49510,7 +49245,7 @@ - + @@ -49518,10 +49253,10 @@ - + - + @@ -49538,16 +49273,16 @@ - + - + - + @@ -49567,7 +49302,7 @@ - + @@ -49575,7 +49310,7 @@ - + @@ -49595,12 +49330,12 @@ - + - + - + @@ -49609,7 +49344,7 @@ - + @@ -49629,7 +49364,7 @@ - + @@ -49640,10 +49375,10 @@ - + - + @@ -49652,30 +49387,30 @@ - + - + - + - + - + - + - + - + @@ -49687,7 +49422,7 @@ - + @@ -49696,7 +49431,7 @@ - + @@ -49707,9 +49442,9 @@ - + - + @@ -49724,89 +49459,89 @@ - + - + - + - + - + - + - + - + - + - + - + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - + - + - + - + @@ -49815,104 +49550,104 @@ - + - + - + - - + + - - - + + + - - - - + + + + - - + + - - - + + + - - + + - - + + - - + + - - - - + + + + - - + + - - - + + + - - + + - - + + - - + + - - + + - - - + + + - - + + @@ -49921,14 +49656,14 @@ - + - + @@ -49957,19 +49692,19 @@ - + - + - + - + @@ -49980,13 +49715,13 @@ - + - + @@ -50013,10 +49748,10 @@ - + - + @@ -50046,7 +49781,7 @@ - + @@ -50058,36 +49793,36 @@ - + - + - + - + - + - + - + - + - + - + - + @@ -50095,10 +49830,10 @@ - + - + @@ -50106,90 +49841,90 @@ - + - + - - - - - - - - - - - - - - + + + + + + + + + + + + + + - + - + - + - + - + - + - + - + - + - + - + - + - - + + - + - - + + @@ -50200,64 +49935,64 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -50266,7 +50001,7 @@ - + @@ -50283,10 +50018,10 @@ - + - + @@ -50310,7 +50045,7 @@ - + @@ -50322,19 +50057,19 @@ - + - + - + - + - + @@ -50379,22 +50114,22 @@ - + - + - + - + - + @@ -50405,30 +50140,30 @@ - + - + - + - + - + - + - + - + @@ -50440,37 +50175,37 @@ - + - + - + - + - + - + - + - + - + - + @@ -50481,13 +50216,13 @@ - + - + @@ -50495,18 +50230,18 @@ - + - + - + @@ -50515,7 +50250,7 @@ - + @@ -50530,32 +50265,32 @@ - + - + - + - + - + - + - + - + - + @@ -50564,13 +50299,13 @@ - + - + - + @@ -50612,13 +50347,13 @@ - + - + - + @@ -50627,7 +50362,7 @@ - + @@ -50639,77 +50374,77 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -50717,9 +50452,9 @@ - + - + @@ -50728,7 +50463,7 @@ - + @@ -50755,9 +50490,9 @@ - + - + @@ -50769,27 +50504,27 @@ - + - + - + - + - + - + - + @@ -50800,220 +50535,220 @@ - + - + - + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - + - + - - + + - - + + - - + + - + - - - + + + - - - + + + - - - + + + - - + + - - + + - - + + - - - + + + - - + + - - + + - - + + - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + - - + + - - + + - - + + - - + + - + - + - + - + @@ -51022,22 +50757,22 @@ - + - + - + - + - + - + @@ -51067,7 +50802,7 @@ - + @@ -51079,7 +50814,7 @@ - + @@ -51094,14 +50829,14 @@ - + - + @@ -51109,15 +50844,15 @@ - + - + - + @@ -51132,7 +50867,7 @@ - + @@ -51146,96 +50881,96 @@ - - - - - - - - - - - + + + + + + + + + + + - - + + - + - - - + + + - - + + - - - + + + - - - + + + - - + + - - + + - - - + + + - + - - + + - + - + - + - + - + - + @@ -51246,20 +50981,20 @@ - + - + - - + + @@ -51270,7 +51005,7 @@ - + @@ -51281,7 +51016,7 @@ - + @@ -51293,7 +51028,7 @@ - + @@ -51307,10 +51042,10 @@ - + - + @@ -51318,34 +51053,34 @@ - + - + - + - + - + - - + + - + - + @@ -51353,21 +51088,21 @@ - + - + - + - + - + - + @@ -51387,7 +51122,7 @@ - + @@ -51407,7 +51142,7 @@ - + @@ -51424,9 +51159,9 @@ - + - + @@ -51438,32 +51173,37 @@ - - - + + + + + + + + + - - + + - + - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + @@ -51471,166 +51211,161 @@ - - - - - - + - + - + - - + + - + - + - + - + - - + + - - + + - - - + + + - - + + - - + + - - + + - - + + - + - + - + - + - + - + - + - - + + - - + + - - + + - - + + - + - + + - - + - - + + - - + + - - - + + + - - + + - - + + - + - + @@ -51638,7 +51373,7 @@ - + @@ -51646,53 +51381,53 @@ - + - + - - - + + + - + - + - + - + - + - + - - + + - + @@ -51700,7 +51435,7 @@ - + @@ -51708,7 +51443,7 @@ - + @@ -51719,7 +51454,7 @@ - + @@ -51733,12 +51468,12 @@ - + - + @@ -51746,7 +51481,7 @@ - + @@ -51754,10 +51489,10 @@ - + - + @@ -51765,7 +51500,7 @@ - + @@ -51800,22 +51535,22 @@ - + - + - + - + - + - + @@ -51836,10 +51571,10 @@ - + - + @@ -51848,7 +51583,7 @@ - + @@ -51856,12 +51591,12 @@ - + - + - + @@ -51894,8 +51629,8 @@ - - + + @@ -51906,7 +51641,7 @@ - + @@ -51932,7 +51667,7 @@ - + @@ -51946,7 +51681,7 @@ - + @@ -51954,7 +51689,7 @@ - + @@ -51965,7 +51700,7 @@ - + @@ -51982,12 +51717,12 @@ - + - + - + @@ -52017,9 +51752,9 @@ - + - + @@ -52028,12 +51763,12 @@ - + - + @@ -52048,13 +51783,13 @@ - + - + - + @@ -52066,55 +51801,55 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -52129,16 +51864,16 @@ - + - + - + @@ -52150,28 +51885,28 @@ - + - + - + - + - + - + - + @@ -52188,15 +51923,15 @@ - + - + - + - + @@ -52208,7 +51943,7 @@ - + @@ -52234,7 +51969,7 @@ - + @@ -52246,10 +51981,10 @@ - + - + @@ -52263,7 +51998,7 @@ - + @@ -52274,7 +52009,7 @@ - + @@ -52288,7 +52023,7 @@ - + @@ -52299,10 +52034,10 @@ - + - + @@ -52313,7 +52048,7 @@ - + @@ -52336,24 +52071,24 @@ - + - + - + - + - + - + - + @@ -52364,7 +52099,7 @@ - + @@ -52378,7 +52113,7 @@ - + @@ -52392,25 +52127,25 @@ - + - + - + - + - + - + - + @@ -52421,7 +52156,7 @@ - + @@ -52432,16 +52167,16 @@ - + - + - + - + @@ -52450,7 +52185,7 @@ - + @@ -52461,7 +52196,7 @@ - + @@ -52502,15 +52237,15 @@ - + - + - + - + @@ -52531,7 +52266,7 @@ - + @@ -52539,15 +52274,15 @@ - + - + - + @@ -52555,21 +52290,21 @@ - + - + - + - + - + @@ -52589,10 +52324,10 @@ - + - + @@ -52601,16 +52336,16 @@ - + - + - + - + @@ -52618,9 +52353,9 @@ - + - + @@ -52644,7 +52379,7 @@ - + @@ -52656,10 +52391,10 @@ - + - + @@ -52667,14 +52402,14 @@ - + - + - + - + @@ -52695,10 +52430,10 @@ - + - + @@ -52707,24 +52442,24 @@ - + - + - + - + - + @@ -52733,13 +52468,13 @@ - + - + @@ -52747,10 +52482,10 @@ - + - + @@ -52789,7 +52524,7 @@ - + @@ -52816,13 +52551,13 @@ - + - + @@ -52840,21 +52575,21 @@ - + - + - + - + - + @@ -52887,9 +52622,9 @@ - + - + @@ -52907,7 +52642,7 @@ - + @@ -52915,7 +52650,7 @@ - + @@ -52936,7 +52671,7 @@ - + @@ -52947,10 +52682,10 @@ - + - + @@ -52968,21 +52703,21 @@ - + - + - + - + - + @@ -53036,10 +52771,10 @@ - + - + @@ -53051,10 +52786,10 @@ - + - + @@ -53068,31 +52803,31 @@ - + - + - + - + - + - + - + @@ -53137,18 +52872,18 @@ - + - + - + - + - + @@ -53163,10 +52898,10 @@ - + - + @@ -53204,38 +52939,38 @@ - + - + - + - + - + - + - + - + - + @@ -53243,23 +52978,23 @@ - + - + - + - + - + - + @@ -53267,108 +53002,108 @@ - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + - + - + - + - + - + @@ -53376,11 +53111,11 @@ - + - + @@ -53388,59 +53123,59 @@ - + - + - + - - + + - - + + - + - + - + - + - + @@ -53448,7 +53183,7 @@ - + @@ -53457,12 +53192,12 @@ - + - + @@ -53470,164 +53205,164 @@ - - + + - + - + - + - + - - + + - - + + - - + + - - + + - - + + - - + + - + - - + + - - + + - - + + - + - + - + - - + + - + - - + + - + - - + + - - + + - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - + - - + + - - + + - - + + - - + + - + @@ -53637,43 +53372,43 @@ - - + + - + - + - - + + - + - + - + - + - + @@ -53682,15 +53417,15 @@ - + - + - + @@ -53698,27 +53433,27 @@ - - - - + + + + - + - + - + - + - + @@ -53726,65 +53461,65 @@ - - + + - - - + + + - + - + - + - - + + - + - + - + - + - + - + - + - - + + - + @@ -53792,24 +53527,24 @@ - + - + - + - + - + - + - + @@ -53841,21 +53576,21 @@ - + - + - - + + - + @@ -53864,7 +53599,7 @@ - + @@ -53873,19 +53608,19 @@ - - + + - + - + - + @@ -53924,13 +53659,13 @@ - + - + @@ -53972,7 +53707,7 @@ - + @@ -53984,18 +53719,18 @@ - + - + - + @@ -54004,10 +53739,10 @@ - + - + @@ -54016,10 +53751,10 @@ - + - + @@ -54028,7 +53763,7 @@ - + @@ -54036,19 +53771,19 @@ - + - + - + - + @@ -54057,13 +53792,13 @@ - + - + @@ -54072,25 +53807,25 @@ - + - + - + - + - + @@ -54102,7 +53837,7 @@ - + @@ -54111,109 +53846,109 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -54221,15 +53956,15 @@ - + - + - + @@ -54238,8 +53973,8 @@ - - + + @@ -54267,7 +54002,7 @@ - + @@ -54278,24 +54013,24 @@ - + - + - + - + - + - + @@ -54304,9 +54039,9 @@ - + - + @@ -54315,7 +54050,7 @@ - + @@ -54330,59 +54065,59 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -54391,148 +54126,148 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - - + + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - - + + - + @@ -54541,53 +54276,53 @@ - + - + - + - + - + - + - + - + - + - + - + - + - - + + - + - + @@ -54601,67 +54336,67 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -54669,9 +54404,9 @@ - + - + @@ -54680,7 +54415,7 @@ - + @@ -54688,19 +54423,19 @@ - + - + - + - + - + @@ -54715,108 +54450,108 @@ - + - + - + - + - + - + - + - + - + - + - + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - - - - - - - - + + + + + + - + - + @@ -54825,147 +54560,147 @@ - + - - + + - + - + - + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - - + + + - - + + - - + + - - + + - - + + - - + + - - + + - + - + - + - + - + - - + + - + - + - + - + - + - + - - + + - + - - - + + + @@ -54975,67 +54710,67 @@ - - - + + + - - + + - - + + - - + + - - + + - - + + - - + + - + - - - + + + - + - + - - + + @@ -55043,48 +54778,48 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -55093,13 +54828,13 @@ - + - + - + @@ -55110,7 +54845,7 @@ - + @@ -55124,7 +54859,7 @@ - + @@ -55138,7 +54873,7 @@ - + @@ -55152,7 +54887,7 @@ - + @@ -55166,21 +54901,21 @@ - + - + - + - + - + @@ -55197,33 +54932,33 @@ - + - + - + - + - + - + - + - - + + @@ -55234,13 +54969,13 @@ - + - + - + @@ -55249,7 +54984,7 @@ - + @@ -55282,7 +55017,7 @@ - + @@ -55291,15 +55026,15 @@ - + - + - + @@ -55344,9 +55079,9 @@ - + - + @@ -55355,7 +55090,7 @@ - + @@ -55363,10 +55098,10 @@ - + - + @@ -55374,7 +55109,7 @@ - + @@ -55382,7 +55117,7 @@ - + @@ -55393,13 +55128,13 @@ - + - + - + @@ -55408,7 +55143,7 @@ - + @@ -55422,7 +55157,7 @@ - + @@ -55433,7 +55168,7 @@ - + @@ -55441,21 +55176,21 @@ - + - + - + - + - - + + @@ -55464,26 +55199,26 @@ - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + @@ -55500,8 +55235,8 @@ - - + + @@ -55525,11 +55260,11 @@ - + - + @@ -55584,7 +55319,7 @@ - + @@ -55599,7 +55334,7 @@ - + @@ -55649,33 +55384,33 @@ - + - + - + - + - + - + - + - + @@ -55754,44 +55489,41 @@ - + - - + + - + - + - + - + - - - + + + - + - + - - - - - - - + + + + @@ -55814,7 +55546,7 @@ - + @@ -55831,11 +55563,11 @@ - + - - + + @@ -55845,21 +55577,21 @@ - + - + - + - + - + @@ -55873,45 +55605,45 @@ - + - + - + - + - - - - + + + + - - + + - + - + - + - + - + @@ -55947,7 +55679,7 @@ - + @@ -55955,7 +55687,7 @@ - + @@ -55963,9 +55695,9 @@ - + - + @@ -55974,7 +55706,7 @@ - + @@ -55982,10 +55714,10 @@ - + - + @@ -56003,125 +55735,125 @@ - + - + - + - - + + - + - + - + - - + + - + - + - + - + - + - + - + - + - + - + - + - + - - + + - - + + - + - + - + - + - + - + - + - + - + @@ -56148,13 +55880,13 @@ - + - + @@ -56220,10 +55952,10 @@ - + - + @@ -56250,7 +55982,7 @@ - + @@ -56259,13 +55991,13 @@ - + - + - + @@ -56277,10 +56009,10 @@ - + - + @@ -56289,13 +56021,13 @@ - + - + - + @@ -56334,18 +56066,18 @@ - + - + - + - + @@ -56353,9 +56085,9 @@ - + - + @@ -56379,7 +56111,7 @@ - + @@ -56387,13 +56119,13 @@ - + - + @@ -56413,12 +56145,12 @@ - + - + - + @@ -56433,7 +56165,7 @@ - + @@ -56444,7 +56176,7 @@ - + @@ -56467,19 +56199,19 @@ - + - + - + - + @@ -56493,9 +56225,9 @@ - + - + @@ -56510,7 +56242,7 @@ - + @@ -56519,7 +56251,7 @@ - + @@ -56537,10 +56269,10 @@ - + - + @@ -56551,7 +56283,7 @@ - + @@ -56562,12 +56294,12 @@ - + - + @@ -56576,7 +56308,7 @@ - + @@ -56588,7 +56320,7 @@ - + @@ -56600,16 +56332,16 @@ - + - + - + @@ -56618,10 +56350,10 @@ - + - + @@ -56645,38 +56377,38 @@ - + - + - + - + - + - + - + - + - + @@ -56684,15 +56416,15 @@ - + - + - + @@ -56700,12 +56432,12 @@ - + - + - + @@ -56732,10 +56464,10 @@ - + - + @@ -56747,13 +56479,13 @@ - + - + - + @@ -56762,7 +56494,7 @@ - + @@ -56771,13 +56503,13 @@ - + - + - + @@ -56813,7 +56545,7 @@ - + @@ -56825,7 +56557,7 @@ - + @@ -56843,18 +56575,18 @@ - + - + - + - + @@ -56871,16 +56603,16 @@ - + - + - + @@ -56888,7 +56620,7 @@ - + @@ -56914,33 +56646,33 @@ - + - + - + - + - + - + - + - + - + - + @@ -56955,13 +56687,13 @@ - + - + @@ -56970,12 +56702,12 @@ - + - + - + @@ -57008,7 +56740,7 @@ - + @@ -57017,13 +56749,13 @@ - + - + @@ -57037,15 +56769,15 @@ - + - + - + - + @@ -57054,10 +56786,10 @@ - + - + @@ -57068,15 +56800,15 @@ - + - + - + @@ -57091,7 +56823,7 @@ - + @@ -57102,12 +56834,12 @@ - + - + - + @@ -57119,15 +56851,15 @@ - + - + - + - + @@ -57136,22 +56868,22 @@ - + - + - + - + - + @@ -57165,7 +56897,7 @@ - + @@ -57179,15 +56911,15 @@ - + - + - + - + @@ -57196,7 +56928,7 @@ - + @@ -57204,74 +56936,74 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -57279,16 +57011,16 @@ - + - + - + - + @@ -57297,39 +57029,39 @@ - + - + - + - + - + - + @@ -57337,24 +57069,24 @@ - + - + - + - + - + - + - + @@ -57366,7 +57098,7 @@ - + @@ -57377,7 +57109,7 @@ - + @@ -57385,15 +57117,15 @@ - + - + - + - + @@ -57407,18 +57139,18 @@ - + - + - + - + @@ -57427,15 +57159,15 @@ - + - + - + @@ -57455,18 +57187,18 @@ - + - + - + - + @@ -57475,7 +57207,7 @@ - + @@ -57483,9 +57215,9 @@ - + - + @@ -57494,7 +57226,7 @@ - + @@ -57502,10 +57234,10 @@ - + - + @@ -57513,7 +57245,7 @@ - + @@ -57524,10 +57256,10 @@ - + - + @@ -57556,333 +57288,333 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -57890,7 +57622,7 @@ - + @@ -57899,12 +57631,12 @@ - + - + - + @@ -57919,19 +57651,19 @@ - + - + - + - + @@ -57943,22 +57675,22 @@ - + - + - + - + - + @@ -57973,10 +57705,10 @@ - + - + @@ -58029,13 +57761,13 @@ - + - + @@ -58052,7 +57784,7 @@ - + @@ -58067,7 +57799,7 @@ - + @@ -58082,7 +57814,7 @@ - + @@ -58096,7 +57828,7 @@ - + @@ -58104,48 +57836,48 @@ - + - + - + - + - + - + - + - + - + - + - + - + @@ -58156,7 +57888,7 @@ - + @@ -58191,10 +57923,10 @@ - + - + @@ -58203,7 +57935,7 @@ - + @@ -58236,7 +57968,7 @@ - + @@ -58245,10 +57977,10 @@ - + - + @@ -58257,21 +57989,21 @@ - + - + - + - + @@ -58282,7 +58014,7 @@ - + @@ -58320,16 +58052,16 @@ - + - + - + - + @@ -58347,10 +58079,10 @@ - + - + @@ -58368,13 +58100,13 @@ - + - + - + @@ -58395,7 +58127,7 @@ - + @@ -58416,7 +58148,7 @@ - + @@ -58445,7 +58177,7 @@ - + @@ -58456,7 +58188,7 @@ - + @@ -58464,7 +58196,7 @@ - + @@ -58481,10 +58213,10 @@ - + - + @@ -58522,7 +58254,7 @@ - + @@ -58557,7 +58289,7 @@ - + @@ -58637,7 +58369,7 @@ - + @@ -58646,9 +58378,9 @@ - + - + @@ -58696,7 +58428,7 @@ - + @@ -58705,12 +58437,12 @@ - + - + - + @@ -58739,9 +58471,9 @@ - + - + @@ -58756,7 +58488,7 @@ - + @@ -58764,9 +58496,9 @@ - + - + @@ -58775,7 +58507,7 @@ - + @@ -58793,28 +58525,28 @@ - + - + - + - + - + - + - + @@ -58823,23 +58555,23 @@ - + - + - + - + - + @@ -58848,7 +58580,7 @@ - + @@ -58866,9 +58598,9 @@ - + - + @@ -58880,7 +58612,7 @@ - + @@ -58889,16 +58621,16 @@ - + - + - + - + @@ -58910,7 +58642,7 @@ - + @@ -58927,9 +58659,9 @@ - + - + @@ -58944,12 +58676,12 @@ - + - + - + @@ -58964,7 +58696,7 @@ - + @@ -58976,10 +58708,10 @@ - + - + @@ -58997,22 +58729,22 @@ - + - + - + - + - + @@ -59042,24 +58774,24 @@ - + - + - + - + - + - + @@ -59068,15 +58800,15 @@ - + - + - + - + @@ -59084,7 +58816,7 @@ - + @@ -59120,21 +58852,21 @@ - + - + - + - + - + @@ -59164,7 +58896,7 @@ - + @@ -59187,9 +58919,9 @@ - + - + @@ -59213,10 +58945,10 @@ - + - + @@ -59227,7 +58959,7 @@ - + @@ -59235,7 +58967,7 @@ - + @@ -59246,7 +58978,7 @@ - + @@ -59254,45 +58986,45 @@ - + - + - + - + - + - + - + - + - + - + - + @@ -59301,7 +59033,7 @@ - + @@ -59313,18 +59045,18 @@ - + - + - + - + @@ -59332,7 +59064,7 @@ - + @@ -59340,7 +59072,7 @@ - + @@ -59348,12 +59080,12 @@ - + - + - + @@ -59374,7 +59106,7 @@ - + @@ -59398,16 +59130,16 @@ - + - + - + @@ -59422,7 +59154,7 @@ - + @@ -59430,7 +59162,7 @@ - + @@ -59438,7 +59170,7 @@ - + @@ -59452,15 +59184,15 @@ - + - + - + - + @@ -59472,12 +59204,12 @@ - + - + @@ -59509,7 +59241,7 @@ - + @@ -59526,35 +59258,35 @@ - + - + - + - + - + - + - + - + - + @@ -59584,16 +59316,16 @@ - + - + - + @@ -59605,473 +59337,473 @@ - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - + - + - + - + - + - - + + - - - + + + - - - + + + - - - + + + - - - + + + - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - + + + + + + + - - + + - - - - - - - + - + - + - + - + - + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + - - + + - - + + - + - - + + - + - - - - - - - - - - - - - - + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + - - - - - - - - - + + - - + + - + - - + + - - + + - + - - + + - + - - + + - + - - + + - - + + - - + + - - + + - + - - + + @@ -60083,143 +59815,143 @@ - - + + - - + + - - + + - - + + - + - - + + - + - - + + - - + + - + - - + + - + - - + + - + - - + + - + - - + + - + - - + + - + - - + + - - + + - - + + - - + + - - + + - + - - + + - - + + - - + + - - + + - - + + @@ -60227,230 +59959,230 @@ - - + + - - + + - + - - + + - + - - + + - - + + - + - - + + - - + + - - + + - - - + + + - - - - + + + + - - - - + + + + - - - - + + + + - - - - + + + + - - - - + + + + - - - - + + + + - - - - + + + + - - - - + + + + - - - - + + + + - - - - + + + + - - - + + + - - - + + + - - - + + + - - - + + + - + - + - - + + - - + + - - - + + + - - + + - - - + + + - - - - + + + + - - - + + + - - - + + + - - + + - - - + + + - + - + - + @@ -60462,32 +60194,32 @@ - + - + - + - - + + - - + + - - - + + + @@ -60495,16 +60227,16 @@ - + - - + + - + @@ -60519,7 +60251,7 @@ - + @@ -60528,12 +60260,12 @@ - + - + @@ -60547,7 +60279,7 @@ - + @@ -60555,7 +60287,7 @@ - + @@ -60570,13 +60302,13 @@ - + - + - + @@ -60599,17 +60331,17 @@ - + - + - + @@ -60632,40 +60364,40 @@ - + - + - + - + - + - + - + @@ -60688,9 +60420,9 @@ - + - + @@ -60698,7 +60430,7 @@ - + @@ -60730,81 +60462,87 @@ - - - + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + - + - + - + - + - - - - - - - - - - - - - + - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/block/Kconfig b/block/Kconfig index 606a67e47e68..e9ce5e9bc9c6 100644 --- a/block/Kconfig +++ b/block/Kconfig @@ -179,13 +179,20 @@ config BLK_SED_OPAL config BLK_INLINE_ENCRYPTION bool "Enable inline encryption support in block layer" + help + Build the blk-crypto subsystem. Enabling this lets the + block layer handle encryption, so users can take + advantage of inline encryption hardware if present. + +config BLK_INLINE_ENCRYPTION_FALLBACK + bool "Enable crypto API fallback for blk-crypto" + depends on BLK_INLINE_ENCRYPTION select CRYPTO select CRYPTO_BLKCIPHER help - Build the blk-crypto subsystem. - Enabling this lets the block layer handle encryption, - so users can take advantage of inline encryption - hardware if present. + Enabling this lets the block layer handle inline encryption + by falling back to the kernel crypto API when inline + encryption hardware is not present. menu "Partition Types" diff --git a/block/Makefile b/block/Makefile index 8932c7e4fd07..9bfd4c9f9e72 100644 --- a/block/Makefile +++ b/block/Makefile @@ -38,3 +38,4 @@ obj-$(CONFIG_BLK_SED_OPAL) += sed-opal.o obj-$(CONFIG_BLK_PM) += blk-pm.o obj-$(CONFIG_BLK_INLINE_ENCRYPTION) += keyslot-manager.o bio-crypt-ctx.o \ blk-crypto.o +obj-$(CONFIG_BLK_INLINE_ENCRYPTION_FALLBACK) += blk-crypto-fallback.o \ No newline at end of file diff --git a/block/bio-crypt-ctx.c b/block/bio-crypt-ctx.c index 0f7641b875e9..75982dabc7a3 100644 --- a/block/bio-crypt-ctx.c +++ b/block/bio-crypt-ctx.c @@ -5,26 +5,43 @@ #include #include -#include #include +#include +#include + +#include "blk-crypto-internal.h" static int num_prealloc_crypt_ctxs = 128; + +module_param(num_prealloc_crypt_ctxs, int, 0444); +MODULE_PARM_DESC(num_prealloc_crypt_ctxs, + "Number of bio crypto contexts to preallocate"); + static struct kmem_cache *bio_crypt_ctx_cache; static mempool_t *bio_crypt_ctx_pool; -int bio_crypt_ctx_init(void) +int __init bio_crypt_ctx_init(void) { + size_t i; + bio_crypt_ctx_cache = KMEM_CACHE(bio_crypt_ctx, 0); if (!bio_crypt_ctx_cache) return -ENOMEM; - bio_crypt_ctx_pool = mempool_create_slab_pool( - num_prealloc_crypt_ctxs, - bio_crypt_ctx_cache); - + bio_crypt_ctx_pool = mempool_create_slab_pool(num_prealloc_crypt_ctxs, + bio_crypt_ctx_cache); if (!bio_crypt_ctx_pool) return -ENOMEM; + /* This is assumed in various places. */ + BUILD_BUG_ON(BLK_ENCRYPTION_MODE_INVALID != 0); + + /* Sanity check that no algorithm exceeds the defined limits. */ + for (i = 0; i < BLK_ENCRYPTION_MODE_MAX; i++) { + BUG_ON(blk_crypto_modes[i].keysize > BLK_CRYPTO_MAX_KEY_SIZE); + BUG_ON(blk_crypto_modes[i].ivsize > BLK_CRYPTO_MAX_IV_SIZE); + } + return 0; } @@ -32,51 +49,43 @@ struct bio_crypt_ctx *bio_crypt_alloc_ctx(gfp_t gfp_mask) { return mempool_alloc(bio_crypt_ctx_pool, gfp_mask); } -EXPORT_SYMBOL(bio_crypt_alloc_ctx); void bio_crypt_free_ctx(struct bio *bio) { mempool_free(bio->bi_crypt_context, bio_crypt_ctx_pool); bio->bi_crypt_context = NULL; } -EXPORT_SYMBOL(bio_crypt_free_ctx); -int bio_crypt_clone(struct bio *dst, struct bio *src, gfp_t gfp_mask) +void bio_crypt_clone(struct bio *dst, struct bio *src, gfp_t gfp_mask) { + const struct bio_crypt_ctx *src_bc = src->bi_crypt_context; + /* - * If a bio is swhandled, then it will be decrypted when bio_endio - * is called. As we only want the data to be decrypted once, copies - * of the bio must not have have a crypt context. + * If a bio is fallback_crypted, then it will be decrypted when + * bio_endio is called. As we only want the data to be decrypted once, + * copies of the bio must not have have a crypt context. */ - if (!bio_has_crypt_ctx(src) || bio_crypt_swhandled(src)) - return 0; + if (!src_bc || bio_crypt_fallback_crypted(src_bc)) + return; dst->bi_crypt_context = bio_crypt_alloc_ctx(gfp_mask); - if (!dst->bi_crypt_context) - return -ENOMEM; + *dst->bi_crypt_context = *src_bc; - *dst->bi_crypt_context = *src->bi_crypt_context; - - if (bio_crypt_has_keyslot(src)) - keyslot_manager_get_slot(src->bi_crypt_context->processing_ksm, - src->bi_crypt_context->keyslot); - - return 0; + if (src_bc->bc_keyslot >= 0) + keyslot_manager_get_slot(src_bc->bc_ksm, src_bc->bc_keyslot); } -EXPORT_SYMBOL(bio_crypt_clone); +EXPORT_SYMBOL_GPL(bio_crypt_clone); -bool bio_crypt_should_process(struct bio *bio, struct request_queue *q) +bool bio_crypt_should_process(struct request *rq) { - if (!bio_has_crypt_ctx(bio)) + struct bio *bio = rq->bio; + + if (!bio || !bio->bi_crypt_context) return false; - if (q->ksm != bio->bi_crypt_context->processing_ksm) - return false; - - WARN_ON(!bio_crypt_has_keyslot(bio)); - return true; + return rq->q->ksm == bio->bi_crypt_context->bc_ksm; } -EXPORT_SYMBOL(bio_crypt_should_process); +EXPORT_SYMBOL_GPL(bio_crypt_should_process); /* * Checks that two bio crypt contexts are compatible - i.e. that @@ -87,23 +96,19 @@ bool bio_crypt_ctx_compatible(struct bio *b_1, struct bio *b_2) struct bio_crypt_ctx *bc1 = b_1->bi_crypt_context; struct bio_crypt_ctx *bc2 = b_2->bi_crypt_context; - if (bio_has_crypt_ctx(b_1) != bio_has_crypt_ctx(b_2)) + if (bc1 != bc2) return false; - if (!bio_has_crypt_ctx(b_1)) - return true; - - return bc1->keyslot == bc2->keyslot && - bc1->data_unit_size_bits == bc2->data_unit_size_bits; + return !bc1 || bc1->bc_key == bc2->bc_key; } /* * Checks that two bio crypt contexts are compatible, and also * that their data_unit_nums are continuous (and can hence be merged) + * in the order b_1 followed by b_2. */ -bool bio_crypt_ctx_back_mergeable(struct bio *b_1, - unsigned int b1_sectors, - struct bio *b_2) +bool bio_crypt_ctx_mergeable(struct bio *b_1, unsigned int b1_bytes, + struct bio *b_2) { struct bio_crypt_ctx *bc1 = b_1->bi_crypt_context; struct bio_crypt_ctx *bc2 = b_2->bi_crypt_context; @@ -111,35 +116,25 @@ bool bio_crypt_ctx_back_mergeable(struct bio *b_1, if (!bio_crypt_ctx_compatible(b_1, b_2)) return false; - return !bio_has_crypt_ctx(b_1) || - (bc1->data_unit_num + - (b1_sectors >> (bc1->data_unit_size_bits - 9)) == - bc2->data_unit_num); + return !bc1 || bio_crypt_dun_is_contiguous(bc1, b1_bytes, bc2->bc_dun); } -void bio_crypt_ctx_release_keyslot(struct bio *bio) +void bio_crypt_ctx_release_keyslot(struct bio_crypt_ctx *bc) { - struct bio_crypt_ctx *crypt_ctx = bio->bi_crypt_context; - - keyslot_manager_put_slot(crypt_ctx->processing_ksm, crypt_ctx->keyslot); - bio->bi_crypt_context->processing_ksm = NULL; - bio->bi_crypt_context->keyslot = -1; + keyslot_manager_put_slot(bc->bc_ksm, bc->bc_keyslot); + bc->bc_ksm = NULL; + bc->bc_keyslot = -1; } -int bio_crypt_ctx_acquire_keyslot(struct bio *bio, struct keyslot_manager *ksm) +int bio_crypt_ctx_acquire_keyslot(struct bio_crypt_ctx *bc, + struct keyslot_manager *ksm) { - int slot; - enum blk_crypto_mode_num crypto_mode = bio_crypto_mode(bio); + int slot = keyslot_manager_get_slot_for_key(ksm, bc->bc_key); - if (!ksm) - return -ENOMEM; - - slot = keyslot_manager_get_slot_for_key(ksm, - bio_crypt_raw_key(bio), crypto_mode, - 1 << bio->bi_crypt_context->data_unit_size_bits); if (slot < 0) return slot; - bio_crypt_set_keyslot(bio, slot, ksm); + bc->bc_keyslot = slot; + bc->bc_ksm = ksm; return 0; } diff --git a/block/bio.c b/block/bio.c index 2d6a840bb67f..d2f552ae0753 100644 --- a/block/bio.c +++ b/block/bio.c @@ -234,6 +234,8 @@ fallback: void bio_uninit(struct bio *bio) { bio_disassociate_blkg(bio); + + bio_crypt_free_ctx(bio); } EXPORT_SYMBOL(bio_uninit); @@ -242,7 +244,6 @@ static void bio_free(struct bio *bio) struct bio_set *bs = bio->bi_pool; void *p; - bio_crypt_free_ctx(bio); bio_uninit(bio); if (bs) { @@ -653,10 +654,7 @@ struct bio *bio_clone_fast(struct bio *bio, gfp_t gfp_mask, struct bio_set *bs) __bio_clone_fast(b, bio); - if (bio_crypt_clone(b, bio, gfp_mask) < 0) { - bio_put(b); - return NULL; - } + bio_crypt_clone(b, bio, gfp_mask); if (bio_integrity(bio) && bio_integrity_clone(b, bio, gfp_mask) < 0) { diff --git a/block/blk-core.c b/block/blk-core.c index 808adf0bb44b..9e10ad79df34 100644 --- a/block/blk-core.c +++ b/block/blk-core.c @@ -1115,14 +1115,11 @@ blk_qc_t direct_make_request(struct bio *bio) { struct request_queue *q = bio->bi_disk->queue; bool nowait = bio->bi_opf & REQ_NOWAIT; - blk_qc_t ret; + blk_qc_t ret = BLK_QC_T_NONE; if (!generic_make_request_checks(bio)) return BLK_QC_T_NONE; - if (blk_crypto_submit_bio(&bio)) - return BLK_QC_T_NONE; - if (unlikely(blk_queue_enter(q, nowait ? BLK_MQ_REQ_NOWAIT : 0))) { if (nowait && !blk_queue_dying(q)) bio->bi_status = BLK_STS_AGAIN; @@ -1132,7 +1129,8 @@ blk_qc_t direct_make_request(struct bio *bio) return BLK_QC_T_NONE; } - ret = q->make_request_fn(q, bio); + if (!blk_crypto_submit_bio(&bio)) + ret = q->make_request_fn(q, bio); blk_queue_exit(q); return ret; } @@ -1819,8 +1817,8 @@ int __init blk_dev_init(void) if (bio_crypt_ctx_init() < 0) panic("Failed to allocate mem for bio crypt ctxs\n"); - if (blk_crypto_init() < 0) - panic("Failed to init blk-crypto\n"); + if (blk_crypto_fallback_init() < 0) + panic("Failed to init blk-crypto-fallback\n"); return 0; } diff --git a/block/blk-crypto-fallback.c b/block/blk-crypto-fallback.c new file mode 100644 index 000000000000..9773bd6a5186 --- /dev/null +++ b/block/blk-crypto-fallback.c @@ -0,0 +1,648 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright 2019 Google LLC + */ + +/* + * Refer to Documentation/block/inline-encryption.rst for detailed explanation. + */ + +#define pr_fmt(fmt) "blk-crypto-fallback: " fmt + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "blk-crypto-internal.h" + +static unsigned int num_prealloc_bounce_pg = 32; +module_param(num_prealloc_bounce_pg, uint, 0); +MODULE_PARM_DESC(num_prealloc_bounce_pg, + "Number of preallocated bounce pages for the blk-crypto crypto API fallback"); + +static unsigned int blk_crypto_num_keyslots = 100; +module_param_named(num_keyslots, blk_crypto_num_keyslots, uint, 0); +MODULE_PARM_DESC(num_keyslots, + "Number of keyslots for the blk-crypto crypto API fallback"); + +static unsigned int num_prealloc_fallback_crypt_ctxs = 128; +module_param(num_prealloc_fallback_crypt_ctxs, uint, 0); +MODULE_PARM_DESC(num_prealloc_crypt_fallback_ctxs, + "Number of preallocated bio fallback crypto contexts for blk-crypto to use during crypto API fallback"); + +struct bio_fallback_crypt_ctx { + struct bio_crypt_ctx crypt_ctx; + /* + * Copy of the bvec_iter when this bio was submitted. + * We only want to en/decrypt the part of the bio as described by the + * bvec_iter upon submission because bio might be split before being + * resubmitted + */ + struct bvec_iter crypt_iter; + u64 fallback_dun[BLK_CRYPTO_DUN_ARRAY_SIZE]; +}; + +/* The following few vars are only used during the crypto API fallback */ +static struct kmem_cache *bio_fallback_crypt_ctx_cache; +static mempool_t *bio_fallback_crypt_ctx_pool; + +/* + * Allocating a crypto tfm during I/O can deadlock, so we have to preallocate + * all of a mode's tfms when that mode starts being used. Since each mode may + * need all the keyslots at some point, each mode needs its own tfm for each + * keyslot; thus, a keyslot may contain tfms for multiple modes. However, to + * match the behavior of real inline encryption hardware (which only supports a + * single encryption context per keyslot), we only allow one tfm per keyslot to + * be used at a time - the rest of the unused tfms have their keys cleared. + */ +static DEFINE_MUTEX(tfms_init_lock); +static bool tfms_inited[BLK_ENCRYPTION_MODE_MAX]; + +struct blk_crypto_decrypt_work { + struct work_struct work; + struct bio *bio; +}; + +static struct blk_crypto_keyslot { + struct crypto_skcipher *tfm; + enum blk_crypto_mode_num crypto_mode; + struct crypto_skcipher *tfms[BLK_ENCRYPTION_MODE_MAX]; +} *blk_crypto_keyslots; + +/* The following few vars are only used during the crypto API fallback */ +static struct keyslot_manager *blk_crypto_ksm; +static struct workqueue_struct *blk_crypto_wq; +static mempool_t *blk_crypto_bounce_page_pool; +static struct kmem_cache *blk_crypto_decrypt_work_cache; + +bool bio_crypt_fallback_crypted(const struct bio_crypt_ctx *bc) +{ + return bc && bc->bc_ksm == blk_crypto_ksm; +} + +/* + * This is the key we set when evicting a keyslot. This *should* be the all 0's + * key, but AES-XTS rejects that key, so we use some random bytes instead. + */ +static u8 blank_key[BLK_CRYPTO_MAX_KEY_SIZE]; + +static void blk_crypto_evict_keyslot(unsigned int slot) +{ + struct blk_crypto_keyslot *slotp = &blk_crypto_keyslots[slot]; + enum blk_crypto_mode_num crypto_mode = slotp->crypto_mode; + int err; + + WARN_ON(slotp->crypto_mode == BLK_ENCRYPTION_MODE_INVALID); + + /* Clear the key in the skcipher */ + err = crypto_skcipher_setkey(slotp->tfms[crypto_mode], blank_key, + blk_crypto_modes[crypto_mode].keysize); + WARN_ON(err); + slotp->crypto_mode = BLK_ENCRYPTION_MODE_INVALID; +} + +static int blk_crypto_keyslot_program(struct keyslot_manager *ksm, + const struct blk_crypto_key *key, + unsigned int slot) +{ + struct blk_crypto_keyslot *slotp = &blk_crypto_keyslots[slot]; + const enum blk_crypto_mode_num crypto_mode = key->crypto_mode; + int err; + + if (crypto_mode != slotp->crypto_mode && + slotp->crypto_mode != BLK_ENCRYPTION_MODE_INVALID) { + blk_crypto_evict_keyslot(slot); + } + + if (!slotp->tfms[crypto_mode]) + return -ENOMEM; + slotp->crypto_mode = crypto_mode; + err = crypto_skcipher_setkey(slotp->tfms[crypto_mode], key->raw, + key->size); + if (err) { + blk_crypto_evict_keyslot(slot); + return err; + } + return 0; +} + +static int blk_crypto_keyslot_evict(struct keyslot_manager *ksm, + const struct blk_crypto_key *key, + unsigned int slot) +{ + blk_crypto_evict_keyslot(slot); + return 0; +} + +/* + * The crypto API fallback KSM ops - only used for a bio when it specifies a + * blk_crypto_mode for which we failed to get a keyslot in the device's inline + * encryption hardware (which probably means the device doesn't have inline + * encryption hardware that supports that crypto mode). + */ +static const struct keyslot_mgmt_ll_ops blk_crypto_ksm_ll_ops = { + .keyslot_program = blk_crypto_keyslot_program, + .keyslot_evict = blk_crypto_keyslot_evict, +}; + +static void blk_crypto_encrypt_endio(struct bio *enc_bio) +{ + struct bio *src_bio = enc_bio->bi_private; + int i; + + for (i = 0; i < enc_bio->bi_vcnt; i++) + mempool_free(enc_bio->bi_io_vec[i].bv_page, + blk_crypto_bounce_page_pool); + + src_bio->bi_status = enc_bio->bi_status; + + bio_put(enc_bio); + bio_endio(src_bio); +} + +static struct bio *blk_crypto_clone_bio(struct bio *bio_src) +{ + struct bvec_iter iter; + struct bio_vec bv; + struct bio *bio; + + bio = bio_alloc_bioset(GFP_NOIO, bio_segments(bio_src), NULL); + if (!bio) + return NULL; + bio->bi_disk = bio_src->bi_disk; + bio->bi_opf = bio_src->bi_opf; + bio->bi_ioprio = bio_src->bi_ioprio; + bio->bi_write_hint = bio_src->bi_write_hint; + bio->bi_iter.bi_sector = bio_src->bi_iter.bi_sector; + bio->bi_iter.bi_size = bio_src->bi_iter.bi_size; + + bio_for_each_segment(bv, bio_src, iter) + bio->bi_io_vec[bio->bi_vcnt++] = bv; + + if (bio_integrity(bio_src) && + bio_integrity_clone(bio, bio_src, GFP_NOIO) < 0) { + bio_put(bio); + return NULL; + } + + bio_clone_blkg_association(bio, bio_src); + blkcg_bio_issue_init(bio); + + return bio; +} + +static int blk_crypto_alloc_cipher_req(struct bio *src_bio, + struct skcipher_request **ciph_req_ret, + struct crypto_wait *wait) +{ + struct skcipher_request *ciph_req; + const struct blk_crypto_keyslot *slotp; + + slotp = &blk_crypto_keyslots[src_bio->bi_crypt_context->bc_keyslot]; + ciph_req = skcipher_request_alloc(slotp->tfms[slotp->crypto_mode], + GFP_NOIO); + if (!ciph_req) { + src_bio->bi_status = BLK_STS_RESOURCE; + return -ENOMEM; + } + + skcipher_request_set_callback(ciph_req, + CRYPTO_TFM_REQ_MAY_BACKLOG | + CRYPTO_TFM_REQ_MAY_SLEEP, + crypto_req_done, wait); + *ciph_req_ret = ciph_req; + return 0; +} + +static int blk_crypto_split_bio_if_needed(struct bio **bio_ptr) +{ + struct bio *bio = *bio_ptr; + unsigned int i = 0; + unsigned int num_sectors = 0; + struct bio_vec bv; + struct bvec_iter iter; + + bio_for_each_segment(bv, bio, iter) { + num_sectors += bv.bv_len >> SECTOR_SHIFT; + if (++i == BIO_MAX_PAGES) + break; + } + if (num_sectors < bio_sectors(bio)) { + struct bio *split_bio; + + split_bio = bio_split(bio, num_sectors, GFP_NOIO, NULL); + if (!split_bio) { + bio->bi_status = BLK_STS_RESOURCE; + return -ENOMEM; + } + bio_chain(split_bio, bio); + generic_make_request(bio); + *bio_ptr = split_bio; + } + return 0; +} + +union blk_crypto_iv { + __le64 dun[BLK_CRYPTO_DUN_ARRAY_SIZE]; + u8 bytes[BLK_CRYPTO_MAX_IV_SIZE]; +}; + +static void blk_crypto_dun_to_iv(const u64 dun[BLK_CRYPTO_DUN_ARRAY_SIZE], + union blk_crypto_iv *iv) +{ + int i; + + for (i = 0; i < BLK_CRYPTO_DUN_ARRAY_SIZE; i++) + iv->dun[i] = cpu_to_le64(dun[i]); +} + +/* + * The crypto API fallback's encryption routine. + * Allocate a bounce bio for encryption, encrypt the input bio using crypto API, + * and replace *bio_ptr with the bounce bio. May split input bio if it's too + * large. + */ +static int blk_crypto_encrypt_bio(struct bio **bio_ptr) +{ + struct bio *src_bio; + struct skcipher_request *ciph_req = NULL; + DECLARE_CRYPTO_WAIT(wait); + u64 curr_dun[BLK_CRYPTO_DUN_ARRAY_SIZE]; + union blk_crypto_iv iv; + struct scatterlist src, dst; + struct bio *enc_bio; + unsigned int i, j; + int data_unit_size; + struct bio_crypt_ctx *bc; + int err = 0; + + /* Split the bio if it's too big for single page bvec */ + err = blk_crypto_split_bio_if_needed(bio_ptr); + if (err) + return err; + + src_bio = *bio_ptr; + bc = src_bio->bi_crypt_context; + data_unit_size = bc->bc_key->data_unit_size; + + /* Allocate bounce bio for encryption */ + enc_bio = blk_crypto_clone_bio(src_bio); + if (!enc_bio) { + src_bio->bi_status = BLK_STS_RESOURCE; + return -ENOMEM; + } + + /* + * Use the crypto API fallback keyslot manager to get a crypto_skcipher + * for the algorithm and key specified for this bio. + */ + err = bio_crypt_ctx_acquire_keyslot(bc, blk_crypto_ksm); + if (err) { + src_bio->bi_status = BLK_STS_IOERR; + goto out_put_enc_bio; + } + + /* and then allocate an skcipher_request for it */ + err = blk_crypto_alloc_cipher_req(src_bio, &ciph_req, &wait); + if (err) + goto out_release_keyslot; + + memcpy(curr_dun, bc->bc_dun, sizeof(curr_dun)); + sg_init_table(&src, 1); + sg_init_table(&dst, 1); + + skcipher_request_set_crypt(ciph_req, &src, &dst, data_unit_size, + iv.bytes); + + /* Encrypt each page in the bounce bio */ + for (i = 0; i < enc_bio->bi_vcnt; i++) { + struct bio_vec *enc_bvec = &enc_bio->bi_io_vec[i]; + struct page *plaintext_page = enc_bvec->bv_page; + struct page *ciphertext_page = + mempool_alloc(blk_crypto_bounce_page_pool, GFP_NOIO); + + enc_bvec->bv_page = ciphertext_page; + + if (!ciphertext_page) { + src_bio->bi_status = BLK_STS_RESOURCE; + err = -ENOMEM; + goto out_free_bounce_pages; + } + + sg_set_page(&src, plaintext_page, data_unit_size, + enc_bvec->bv_offset); + sg_set_page(&dst, ciphertext_page, data_unit_size, + enc_bvec->bv_offset); + + /* Encrypt each data unit in this page */ + for (j = 0; j < enc_bvec->bv_len; j += data_unit_size) { + blk_crypto_dun_to_iv(curr_dun, &iv); + err = crypto_wait_req(crypto_skcipher_encrypt(ciph_req), + &wait); + if (err) { + i++; + src_bio->bi_status = BLK_STS_RESOURCE; + goto out_free_bounce_pages; + } + bio_crypt_dun_increment(curr_dun, 1); + src.offset += data_unit_size; + dst.offset += data_unit_size; + } + } + + enc_bio->bi_private = src_bio; + enc_bio->bi_end_io = blk_crypto_encrypt_endio; + *bio_ptr = enc_bio; + + enc_bio = NULL; + err = 0; + goto out_free_ciph_req; + +out_free_bounce_pages: + while (i > 0) + mempool_free(enc_bio->bi_io_vec[--i].bv_page, + blk_crypto_bounce_page_pool); +out_free_ciph_req: + skcipher_request_free(ciph_req); +out_release_keyslot: + bio_crypt_ctx_release_keyslot(bc); +out_put_enc_bio: + if (enc_bio) + bio_put(enc_bio); + + return err; +} + +static void blk_crypto_free_fallback_crypt_ctx(struct bio *bio) +{ + mempool_free(container_of(bio->bi_crypt_context, + struct bio_fallback_crypt_ctx, + crypt_ctx), + bio_fallback_crypt_ctx_pool); + bio->bi_crypt_context = NULL; +} + +/* + * The crypto API fallback's main decryption routine. + * Decrypts input bio in place. + */ +static void blk_crypto_decrypt_bio(struct work_struct *work) +{ + struct blk_crypto_decrypt_work *decrypt_work = + container_of(work, struct blk_crypto_decrypt_work, work); + struct bio *bio = decrypt_work->bio; + struct skcipher_request *ciph_req = NULL; + DECLARE_CRYPTO_WAIT(wait); + struct bio_vec bv; + struct bvec_iter iter; + u64 curr_dun[BLK_CRYPTO_DUN_ARRAY_SIZE]; + union blk_crypto_iv iv; + struct scatterlist sg; + struct bio_crypt_ctx *bc = bio->bi_crypt_context; + struct bio_fallback_crypt_ctx *f_ctx = + container_of(bc, struct bio_fallback_crypt_ctx, crypt_ctx); + const int data_unit_size = bc->bc_key->data_unit_size; + unsigned int i; + int err; + + /* + * Use the crypto API fallback keyslot manager to get a crypto_skcipher + * for the algorithm and key specified for this bio. + */ + if (bio_crypt_ctx_acquire_keyslot(bc, blk_crypto_ksm)) { + bio->bi_status = BLK_STS_RESOURCE; + goto out_no_keyslot; + } + + /* and then allocate an skcipher_request for it */ + err = blk_crypto_alloc_cipher_req(bio, &ciph_req, &wait); + if (err) + goto out; + + memcpy(curr_dun, f_ctx->fallback_dun, sizeof(curr_dun)); + sg_init_table(&sg, 1); + skcipher_request_set_crypt(ciph_req, &sg, &sg, data_unit_size, + iv.bytes); + + /* Decrypt each segment in the bio */ + __bio_for_each_segment(bv, bio, iter, f_ctx->crypt_iter) { + struct page *page = bv.bv_page; + + sg_set_page(&sg, page, data_unit_size, bv.bv_offset); + + /* Decrypt each data unit in the segment */ + for (i = 0; i < bv.bv_len; i += data_unit_size) { + blk_crypto_dun_to_iv(curr_dun, &iv); + if (crypto_wait_req(crypto_skcipher_decrypt(ciph_req), + &wait)) { + bio->bi_status = BLK_STS_IOERR; + goto out; + } + bio_crypt_dun_increment(curr_dun, 1); + sg.offset += data_unit_size; + } + } + +out: + skcipher_request_free(ciph_req); + bio_crypt_ctx_release_keyslot(bc); +out_no_keyslot: + kmem_cache_free(blk_crypto_decrypt_work_cache, decrypt_work); + blk_crypto_free_fallback_crypt_ctx(bio); + bio_endio(bio); +} + +/* + * Queue bio for decryption. + * Returns true iff bio was queued for decryption. + */ +bool blk_crypto_queue_decrypt_bio(struct bio *bio) +{ + struct blk_crypto_decrypt_work *decrypt_work; + + /* If there was an IO error, don't queue for decrypt. */ + if (bio->bi_status) + goto out; + + decrypt_work = kmem_cache_zalloc(blk_crypto_decrypt_work_cache, + GFP_ATOMIC); + if (!decrypt_work) { + bio->bi_status = BLK_STS_RESOURCE; + goto out; + } + + INIT_WORK(&decrypt_work->work, blk_crypto_decrypt_bio); + decrypt_work->bio = bio; + queue_work(blk_crypto_wq, &decrypt_work->work); + + return true; +out: + blk_crypto_free_fallback_crypt_ctx(bio); + return false; +} + +/** + * blk_crypto_start_using_mode() - Start using a crypto algorithm on a device + * @mode_num: the blk_crypto_mode we want to allocate ciphers for. + * @data_unit_size: the data unit size that will be used + * @q: the request queue for the device + * + * Upper layers must call this function to ensure that a the crypto API fallback + * has transforms for this algorithm, if they become necessary. + * + * Return: 0 on success and -err on error. + */ +int blk_crypto_start_using_mode(enum blk_crypto_mode_num mode_num, + unsigned int data_unit_size, + struct request_queue *q) +{ + struct blk_crypto_keyslot *slotp; + unsigned int i; + int err = 0; + + /* + * Fast path + * Ensure that updates to blk_crypto_keyslots[i].tfms[mode_num] + * for each i are visible before we try to access them. + */ + if (likely(smp_load_acquire(&tfms_inited[mode_num]))) + return 0; + + /* + * If the keyslot manager of the request queue supports this + * crypto mode, then we don't need to allocate this mode. + */ + if (keyslot_manager_crypto_mode_supported(q->ksm, mode_num, + data_unit_size)) + return 0; + + mutex_lock(&tfms_init_lock); + if (likely(tfms_inited[mode_num])) + goto out; + + for (i = 0; i < blk_crypto_num_keyslots; i++) { + slotp = &blk_crypto_keyslots[i]; + slotp->tfms[mode_num] = crypto_alloc_skcipher( + blk_crypto_modes[mode_num].cipher_str, + 0, 0); + if (IS_ERR(slotp->tfms[mode_num])) { + err = PTR_ERR(slotp->tfms[mode_num]); + slotp->tfms[mode_num] = NULL; + goto out_free_tfms; + } + + crypto_skcipher_set_flags(slotp->tfms[mode_num], + CRYPTO_TFM_REQ_FORBID_WEAK_KEYS); + } + + /* + * Ensure that updates to blk_crypto_keyslots[i].tfms[mode_num] + * for each i are visible before we set tfms_inited[mode_num]. + */ + smp_store_release(&tfms_inited[mode_num], true); + goto out; + +out_free_tfms: + for (i = 0; i < blk_crypto_num_keyslots; i++) { + slotp = &blk_crypto_keyslots[i]; + crypto_free_skcipher(slotp->tfms[mode_num]); + slotp->tfms[mode_num] = NULL; + } +out: + mutex_unlock(&tfms_init_lock); + return err; +} + +int blk_crypto_fallback_evict_key(const struct blk_crypto_key *key) +{ + return keyslot_manager_evict_key(blk_crypto_ksm, key); +} + +int blk_crypto_fallback_submit_bio(struct bio **bio_ptr) +{ + struct bio *bio = *bio_ptr; + struct bio_crypt_ctx *bc = bio->bi_crypt_context; + struct bio_fallback_crypt_ctx *f_ctx; + + if (WARN_ON_ONCE(!tfms_inited[bc->bc_key->crypto_mode])) { + bio->bi_status = BLK_STS_IOERR; + return -EIO; + } + + if (bio_data_dir(bio) == WRITE) + return blk_crypto_encrypt_bio(bio_ptr); + + /* + * Mark bio as fallback crypted and replace the bio_crypt_ctx with + * another one contained in a bio_fallback_crypt_ctx, so that the + * fallback has space to store the info it needs for decryption. + */ + bc->bc_ksm = blk_crypto_ksm; + f_ctx = mempool_alloc(bio_fallback_crypt_ctx_pool, GFP_NOIO); + f_ctx->crypt_ctx = *bc; + memcpy(f_ctx->fallback_dun, bc->bc_dun, sizeof(f_ctx->fallback_dun)); + f_ctx->crypt_iter = bio->bi_iter; + + bio_crypt_free_ctx(bio); + bio->bi_crypt_context = &f_ctx->crypt_ctx; + + return 0; +} + +int __init blk_crypto_fallback_init(void) +{ + int i; + unsigned int crypto_mode_supported[BLK_ENCRYPTION_MODE_MAX]; + + prandom_bytes(blank_key, BLK_CRYPTO_MAX_KEY_SIZE); + + /* All blk-crypto modes have a crypto API fallback. */ + for (i = 0; i < BLK_ENCRYPTION_MODE_MAX; i++) + crypto_mode_supported[i] = 0xFFFFFFFF; + crypto_mode_supported[BLK_ENCRYPTION_MODE_INVALID] = 0; + + blk_crypto_ksm = keyslot_manager_create(blk_crypto_num_keyslots, + &blk_crypto_ksm_ll_ops, + crypto_mode_supported, NULL); + if (!blk_crypto_ksm) + return -ENOMEM; + + blk_crypto_wq = alloc_workqueue("blk_crypto_wq", + WQ_UNBOUND | WQ_HIGHPRI | + WQ_MEM_RECLAIM, num_online_cpus()); + if (!blk_crypto_wq) + return -ENOMEM; + + blk_crypto_keyslots = kcalloc(blk_crypto_num_keyslots, + sizeof(blk_crypto_keyslots[0]), + GFP_KERNEL); + if (!blk_crypto_keyslots) + return -ENOMEM; + + blk_crypto_bounce_page_pool = + mempool_create_page_pool(num_prealloc_bounce_pg, 0); + if (!blk_crypto_bounce_page_pool) + return -ENOMEM; + + blk_crypto_decrypt_work_cache = KMEM_CACHE(blk_crypto_decrypt_work, + SLAB_RECLAIM_ACCOUNT); + if (!blk_crypto_decrypt_work_cache) + return -ENOMEM; + + bio_fallback_crypt_ctx_cache = KMEM_CACHE(bio_fallback_crypt_ctx, 0); + if (!bio_fallback_crypt_ctx_cache) + return -ENOMEM; + + bio_fallback_crypt_ctx_pool = + mempool_create_slab_pool(num_prealloc_fallback_crypt_ctxs, + bio_fallback_crypt_ctx_cache); + if (!bio_fallback_crypt_ctx_pool) + return -ENOMEM; + + return 0; +} diff --git a/block/blk-crypto-internal.h b/block/blk-crypto-internal.h new file mode 100644 index 000000000000..43351eecc97a --- /dev/null +++ b/block/blk-crypto-internal.h @@ -0,0 +1,58 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright 2019 Google LLC + */ + +#ifndef __LINUX_BLK_CRYPTO_INTERNAL_H +#define __LINUX_BLK_CRYPTO_INTERNAL_H + +#include + +/* Represents a crypto mode supported by blk-crypto */ +struct blk_crypto_mode { + const char *cipher_str; /* crypto API name (for fallback case) */ + unsigned int keysize; /* key size in bytes */ + unsigned int ivsize; /* iv size in bytes */ +}; + +extern const struct blk_crypto_mode blk_crypto_modes[]; + +#ifdef CONFIG_BLK_INLINE_ENCRYPTION_FALLBACK + +int blk_crypto_fallback_submit_bio(struct bio **bio_ptr); + +bool blk_crypto_queue_decrypt_bio(struct bio *bio); + +int blk_crypto_fallback_evict_key(const struct blk_crypto_key *key); + +bool bio_crypt_fallback_crypted(const struct bio_crypt_ctx *bc); + +#else /* CONFIG_BLK_INLINE_ENCRYPTION_FALLBACK */ + +static inline bool bio_crypt_fallback_crypted(const struct bio_crypt_ctx *bc) +{ + return false; +} + +static inline int blk_crypto_fallback_submit_bio(struct bio **bio_ptr) +{ + pr_warn_once("blk-crypto crypto API fallback disabled; failing request"); + (*bio_ptr)->bi_status = BLK_STS_NOTSUPP; + return -EIO; +} + +static inline bool blk_crypto_queue_decrypt_bio(struct bio *bio) +{ + WARN_ON(1); + return false; +} + +static inline int +blk_crypto_fallback_evict_key(const struct blk_crypto_key *key) +{ + return 0; +} + +#endif /* CONFIG_BLK_INLINE_ENCRYPTION_FALLBACK */ + +#endif /* __LINUX_BLK_CRYPTO_INTERNAL_H */ diff --git a/block/blk-crypto.c b/block/blk-crypto.c index 89649655bf4b..2c47e8eec865 100644 --- a/block/blk-crypto.c +++ b/block/blk-crypto.c @@ -10,219 +10,36 @@ #define pr_fmt(fmt) "blk-crypto: " fmt #include +#include #include -#include -#include -#include -#include -#include -#include -#include +#include +#include -/* Represents a crypto mode supported by blk-crypto */ -struct blk_crypto_mode { - const char *cipher_str; /* crypto API name (for fallback case) */ - size_t keysize; /* key size in bytes */ -}; +#include "blk-crypto-internal.h" -static const struct blk_crypto_mode blk_crypto_modes[] = { +const struct blk_crypto_mode blk_crypto_modes[] = { [BLK_ENCRYPTION_MODE_AES_256_XTS] = { .cipher_str = "xts(aes)", .keysize = 64, + .ivsize = 16, + }, + [BLK_ENCRYPTION_MODE_AES_128_CBC_ESSIV] = { + .cipher_str = "essiv(cbc(aes),sha256)", + .keysize = 16, + .ivsize = 16, + }, + [BLK_ENCRYPTION_MODE_ADIANTUM] = { + .cipher_str = "adiantum(xchacha12,aes)", + .keysize = 32, + .ivsize = 32, }, }; -static unsigned int num_prealloc_bounce_pg = 32; -module_param(num_prealloc_bounce_pg, uint, 0); -MODULE_PARM_DESC(num_prealloc_bounce_pg, - "Number of preallocated bounce pages for blk-crypto to use during crypto API fallback encryption"); - -#define BLK_CRYPTO_MAX_KEY_SIZE 64 -static int blk_crypto_num_keyslots = 100; -module_param_named(num_keyslots, blk_crypto_num_keyslots, int, 0); -MODULE_PARM_DESC(num_keyslots, - "Number of keyslots for crypto API fallback in blk-crypto."); - -static struct blk_crypto_keyslot { - struct crypto_skcipher *tfm; - enum blk_crypto_mode_num crypto_mode; - u8 key[BLK_CRYPTO_MAX_KEY_SIZE]; - struct crypto_skcipher *tfms[ARRAY_SIZE(blk_crypto_modes)]; -} *blk_crypto_keyslots; - -/* - * Allocating a crypto tfm during I/O can deadlock, so we have to preallocate - * all of a mode's tfms when that mode starts being used. Since each mode may - * need all the keyslots at some point, each mode needs its own tfm for each - * keyslot; thus, a keyslot may contain tfms for multiple modes. However, to - * match the behavior of real inline encryption hardware (which only supports a - * single encryption context per keyslot), we only allow one tfm per keyslot to - * be used at a time - the rest of the unused tfms have their keys cleared. - */ -static struct mutex tfms_lock[ARRAY_SIZE(blk_crypto_modes)]; -static bool tfms_inited[ARRAY_SIZE(blk_crypto_modes)]; - -struct work_mem { - struct work_struct crypto_work; - struct bio *bio; -}; - -/* The following few vars are only used during the crypto API fallback */ -static struct keyslot_manager *blk_crypto_ksm; -static struct workqueue_struct *blk_crypto_wq; -static mempool_t *blk_crypto_page_pool; -static struct kmem_cache *blk_crypto_work_mem_cache; - -bool bio_crypt_swhandled(struct bio *bio) -{ - return bio_has_crypt_ctx(bio) && - bio->bi_crypt_context->processing_ksm == blk_crypto_ksm; -} - -static u8 blank_key[BLK_CRYPTO_MAX_KEY_SIZE]; -static void evict_keyslot(unsigned int slot) -{ - struct blk_crypto_keyslot *slotp = &blk_crypto_keyslots[slot]; - enum blk_crypto_mode_num crypto_mode = slotp->crypto_mode; - int err; - - WARN_ON(slotp->crypto_mode == BLK_ENCRYPTION_MODE_INVALID); - - /* Clear the key in the skcipher */ - err = crypto_skcipher_setkey(slotp->tfms[crypto_mode], blank_key, - blk_crypto_modes[crypto_mode].keysize); - WARN_ON(err); - memzero_explicit(slotp->key, BLK_CRYPTO_MAX_KEY_SIZE); - slotp->crypto_mode = BLK_ENCRYPTION_MODE_INVALID; -} - -static int blk_crypto_keyslot_program(void *priv, const u8 *key, - enum blk_crypto_mode_num crypto_mode, - unsigned int data_unit_size, - unsigned int slot) -{ - struct blk_crypto_keyslot *slotp = &blk_crypto_keyslots[slot]; - const struct blk_crypto_mode *mode = &blk_crypto_modes[crypto_mode]; - size_t keysize = mode->keysize; - int err; - - if (crypto_mode != slotp->crypto_mode && - slotp->crypto_mode != BLK_ENCRYPTION_MODE_INVALID) { - evict_keyslot(slot); - } - - if (!slotp->tfms[crypto_mode]) - return -ENOMEM; - slotp->crypto_mode = crypto_mode; - err = crypto_skcipher_setkey(slotp->tfms[crypto_mode], key, keysize); - - if (err) { - evict_keyslot(slot); - return err; - } - - memcpy(slotp->key, key, keysize); - - return 0; -} - -static int blk_crypto_keyslot_evict(void *priv, const u8 *key, - enum blk_crypto_mode_num crypto_mode, - unsigned int data_unit_size, - unsigned int slot) -{ - evict_keyslot(slot); - return 0; -} - -static int blk_crypto_keyslot_find(void *priv, - const u8 *key, - enum blk_crypto_mode_num crypto_mode, - unsigned int data_unit_size_bytes) -{ - int slot; - const size_t keysize = blk_crypto_modes[crypto_mode].keysize; - - for (slot = 0; slot < blk_crypto_num_keyslots; slot++) { - if (blk_crypto_keyslots[slot].crypto_mode == crypto_mode && - !crypto_memneq(blk_crypto_keyslots[slot].key, key, keysize)) - return slot; - } - - return -ENOKEY; -} - -static bool blk_crypto_mode_supported(void *priv, - enum blk_crypto_mode_num crypt_mode, - unsigned int data_unit_size) -{ - /* All blk_crypto_modes are required to have a crypto API fallback. */ - return true; -} - -/* - * The crypto API fallback KSM ops - only used for a bio when it specifies a - * blk_crypto_mode for which we failed to get a keyslot in the device's inline - * encryption hardware (which probably means the device doesn't have inline - * encryption hardware that supports that crypto mode). - */ -static const struct keyslot_mgmt_ll_ops blk_crypto_ksm_ll_ops = { - .keyslot_program = blk_crypto_keyslot_program, - .keyslot_evict = blk_crypto_keyslot_evict, - .keyslot_find = blk_crypto_keyslot_find, - .crypto_mode_supported = blk_crypto_mode_supported, -}; - -static void blk_crypto_encrypt_endio(struct bio *enc_bio) -{ - struct bio *src_bio = enc_bio->bi_private; - int i; - - for (i = 0; i < enc_bio->bi_vcnt; i++) - mempool_free(enc_bio->bi_io_vec[i].bv_page, - blk_crypto_page_pool); - - src_bio->bi_status = enc_bio->bi_status; - - bio_put(enc_bio); - bio_endio(src_bio); -} - -static struct bio *blk_crypto_clone_bio(struct bio *bio_src) -{ - struct bvec_iter iter; - struct bio_vec bv; - struct bio *bio; - - bio = bio_alloc_bioset(GFP_NOIO, bio_segments(bio_src), NULL); - if (!bio) - return NULL; - bio->bi_disk = bio_src->bi_disk; - bio->bi_opf = bio_src->bi_opf; - bio->bi_ioprio = bio_src->bi_ioprio; - bio->bi_write_hint = bio_src->bi_write_hint; - bio->bi_iter.bi_sector = bio_src->bi_iter.bi_sector; - bio->bi_iter.bi_size = bio_src->bi_iter.bi_size; - - bio_for_each_segment(bv, bio_src, iter) - bio->bi_io_vec[bio->bi_vcnt++] = bv; - - if (bio_integrity(bio_src) && - bio_integrity_clone(bio, bio_src, GFP_NOIO) < 0) { - bio_put(bio); - return NULL; - } - - bio_clone_blkg_association(bio, bio_src); - blkcg_bio_issue_init(bio); - - return bio; -} - /* Check that all I/O segments are data unit aligned */ static int bio_crypt_check_alignment(struct bio *bio) { - int data_unit_size = 1 << bio->bi_crypt_context->data_unit_size_bits; + const unsigned int data_unit_size = + bio->bi_crypt_context->bc_key->data_unit_size; struct bvec_iter iter; struct bio_vec bv; @@ -233,268 +50,6 @@ static int bio_crypt_check_alignment(struct bio *bio) return 0; } -static int blk_crypto_alloc_cipher_req(struct bio *src_bio, - struct skcipher_request **ciph_req_ptr, - struct crypto_wait *wait) -{ - int slot; - struct skcipher_request *ciph_req; - struct blk_crypto_keyslot *slotp; - - slot = bio_crypt_get_keyslot(src_bio); - slotp = &blk_crypto_keyslots[slot]; - ciph_req = skcipher_request_alloc(slotp->tfms[slotp->crypto_mode], - GFP_NOIO); - if (!ciph_req) { - src_bio->bi_status = BLK_STS_RESOURCE; - return -ENOMEM; - } - - skcipher_request_set_callback(ciph_req, - CRYPTO_TFM_REQ_MAY_BACKLOG | - CRYPTO_TFM_REQ_MAY_SLEEP, - crypto_req_done, wait); - *ciph_req_ptr = ciph_req; - return 0; -} - -static int blk_crypto_split_bio_if_needed(struct bio **bio_ptr) -{ - struct bio *bio = *bio_ptr; - unsigned int i = 0; - unsigned int num_sectors = 0; - struct bio_vec bv; - struct bvec_iter iter; - - bio_for_each_segment(bv, bio, iter) { - num_sectors += bv.bv_len >> SECTOR_SHIFT; - if (++i == BIO_MAX_PAGES) - break; - } - if (num_sectors < bio_sectors(bio)) { - struct bio *split_bio; - - split_bio = bio_split(bio, num_sectors, GFP_NOIO, NULL); - if (!split_bio) { - bio->bi_status = BLK_STS_RESOURCE; - return -ENOMEM; - } - bio_chain(split_bio, bio); - generic_make_request(bio); - *bio_ptr = split_bio; - } - return 0; -} - -/* - * The crypto API fallback's encryption routine. - * Allocate a bounce bio for encryption, encrypt the input bio using - * crypto API, and replace *bio_ptr with the bounce bio. May split input - * bio if it's too large. - */ -static int blk_crypto_encrypt_bio(struct bio **bio_ptr) -{ - struct bio *src_bio; - struct skcipher_request *ciph_req = NULL; - DECLARE_CRYPTO_WAIT(wait); - int err = 0; - u64 curr_dun; - union { - __le64 dun; - u8 bytes[16]; - } iv; - struct scatterlist src, dst; - struct bio *enc_bio; - struct bio_vec *enc_bvec; - int i, j; - int data_unit_size; - - /* Split the bio if it's too big for single page bvec */ - err = blk_crypto_split_bio_if_needed(bio_ptr); - if (err) - return err; - - src_bio = *bio_ptr; - data_unit_size = 1 << src_bio->bi_crypt_context->data_unit_size_bits; - - /* Allocate bounce bio for encryption */ - enc_bio = blk_crypto_clone_bio(src_bio); - if (!enc_bio) { - src_bio->bi_status = BLK_STS_RESOURCE; - return -ENOMEM; - } - - /* - * Use the crypto API fallback keyslot manager to get a crypto_skcipher - * for the algorithm and key specified for this bio. - */ - err = bio_crypt_ctx_acquire_keyslot(src_bio, blk_crypto_ksm); - if (err) { - src_bio->bi_status = BLK_STS_IOERR; - goto out_put_enc_bio; - } - - /* and then allocate an skcipher_request for it */ - err = blk_crypto_alloc_cipher_req(src_bio, &ciph_req, &wait); - if (err) - goto out_release_keyslot; - - curr_dun = bio_crypt_data_unit_num(src_bio); - sg_init_table(&src, 1); - sg_init_table(&dst, 1); - - skcipher_request_set_crypt(ciph_req, &src, &dst, - data_unit_size, iv.bytes); - - /* Encrypt each page in the bounce bio */ - for (i = 0, enc_bvec = enc_bio->bi_io_vec; i < enc_bio->bi_vcnt; - enc_bvec++, i++) { - struct page *plaintext_page = enc_bvec->bv_page; - struct page *ciphertext_page = - mempool_alloc(blk_crypto_page_pool, GFP_NOIO); - - enc_bvec->bv_page = ciphertext_page; - - if (!ciphertext_page) { - src_bio->bi_status = BLK_STS_RESOURCE; - err = -ENOMEM; - goto out_free_bounce_pages; - } - - sg_set_page(&src, plaintext_page, data_unit_size, - enc_bvec->bv_offset); - sg_set_page(&dst, ciphertext_page, data_unit_size, - enc_bvec->bv_offset); - - /* Encrypt each data unit in this page */ - for (j = 0; j < enc_bvec->bv_len; j += data_unit_size) { - memset(&iv, 0, sizeof(iv)); - iv.dun = cpu_to_le64(curr_dun); - - err = crypto_wait_req(crypto_skcipher_encrypt(ciph_req), - &wait); - if (err) { - i++; - src_bio->bi_status = BLK_STS_RESOURCE; - goto out_free_bounce_pages; - } - curr_dun++; - src.offset += data_unit_size; - dst.offset += data_unit_size; - } - } - - enc_bio->bi_private = src_bio; - enc_bio->bi_end_io = blk_crypto_encrypt_endio; - *bio_ptr = enc_bio; - - enc_bio = NULL; - err = 0; - goto out_free_ciph_req; - -out_free_bounce_pages: - while (i > 0) - mempool_free(enc_bio->bi_io_vec[--i].bv_page, - blk_crypto_page_pool); -out_free_ciph_req: - skcipher_request_free(ciph_req); -out_release_keyslot: - bio_crypt_ctx_release_keyslot(src_bio); -out_put_enc_bio: - if (enc_bio) - bio_put(enc_bio); - - return err; -} - -/* - * The crypto API fallback's main decryption routine. - * Decrypts input bio in place. - */ -static void blk_crypto_decrypt_bio(struct work_struct *w) -{ - struct work_mem *work_mem = - container_of(w, struct work_mem, crypto_work); - struct bio *bio = work_mem->bio; - struct skcipher_request *ciph_req = NULL; - DECLARE_CRYPTO_WAIT(wait); - struct bio_vec bv; - struct bvec_iter iter; - u64 curr_dun; - union { - __le64 dun; - u8 bytes[16]; - } iv; - struct scatterlist sg; - int data_unit_size = 1 << bio->bi_crypt_context->data_unit_size_bits; - int i; - int err; - - /* - * Use the crypto API fallback keyslot manager to get a crypto_skcipher - * for the algorithm and key specified for this bio. - */ - if (bio_crypt_ctx_acquire_keyslot(bio, blk_crypto_ksm)) { - bio->bi_status = BLK_STS_RESOURCE; - goto out_no_keyslot; - } - - /* and then allocate an skcipher_request for it */ - err = blk_crypto_alloc_cipher_req(bio, &ciph_req, &wait); - if (err) - goto out; - - curr_dun = bio_crypt_sw_data_unit_num(bio); - sg_init_table(&sg, 1); - skcipher_request_set_crypt(ciph_req, &sg, &sg, data_unit_size, - iv.bytes); - - /* Decrypt each segment in the bio */ - __bio_for_each_segment(bv, bio, iter, - bio->bi_crypt_context->crypt_iter) { - struct page *page = bv.bv_page; - - sg_set_page(&sg, page, data_unit_size, bv.bv_offset); - - /* Decrypt each data unit in the segment */ - for (i = 0; i < bv.bv_len; i += data_unit_size) { - memset(&iv, 0, sizeof(iv)); - iv.dun = cpu_to_le64(curr_dun); - if (crypto_wait_req(crypto_skcipher_decrypt(ciph_req), - &wait)) { - bio->bi_status = BLK_STS_IOERR; - goto out; - } - curr_dun++; - sg.offset += data_unit_size; - } - } - -out: - skcipher_request_free(ciph_req); - bio_crypt_ctx_release_keyslot(bio); -out_no_keyslot: - kmem_cache_free(blk_crypto_work_mem_cache, work_mem); - bio_endio(bio); -} - -/* Queue bio for decryption */ -static void blk_crypto_queue_decrypt_bio(struct bio *bio) -{ - struct work_mem *work_mem = - kmem_cache_zalloc(blk_crypto_work_mem_cache, GFP_ATOMIC); - - if (!work_mem) { - bio->bi_status = BLK_STS_RESOURCE; - bio_endio(bio); - return; - } - - INIT_WORK(&work_mem->crypto_work, blk_crypto_decrypt_bio); - work_mem->bio = bio; - queue_work(blk_crypto_wq, &work_mem->crypto_work); -} - /** * blk_crypto_submit_bio - handle submitting bio for inline encryption * @@ -518,20 +73,20 @@ int blk_crypto_submit_bio(struct bio **bio_ptr) { struct bio *bio = *bio_ptr; struct request_queue *q; + struct bio_crypt_ctx *bc = bio->bi_crypt_context; int err; - struct bio_crypt_ctx *crypt_ctx; - if (!bio_has_crypt_ctx(bio) || !bio_has_data(bio)) + if (!bc || !bio_has_data(bio)) return 0; /* - * When a read bio is marked for sw decryption, its bi_iter is saved - * so that when we decrypt the bio later, we know what part of it was - * marked for sw decryption (when the bio is passed down after + * When a read bio is marked for fallback decryption, its bi_iter is + * saved so that when we decrypt the bio later, we know what part of it + * was marked for fallback decryption (when the bio is passed down after * blk_crypto_submit bio, it may be split or advanced so we cannot rely * on the bi_iter while decrypting in blk_crypto_endio) */ - if (bio_crypt_swhandled(bio)) + if (bio_crypt_fallback_crypted(bc)) return 0; err = bio_crypt_check_alignment(bio); @@ -540,21 +95,22 @@ int blk_crypto_submit_bio(struct bio **bio_ptr) goto out; } - crypt_ctx = bio->bi_crypt_context; q = bio->bi_disk->queue; - if (bio_crypt_has_keyslot(bio)) { + if (bc->bc_ksm) { /* Key already programmed into device? */ - if (q->ksm == crypt_ctx->processing_ksm) + if (q->ksm == bc->bc_ksm) return 0; /* Nope, release the existing keyslot. */ - bio_crypt_ctx_release_keyslot(bio); + bio_crypt_ctx_release_keyslot(bc); } /* Get device keyslot if supported */ - if (q->ksm) { - err = bio_crypt_ctx_acquire_keyslot(bio, q->ksm); + if (keyslot_manager_crypto_mode_supported(q->ksm, + bc->bc_key->crypto_mode, + bc->bc_key->data_unit_size)) { + err = bio_crypt_ctx_acquire_keyslot(bc, q->ksm); if (!err) return 0; @@ -563,24 +119,10 @@ int blk_crypto_submit_bio(struct bio **bio_ptr) } /* Fallback to crypto API */ - if (!READ_ONCE(tfms_inited[bio->bi_crypt_context->crypto_mode])) { - err = -EIO; - bio->bi_status = BLK_STS_IOERR; + err = blk_crypto_fallback_submit_bio(bio_ptr); + if (err) goto out; - } - if (bio_data_dir(bio) == WRITE) { - /* Encrypt the data now */ - err = blk_crypto_encrypt_bio(bio_ptr); - if (err) - goto out; - } else { - /* Mark bio as swhandled */ - bio->bi_crypt_context->processing_ksm = blk_crypto_ksm; - bio->bi_crypt_context->crypt_iter = bio->bi_iter; - bio->bi_crypt_context->sw_data_unit_num = - bio->bi_crypt_context->data_unit_num; - } return 0; out: bio_endio(*bio_ptr); @@ -590,10 +132,10 @@ out: /** * blk_crypto_endio - clean up bio w.r.t inline encryption during bio_endio * - * @bio - the bio to clean up + * @bio: the bio to clean up * - * If blk_crypto_submit_bio decided to fallback to crypto API for this - * bio, we queue the bio for decryption into a workqueue and return false, + * If blk_crypto_submit_bio decided to fallback to crypto API for this bio, + * we queue the bio for decryption into a workqueue and return false, * and call bio_endio(bio) at a later time (after the bio has been decrypted). * * If the bio is not to be decrypted by the crypto API, this function releases @@ -604,195 +146,97 @@ out: */ bool blk_crypto_endio(struct bio *bio) { - if (!bio_has_crypt_ctx(bio)) + struct bio_crypt_ctx *bc = bio->bi_crypt_context; + + if (!bc) return true; - if (bio_crypt_swhandled(bio)) { + if (bio_crypt_fallback_crypted(bc)) { /* - * The only bios that are swhandled when they reach here - * are those with bio_data_dir(bio) == READ, since WRITE - * bios that are encrypted by the crypto API fallback are - * handled by blk_crypto_encrypt_endio. + * The only bios who's crypto is handled by the blk-crypto + * fallback when they reach here are those with + * bio_data_dir(bio) == READ, since WRITE bios that are + * encrypted by the crypto API fallback are handled by + * blk_crypto_encrypt_endio(). */ - - /* If there was an IO error, don't decrypt. */ - if (bio->bi_status) - return true; - - blk_crypto_queue_decrypt_bio(bio); - return false; + return !blk_crypto_queue_decrypt_bio(bio); } - if (bio_crypt_has_keyslot(bio)) - bio_crypt_ctx_release_keyslot(bio); + if (bc->bc_keyslot >= 0) + bio_crypt_ctx_release_keyslot(bc); return true; } /** - * blk_crypto_start_using_mode() - Allocate skciphers for a - * mode_num for all keyslots - * @mode_num - the blk_crypto_mode we want to allocate ciphers for. + * blk_crypto_init_key() - Prepare a key for use with blk-crypto + * @blk_key: Pointer to the blk_crypto_key to initialize. + * @raw_key: Pointer to the raw key. Must be the correct length for the chosen + * @crypto_mode; see blk_crypto_modes[]. + * @crypto_mode: identifier for the encryption algorithm to use + * @data_unit_size: the data unit size to use for en/decryption * - * Upper layers (filesystems) should call this function to ensure that a - * the crypto API fallback has transforms for this algorithm, if they become - * necessary. - * - * Return: 0 on success and -err on error. + * Return: The blk_crypto_key that was prepared, or an ERR_PTR() on error. When + * done using the key, it must be freed with blk_crypto_free_key(). */ -int blk_crypto_start_using_mode(enum blk_crypto_mode_num mode_num, - unsigned int data_unit_size, - struct request_queue *q) +int blk_crypto_init_key(struct blk_crypto_key *blk_key, const u8 *raw_key, + enum blk_crypto_mode_num crypto_mode, + unsigned int data_unit_size) { - struct blk_crypto_keyslot *slotp; - int err = 0; - int i; + const struct blk_crypto_mode *mode; + static siphash_key_t hash_key; + + memset(blk_key, 0, sizeof(*blk_key)); + + if (crypto_mode >= ARRAY_SIZE(blk_crypto_modes)) + return -EINVAL; + + mode = &blk_crypto_modes[crypto_mode]; + if (mode->keysize == 0) + return -EINVAL; + + if (!is_power_of_2(data_unit_size)) + return -EINVAL; + + blk_key->crypto_mode = crypto_mode; + blk_key->data_unit_size = data_unit_size; + blk_key->data_unit_size_bits = ilog2(data_unit_size); + blk_key->size = mode->keysize; + memcpy(blk_key->raw, raw_key, mode->keysize); /* - * Fast path - * Ensure that updates to blk_crypto_keyslots[i].tfms[mode_num] - * for each i are visible before we try to access them. + * The keyslot manager uses the SipHash of the key to implement O(1) key + * lookups while avoiding leaking information about the keys. It's + * precomputed here so that it only needs to be computed once per key. */ - if (likely(smp_load_acquire(&tfms_inited[mode_num]))) - return 0; + get_random_once(&hash_key, sizeof(hash_key)); + blk_key->hash = siphash(raw_key, mode->keysize, &hash_key); - /* - * If the keyslot manager of the request queue supports this - * crypto mode, then we don't need to allocate this mode. - */ - if (keyslot_manager_crypto_mode_supported(q->ksm, mode_num, - data_unit_size)) { - return 0; - } - - mutex_lock(&tfms_lock[mode_num]); - if (likely(tfms_inited[mode_num])) - goto out; - - for (i = 0; i < blk_crypto_num_keyslots; i++) { - slotp = &blk_crypto_keyslots[i]; - slotp->tfms[mode_num] = crypto_alloc_skcipher( - blk_crypto_modes[mode_num].cipher_str, - 0, 0); - if (IS_ERR(slotp->tfms[mode_num])) { - err = PTR_ERR(slotp->tfms[mode_num]); - slotp->tfms[mode_num] = NULL; - goto out_free_tfms; - } - - crypto_skcipher_set_flags(slotp->tfms[mode_num], - CRYPTO_TFM_REQ_FORBID_WEAK_KEYS); - } - - /* - * Ensure that updates to blk_crypto_keyslots[i].tfms[mode_num] - * for each i are visible before we set tfms_inited[mode_num]. - */ - smp_store_release(&tfms_inited[mode_num], true); - goto out; - -out_free_tfms: - for (i = 0; i < blk_crypto_num_keyslots; i++) { - slotp = &blk_crypto_keyslots[i]; - crypto_free_skcipher(slotp->tfms[mode_num]); - slotp->tfms[mode_num] = NULL; - } -out: - mutex_unlock(&tfms_lock[mode_num]); - return err; + return 0; } -EXPORT_SYMBOL(blk_crypto_start_using_mode); /** * blk_crypto_evict_key() - Evict a key from any inline encryption hardware * it may have been programmed into - * @q - The request queue who's keyslot manager this key might have been - * programmed into - * @key - The key to evict - * @mode - The blk_crypto_mode_num used with this key - * @data_unit_size - The data unit size used with this key + * @q: The request queue who's keyslot manager this key might have been + * programmed into + * @key: The key to evict * * Upper layers (filesystems) should call this function to ensure that a key * is evicted from hardware that it might have been programmed into. This * will call keyslot_manager_evict_key on the queue's keyslot manager, if one * exists, and supports the crypto algorithm with the specified data unit size. - * Otherwise, it will evict the key from the blk_crypto_ksm. + * Otherwise, it will evict the key from the blk-crypto-fallback's ksm. * * Return: 0 on success, -err on error. */ -int blk_crypto_evict_key(struct request_queue *q, const u8 *key, - enum blk_crypto_mode_num mode, - unsigned int data_unit_size) +int blk_crypto_evict_key(struct request_queue *q, + const struct blk_crypto_key *key) { - struct keyslot_manager *ksm = blk_crypto_ksm; + if (q->ksm && + keyslot_manager_crypto_mode_supported(q->ksm, key->crypto_mode, + key->data_unit_size)) + return keyslot_manager_evict_key(q->ksm, key); - if (q && q->ksm && keyslot_manager_crypto_mode_supported(q->ksm, mode, - data_unit_size)) { - ksm = q->ksm; - } - - return keyslot_manager_evict_key(ksm, key, mode, data_unit_size); -} -EXPORT_SYMBOL(blk_crypto_evict_key); - -int __init blk_crypto_init(void) -{ - int i; - int err = -ENOMEM; - - prandom_bytes(blank_key, BLK_CRYPTO_MAX_KEY_SIZE); - - blk_crypto_ksm = keyslot_manager_create(blk_crypto_num_keyslots, - &blk_crypto_ksm_ll_ops, - NULL); - if (!blk_crypto_ksm) - goto out; - - blk_crypto_wq = alloc_workqueue("blk_crypto_wq", - WQ_UNBOUND | WQ_HIGHPRI | - WQ_MEM_RECLAIM, - num_online_cpus()); - if (!blk_crypto_wq) - goto out_free_ksm; - - blk_crypto_keyslots = kcalloc(blk_crypto_num_keyslots, - sizeof(*blk_crypto_keyslots), - GFP_KERNEL); - if (!blk_crypto_keyslots) - goto out_free_workqueue; - - for (i = 0; i < blk_crypto_num_keyslots; i++) { - blk_crypto_keyslots[i].crypto_mode = - BLK_ENCRYPTION_MODE_INVALID; - } - - for (i = 0; i < ARRAY_SIZE(blk_crypto_modes); i++) - mutex_init(&tfms_lock[i]); - - blk_crypto_page_pool = - mempool_create_page_pool(num_prealloc_bounce_pg, 0); - if (!blk_crypto_page_pool) - goto out_free_keyslots; - - blk_crypto_work_mem_cache = KMEM_CACHE(work_mem, SLAB_RECLAIM_ACCOUNT); - if (!blk_crypto_work_mem_cache) - goto out_free_page_pool; - - return 0; - -out_free_page_pool: - mempool_destroy(blk_crypto_page_pool); - blk_crypto_page_pool = NULL; -out_free_keyslots: - kzfree(blk_crypto_keyslots); - blk_crypto_keyslots = NULL; -out_free_workqueue: - destroy_workqueue(blk_crypto_wq); - blk_crypto_wq = NULL; -out_free_ksm: - keyslot_manager_destroy(blk_crypto_ksm); - blk_crypto_ksm = NULL; -out: - pr_warn("No memory for blk-crypto crypto API fallback."); - return err; + return blk_crypto_fallback_evict_key(key); } diff --git a/block/blk-merge.c b/block/blk-merge.c index c97c02a20c6a..50dce8cf4570 100644 --- a/block/blk-merge.c +++ b/block/blk-merge.c @@ -557,9 +557,6 @@ static inline int ll_new_hw_segment(struct request *req, struct bio *bio, if (blk_integrity_merge_bio(req->q, req, bio) == false) goto no_merge; - if (WARN_ON_ONCE(!bio_crypt_ctx_compatible(bio, req->bio))) - goto no_merge; - /* * This will form the start of a new hw segment. Bump both * counters. @@ -584,6 +581,8 @@ int ll_back_merge_fn(struct request *req, struct bio *bio, unsigned int nr_segs) req_set_nomerge(req->q, req); return 0; } + if (!bio_crypt_ctx_mergeable(req->bio, blk_rq_bytes(req), bio)) + return 0; return ll_new_hw_segment(req, bio, nr_segs); } @@ -600,6 +599,8 @@ int ll_front_merge_fn(struct request *req, struct bio *bio, unsigned int nr_segs req_set_nomerge(req->q, req); return 0; } + if (!bio_crypt_ctx_mergeable(bio, bio->bi_iter.bi_size, req->bio)) + return 0; return ll_new_hw_segment(req, bio, nr_segs); } @@ -644,6 +645,9 @@ static int ll_merge_requests_fn(struct request_queue *q, struct request *req, if (blk_integrity_merge_rq(q, req, next) == false) return 0; + if (!bio_crypt_ctx_mergeable(req->bio, blk_rq_bytes(req), next->bio)) + return 0; + /* Merge is OK... */ req->nr_phys_segments = total_phys_segments; return 1; @@ -714,14 +718,8 @@ static enum elv_merge blk_try_req_merge(struct request *req, { if (blk_discard_mergable(req)) return ELEVATOR_DISCARD_MERGE; - else if (blk_rq_pos(req) + blk_rq_sectors(req) == blk_rq_pos(next)) { - if (!bio_crypt_ctx_back_mergeable(req->bio, - blk_rq_sectors(req), - next->bio)) { - return ELEVATOR_NO_MERGE; - } + else if (blk_rq_pos(req) + blk_rq_sectors(req) == blk_rq_pos(next)) return ELEVATOR_BACK_MERGE; - } return ELEVATOR_NO_MERGE; } @@ -757,9 +755,6 @@ static struct request *attempt_merge(struct request_queue *q, if (req->ioprio != next->ioprio) return NULL; - if (!bio_crypt_ctx_compatible(req->bio, next->bio)) - return NULL; - /* * If we are allowed to merge, then append bio list * from next to rq and release next. merge_requests_fn @@ -901,22 +896,11 @@ bool blk_rq_merge_ok(struct request *rq, struct bio *bio) enum elv_merge blk_try_merge(struct request *rq, struct bio *bio) { - if (blk_discard_mergable(rq)) { + if (blk_discard_mergable(rq)) return ELEVATOR_DISCARD_MERGE; - } else if (blk_rq_pos(rq) + blk_rq_sectors(rq) == - bio->bi_iter.bi_sector) { - if (!bio_crypt_ctx_back_mergeable(rq->bio, - blk_rq_sectors(rq), bio)) { - return ELEVATOR_NO_MERGE; - } + else if (blk_rq_pos(rq) + blk_rq_sectors(rq) == bio->bi_iter.bi_sector) return ELEVATOR_BACK_MERGE; - } else if (blk_rq_pos(rq) - bio_sectors(bio) == - bio->bi_iter.bi_sector) { - if (!bio_crypt_ctx_back_mergeable(bio, - bio_sectors(bio), rq->bio)) { - return ELEVATOR_NO_MERGE; - } + else if (blk_rq_pos(rq) - bio_sectors(bio) == bio->bi_iter.bi_sector) return ELEVATOR_FRONT_MERGE; - } return ELEVATOR_NO_MERGE; } diff --git a/block/bounce.c b/block/bounce.c index 6f9a2359b22a..aa57ccc6ced3 100644 --- a/block/bounce.c +++ b/block/bounce.c @@ -267,10 +267,7 @@ static struct bio *bounce_clone_bio(struct bio *bio_src, gfp_t gfp_mask, break; } - if (bio_crypt_clone(bio, bio_src, gfp_mask) < 0) { - bio_put(bio); - return NULL; - } + bio_crypt_clone(bio, bio_src, gfp_mask); if (bio_integrity(bio_src) && bio_integrity_clone(bio, bio_src, gfp_mask) < 0) { diff --git a/block/keyslot-manager.c b/block/keyslot-manager.c index 020931fc9f7d..cdf0607019d4 100644 --- a/block/keyslot-manager.c +++ b/block/keyslot-manager.c @@ -1,7 +1,5 @@ // SPDX-License-Identifier: GPL-2.0 /* - * keyslot-manager.c - * * Copyright 2019 Google LLC */ @@ -27,6 +25,7 @@ * Upper layers will call keyslot_manager_get_slot_for_key() to program a * key into some slot in the inline encryption hardware. */ +#include #include #include #include @@ -36,12 +35,14 @@ struct keyslot { atomic_t slot_refs; struct list_head idle_slot_node; + struct hlist_node hash_node; + struct blk_crypto_key key; }; struct keyslot_manager { unsigned int num_slots; - atomic_t num_idle_slots; struct keyslot_mgmt_ll_ops ksm_ll_ops; + unsigned int crypto_mode_supported[BLK_ENCRYPTION_MODE_MAX]; void *ll_priv_data; /* Protects programming and evicting keys from the device */ @@ -52,6 +53,15 @@ struct keyslot_manager { struct list_head idle_slots; spinlock_t idle_slots_lock; + /* + * Hash table which maps key hashes to keyslots, so that we can find a + * key's keyslot in O(1) time rather than O(num_slots). Protected by + * 'lock'. A cryptographic hash function is used so that timing attacks + * can't leak information about the raw keys. + */ + struct hlist_head *slot_hashtable; + unsigned int slot_hashtable_size; + /* Per-keyslot data */ struct keyslot slots[]; }; @@ -62,6 +72,13 @@ struct keyslot_manager { * @ksm_ll_ops: The struct keyslot_mgmt_ll_ops for the device that this keyslot * manager will use to perform operations like programming and * evicting keys. + * @crypto_mode_supported: Array of size BLK_ENCRYPTION_MODE_MAX of + * bitmasks that represents whether a crypto mode + * and data unit size are supported. The i'th bit + * of crypto_mode_supported[crypto_mode] is set iff + * a data unit size of (1 << i) is supported. We + * only support data unit sizes that are powers of + * 2. * @ll_priv_data: Private data passed as is to the functions in ksm_ll_ops. * * Allocate memory for and initialize a keyslot manager. Called by e.g. @@ -71,20 +88,20 @@ struct keyslot_manager { * Return: Pointer to constructed keyslot manager or NULL on error. */ struct keyslot_manager *keyslot_manager_create(unsigned int num_slots, - const struct keyslot_mgmt_ll_ops *ksm_ll_ops, - void *ll_priv_data) + const struct keyslot_mgmt_ll_ops *ksm_ll_ops, + const unsigned int crypto_mode_supported[BLK_ENCRYPTION_MODE_MAX], + void *ll_priv_data) { struct keyslot_manager *ksm; - int slot; + unsigned int slot; + unsigned int i; if (num_slots == 0) return NULL; /* Check that all ops are specified */ if (ksm_ll_ops->keyslot_program == NULL || - ksm_ll_ops->keyslot_evict == NULL || - ksm_ll_ops->crypto_mode_supported == NULL || - ksm_ll_ops->keyslot_find == NULL) + ksm_ll_ops->keyslot_evict == NULL) return NULL; ksm = kvzalloc(struct_size(ksm, slots, num_slots), GFP_KERNEL); @@ -92,8 +109,9 @@ struct keyslot_manager *keyslot_manager_create(unsigned int num_slots, return NULL; ksm->num_slots = num_slots; - atomic_set(&ksm->num_idle_slots, num_slots); ksm->ksm_ll_ops = *ksm_ll_ops; + memcpy(ksm->crypto_mode_supported, crypto_mode_supported, + sizeof(ksm->crypto_mode_supported)); ksm->ll_priv_data = ll_priv_data; init_rwsem(&ksm->lock); @@ -108,9 +126,29 @@ struct keyslot_manager *keyslot_manager_create(unsigned int num_slots, spin_lock_init(&ksm->idle_slots_lock); + ksm->slot_hashtable_size = roundup_pow_of_two(num_slots); + ksm->slot_hashtable = kvmalloc_array(ksm->slot_hashtable_size, + sizeof(ksm->slot_hashtable[0]), + GFP_KERNEL); + if (!ksm->slot_hashtable) + goto err_free_ksm; + for (i = 0; i < ksm->slot_hashtable_size; i++) + INIT_HLIST_HEAD(&ksm->slot_hashtable[i]); + return ksm; + +err_free_ksm: + keyslot_manager_destroy(ksm); + return NULL; +} +EXPORT_SYMBOL_GPL(keyslot_manager_create); + +static inline struct hlist_head * +hash_bucket_for_key(struct keyslot_manager *ksm, + const struct blk_crypto_key *key) +{ + return &ksm->slot_hashtable[key->hash & (ksm->slot_hashtable_size - 1)]; } -EXPORT_SYMBOL(keyslot_manager_create); static void remove_slot_from_lru_list(struct keyslot_manager *ksm, int slot) { @@ -119,22 +157,32 @@ static void remove_slot_from_lru_list(struct keyslot_manager *ksm, int slot) spin_lock_irqsave(&ksm->idle_slots_lock, flags); list_del(&ksm->slots[slot].idle_slot_node); spin_unlock_irqrestore(&ksm->idle_slots_lock, flags); - - atomic_dec(&ksm->num_idle_slots); } -static int find_and_grab_keyslot(struct keyslot_manager *ksm, const u8 *key, - enum blk_crypto_mode_num crypto_mode, - unsigned int data_unit_size) +static int find_keyslot(struct keyslot_manager *ksm, + const struct blk_crypto_key *key) +{ + const struct hlist_head *head = hash_bucket_for_key(ksm, key); + const struct keyslot *slotp; + + hlist_for_each_entry(slotp, head, hash_node) { + if (slotp->key.hash == key->hash && + slotp->key.crypto_mode == key->crypto_mode && + slotp->key.data_unit_size == key->data_unit_size && + !crypto_memneq(slotp->key.raw, key->raw, key->size)) + return slotp - ksm->slots; + } + return -ENOKEY; +} + +static int find_and_grab_keyslot(struct keyslot_manager *ksm, + const struct blk_crypto_key *key) { int slot; - slot = ksm->ksm_ll_ops.keyslot_find(ksm->ll_priv_data, key, - crypto_mode, data_unit_size); + slot = find_keyslot(ksm, key); if (slot < 0) return slot; - if (WARN_ON(slot >= ksm->num_slots)) - return -EINVAL; if (atomic_inc_return(&ksm->slots[slot].slot_refs) == 1) { /* Took first reference to this slot; remove it from LRU list */ remove_slot_from_lru_list(ksm, slot); @@ -145,37 +193,32 @@ static int find_and_grab_keyslot(struct keyslot_manager *ksm, const u8 *key, /** * keyslot_manager_get_slot_for_key() - Program a key into a keyslot. * @ksm: The keyslot manager to program the key into. - * @key: Pointer to the bytes of the key to program. Must be the correct length - * for the chosen @crypto_mode; see blk_crypto_modes in blk-crypto.c. - * @crypto_mode: Identifier for the encryption algorithm to use. - * @data_unit_size: The data unit size to use for en/decryption. + * @key: Pointer to the key object to program, including the raw key, crypto + * mode, and data unit size. * - * Get a keyslot that's been programmed with the specified key, crypto_mode, and - * data_unit_size. If one already exists, return it with incremented refcount. - * Otherwise, wait for a keyslot to become idle and program it. + * Get a keyslot that's been programmed with the specified key. If one already + * exists, return it with incremented refcount. Otherwise, wait for a keyslot + * to become idle and program it. * * Context: Process context. Takes and releases ksm->lock. * Return: The keyslot on success, else a -errno value. */ int keyslot_manager_get_slot_for_key(struct keyslot_manager *ksm, - const u8 *key, - enum blk_crypto_mode_num crypto_mode, - unsigned int data_unit_size) + const struct blk_crypto_key *key) { int slot; int err; struct keyslot *idle_slot; down_read(&ksm->lock); - slot = find_and_grab_keyslot(ksm, key, crypto_mode, data_unit_size); + slot = find_and_grab_keyslot(ksm, key); up_read(&ksm->lock); if (slot != -ENOKEY) return slot; for (;;) { down_write(&ksm->lock); - slot = find_and_grab_keyslot(ksm, key, crypto_mode, - data_unit_size); + slot = find_and_grab_keyslot(ksm, key); if (slot != -ENOKEY) { up_write(&ksm->lock); return slot; @@ -185,42 +228,43 @@ int keyslot_manager_get_slot_for_key(struct keyslot_manager *ksm, * If we're here, that means there wasn't a slot that was * already programmed with the key. So try to program it. */ - if (atomic_read(&ksm->num_idle_slots) > 0) + if (!list_empty(&ksm->idle_slots)) break; up_write(&ksm->lock); wait_event(ksm->idle_slots_wait_queue, - (atomic_read(&ksm->num_idle_slots) > 0)); + !list_empty(&ksm->idle_slots)); } idle_slot = list_first_entry(&ksm->idle_slots, struct keyslot, idle_slot_node); slot = idle_slot - ksm->slots; - err = ksm->ksm_ll_ops.keyslot_program(ksm->ll_priv_data, key, - crypto_mode, - data_unit_size, - slot); - + err = ksm->ksm_ll_ops.keyslot_program(ksm, key, slot); if (err) { wake_up(&ksm->idle_slots_wait_queue); up_write(&ksm->lock); return err; } - atomic_set(&ksm->slots[slot].slot_refs, 1); + /* Move this slot to the hash list for the new key. */ + if (idle_slot->key.crypto_mode != BLK_ENCRYPTION_MODE_INVALID) + hlist_del(&idle_slot->hash_node); + hlist_add_head(&idle_slot->hash_node, hash_bucket_for_key(ksm, key)); + + atomic_set(&idle_slot->slot_refs, 1); + idle_slot->key = *key; + remove_slot_from_lru_list(ksm, slot); up_write(&ksm->lock); return slot; - } -EXPORT_SYMBOL(keyslot_manager_get_slot_for_key); /** * keyslot_manager_get_slot() - Increment the refcount on the specified slot. - * @ksm - The keyslot manager that we want to modify. - * @slot - The slot to increment the refcount of. + * @ksm: The keyslot manager that we want to modify. + * @slot: The slot to increment the refcount of. * * This function assumes that there is already an active reference to that slot * and simply increments the refcount. This is useful when cloning a bio that @@ -236,7 +280,6 @@ void keyslot_manager_get_slot(struct keyslot_manager *ksm, unsigned int slot) WARN_ON(atomic_inc_return(&ksm->slots[slot].slot_refs) < 2); } -EXPORT_SYMBOL(keyslot_manager_get_slot); /** * keyslot_manager_put_slot() - Release a reference to a slot @@ -257,19 +300,17 @@ void keyslot_manager_put_slot(struct keyslot_manager *ksm, unsigned int slot) list_add_tail(&ksm->slots[slot].idle_slot_node, &ksm->idle_slots); spin_unlock_irqrestore(&ksm->idle_slots_lock, flags); - atomic_inc(&ksm->num_idle_slots); wake_up(&ksm->idle_slots_wait_queue); } } -EXPORT_SYMBOL(keyslot_manager_put_slot); /** * keyslot_manager_crypto_mode_supported() - Find out if a crypto_mode/data * unit size combination is supported * by a ksm. - * @ksm - The keyslot manager to check - * @crypto_mode - The crypto mode to check for. - * @data_unit_size - The data_unit_size for the mode. + * @ksm: The keyslot manager to check + * @crypto_mode: The crypto mode to check for. + * @data_unit_size: The data_unit_size for the mode. * * Calls and returns the result of the crypto_mode_supported function specified * by the ksm. @@ -284,69 +325,102 @@ bool keyslot_manager_crypto_mode_supported(struct keyslot_manager *ksm, { if (!ksm) return false; - return ksm->ksm_ll_ops.crypto_mode_supported(ksm->ll_priv_data, - crypto_mode, - data_unit_size); + if (WARN_ON(crypto_mode >= BLK_ENCRYPTION_MODE_MAX)) + return false; + if (WARN_ON(!is_power_of_2(data_unit_size))) + return false; + return ksm->crypto_mode_supported[crypto_mode] & data_unit_size; } -EXPORT_SYMBOL(keyslot_manager_crypto_mode_supported); - -bool keyslot_manager_rq_crypto_mode_supported(struct request_queue *q, - enum blk_crypto_mode_num crypto_mode, - unsigned int data_unit_size) -{ - return keyslot_manager_crypto_mode_supported(q->ksm, crypto_mode, - data_unit_size); -} -EXPORT_SYMBOL(keyslot_manager_rq_crypto_mode_supported); /** * keyslot_manager_evict_key() - Evict a key from the lower layer device. - * @ksm - The keyslot manager to evict from - * @key - The key to evict - * @crypto_mode - The crypto algorithm the key was programmed with. - * @data_unit_size - The data_unit_size the key was programmed with. + * @ksm: The keyslot manager to evict from + * @key: The key to evict * - * Finds the slot that the specified key, crypto_mode, data_unit_size combo - * was programmed into, and evicts that slot from the lower layer device if - * the refcount on the slot is 0. Returns -EBUSY if the refcount is not 0, and - * -errno on error. + * Find the keyslot that the specified key was programmed into, and evict that + * slot from the lower layer device if that slot is not currently in use. * * Context: Process context. Takes and releases ksm->lock. + * Return: 0 on success, -EBUSY if the key is still in use, or another + * -errno value on other error. */ int keyslot_manager_evict_key(struct keyslot_manager *ksm, - const u8 *key, - enum blk_crypto_mode_num crypto_mode, - unsigned int data_unit_size) + const struct blk_crypto_key *key) { int slot; - int err = 0; + int err; + struct keyslot *slotp; down_write(&ksm->lock); - slot = ksm->ksm_ll_ops.keyslot_find(ksm->ll_priv_data, key, - crypto_mode, - data_unit_size); - + slot = find_keyslot(ksm, key); if (slot < 0) { - up_write(&ksm->lock); - return slot; + err = slot; + goto out_unlock; } + slotp = &ksm->slots[slot]; - if (atomic_read(&ksm->slots[slot].slot_refs) == 0) { - err = ksm->ksm_ll_ops.keyslot_evict(ksm->ll_priv_data, key, - crypto_mode, - data_unit_size, - slot); - } else { + if (atomic_read(&slotp->slot_refs) != 0) { err = -EBUSY; + goto out_unlock; } + err = ksm->ksm_ll_ops.keyslot_evict(ksm, key, slot); + if (err) + goto out_unlock; + hlist_del(&slotp->hash_node); + memzero_explicit(&slotp->key, sizeof(slotp->key)); + err = 0; +out_unlock: up_write(&ksm->lock); return err; } -EXPORT_SYMBOL(keyslot_manager_evict_key); + +/** + * keyslot_manager_reprogram_all_keys() - Re-program all keyslots. + * @ksm: The keyslot manager + * + * Re-program all keyslots that are supposed to have a key programmed. This is + * intended only for use by drivers for hardware that loses its keys on reset. + * + * Context: Process context. Takes and releases ksm->lock. + */ +void keyslot_manager_reprogram_all_keys(struct keyslot_manager *ksm) +{ + unsigned int slot; + + down_write(&ksm->lock); + for (slot = 0; slot < ksm->num_slots; slot++) { + const struct keyslot *slotp = &ksm->slots[slot]; + int err; + + if (slotp->key.crypto_mode == BLK_ENCRYPTION_MODE_INVALID) + continue; + + err = ksm->ksm_ll_ops.keyslot_program(ksm, &slotp->key, slot); + WARN_ON(err); + } + up_write(&ksm->lock); +} +EXPORT_SYMBOL_GPL(keyslot_manager_reprogram_all_keys); + +/** + * keyslot_manager_private() - return the private data stored with ksm + * @ksm: The keyslot manager + * + * Returns the private data passed to the ksm when it was created. + */ +void *keyslot_manager_private(struct keyslot_manager *ksm) +{ + return ksm->ll_priv_data; +} +EXPORT_SYMBOL_GPL(keyslot_manager_private); void keyslot_manager_destroy(struct keyslot_manager *ksm) { - kvfree(ksm); + if (ksm) { + kvfree(ksm->slot_hashtable); + memzero_explicit(ksm, struct_size(ksm, slots, ksm->num_slots)); + kvfree(ksm); + } } -EXPORT_SYMBOL(keyslot_manager_destroy); +EXPORT_SYMBOL_GPL(keyslot_manager_destroy); diff --git a/drivers/md/dm.c b/drivers/md/dm.c index 67c24294d7c8..59fe829aac9c 100644 --- a/drivers/md/dm.c +++ b/drivers/md/dm.c @@ -1322,15 +1322,13 @@ static int clone_bio(struct dm_target_io *tio, struct bio *bio, sector_t sector, unsigned len) { struct bio *clone = &tio->clone; - int ret; __bio_clone_fast(clone, bio); - ret = bio_crypt_clone(clone, bio, GFP_NOIO); - if (ret < 0) - return ret; + bio_crypt_clone(clone, bio, GFP_NOIO); if (bio_integrity(bio)) { + int r; if (unlikely(!dm_target_has_integrity(tio->ti->type) && !dm_target_passes_integrity(tio->ti->type))) { DMWARN("%s: the target %s doesn't support integrity data.", @@ -1339,11 +1337,9 @@ static int clone_bio(struct dm_target_io *tio, struct bio *bio, return -EIO; } - ret = bio_integrity_clone(clone, bio, GFP_NOIO); - if (ret < 0) { - bio_crypt_free_ctx(clone); - return ret; - } + r = bio_integrity_clone(clone, bio, GFP_NOIO); + if (r < 0) + return r; } bio_advance(clone, to_bytes(sector - clone->bi_iter.bi_sector)); diff --git a/drivers/scsi/ufs/ufshcd-crypto.c b/drivers/scsi/ufs/ufshcd-crypto.c index 75d834180f8b..1cfd9a327049 100644 --- a/drivers/scsi/ufs/ufshcd-crypto.c +++ b/drivers/scsi/ufs/ufshcd-crypto.c @@ -3,8 +3,7 @@ * Copyright 2019 Google LLC */ -#include - +#include #include "ufshcd.h" #include "ufshcd-crypto.h" @@ -25,19 +24,23 @@ static u8 get_data_unit_size_mask(unsigned int data_unit_size) static size_t get_keysize_bytes(enum ufs_crypto_key_size size) { switch (size) { - case UFS_CRYPTO_KEY_SIZE_128: return 16; - case UFS_CRYPTO_KEY_SIZE_192: return 24; - case UFS_CRYPTO_KEY_SIZE_256: return 32; - case UFS_CRYPTO_KEY_SIZE_512: return 64; - default: return 0; + case UFS_CRYPTO_KEY_SIZE_128: + return 16; + case UFS_CRYPTO_KEY_SIZE_192: + return 24; + case UFS_CRYPTO_KEY_SIZE_256: + return 32; + case UFS_CRYPTO_KEY_SIZE_512: + return 64; + default: + return 0; } } -static int ufshcd_crypto_cap_find(void *hba_p, +static int ufshcd_crypto_cap_find(struct ufs_hba *hba, enum blk_crypto_mode_num crypto_mode, unsigned int data_unit_size) { - struct ufs_hba *hba = hba_p; enum ufs_crypto_alg ufs_alg; u8 data_unit_mask; int cap_idx; @@ -52,7 +55,8 @@ static int ufshcd_crypto_cap_find(void *hba_p, ufs_alg = UFS_CRYPTO_ALG_AES_XTS; ufs_key_size = UFS_CRYPTO_KEY_SIZE_256; break; - default: return -EINVAL; + default: + return -EINVAL; } data_unit_mask = get_data_unit_size_mask(data_unit_size); @@ -101,8 +105,10 @@ static int ufshcd_crypto_cfg_entry_write_key(union ufs_crypto_cfg_entry *cfg, memcpy(cfg->crypto_key + UFS_CRYPTO_KEY_MAX_SIZE/2, key + key_size_bytes/2, key_size_bytes/2); return 0; - case UFS_CRYPTO_ALG_BITLOCKER_AES_CBC: // fallthrough - case UFS_CRYPTO_ALG_AES_ECB: // fallthrough + case UFS_CRYPTO_ALG_BITLOCKER_AES_CBC: + /* fall through */ + case UFS_CRYPTO_ALG_AES_ECB: + /* fall through */ case UFS_CRYPTO_ALG_ESSIV_AES_CBC: memcpy(cfg->crypto_key, key, key_size_bytes); return 0; @@ -111,13 +117,15 @@ static int ufshcd_crypto_cfg_entry_write_key(union ufs_crypto_cfg_entry *cfg, return -EINVAL; } -static void program_key(struct ufs_hba *hba, - const union ufs_crypto_cfg_entry *cfg, - int slot) +static void ufshcd_program_key(struct ufs_hba *hba, + const union ufs_crypto_cfg_entry *cfg, + int slot) { int i; u32 slot_offset = hba->crypto_cfg_register + slot * sizeof(*cfg); + pm_runtime_get_sync(hba->dev); + ufshcd_hold(hba, false); /* Clear the dword 16 */ ufshcd_writel(hba, 0, slot_offset + 16 * sizeof(cfg->reg_val[0])); /* Ensure that CFGE is cleared before programming the key */ @@ -137,29 +145,45 @@ static void program_key(struct ufs_hba *hba, ufshcd_writel(hba, le32_to_cpu(cfg->reg_val[16]), slot_offset + 16 * sizeof(cfg->reg_val[0])); wmb(); + ufshcd_release(hba); + pm_runtime_put_sync(hba->dev); } -static int ufshcd_crypto_keyslot_program(void *hba_p, const u8 *key, - enum blk_crypto_mode_num crypto_mode, - unsigned int data_unit_size, +static void ufshcd_clear_keyslot(struct ufs_hba *hba, int slot) +{ + union ufs_crypto_cfg_entry cfg = { 0 }; + + ufshcd_program_key(hba, &cfg, slot); +} + +/* Clear all keyslots at driver init time */ +static void ufshcd_clear_all_keyslots(struct ufs_hba *hba) +{ + int slot; + + for (slot = 0; slot < ufshcd_num_keyslots(hba); slot++) + ufshcd_clear_keyslot(hba, slot); +} + +static int ufshcd_crypto_keyslot_program(struct keyslot_manager *ksm, + const struct blk_crypto_key *key, unsigned int slot) { - struct ufs_hba *hba = hba_p; + struct ufs_hba *hba = keyslot_manager_private(ksm); int err = 0; u8 data_unit_mask; union ufs_crypto_cfg_entry cfg; - union ufs_crypto_cfg_entry *cfg_arr = hba->crypto_cfgs; int cap_idx; - cap_idx = ufshcd_crypto_cap_find(hba_p, crypto_mode, - data_unit_size); + cap_idx = ufshcd_crypto_cap_find(hba, key->crypto_mode, + key->data_unit_size); if (!ufshcd_is_crypto_enabled(hba) || !ufshcd_keyslot_valid(hba, slot) || !ufshcd_cap_idx_valid(hba, cap_idx)) return -EINVAL; - data_unit_mask = get_data_unit_size_mask(data_unit_size); + data_unit_mask = get_data_unit_size_mask(key->data_unit_size); if (!(data_unit_mask & hba->crypto_cap_array[cap_idx].sdus_mask)) return -EINVAL; @@ -169,134 +193,74 @@ static int ufshcd_crypto_keyslot_program(void *hba_p, const u8 *key, cfg.crypto_cap_idx = cap_idx; cfg.config_enable |= UFS_CRYPTO_CONFIGURATION_ENABLE; - err = ufshcd_crypto_cfg_entry_write_key(&cfg, key, - hba->crypto_cap_array[cap_idx]); + err = ufshcd_crypto_cfg_entry_write_key(&cfg, key->raw, + hba->crypto_cap_array[cap_idx]); if (err) return err; - program_key(hba, &cfg, slot); + ufshcd_program_key(hba, &cfg, slot); - memcpy(&cfg_arr[slot], &cfg, sizeof(cfg)); memzero_explicit(&cfg, sizeof(cfg)); - return 0; } -static int ufshcd_crypto_keyslot_find(void *hba_p, - const u8 *key, - enum blk_crypto_mode_num crypto_mode, - unsigned int data_unit_size) -{ - struct ufs_hba *hba = hba_p; - int err = 0; - int slot; - u8 data_unit_mask; - union ufs_crypto_cfg_entry cfg; - union ufs_crypto_cfg_entry *cfg_arr = hba->crypto_cfgs; - int cap_idx; - - cap_idx = ufshcd_crypto_cap_find(hba_p, crypto_mode, - data_unit_size); - - if (!ufshcd_is_crypto_enabled(hba) || - !ufshcd_cap_idx_valid(hba, cap_idx)) - return -EINVAL; - - data_unit_mask = get_data_unit_size_mask(data_unit_size); - - if (!(data_unit_mask & hba->crypto_cap_array[cap_idx].sdus_mask)) - return -EINVAL; - - memset(&cfg, 0, sizeof(cfg)); - err = ufshcd_crypto_cfg_entry_write_key(&cfg, key, - hba->crypto_cap_array[cap_idx]); - - if (err) - return -EINVAL; - - for (slot = 0; slot < NUM_KEYSLOTS(hba); slot++) { - if ((cfg_arr[slot].config_enable & - UFS_CRYPTO_CONFIGURATION_ENABLE) && - data_unit_mask == cfg_arr[slot].data_unit_size && - cap_idx == cfg_arr[slot].crypto_cap_idx && - !crypto_memneq(&cfg.crypto_key, cfg_arr[slot].crypto_key, - UFS_CRYPTO_KEY_MAX_SIZE)) { - memzero_explicit(&cfg, sizeof(cfg)); - return slot; - } - } - - memzero_explicit(&cfg, sizeof(cfg)); - return -ENOKEY; -} - -static int ufshcd_crypto_keyslot_evict(void *hba_p, const u8 *key, - enum blk_crypto_mode_num crypto_mode, - unsigned int data_unit_size, +static int ufshcd_crypto_keyslot_evict(struct keyslot_manager *ksm, + const struct blk_crypto_key *key, unsigned int slot) { - struct ufs_hba *hba = hba_p; - int i = 0; - u32 reg_base; - union ufs_crypto_cfg_entry *cfg_arr = hba->crypto_cfgs; + struct ufs_hba *hba = keyslot_manager_private(ksm); if (!ufshcd_is_crypto_enabled(hba) || !ufshcd_keyslot_valid(hba, slot)) return -EINVAL; - memset(&cfg_arr[slot], 0, sizeof(cfg_arr[slot])); - reg_base = hba->crypto_cfg_register + slot * sizeof(cfg_arr[0]); - /* * Clear the crypto cfg on the device. Clearing CFGE * might not be sufficient, so just clear the entire cfg. */ - for (i = 0; i < sizeof(cfg_arr[0]); i += sizeof(__le32)) - ufshcd_writel(hba, 0, reg_base + i); - wmb(); + ufshcd_clear_keyslot(hba, slot); return 0; } -static bool ufshcd_crypto_mode_supported(void *hba_p, - enum blk_crypto_mode_num crypto_mode, - unsigned int data_unit_size) -{ - return ufshcd_crypto_cap_find(hba_p, crypto_mode, data_unit_size) >= 0; -} - /* Functions implementing UFSHCI v2.1 specification behaviour */ void ufshcd_crypto_enable_spec(struct ufs_hba *hba) { - union ufs_crypto_cfg_entry *cfg_arr = hba->crypto_cfgs; - int slot; - if (!ufshcd_hba_is_crypto_supported(hba)) return; hba->caps |= UFSHCD_CAP_CRYPTO; - /* - * Reset might clear all keys, so reprogram all the keys. - * Also serves to clear keys on driver init. - */ - for (slot = 0; slot < NUM_KEYSLOTS(hba); slot++) - program_key(hba, &cfg_arr[slot], slot); + + /* Reset might clear all keys, so reprogram all the keys. */ + keyslot_manager_reprogram_all_keys(hba->ksm); } -EXPORT_SYMBOL(ufshcd_crypto_enable_spec); +EXPORT_SYMBOL_GPL(ufshcd_crypto_enable_spec); void ufshcd_crypto_disable_spec(struct ufs_hba *hba) { hba->caps &= ~UFSHCD_CAP_CRYPTO; } -EXPORT_SYMBOL(ufshcd_crypto_disable_spec); +EXPORT_SYMBOL_GPL(ufshcd_crypto_disable_spec); static const struct keyslot_mgmt_ll_ops ufshcd_ksm_ops = { .keyslot_program = ufshcd_crypto_keyslot_program, .keyslot_evict = ufshcd_crypto_keyslot_evict, - .keyslot_find = ufshcd_crypto_keyslot_find, - .crypto_mode_supported = ufshcd_crypto_mode_supported, }; +enum blk_crypto_mode_num ufshcd_blk_crypto_mode_num_for_alg_dusize( + enum ufs_crypto_alg ufs_crypto_alg, + enum ufs_crypto_key_size key_size) +{ + /* + * This is currently the only mode that UFS and blk-crypto both support. + */ + if (ufs_crypto_alg == UFS_CRYPTO_ALG_AES_XTS && + key_size == UFS_CRYPTO_KEY_SIZE_256) + return BLK_ENCRYPTION_MODE_AES_256_XTS; + + return BLK_ENCRYPTION_MODE_INVALID; +} + /** * ufshcd_hba_init_crypto - Read crypto capabilities, init crypto fields in hba * @hba: Per adapter instance @@ -308,6 +272,8 @@ int ufshcd_hba_init_crypto_spec(struct ufs_hba *hba, { int cap_idx = 0; int err = 0; + unsigned int crypto_modes_supported[BLK_ENCRYPTION_MODE_MAX]; + enum blk_crypto_mode_num blk_mode_num; /* Default to disabling crypto */ hba->caps &= ~UFSHCD_CAP_CRYPTO; @@ -336,16 +302,7 @@ int ufshcd_hba_init_crypto_spec(struct ufs_hba *hba, goto out; } - hba->crypto_cfgs = - devm_kcalloc(hba->dev, - NUM_KEYSLOTS(hba), - sizeof(hba->crypto_cfgs[0]), - GFP_KERNEL); - if (!hba->crypto_cfgs) { - err = -ENOMEM; - goto out_free_cfg_mem; - } - + memset(crypto_modes_supported, 0, sizeof(crypto_modes_supported)); /* * Store all the capabilities now so that we don't need to repeatedly * access the device each time we want to know its capabilities @@ -356,26 +313,35 @@ int ufshcd_hba_init_crypto_spec(struct ufs_hba *hba, cpu_to_le32(ufshcd_readl(hba, REG_UFS_CRYPTOCAP + cap_idx * sizeof(__le32))); + blk_mode_num = ufshcd_blk_crypto_mode_num_for_alg_dusize( + hba->crypto_cap_array[cap_idx].algorithm_id, + hba->crypto_cap_array[cap_idx].key_size); + if (blk_mode_num == BLK_ENCRYPTION_MODE_INVALID) + continue; + crypto_modes_supported[blk_mode_num] |= + hba->crypto_cap_array[cap_idx].sdus_mask * 512; } - hba->ksm = keyslot_manager_create(NUM_KEYSLOTS(hba), ksm_ops, hba); + ufshcd_clear_all_keyslots(hba); + + hba->ksm = keyslot_manager_create(ufshcd_num_keyslots(hba), ksm_ops, + crypto_modes_supported, hba); if (!hba->ksm) { err = -ENOMEM; - goto out_free_crypto_cfgs; + goto out_free_caps; } return 0; -out_free_crypto_cfgs: - devm_kfree(hba->dev, hba->crypto_cfgs); -out_free_cfg_mem: + +out_free_caps: devm_kfree(hba->dev, hba->crypto_cap_array); out: /* Indicate that init failed by setting crypto_capabilities to 0 */ hba->crypto_capabilities.reg_val = 0; return err; } -EXPORT_SYMBOL(ufshcd_hba_init_crypto_spec); +EXPORT_SYMBOL_GPL(ufshcd_hba_init_crypto_spec); void ufshcd_crypto_setup_rq_keyslot_manager_spec(struct ufs_hba *hba, struct request_queue *q) @@ -385,26 +351,26 @@ void ufshcd_crypto_setup_rq_keyslot_manager_spec(struct ufs_hba *hba, q->ksm = hba->ksm; } -EXPORT_SYMBOL(ufshcd_crypto_setup_rq_keyslot_manager_spec); +EXPORT_SYMBOL_GPL(ufshcd_crypto_setup_rq_keyslot_manager_spec); void ufshcd_crypto_destroy_rq_keyslot_manager_spec(struct ufs_hba *hba, struct request_queue *q) { keyslot_manager_destroy(hba->ksm); } -EXPORT_SYMBOL(ufshcd_crypto_destroy_rq_keyslot_manager_spec); +EXPORT_SYMBOL_GPL(ufshcd_crypto_destroy_rq_keyslot_manager_spec); int ufshcd_prepare_lrbp_crypto_spec(struct ufs_hba *hba, struct scsi_cmnd *cmd, struct ufshcd_lrb *lrbp) { - int key_slot; + struct bio_crypt_ctx *bc; - if (!cmd->request->bio || - !bio_crypt_should_process(cmd->request->bio, cmd->request->q)) { + if (!bio_crypt_should_process(cmd->request)) { lrbp->crypto_enable = false; return 0; } + bc = cmd->request->bio->bi_crypt_context; if (WARN_ON(!ufshcd_is_crypto_enabled(hba))) { /* @@ -413,17 +379,16 @@ int ufshcd_prepare_lrbp_crypto_spec(struct ufs_hba *hba, */ return -EINVAL; } - key_slot = bio_crypt_get_keyslot(cmd->request->bio); - if (!ufshcd_keyslot_valid(hba, key_slot)) + if (!ufshcd_keyslot_valid(hba, bc->bc_keyslot)) return -EINVAL; lrbp->crypto_enable = true; - lrbp->crypto_key_slot = key_slot; - lrbp->data_unit_num = bio_crypt_data_unit_num(cmd->request->bio); + lrbp->crypto_key_slot = bc->bc_keyslot; + lrbp->data_unit_num = bc->bc_dun[0]; return 0; } -EXPORT_SYMBOL(ufshcd_prepare_lrbp_crypto_spec); +EXPORT_SYMBOL_GPL(ufshcd_prepare_lrbp_crypto_spec); /* Crypto Variant Ops Support */ diff --git a/drivers/scsi/ufs/ufshcd-crypto.h b/drivers/scsi/ufs/ufshcd-crypto.h index 3c03d0e23e87..24a587c7a94e 100644 --- a/drivers/scsi/ufs/ufshcd-crypto.h +++ b/drivers/scsi/ufs/ufshcd-crypto.h @@ -11,7 +11,10 @@ #include "ufshcd.h" #include "ufshci.h" -#define NUM_KEYSLOTS(hba) (hba->crypto_capabilities.config_count + 1) +static inline int ufshcd_num_keyslots(struct ufs_hba *hba) +{ + return hba->crypto_capabilities.config_count + 1; +} static inline bool ufshcd_keyslot_valid(struct ufs_hba *hba, unsigned int slot) { @@ -19,7 +22,7 @@ static inline bool ufshcd_keyslot_valid(struct ufs_hba *hba, unsigned int slot) * The actual number of configurations supported is (CFGC+1), so slot * numbers range from 0 to config_count inclusive. */ - return slot < NUM_KEYSLOTS(hba); + return slot < ufshcd_num_keyslots(hba); } static inline bool ufshcd_hba_is_crypto_supported(struct ufs_hba *hba) @@ -51,6 +54,11 @@ void ufshcd_crypto_setup_rq_keyslot_manager_spec(struct ufs_hba *hba, void ufshcd_crypto_destroy_rq_keyslot_manager_spec(struct ufs_hba *hba, struct request_queue *q); +static inline bool ufshcd_lrbp_crypto_enabled(struct ufshcd_lrb *lrbp) +{ + return lrbp->crypto_enable; +} + /* Crypto Variant Ops Support */ void ufshcd_crypto_enable(struct ufs_hba *hba); @@ -118,10 +126,14 @@ static inline int ufshcd_prepare_lrbp_crypto(struct ufs_hba *hba, struct scsi_cmnd *cmd, struct ufshcd_lrb *lrbp) { - lrbp->crypto_enable = false; return 0; } +static inline bool ufshcd_lrbp_crypto_enabled(struct ufshcd_lrb *lrbp) +{ + return false; +} + static inline int ufshcd_complete_lrbp_crypto(struct ufs_hba *hba, struct scsi_cmnd *cmd, struct ufshcd_lrb *lrbp) diff --git a/drivers/scsi/ufs/ufshcd.c b/drivers/scsi/ufs/ufshcd.c index 9216198da200..308be8685e86 100644 --- a/drivers/scsi/ufs/ufshcd.c +++ b/drivers/scsi/ufs/ufshcd.c @@ -2246,13 +2246,15 @@ static void ufshcd_prepare_req_desc_hdr(struct ufshcd_lrb *lrbp, dword_0 |= UTP_REQ_DESC_INT_CMD; /* Transfer request descriptor header fields */ - if (lrbp->crypto_enable) { + if (ufshcd_lrbp_crypto_enabled(lrbp)) { +#if IS_ENABLED(CONFIG_SCSI_UFS_CRYPTO) dword_0 |= UTP_REQ_DESC_CRYPTO_ENABLE_CMD; dword_0 |= lrbp->crypto_key_slot; req_desc->header.dword_1 = - cpu_to_le32((u32)lrbp->data_unit_num); + cpu_to_le32(lower_32_bits(lrbp->data_unit_num)); req_desc->header.dword_3 = - cpu_to_le32((u32)(lrbp->data_unit_num >> 32)); + cpu_to_le32(upper_32_bits(lrbp->data_unit_num)); +#endif /* CONFIG_SCSI_UFS_CRYPTO */ } else { /* dword_1 and dword_3 are reserved, hence they are set to 0 */ req_desc->header.dword_1 = 0; @@ -2555,7 +2557,9 @@ static int ufshcd_compose_dev_cmd(struct ufs_hba *hba, lrbp->task_tag = tag; lrbp->lun = 0; /* device management cmd is not specific to any LUN */ lrbp->intr_cmd = true; /* No interrupt aggregation */ +#if IS_ENABLED(CONFIG_SCSI_UFS_CRYPTO) lrbp->crypto_enable = false; /* No crypto operations */ +#endif hba->dev_cmd.type = cmd_type; return ufshcd_comp_devman_upiu(hba, lrbp); diff --git a/drivers/scsi/ufs/ufshcd.h b/drivers/scsi/ufs/ufshcd.h index 6fe75302df59..9f1452299803 100644 --- a/drivers/scsi/ufs/ufshcd.h +++ b/drivers/scsi/ufs/ufshcd.h @@ -194,9 +194,11 @@ struct ufshcd_lrb { bool intr_cmd; ktime_t issue_time_stamp; ktime_t compl_time_stamp; +#if IS_ENABLED(CONFIG_SCSI_UFS_CRYPTO) bool crypto_enable; u8 crypto_key_slot; u64 data_unit_num; +#endif /* CONFIG_SCSI_UFS_CRYPTO */ bool req_abort_skip; }; @@ -546,7 +548,6 @@ struct ufs_stats { * @crypto_capabilities: Content of crypto capabilities register (0x100) * @crypto_cap_array: Array of crypto capabilities * @crypto_cfg_register: Start of the crypto cfg array - * @crypto_cfgs: Array of crypto configurations (i.e. config for each slot) * @ksm: the keyslot manager tied to this hba */ struct ufs_hba { @@ -777,7 +778,6 @@ struct ufs_hba { union ufs_crypto_capabilities crypto_capabilities; union ufs_crypto_cap_entry *crypto_cap_array; u32 crypto_cfg_register; - union ufs_crypto_cfg_entry *crypto_cfgs; struct keyslot_manager *ksm; #endif /* CONFIG_SCSI_UFS_CRYPTO */ }; diff --git a/fs/buffer.c b/fs/buffer.c index 2e64fa95a1ac..32141c922fed 100644 --- a/fs/buffer.c +++ b/fs/buffer.c @@ -3046,7 +3046,7 @@ static int submit_bh_wbc(int op, int op_flags, struct buffer_head *bh, */ bio = bio_alloc(GFP_NOIO, 1); - fscrypt_set_bio_crypt_ctx_bh(bio, bh, GFP_NOIO | __GFP_NOFAIL); + fscrypt_set_bio_crypt_ctx_bh(bio, bh, GFP_NOIO); bio->bi_iter.bi_sector = bh->b_blocknr * (bh->b_size >> 9); bio_set_dev(bio, bh->b_bdev); diff --git a/fs/crypto/bio.c b/fs/crypto/bio.c index 956798debf71..d28d8e803554 100644 --- a/fs/crypto/bio.c +++ b/fs/crypto/bio.c @@ -73,11 +73,8 @@ int fscrypt_zeroout_range(const struct inode *inode, pgoff_t lblk, err = -ENOMEM; goto errout; } - err = fscrypt_set_bio_crypt_ctx(bio, inode, lblk, GFP_NOIO); - if (err) { - bio_put(bio); - goto errout; - } + fscrypt_set_bio_crypt_ctx(bio, inode, lblk, GFP_NOIO); + bio_set_dev(bio, inode->i_sb->s_bdev); bio->bi_iter.bi_sector = pblk << (blockbits - 9); bio_set_op_attrs(bio, REQ_OP_WRITE, 0); diff --git a/fs/crypto/crypto.c b/fs/crypto/crypto.c index 3719efa546c6..7549d76a0aba 100644 --- a/fs/crypto/crypto.c +++ b/fs/crypto/crypto.c @@ -97,7 +97,7 @@ int fscrypt_crypt_block(const struct inode *inode, fscrypt_direction_t rw, DECLARE_CRYPTO_WAIT(wait); struct scatterlist dst, src; struct fscrypt_info *ci = inode->i_crypt_info; - struct crypto_skcipher *tfm = ci->ci_ctfm; + struct crypto_skcipher *tfm = ci->ci_key.tfm; int res = 0; if (WARN_ON_ONCE(len <= 0)) diff --git a/fs/crypto/fname.c b/fs/crypto/fname.c index 3da3707c10e3..3aafddaab703 100644 --- a/fs/crypto/fname.c +++ b/fs/crypto/fname.c @@ -40,7 +40,7 @@ int fname_encrypt(struct inode *inode, const struct qstr *iname, struct skcipher_request *req = NULL; DECLARE_CRYPTO_WAIT(wait); struct fscrypt_info *ci = inode->i_crypt_info; - struct crypto_skcipher *tfm = ci->ci_ctfm; + struct crypto_skcipher *tfm = ci->ci_key.tfm; union fscrypt_iv iv; struct scatterlist sg; int res; @@ -93,7 +93,7 @@ static int fname_decrypt(struct inode *inode, DECLARE_CRYPTO_WAIT(wait); struct scatterlist src_sg, dst_sg; struct fscrypt_info *ci = inode->i_crypt_info; - struct crypto_skcipher *tfm = ci->ci_ctfm; + struct crypto_skcipher *tfm = ci->ci_key.tfm; union fscrypt_iv iv; int res; diff --git a/fs/crypto/fscrypt_private.h b/fs/crypto/fscrypt_private.h index c731bd4245c5..19b7a8ea5954 100644 --- a/fs/crypto/fscrypt_private.h +++ b/fs/crypto/fscrypt_private.h @@ -15,8 +15,6 @@ #include #include -struct fscrypt_master_key; - #define CONST_STRLEN(str) (sizeof(str) - 1) #define FS_KEY_DERIVATION_NONCE_SIZE 16 @@ -154,6 +152,20 @@ struct fscrypt_symlink_data { char encrypted_path[1]; } __packed; +/** + * struct fscrypt_prepared_key - a key prepared for actual encryption/decryption + * @tfm: crypto API transform object + * @blk_key: key for blk-crypto + * + * Normally only one of the fields will be non-NULL. + */ +struct fscrypt_prepared_key { + struct crypto_skcipher *tfm; +#ifdef CONFIG_FS_ENCRYPTION_INLINE_CRYPT + struct fscrypt_blk_crypto_key *blk_key; +#endif +}; + /* * fscrypt_info - the "encryption key" for an inode * @@ -163,20 +175,20 @@ struct fscrypt_symlink_data { */ struct fscrypt_info { - /* The actual crypto transform used for encryption and decryption */ - struct crypto_skcipher *ci_ctfm; - -#ifdef CONFIG_FS_ENCRYPTION_INLINE_CRYPT - /* - * The raw key for inline encryption, if this file is using inline - * encryption rather than the traditional filesystem layer encryption. - */ - const u8 *ci_inline_crypt_key; -#endif + /* The key in a form prepared for actual encryption/decryption */ + struct fscrypt_prepared_key ci_key; /* True if the key should be freed when this fscrypt_info is freed */ bool ci_owns_key; +#ifdef CONFIG_FS_ENCRYPTION_INLINE_CRYPT + /* + * True if this inode will use inline encryption (blk-crypto) instead of + * the traditional filesystem-layer encryption. + */ + bool ci_inlinecrypt; +#endif + /* * Encryption mode used for this inode. It corresponds to either the * contents or filenames encryption mode, depending on the inode type. @@ -201,7 +213,7 @@ struct fscrypt_info { /* * If non-NULL, then encryption is done using the master key directly - * and ci_ctfm will equal ci_direct_key->dk_ctfm. + * and ci_key will equal ci_direct_key->dk_key. */ struct fscrypt_direct_key *ci_direct_key; @@ -265,6 +277,7 @@ union fscrypt_iv { u8 nonce[FS_KEY_DERIVATION_NONCE_SIZE]; }; u8 raw[FSCRYPT_MAX_IV_SIZE]; + __le64 dun[FSCRYPT_MAX_IV_SIZE / sizeof(__le64)]; }; void fscrypt_generate_iv(union fscrypt_iv *iv, u64 lblk_num, @@ -306,49 +319,71 @@ extern void fscrypt_destroy_hkdf(struct fscrypt_hkdf *hkdf); /* inline_crypt.c */ #ifdef CONFIG_FS_ENCRYPTION_INLINE_CRYPT -extern bool fscrypt_should_use_inline_encryption(const struct fscrypt_info *ci); +extern void fscrypt_select_encryption_impl(struct fscrypt_info *ci); -extern int fscrypt_set_inline_crypt_key(struct fscrypt_info *ci, - const u8 *derived_key); +static inline bool +fscrypt_using_inline_encryption(const struct fscrypt_info *ci) +{ + return ci->ci_inlinecrypt; +} -extern void fscrypt_free_inline_crypt_key(struct fscrypt_info *ci); +extern int fscrypt_prepare_inline_crypt_key( + struct fscrypt_prepared_key *prep_key, + const u8 *raw_key, + const struct fscrypt_info *ci); -extern int fscrypt_setup_per_mode_inline_crypt_key( - struct fscrypt_info *ci, - struct fscrypt_master_key *mk); +extern void fscrypt_destroy_inline_crypt_key( + struct fscrypt_prepared_key *prep_key); -extern void fscrypt_evict_inline_crypt_keys(struct fscrypt_master_key *mk); +/* + * Check whether the crypto transform or blk-crypto key has been allocated in + * @prep_key, depending on which encryption implementation the file will use. + */ +static inline bool +fscrypt_is_key_prepared(struct fscrypt_prepared_key *prep_key, + const struct fscrypt_info *ci) +{ + /* + * The READ_ONCE() here pairs with the smp_store_release() in + * fscrypt_prepare_key(). (This only matters for the per-mode keys, + * which are shared by multiple inodes.) + */ + if (fscrypt_using_inline_encryption(ci)) + return READ_ONCE(prep_key->blk_key) != NULL; + return READ_ONCE(prep_key->tfm) != NULL; +} #else /* CONFIG_FS_ENCRYPTION_INLINE_CRYPT */ -static inline bool fscrypt_should_use_inline_encryption( +static inline void fscrypt_select_encryption_impl(struct fscrypt_info *ci) +{ +} + +static inline bool fscrypt_using_inline_encryption( const struct fscrypt_info *ci) { return false; } -static inline int fscrypt_set_inline_crypt_key(struct fscrypt_info *ci, - const u8 *derived_key) +static inline int +fscrypt_prepare_inline_crypt_key(struct fscrypt_prepared_key *prep_key, + const u8 *raw_key, + const struct fscrypt_info *ci) { WARN_ON(1); return -EOPNOTSUPP; } -static inline void fscrypt_free_inline_crypt_key(struct fscrypt_info *ci) +static inline void +fscrypt_destroy_inline_crypt_key(struct fscrypt_prepared_key *prep_key) { } -static inline int fscrypt_setup_per_mode_inline_crypt_key( - struct fscrypt_info *ci, - struct fscrypt_master_key *mk) -{ - WARN_ON(1); - return -EOPNOTSUPP; -} - -static inline void fscrypt_evict_inline_crypt_keys( - struct fscrypt_master_key *mk) +static inline bool +fscrypt_is_key_prepared(struct fscrypt_prepared_key *prep_key, + const struct fscrypt_info *ci) { + return READ_ONCE(prep_key->tfm) != NULL; } #endif /* !CONFIG_FS_ENCRYPTION_INLINE_CRYPT */ @@ -441,25 +476,12 @@ struct fscrypt_master_key { struct list_head mk_decrypted_inodes; spinlock_t mk_decrypted_inodes_lock; - /* Crypto API transforms for DIRECT_KEY policies, allocated on-demand */ - struct crypto_skcipher *mk_direct_tfms[__FSCRYPT_MODE_MAX + 1]; + /* Per-mode keys for DIRECT_KEY policies, allocated on-demand */ + struct fscrypt_prepared_key mk_direct_keys[__FSCRYPT_MODE_MAX + 1]; - /* - * Crypto API transforms for filesystem-layer implementation of - * IV_INO_LBLK_64 policies, allocated on-demand. - */ - struct crypto_skcipher *mk_iv_ino_lblk_64_tfms[__FSCRYPT_MODE_MAX + 1]; + /* Per-mode keys for IV_INO_LBLK_64 policies, allocated on-demand */ + struct fscrypt_prepared_key mk_iv_ino_lblk_64_keys[__FSCRYPT_MODE_MAX + 1]; -#ifdef CONFIG_FS_ENCRYPTION_INLINE_CRYPT - /* Raw keys for IV_INO_LBLK_64 policies, allocated on-demand */ - u8 *mk_iv_ino_lblk_64_raw_keys[__FSCRYPT_MODE_MAX + 1]; - - /* The data unit size being used for inline encryption */ - unsigned int mk_data_unit_size; - - /* The filesystem's block device */ - struct block_device *mk_bdev; -#endif } __randomize_layout; static inline bool @@ -526,9 +548,11 @@ fscrypt_mode_supports_direct_key(const struct fscrypt_mode *mode) return mode->ivsize >= offsetofend(union fscrypt_iv, nonce); } -extern struct crypto_skcipher * -fscrypt_allocate_skcipher(struct fscrypt_mode *mode, const u8 *raw_key, - const struct inode *inode); +extern int fscrypt_prepare_key(struct fscrypt_prepared_key *prep_key, + const u8 *raw_key, + const struct fscrypt_info *ci); + +extern void fscrypt_destroy_prepared_key(struct fscrypt_prepared_key *prep_key); extern int fscrypt_set_derived_key(struct fscrypt_info *ci, const u8 *derived_key); diff --git a/fs/crypto/inline_crypt.c b/fs/crypto/inline_crypt.c index e41c6d66ff0d..aa04e8a7d674 100644 --- a/fs/crypto/inline_crypt.c +++ b/fs/crypto/inline_crypt.c @@ -15,187 +15,126 @@ #include #include #include -#include #include "fscrypt_private.h" -/* Return true iff inline encryption should be used for this file */ -bool fscrypt_should_use_inline_encryption(const struct fscrypt_info *ci) +struct fscrypt_blk_crypto_key { + struct blk_crypto_key base; + int num_devs; + struct request_queue *devs[]; +}; + +/* Enable inline encryption for this file if supported. */ +void fscrypt_select_encryption_impl(struct fscrypt_info *ci) { const struct inode *inode = ci->ci_inode; struct super_block *sb = inode->i_sb; /* The file must need contents encryption, not filenames encryption */ if (!S_ISREG(inode->i_mode)) - return false; + return; /* blk-crypto must implement the needed encryption algorithm */ if (ci->ci_mode->blk_crypto_mode == BLK_ENCRYPTION_MODE_INVALID) - return false; - - /* DIRECT_KEY needs a 24+ byte IV, so it can't work with 8-byte DUNs */ - if (fscrypt_is_direct_key_policy(&ci->ci_policy)) - return false; + return; /* The filesystem must be mounted with -o inlinecrypt */ if (!sb->s_cop->inline_crypt_enabled || !sb->s_cop->inline_crypt_enabled(sb)) - return false; + return; - return true; + ci->ci_inlinecrypt = true; } -/* Set a per-file inline encryption key (for passing to blk-crypto) */ -int fscrypt_set_inline_crypt_key(struct fscrypt_info *ci, const u8 *derived_key) +int fscrypt_prepare_inline_crypt_key(struct fscrypt_prepared_key *prep_key, + const u8 *raw_key, + const struct fscrypt_info *ci) { - const struct fscrypt_mode *mode = ci->ci_mode; - const struct super_block *sb = ci->ci_inode->i_sb; - - ci->ci_inline_crypt_key = kmemdup(derived_key, mode->keysize, GFP_NOFS); - if (!ci->ci_inline_crypt_key) - return -ENOMEM; - ci->ci_owns_key = true; - - return blk_crypto_start_using_mode(mode->blk_crypto_mode, - sb->s_blocksize, - sb->s_bdev->bd_queue); -} - -/* Free a per-file inline encryption key and evict it from blk-crypto */ -void fscrypt_free_inline_crypt_key(struct fscrypt_info *ci) -{ - if (ci->ci_inline_crypt_key != NULL) { - const struct fscrypt_mode *mode = ci->ci_mode; - const struct super_block *sb = ci->ci_inode->i_sb; - - blk_crypto_evict_key(sb->s_bdev->bd_queue, - ci->ci_inline_crypt_key, - mode->blk_crypto_mode, sb->s_blocksize); - kzfree(ci->ci_inline_crypt_key); - } -} - -/* - * Set up ->inline_crypt_key (for passing to blk-crypto) for inodes which use an - * IV_INO_LBLK_64 encryption policy. - * - * Return: 0 on success, -errno on failure - */ -int fscrypt_setup_per_mode_inline_crypt_key(struct fscrypt_info *ci, - struct fscrypt_master_key *mk) -{ - static DEFINE_MUTEX(inline_crypt_setup_mutex); - const struct super_block *sb = ci->ci_inode->i_sb; - struct block_device *bdev = sb->s_bdev; - const struct fscrypt_mode *mode = ci->ci_mode; - const u8 mode_num = mode - fscrypt_modes; - u8 *raw_key; - u8 hkdf_info[sizeof(mode_num) + sizeof(sb->s_uuid)]; + const struct inode *inode = ci->ci_inode; + struct super_block *sb = inode->i_sb; + enum blk_crypto_mode_num crypto_mode = ci->ci_mode->blk_crypto_mode; + int num_devs = 1; + int queue_refs = 0; + struct fscrypt_blk_crypto_key *blk_key; int err; + int i; - if (WARN_ON(mode_num > __FSCRYPT_MODE_MAX)) + if (sb->s_cop->get_num_devices) + num_devs = sb->s_cop->get_num_devices(sb); + if (WARN_ON(num_devs < 1)) return -EINVAL; - /* pairs with smp_store_release() below */ - raw_key = smp_load_acquire(&mk->mk_iv_ino_lblk_64_raw_keys[mode_num]); - if (raw_key) { - err = 0; - goto out; + blk_key = kzalloc(struct_size(blk_key, devs, num_devs), GFP_NOFS); + if (!blk_key) + return -ENOMEM; + + blk_key->num_devs = num_devs; + if (num_devs == 1) + blk_key->devs[0] = bdev_get_queue(sb->s_bdev); + else + sb->s_cop->get_devices(sb, blk_key->devs); + + err = blk_crypto_init_key(&blk_key->base, raw_key, crypto_mode, + sb->s_blocksize); + if (err) { + fscrypt_err(inode, "error %d initializing blk-crypto key", err); + goto fail; } - mutex_lock(&inline_crypt_setup_mutex); - - raw_key = mk->mk_iv_ino_lblk_64_raw_keys[mode_num]; - if (raw_key) { - err = 0; - goto out_unlock; - } - - raw_key = kmalloc(mode->keysize, GFP_NOFS); - if (!raw_key) { - err = -ENOMEM; - goto out_unlock; - } - - BUILD_BUG_ON(sizeof(mode_num) != 1); - BUILD_BUG_ON(sizeof(sb->s_uuid) != 16); - BUILD_BUG_ON(sizeof(hkdf_info) != 17); - hkdf_info[0] = mode_num; - memcpy(&hkdf_info[1], &sb->s_uuid, sizeof(sb->s_uuid)); - - err = fscrypt_hkdf_expand(&mk->mk_secret.hkdf, - HKDF_CONTEXT_IV_INO_LBLK_64_KEY, - hkdf_info, sizeof(hkdf_info), - raw_key, mode->keysize); - if (err) - goto out_unlock; - - err = blk_crypto_start_using_mode(mode->blk_crypto_mode, - sb->s_blocksize, bdev->bd_queue); - if (err) - goto out_unlock; - /* - * When a master key's first inline encryption key is set up, save a - * reference to the filesystem's block device so that the inline - * encryption keys can be evicted when the master key is destroyed. + * We have to start using blk-crypto on all the filesystem's devices. + * We also have to save all the request_queue's for later so that the + * key can be evicted from them. This is needed because some keys + * aren't destroyed until after the filesystem was already unmounted + * (namely, the per-mode keys in struct fscrypt_master_key). */ - if (!mk->mk_bdev) { - mk->mk_bdev = bdgrab(bdev); - mk->mk_data_unit_size = sb->s_blocksize; - } + for (i = 0; i < num_devs; i++) { + if (!blk_get_queue(blk_key->devs[i])) { + fscrypt_err(inode, "couldn't get request_queue"); + err = -EAGAIN; + goto fail; + } + queue_refs++; - /* pairs with smp_load_acquire() above */ - smp_store_release(&mk->mk_iv_ino_lblk_64_raw_keys[mode_num], raw_key); - err = 0; -out_unlock: - mutex_unlock(&inline_crypt_setup_mutex); -out: - if (err == 0) { - ci->ci_inline_crypt_key = raw_key; - /* - * Since each struct fscrypt_master_key belongs to a particular - * filesystem (a struct super_block), there should be only one - * block device, and only one data unit size as it should equal - * the filesystem's blocksize (i.e. s_blocksize). - */ - if (WARN_ON(mk->mk_bdev != bdev)) - err = -EINVAL; - if (WARN_ON(mk->mk_data_unit_size != sb->s_blocksize)) - err = -EINVAL; - } else { - kzfree(raw_key); + err = blk_crypto_start_using_mode(crypto_mode, sb->s_blocksize, + blk_key->devs[i]); + if (err) { + fscrypt_err(inode, + "error %d starting to use blk-crypto", err); + goto fail; + } } + /* + * Pairs with READ_ONCE() in fscrypt_is_key_prepared(). (Only matters + * for the per-mode keys, which are shared by multiple inodes.) + */ + smp_store_release(&prep_key->blk_key, blk_key); + return 0; + +fail: + for (i = 0; i < queue_refs; i++) + blk_put_queue(blk_key->devs[i]); + kzfree(blk_key); return err; } -/* - * Evict per-mode inline encryption keys from blk-crypto when a master key is - * destroyed. - */ -void fscrypt_evict_inline_crypt_keys(struct fscrypt_master_key *mk) +void fscrypt_destroy_inline_crypt_key(struct fscrypt_prepared_key *prep_key) { - struct block_device *bdev = mk->mk_bdev; - size_t i; + struct fscrypt_blk_crypto_key *blk_key = prep_key->blk_key; + int i; - if (!bdev) /* No inline encryption keys? */ - return; - - for (i = 0; i < ARRAY_SIZE(mk->mk_iv_ino_lblk_64_raw_keys); i++) { - u8 *raw_key = mk->mk_iv_ino_lblk_64_raw_keys[i]; - - if (raw_key != NULL) { - blk_crypto_evict_key(bdev->bd_queue, raw_key, - fscrypt_modes[i].blk_crypto_mode, - mk->mk_data_unit_size); - kzfree(raw_key); + if (blk_key) { + for (i = 0; i < blk_key->num_devs; i++) { + blk_crypto_evict_key(blk_key->devs[i], &blk_key->base); + blk_put_queue(blk_key->devs[i]); } + kzfree(blk_key); } - bdput(bdev); } /** - * fscrypt_inode_uses_inline_crypto - test whether an inode uses inline encryption + * fscrypt_inode_uses_inline_crypto - test whether an inode uses inline + * encryption * @inode: an inode * * Return: true if the inode requires file contents encryption and if the @@ -205,12 +144,13 @@ void fscrypt_evict_inline_crypt_keys(struct fscrypt_master_key *mk) bool fscrypt_inode_uses_inline_crypto(const struct inode *inode) { return IS_ENCRYPTED(inode) && S_ISREG(inode->i_mode) && - inode->i_crypt_info->ci_inline_crypt_key != NULL; + inode->i_crypt_info->ci_inlinecrypt; } EXPORT_SYMBOL_GPL(fscrypt_inode_uses_inline_crypto); /** - * fscrypt_inode_uses_fs_layer_crypto - test whether an inode uses fs-layer encryption + * fscrypt_inode_uses_fs_layer_crypto - test whether an inode uses fs-layer + * encryption * @inode: an inode * * Return: true if the inode requires file contents encryption and if the @@ -220,22 +160,22 @@ EXPORT_SYMBOL_GPL(fscrypt_inode_uses_inline_crypto); bool fscrypt_inode_uses_fs_layer_crypto(const struct inode *inode) { return IS_ENCRYPTED(inode) && S_ISREG(inode->i_mode) && - inode->i_crypt_info->ci_inline_crypt_key == NULL; + !inode->i_crypt_info->ci_inlinecrypt; } EXPORT_SYMBOL_GPL(fscrypt_inode_uses_fs_layer_crypto); -static inline u64 fscrypt_generate_dun(const struct fscrypt_info *ci, - u64 lblk_num) +static void fscrypt_generate_dun(const struct fscrypt_info *ci, u64 lblk_num, + u64 dun[BLK_CRYPTO_DUN_ARRAY_SIZE]) { union fscrypt_iv iv; + int i; fscrypt_generate_iv(&iv, lblk_num, ci); - /* - * fscrypt_should_use_inline_encryption() ensures we never get here if - * more than the first 8 bytes of the IV are nonzero. - */ - BUG_ON(memchr_inv(&iv.raw[8], 0, ci->ci_mode->ivsize - 8)); - return le64_to_cpu(iv.lblk_num); + + BUILD_BUG_ON(FSCRYPT_MAX_IV_SIZE > BLK_CRYPTO_MAX_IV_SIZE); + memset(dun, 0, BLK_CRYPTO_MAX_IV_SIZE); + for (i = 0; i < ci->ci_mode->ivsize/sizeof(dun[0]); i++) + dun[i] = le64_to_cpu(iv.dun[i]); } /** @@ -243,7 +183,8 @@ static inline u64 fscrypt_generate_dun(const struct fscrypt_info *ci, * @bio: a bio which will eventually be submitted to the file * @inode: the file's inode * @first_lblk: the first file logical block number in the I/O - * @gfp_mask: memory allocation flags + * @gfp_mask: memory allocation flags - these must be a waiting mask so that + * bio_crypt_set_ctx can't fail. * * If the contents of the file should be encrypted (or decrypted) with inline * encryption, then assign the appropriate encryption context to the bio. @@ -252,24 +193,18 @@ static inline u64 fscrypt_generate_dun(const struct fscrypt_info *ci, * otherwise fscrypt_mergeable_bio() won't work as intended. * * The encryption context will be freed automatically when the bio is freed. - * - * Return: 0 on success, -errno on failure. If __GFP_NOFAIL is specified, this - * is guaranteed to succeed. */ -int fscrypt_set_bio_crypt_ctx(struct bio *bio, const struct inode *inode, - u64 first_lblk, gfp_t gfp_mask) +void fscrypt_set_bio_crypt_ctx(struct bio *bio, const struct inode *inode, + u64 first_lblk, gfp_t gfp_mask) { const struct fscrypt_info *ci = inode->i_crypt_info; - u64 dun; + u64 dun[BLK_CRYPTO_DUN_ARRAY_SIZE]; if (!fscrypt_inode_uses_inline_crypto(inode)) - return 0; + return; - dun = fscrypt_generate_dun(ci, first_lblk); - - return bio_crypt_set_ctx(bio, ci->ci_inline_crypt_key, - ci->ci_mode->blk_crypto_mode, - dun, inode->i_blkbits, gfp_mask); + fscrypt_generate_dun(ci, first_lblk, dun); + bio_crypt_set_ctx(bio, &ci->ci_key.blk_key->base, dun, gfp_mask); } EXPORT_SYMBOL_GPL(fscrypt_set_bio_crypt_ctx); @@ -298,27 +233,24 @@ static bool bh_get_inode_and_lblk_num(const struct buffer_head *bh, } /** - * fscrypt_set_bio_crypt_ctx_bh - prepare a file contents bio for inline encryption + * fscrypt_set_bio_crypt_ctx_bh - prepare a file contents bio for inline + * encryption * @bio: a bio which will eventually be submitted to the file * @first_bh: the first buffer_head for which I/O will be submitted * @gfp_mask: memory allocation flags * * Same as fscrypt_set_bio_crypt_ctx(), except this takes a buffer_head instead * of an inode and block number directly. - * - * Return: 0 on success, -errno on failure */ -int fscrypt_set_bio_crypt_ctx_bh(struct bio *bio, +void fscrypt_set_bio_crypt_ctx_bh(struct bio *bio, const struct buffer_head *first_bh, gfp_t gfp_mask) { const struct inode *inode; u64 first_lblk; - if (!bh_get_inode_and_lblk_num(first_bh, &inode, &first_lblk)) - return 0; - - return fscrypt_set_bio_crypt_ctx(bio, inode, first_lblk, gfp_mask); + if (bh_get_inode_and_lblk_num(first_bh, &inode, &first_lblk)) + fscrypt_set_bio_crypt_ctx(bio, inode, first_lblk, gfp_mask); } EXPORT_SYMBOL_GPL(fscrypt_set_bio_crypt_ctx_bh); @@ -342,27 +274,24 @@ EXPORT_SYMBOL_GPL(fscrypt_set_bio_crypt_ctx_bh); bool fscrypt_mergeable_bio(struct bio *bio, const struct inode *inode, u64 next_lblk) { - const struct bio_crypt_ctx *bc; - const u8 *next_key; - u64 next_dun; + const struct bio_crypt_ctx *bc = bio->bi_crypt_context; + u64 next_dun[BLK_CRYPTO_DUN_ARRAY_SIZE]; - if (bio_has_crypt_ctx(bio) != fscrypt_inode_uses_inline_crypto(inode)) + if (!!bc != fscrypt_inode_uses_inline_crypto(inode)) return false; - if (!bio_has_crypt_ctx(bio)) + if (!bc) return true; - bc = bio->bi_crypt_context; - next_key = inode->i_crypt_info->ci_inline_crypt_key; - next_dun = fscrypt_generate_dun(inode->i_crypt_info, next_lblk); /* * Comparing the key pointers is good enough, as all I/O for each key * uses the same pointer. I.e., there's currently no need to support * merging requests where the keys are the same but the pointers differ. */ - return next_key == bc->raw_key && - next_dun == bc->data_unit_num + - (bio_sectors(bio) >> - (bc->data_unit_size_bits - SECTOR_SHIFT)); + if (bc->bc_key != &inode->i_crypt_info->ci_key.blk_key->base) + return false; + + fscrypt_generate_dun(inode->i_crypt_info, next_lblk, next_dun); + return bio_crypt_dun_is_contiguous(bc, bio->bi_iter.bi_size, next_dun); } EXPORT_SYMBOL_GPL(fscrypt_mergeable_bio); @@ -383,7 +312,7 @@ bool fscrypt_mergeable_bio_bh(struct bio *bio, u64 next_lblk; if (!bh_get_inode_and_lblk_num(next_bh, &inode, &next_lblk)) - return !bio_has_crypt_ctx(bio); + return !bio->bi_crypt_context; return fscrypt_mergeable_bio(bio, inode, next_lblk); } diff --git a/fs/crypto/keyring.c b/fs/crypto/keyring.c index 05cc274e5c13..e6b0f0da1eaa 100644 --- a/fs/crypto/keyring.c +++ b/fs/crypto/keyring.c @@ -44,12 +44,10 @@ static void free_master_key(struct fscrypt_master_key *mk) wipe_master_key_secret(&mk->mk_secret); for (i = 0; i <= __FSCRYPT_MODE_MAX; i++) { - crypto_free_skcipher(mk->mk_direct_tfms[i]); - crypto_free_skcipher(mk->mk_iv_ino_lblk_64_tfms[i]); + fscrypt_destroy_prepared_key(&mk->mk_direct_keys[i]); + fscrypt_destroy_prepared_key(&mk->mk_iv_ino_lblk_64_keys[i]); } - fscrypt_evict_inline_crypt_keys(mk); - key_put(mk->mk_users); kzfree(mk); } diff --git a/fs/crypto/keysetup.c b/fs/crypto/keysetup.c index 8070dad9a541..f7e909d690b2 100644 --- a/fs/crypto/keysetup.c +++ b/fs/crypto/keysetup.c @@ -32,6 +32,7 @@ struct fscrypt_mode fscrypt_modes[] = { .cipher_str = "essiv(cbc(aes),sha256)", .keysize = 16, .ivsize = 16, + .blk_crypto_mode = BLK_ENCRYPTION_MODE_AES_128_CBC_ESSIV, }, [FSCRYPT_MODE_AES_128_CTS] = { .friendly_name = "AES-128-CTS-CBC", @@ -44,6 +45,7 @@ struct fscrypt_mode fscrypt_modes[] = { .cipher_str = "adiantum(xchacha12,aes)", .keysize = 32, .ivsize = 32, + .blk_crypto_mode = BLK_ENCRYPTION_MODE_ADIANTUM, }, }; @@ -63,9 +65,9 @@ select_encryption_mode(const union fscrypt_policy *policy, } /* Create a symmetric cipher object for the given encryption mode and key */ -struct crypto_skcipher *fscrypt_allocate_skcipher(struct fscrypt_mode *mode, - const u8 *raw_key, - const struct inode *inode) +static struct crypto_skcipher * +fscrypt_allocate_skcipher(struct fscrypt_mode *mode, const u8 *raw_key, + const struct inode *inode) { struct crypto_skcipher *tfm; int err; @@ -107,33 +109,55 @@ err_free_tfm: return ERR_PTR(err); } -/* Given the per-file key, set up the file's crypto transform object */ -int fscrypt_set_derived_key(struct fscrypt_info *ci, const u8 *derived_key) +/* + * Prepare the crypto transform object or blk-crypto key in @prep_key, given the + * raw key, encryption mode, and flag indicating which encryption implementation + * (fs-layer or blk-crypto) will be used. + */ +int fscrypt_prepare_key(struct fscrypt_prepared_key *prep_key, + const u8 *raw_key, const struct fscrypt_info *ci) { struct crypto_skcipher *tfm; - if (fscrypt_should_use_inline_encryption(ci)) - return fscrypt_set_inline_crypt_key(ci, derived_key); + if (fscrypt_using_inline_encryption(ci)) + return fscrypt_prepare_inline_crypt_key(prep_key, raw_key, ci); - tfm = fscrypt_allocate_skcipher(ci->ci_mode, derived_key, ci->ci_inode); + tfm = fscrypt_allocate_skcipher(ci->ci_mode, raw_key, ci->ci_inode); if (IS_ERR(tfm)) return PTR_ERR(tfm); - - ci->ci_ctfm = tfm; - ci->ci_owns_key = true; + /* + * Pairs with READ_ONCE() in fscrypt_is_key_prepared(). (Only matters + * for the per-mode keys, which are shared by multiple inodes.) + */ + smp_store_release(&prep_key->tfm, tfm); return 0; } +/* Destroy a crypto transform object and/or blk-crypto key. */ +void fscrypt_destroy_prepared_key(struct fscrypt_prepared_key *prep_key) +{ + crypto_free_skcipher(prep_key->tfm); + fscrypt_destroy_inline_crypt_key(prep_key); +} + +/* Given the per-file key, set up the file's crypto transform object */ +int fscrypt_set_derived_key(struct fscrypt_info *ci, const u8 *derived_key) +{ + ci->ci_owns_key = true; + return fscrypt_prepare_key(&ci->ci_key, derived_key, ci); +} + static int setup_per_mode_key(struct fscrypt_info *ci, struct fscrypt_master_key *mk, - struct crypto_skcipher **tfms, + struct fscrypt_prepared_key *keys, u8 hkdf_context, bool include_fs_uuid) { + static DEFINE_MUTEX(mode_key_setup_mutex); const struct inode *inode = ci->ci_inode; const struct super_block *sb = inode->i_sb; struct fscrypt_mode *mode = ci->ci_mode; const u8 mode_num = mode - fscrypt_modes; - struct crypto_skcipher *tfm, *prev_tfm; + struct fscrypt_prepared_key *prep_key; u8 mode_key[FSCRYPT_MAX_KEY_SIZE]; u8 hkdf_info[sizeof(mode_num) + sizeof(sb->s_uuid)]; unsigned int hkdf_infolen = 0; @@ -142,10 +166,16 @@ static int setup_per_mode_key(struct fscrypt_info *ci, if (WARN_ON(mode_num > __FSCRYPT_MODE_MAX)) return -EINVAL; - /* pairs with cmpxchg() below */ - tfm = READ_ONCE(tfms[mode_num]); - if (likely(tfm != NULL)) - goto done; + prep_key = &keys[mode_num]; + if (fscrypt_is_key_prepared(prep_key, ci)) { + ci->ci_key = *prep_key; + return 0; + } + + mutex_lock(&mode_key_setup_mutex); + + if (fscrypt_is_key_prepared(prep_key, ci)) + goto done_unlock; BUILD_BUG_ON(sizeof(mode_num) != 1); BUILD_BUG_ON(sizeof(sb->s_uuid) != 16); @@ -160,21 +190,17 @@ static int setup_per_mode_key(struct fscrypt_info *ci, hkdf_context, hkdf_info, hkdf_infolen, mode_key, mode->keysize); if (err) - return err; - tfm = fscrypt_allocate_skcipher(mode, mode_key, inode); + goto out_unlock; + err = fscrypt_prepare_key(prep_key, mode_key, ci); memzero_explicit(mode_key, mode->keysize); - if (IS_ERR(tfm)) - return PTR_ERR(tfm); - - /* pairs with READ_ONCE() above */ - prev_tfm = cmpxchg(&tfms[mode_num], NULL, tfm); - if (prev_tfm != NULL) { - crypto_free_skcipher(tfm); - tfm = prev_tfm; - } -done: - ci->ci_ctfm = tfm; - return 0; + if (err) + goto out_unlock; +done_unlock: + ci->ci_key = *prep_key; + err = 0; +out_unlock: + mutex_unlock(&mode_key_setup_mutex); + return err; } static int fscrypt_setup_v2_file_key(struct fscrypt_info *ci, @@ -198,7 +224,7 @@ static int fscrypt_setup_v2_file_key(struct fscrypt_info *ci, ci->ci_mode->friendly_name); return -EINVAL; } - return setup_per_mode_key(ci, mk, mk->mk_direct_tfms, + return setup_per_mode_key(ci, mk, mk->mk_direct_keys, HKDF_CONTEXT_DIRECT_KEY, false); } else if (ci->ci_policy.v2.flags & FSCRYPT_POLICY_FLAG_IV_INO_LBLK_64) { @@ -208,9 +234,7 @@ static int fscrypt_setup_v2_file_key(struct fscrypt_info *ci, * the IVs. This format is optimized for use with inline * encryption hardware compliant with the UFS or eMMC standards. */ - if (fscrypt_should_use_inline_encryption(ci)) - return fscrypt_setup_per_mode_inline_crypt_key(ci, mk); - return setup_per_mode_key(ci, mk, mk->mk_iv_ino_lblk_64_tfms, + return setup_per_mode_key(ci, mk, mk->mk_iv_ino_lblk_64_keys, HKDF_CONTEXT_IV_INO_LBLK_64_KEY, true); } @@ -245,6 +269,8 @@ static int setup_file_encryption_key(struct fscrypt_info *ci, struct fscrypt_key_specifier mk_spec; int err; + fscrypt_select_encryption_impl(ci); + switch (ci->ci_policy.version) { case FSCRYPT_POLICY_V1: mk_spec.type = FSCRYPT_KEY_SPEC_TYPE_DESCRIPTOR; @@ -336,10 +362,8 @@ static void put_crypt_info(struct fscrypt_info *ci) if (ci->ci_direct_key) fscrypt_put_direct_key(ci->ci_direct_key); - else if (ci->ci_owns_key) { - crypto_free_skcipher(ci->ci_ctfm); - fscrypt_free_inline_crypt_key(ci); - } + else if (ci->ci_owns_key) + fscrypt_destroy_prepared_key(&ci->ci_key); key = ci->ci_master_key; if (key) { diff --git a/fs/crypto/keysetup_v1.c b/fs/crypto/keysetup_v1.c index 5298ef22aa85..423ee9b64108 100644 --- a/fs/crypto/keysetup_v1.c +++ b/fs/crypto/keysetup_v1.c @@ -146,7 +146,7 @@ struct fscrypt_direct_key { struct hlist_node dk_node; refcount_t dk_refcount; const struct fscrypt_mode *dk_mode; - struct crypto_skcipher *dk_ctfm; + struct fscrypt_prepared_key dk_key; u8 dk_descriptor[FSCRYPT_KEY_DESCRIPTOR_SIZE]; u8 dk_raw[FSCRYPT_MAX_KEY_SIZE]; }; @@ -154,7 +154,7 @@ struct fscrypt_direct_key { static void free_direct_key(struct fscrypt_direct_key *dk) { if (dk) { - crypto_free_skcipher(dk->dk_ctfm); + fscrypt_destroy_prepared_key(&dk->dk_key); kzfree(dk); } } @@ -199,6 +199,8 @@ find_or_insert_direct_key(struct fscrypt_direct_key *to_insert, continue; if (ci->ci_mode != dk->dk_mode) continue; + if (!fscrypt_is_key_prepared(&dk->dk_key, ci)) + continue; if (crypto_memneq(raw_key, dk->dk_raw, ci->ci_mode->keysize)) continue; /* using existing tfm with same (descriptor, mode, raw_key) */ @@ -231,13 +233,9 @@ fscrypt_get_direct_key(const struct fscrypt_info *ci, const u8 *raw_key) return ERR_PTR(-ENOMEM); refcount_set(&dk->dk_refcount, 1); dk->dk_mode = ci->ci_mode; - dk->dk_ctfm = fscrypt_allocate_skcipher(ci->ci_mode, raw_key, - ci->ci_inode); - if (IS_ERR(dk->dk_ctfm)) { - err = PTR_ERR(dk->dk_ctfm); - dk->dk_ctfm = NULL; + err = fscrypt_prepare_key(&dk->dk_key, raw_key, ci); + if (err) goto err_free_dk; - } memcpy(dk->dk_descriptor, ci->ci_policy.v1.master_key_descriptor, FSCRYPT_KEY_DESCRIPTOR_SIZE); memcpy(dk->dk_raw, raw_key, ci->ci_mode->keysize); @@ -274,7 +272,7 @@ static int setup_v1_file_key_direct(struct fscrypt_info *ci, if (IS_ERR(dk)) return PTR_ERR(dk); ci->ci_direct_key = dk; - ci->ci_ctfm = dk->dk_ctfm; + ci->ci_key = dk->dk_key; return 0; } diff --git a/fs/ext4/page-io.c b/fs/ext4/page-io.c index 46a4aeef8275..b55038236845 100644 --- a/fs/ext4/page-io.c +++ b/fs/ext4/page-io.c @@ -362,16 +362,11 @@ static int io_submit_init_bio(struct ext4_io_submit *io, struct buffer_head *bh) { struct bio *bio; - int err; bio = bio_alloc(GFP_NOIO, BIO_MAX_PAGES); if (!bio) return -ENOMEM; - err = fscrypt_set_bio_crypt_ctx_bh(bio, bh, GFP_NOIO); - if (err) { - bio_put(bio); - return err; - } + fscrypt_set_bio_crypt_ctx_bh(bio, bh, GFP_NOIO); bio->bi_iter.bi_sector = bh->b_blocknr * (bh->b_size >> 9); bio_set_dev(bio, bh->b_bdev); bio->bi_end_io = ext4_end_bio; diff --git a/fs/ext4/readpage.c b/fs/ext4/readpage.c index e21216e7ae24..522ccc42b7e5 100644 --- a/fs/ext4/readpage.c +++ b/fs/ext4/readpage.c @@ -406,12 +406,8 @@ int ext4_mpage_readpages(struct address_space *mapping, min_t(int, nr_pages, BIO_MAX_PAGES)); if (!bio) goto set_error_page; - if (fscrypt_set_bio_crypt_ctx(bio, inode, next_block, - GFP_KERNEL) != 0) { - bio_put(bio); - bio = NULL; - goto set_error_page; - } + fscrypt_set_bio_crypt_ctx(bio, inode, next_block, + GFP_KERNEL); ctx = get_bio_post_read_ctx(inode, bio, page->index); if (IS_ERR(ctx)) { bio_put(bio); diff --git a/fs/f2fs/data.c b/fs/f2fs/data.c index ce44cda13bfe..b494242e66b1 100644 --- a/fs/f2fs/data.c +++ b/fs/f2fs/data.c @@ -307,7 +307,7 @@ static struct bio *__bio_alloc(struct f2fs_io_info *fio, int npages) return bio; } -static int f2fs_set_bio_crypt_ctx(struct bio *bio, const struct inode *inode, +static void f2fs_set_bio_crypt_ctx(struct bio *bio, const struct inode *inode, pgoff_t first_idx, const struct f2fs_io_info *fio, gfp_t gfp_mask) @@ -316,10 +316,8 @@ static int f2fs_set_bio_crypt_ctx(struct bio *bio, const struct inode *inode, * The f2fs garbage collector sets ->encrypted_page when it wants to * read/write raw data without encryption. */ - if (fio && fio->encrypted_page) - return 0; - - return fscrypt_set_bio_crypt_ctx(bio, inode, first_idx, gfp_mask); + if (!fio || !fio->encrypted_page) + fscrypt_set_bio_crypt_ctx(bio, inode, first_idx, gfp_mask); } static bool f2fs_crypt_mergeable_bio(struct bio *bio, const struct inode *inode, @@ -507,7 +505,6 @@ int f2fs_submit_page_bio(struct f2fs_io_info *fio) struct bio *bio; struct page *page = fio->encrypted_page ? fio->encrypted_page : fio->page; - int err; if (!f2fs_is_valid_blkaddr(fio->sbi, fio->new_blkaddr, fio->is_por ? META_POR : (__is_meta_io(fio) ? @@ -520,12 +517,8 @@ int f2fs_submit_page_bio(struct f2fs_io_info *fio) /* Allocate a new bio */ bio = __bio_alloc(fio, 1); - err = f2fs_set_bio_crypt_ctx(bio, fio->page->mapping->host, - fio->page->index, fio, GFP_NOIO); - if (err) { - bio_put(bio); - return err; - } + f2fs_set_bio_crypt_ctx(bio, fio->page->mapping->host, + fio->page->index, fio, GFP_NOIO); if (bio_add_page(bio, page, PAGE_SIZE, 0) < PAGE_SIZE) { bio_put(bio); @@ -606,7 +599,7 @@ alloc_new: bio = __bio_alloc(fio, BIO_MAX_PAGES); f2fs_set_bio_crypt_ctx(bio, fio->page->mapping->host, fio->page->index, fio, - GFP_NOIO | __GFP_NOFAIL); + GFP_NOIO); bio_set_op_attrs(bio, fio->op, fio->op_flags); } @@ -690,7 +683,7 @@ alloc_new: io->bio = __bio_alloc(fio, BIO_MAX_PAGES); f2fs_set_bio_crypt_ctx(io->bio, fio->page->mapping->host, fio->page->index, fio, - GFP_NOIO | __GFP_NOFAIL); + GFP_NOIO); io->fio = *fio; } @@ -730,17 +723,12 @@ static struct bio *f2fs_grab_read_bio(struct inode *inode, block_t blkaddr, struct bio *bio; struct bio_post_read_ctx *ctx; unsigned int post_read_steps = 0; - int err; bio = f2fs_bio_alloc(sbi, min_t(int, nr_pages, BIO_MAX_PAGES), false); if (!bio) return ERR_PTR(-ENOMEM); - err = f2fs_set_bio_crypt_ctx(bio, inode, first_idx, NULL, GFP_NOFS); - if (err) { - bio_put(bio); - return ERR_PTR(err); - } + f2fs_set_bio_crypt_ctx(bio, inode, first_idx, NULL, GFP_NOFS); f2fs_target_device(sbi, blkaddr, bio); bio->bi_end_io = f2fs_read_end_io; diff --git a/fs/f2fs/super.c b/fs/f2fs/super.c index 850a2a2394d8..eecfe09c52e7 100644 --- a/fs/f2fs/super.c +++ b/fs/f2fs/super.c @@ -2339,6 +2339,25 @@ static bool f2fs_inline_crypt_enabled(struct super_block *sb) return F2FS_OPTION(F2FS_SB(sb)).inlinecrypt; } +static int f2fs_get_num_devices(struct super_block *sb) +{ + struct f2fs_sb_info *sbi = F2FS_SB(sb); + + if (f2fs_is_multi_device(sbi)) + return sbi->s_ndevs; + return 1; +} + +static void f2fs_get_devices(struct super_block *sb, + struct request_queue **devs) +{ + struct f2fs_sb_info *sbi = F2FS_SB(sb); + int i; + + for (i = 0; i < sbi->s_ndevs; i++) + devs[i] = bdev_get_queue(FDEV(i).bdev); +} + static const struct fscrypt_operations f2fs_cryptops = { .key_prefix = "f2fs:", .get_context = f2fs_get_context, @@ -2349,6 +2368,8 @@ static const struct fscrypt_operations f2fs_cryptops = { .has_stable_inodes = f2fs_has_stable_inodes, .get_ino_and_lblk_bits = f2fs_get_ino_and_lblk_bits, .inline_crypt_enabled = f2fs_inline_crypt_enabled, + .get_num_devices = f2fs_get_num_devices, + .get_devices = f2fs_get_devices, }; #endif diff --git a/include/linux/bio-crypt-ctx.h b/include/linux/bio-crypt-ctx.h index 7c389f310bab..4535df0a6349 100644 --- a/include/linux/bio-crypt-ctx.h +++ b/include/linux/bio-crypt-ctx.h @@ -6,221 +6,188 @@ #define __LINUX_BIO_CRYPT_CTX_H enum blk_crypto_mode_num { - BLK_ENCRYPTION_MODE_INVALID = 0, - BLK_ENCRYPTION_MODE_AES_256_XTS = 1, + BLK_ENCRYPTION_MODE_INVALID, + BLK_ENCRYPTION_MODE_AES_256_XTS, + BLK_ENCRYPTION_MODE_AES_128_CBC_ESSIV, + BLK_ENCRYPTION_MODE_ADIANTUM, + BLK_ENCRYPTION_MODE_MAX, }; #ifdef CONFIG_BLOCK #include #ifdef CONFIG_BLK_INLINE_ENCRYPTION -struct bio_crypt_ctx { - int keyslot; - const u8 *raw_key; + +#define BLK_CRYPTO_MAX_KEY_SIZE 64 + +/** + * struct blk_crypto_key - an inline encryption key + * @crypto_mode: encryption algorithm this key is for + * @data_unit_size: the data unit size for all encryption/decryptions with this + * key. This is the size in bytes of each individual plaintext and + * ciphertext. This is always a power of 2. It might be e.g. the + * filesystem block size or the disk sector size. + * @data_unit_size_bits: log2 of data_unit_size + * @size: size of this key in bytes (determined by @crypto_mode) + * @hash: hash of this key, for keyslot manager use only + * @raw: the raw bytes of this key. Only the first @size bytes are used. + * + * A blk_crypto_key is immutable once created, and many bios can reference it at + * the same time. It must not be freed until all bios using it have completed. + */ +struct blk_crypto_key { enum blk_crypto_mode_num crypto_mode; - u64 data_unit_num; + unsigned int data_unit_size; unsigned int data_unit_size_bits; + unsigned int size; + unsigned int hash; + u8 raw[BLK_CRYPTO_MAX_KEY_SIZE]; +}; + +#define BLK_CRYPTO_MAX_IV_SIZE 32 +#define BLK_CRYPTO_DUN_ARRAY_SIZE (BLK_CRYPTO_MAX_IV_SIZE/sizeof(u64)) + +/** + * struct bio_crypt_ctx - an inline encryption context + * @bc_key: the key, algorithm, and data unit size to use + * @bc_keyslot: the keyslot that has been assigned for this key in @bc_ksm, + * or -1 if no keyslot has been assigned yet. + * @bc_dun: the data unit number (starting IV) to use + * @bc_ksm: the keyslot manager into which the key has been programmed with + * @bc_keyslot, or NULL if this key hasn't yet been programmed. + * + * A bio_crypt_ctx specifies that the contents of the bio will be encrypted (for + * write requests) or decrypted (for read requests) inline by the storage device + * or controller, or by the crypto API fallback. + */ +struct bio_crypt_ctx { + const struct blk_crypto_key *bc_key; + int bc_keyslot; + + /* Data unit number */ + u64 bc_dun[BLK_CRYPTO_DUN_ARRAY_SIZE]; /* * The keyslot manager where the key has been programmed * with keyslot. */ - struct keyslot_manager *processing_ksm; - - /* - * Copy of the bvec_iter when this bio was submitted. - * We only want to en/decrypt the part of the bio - * as described by the bvec_iter upon submission because - * bio might be split before being resubmitted - */ - struct bvec_iter crypt_iter; - u64 sw_data_unit_num; + struct keyslot_manager *bc_ksm; }; -extern int bio_crypt_clone(struct bio *dst, struct bio *src, - gfp_t gfp_mask); +int bio_crypt_ctx_init(void); + +struct bio_crypt_ctx *bio_crypt_alloc_ctx(gfp_t gfp_mask); + +void bio_crypt_free_ctx(struct bio *bio); static inline bool bio_has_crypt_ctx(struct bio *bio) { return bio->bi_crypt_context; } -static inline void bio_crypt_advance(struct bio *bio, unsigned int bytes) +void bio_crypt_clone(struct bio *dst, struct bio *src, gfp_t gfp_mask); + +static inline void bio_crypt_set_ctx(struct bio *bio, + const struct blk_crypto_key *key, + u64 dun[BLK_CRYPTO_DUN_ARRAY_SIZE], + gfp_t gfp_mask) { - if (bio_has_crypt_ctx(bio)) { - bio->bi_crypt_context->data_unit_num += - bytes >> bio->bi_crypt_context->data_unit_size_bits; + struct bio_crypt_ctx *bc = bio_crypt_alloc_ctx(gfp_mask); + + bc->bc_key = key; + memcpy(bc->bc_dun, dun, sizeof(bc->bc_dun)); + bc->bc_ksm = NULL; + bc->bc_keyslot = -1; + + bio->bi_crypt_context = bc; +} + +void bio_crypt_ctx_release_keyslot(struct bio_crypt_ctx *bc); + +int bio_crypt_ctx_acquire_keyslot(struct bio_crypt_ctx *bc, + struct keyslot_manager *ksm); + +struct request; +bool bio_crypt_should_process(struct request *rq); + +static inline bool bio_crypt_dun_is_contiguous(const struct bio_crypt_ctx *bc, + unsigned int bytes, + u64 next_dun[BLK_CRYPTO_DUN_ARRAY_SIZE]) +{ + int i = 0; + unsigned int inc = bytes >> bc->bc_key->data_unit_size_bits; + + while (inc && i < BLK_CRYPTO_DUN_ARRAY_SIZE) { + if (bc->bc_dun[i] + inc != next_dun[i]) + return false; + inc = ((bc->bc_dun[i] + inc) < inc); + i++; + } + + return true; +} + + +static inline void bio_crypt_dun_increment(u64 dun[BLK_CRYPTO_DUN_ARRAY_SIZE], + unsigned int inc) +{ + int i = 0; + + while (inc && i < BLK_CRYPTO_DUN_ARRAY_SIZE) { + dun[i] += inc; + inc = (dun[i] < inc); + i++; } } -extern bool bio_crypt_swhandled(struct bio *bio); - -static inline bool bio_crypt_has_keyslot(struct bio *bio) +static inline void bio_crypt_advance(struct bio *bio, unsigned int bytes) { - return bio->bi_crypt_context->keyslot >= 0; + struct bio_crypt_ctx *bc = bio->bi_crypt_context; + + if (!bc) + return; + + bio_crypt_dun_increment(bc->bc_dun, + bytes >> bc->bc_key->data_unit_size_bits); } -extern int bio_crypt_ctx_init(void); +bool bio_crypt_ctx_compatible(struct bio *b_1, struct bio *b_2); -extern struct bio_crypt_ctx *bio_crypt_alloc_ctx(gfp_t gfp_mask); - -extern void bio_crypt_free_ctx(struct bio *bio); - -static inline int bio_crypt_set_ctx(struct bio *bio, - const u8 *raw_key, - enum blk_crypto_mode_num crypto_mode, - u64 dun, - unsigned int dun_bits, - gfp_t gfp_mask) -{ - struct bio_crypt_ctx *crypt_ctx; - - crypt_ctx = bio_crypt_alloc_ctx(gfp_mask); - if (!crypt_ctx) - return -ENOMEM; - - crypt_ctx->raw_key = raw_key; - crypt_ctx->data_unit_num = dun; - crypt_ctx->data_unit_size_bits = dun_bits; - crypt_ctx->crypto_mode = crypto_mode; - crypt_ctx->processing_ksm = NULL; - crypt_ctx->keyslot = -1; - bio->bi_crypt_context = crypt_ctx; - - return 0; -} - -static inline void bio_set_data_unit_num(struct bio *bio, u64 dun) -{ - bio->bi_crypt_context->data_unit_num = dun; -} - -static inline int bio_crypt_get_keyslot(struct bio *bio) -{ - return bio->bi_crypt_context->keyslot; -} - -static inline void bio_crypt_set_keyslot(struct bio *bio, - unsigned int keyslot, - struct keyslot_manager *ksm) -{ - bio->bi_crypt_context->keyslot = keyslot; - bio->bi_crypt_context->processing_ksm = ksm; -} - -extern void bio_crypt_ctx_release_keyslot(struct bio *bio); - -extern int bio_crypt_ctx_acquire_keyslot(struct bio *bio, - struct keyslot_manager *ksm); - -static inline const u8 *bio_crypt_raw_key(struct bio *bio) -{ - return bio->bi_crypt_context->raw_key; -} - -static inline enum blk_crypto_mode_num bio_crypto_mode(struct bio *bio) -{ - return bio->bi_crypt_context->crypto_mode; -} - -static inline u64 bio_crypt_data_unit_num(struct bio *bio) -{ - return bio->bi_crypt_context->data_unit_num; -} - -static inline u64 bio_crypt_sw_data_unit_num(struct bio *bio) -{ - return bio->bi_crypt_context->sw_data_unit_num; -} - -extern bool bio_crypt_should_process(struct bio *bio, struct request_queue *q); - -extern bool bio_crypt_ctx_compatible(struct bio *b_1, struct bio *b_2); - -extern bool bio_crypt_ctx_back_mergeable(struct bio *b_1, - unsigned int b1_sectors, - struct bio *b_2); +bool bio_crypt_ctx_mergeable(struct bio *b_1, unsigned int b1_bytes, + struct bio *b_2); #else /* CONFIG_BLK_INLINE_ENCRYPTION */ -struct keyslot_manager; - static inline int bio_crypt_ctx_init(void) { return 0; } -static inline int bio_crypt_clone(struct bio *dst, struct bio *src, - gfp_t gfp_mask) -{ - return 0; -} - -static inline void bio_crypt_advance(struct bio *bio, - unsigned int bytes) { } - static inline bool bio_has_crypt_ctx(struct bio *bio) { return false; } +static inline void bio_crypt_clone(struct bio *dst, struct bio *src, + gfp_t gfp_mask) { } + static inline void bio_crypt_free_ctx(struct bio *bio) { } -static inline void bio_crypt_set_ctx(struct bio *bio, - u8 *raw_key, - enum blk_crypto_mode_num crypto_mode, - u64 dun, - unsigned int dun_bits, - gfp_t gfp_mask) { } - -static inline bool bio_crypt_swhandled(struct bio *bio) -{ - return false; -} - -static inline void bio_set_data_unit_num(struct bio *bio, u64 dun) { } - -static inline bool bio_crypt_has_keyslot(struct bio *bio) -{ - return false; -} - -static inline void bio_crypt_set_keyslot(struct bio *bio, - unsigned int keyslot, - struct keyslot_manager *ksm) { } - -static inline int bio_crypt_get_keyslot(struct bio *bio) -{ - return -1; -} - -static inline u8 *bio_crypt_raw_key(struct bio *bio) -{ - return NULL; -} - -static inline u64 bio_crypt_data_unit_num(struct bio *bio) -{ - return 0; -} - -static inline bool bio_crypt_should_process(struct bio *bio, - struct request_queue *q) -{ - return false; -} +static inline void bio_crypt_advance(struct bio *bio, unsigned int bytes) { } static inline bool bio_crypt_ctx_compatible(struct bio *b_1, struct bio *b_2) { return true; } -static inline bool bio_crypt_ctx_back_mergeable(struct bio *b_1, - unsigned int b1_sectors, - struct bio *b_2) +static inline bool bio_crypt_ctx_mergeable(struct bio *b_1, + unsigned int b1_bytes, + struct bio *b_2) { return true; } #endif /* CONFIG_BLK_INLINE_ENCRYPTION */ + #endif /* CONFIG_BLOCK */ + #endif /* __LINUX_BIO_CRYPT_CTX_H */ diff --git a/include/linux/blk-crypto.h b/include/linux/blk-crypto.h index 2a07401244a6..07d32e33f0bb 100644 --- a/include/linux/blk-crypto.h +++ b/include/linux/blk-crypto.h @@ -6,32 +6,23 @@ #ifndef __LINUX_BLK_CRYPTO_H #define __LINUX_BLK_CRYPTO_H -#include #include #ifdef CONFIG_BLK_INLINE_ENCRYPTION -int blk_crypto_init(void); - int blk_crypto_submit_bio(struct bio **bio_ptr); bool blk_crypto_endio(struct bio *bio); -int blk_crypto_start_using_mode(enum blk_crypto_mode_num mode_num, - unsigned int data_unit_size, - struct request_queue *q); +int blk_crypto_init_key(struct blk_crypto_key *blk_key, const u8 *raw_key, + enum blk_crypto_mode_num crypto_mode, + unsigned int data_unit_size); -int blk_crypto_evict_key(struct request_queue *q, const u8 *key, - enum blk_crypto_mode_num mode, - unsigned int data_unit_size); +int blk_crypto_evict_key(struct request_queue *q, + const struct blk_crypto_key *key); #else /* CONFIG_BLK_INLINE_ENCRYPTION */ -static inline int blk_crypto_init(void) -{ - return 0; -} - static inline int blk_crypto_submit_bio(struct bio **bio_ptr) { return 0; @@ -42,21 +33,31 @@ static inline bool blk_crypto_endio(struct bio *bio) return true; } +#endif /* CONFIG_BLK_INLINE_ENCRYPTION */ + +#ifdef CONFIG_BLK_INLINE_ENCRYPTION_FALLBACK + +int blk_crypto_start_using_mode(enum blk_crypto_mode_num mode_num, + unsigned int data_unit_size, + struct request_queue *q); + +int blk_crypto_fallback_init(void); + +#else /* CONFIG_BLK_INLINE_ENCRYPTION_FALLBACK */ + static inline int blk_crypto_start_using_mode(enum blk_crypto_mode_num mode_num, unsigned int data_unit_size, struct request_queue *q) { - return -EOPNOTSUPP; + return 0; } -static inline int blk_crypto_evict_key(struct request_queue *q, const u8 *key, - enum blk_crypto_mode_num mode, - unsigned int data_unit_size) +static inline int blk_crypto_fallback_init(void) { return 0; } -#endif /* CONFIG_BLK_INLINE_ENCRYPTION */ +#endif /* CONFIG_BLK_INLINE_ENCRYPTION_FALLBACK */ #endif /* __LINUX_BLK_CRYPTO_H */ diff --git a/include/linux/fscrypt.h b/include/linux/fscrypt.h index 607076207f73..0b15a0a6a354 100644 --- a/include/linux/fscrypt.h +++ b/include/linux/fscrypt.h @@ -65,6 +65,9 @@ struct fscrypt_operations { void (*get_ino_and_lblk_bits)(struct super_block *sb, int *ino_bits_ret, int *lblk_bits_ret); bool (*inline_crypt_enabled)(struct super_block *sb); + int (*get_num_devices)(struct super_block *sb); + void (*get_devices)(struct super_block *sb, + struct request_queue **devs); }; static inline bool fscrypt_has_encryption_key(const struct inode *inode) @@ -550,12 +553,13 @@ extern bool fscrypt_inode_uses_inline_crypto(const struct inode *inode); extern bool fscrypt_inode_uses_fs_layer_crypto(const struct inode *inode); -extern int fscrypt_set_bio_crypt_ctx(struct bio *bio, const struct inode *inode, - u64 first_lblk, gfp_t gfp_mask); +extern void fscrypt_set_bio_crypt_ctx(struct bio *bio, + const struct inode *inode, + u64 first_lblk, gfp_t gfp_mask); -extern int fscrypt_set_bio_crypt_ctx_bh(struct bio *bio, - const struct buffer_head *first_bh, - gfp_t gfp_mask); +extern void fscrypt_set_bio_crypt_ctx_bh(struct bio *bio, + const struct buffer_head *first_bh, + gfp_t gfp_mask); extern bool fscrypt_mergeable_bio(struct bio *bio, const struct inode *inode, u64 next_lblk); @@ -574,20 +578,14 @@ static inline bool fscrypt_inode_uses_fs_layer_crypto(const struct inode *inode) return IS_ENCRYPTED(inode) && S_ISREG(inode->i_mode); } -static inline int fscrypt_set_bio_crypt_ctx(struct bio *bio, - const struct inode *inode, - u64 first_lblk, gfp_t gfp_mask) -{ - return 0; -} +static inline void fscrypt_set_bio_crypt_ctx(struct bio *bio, + const struct inode *inode, + u64 first_lblk, gfp_t gfp_mask) { } -static inline int fscrypt_set_bio_crypt_ctx_bh( - struct bio *bio, - const struct buffer_head *first_bh, - gfp_t gfp_mask) -{ - return 0; -} +static inline void fscrypt_set_bio_crypt_ctx_bh( + struct bio *bio, + const struct buffer_head *first_bh, + gfp_t gfp_mask) { } static inline bool fscrypt_mergeable_bio(struct bio *bio, const struct inode *inode, diff --git a/include/linux/keyslot-manager.h b/include/linux/keyslot-manager.h index 0777ade7907c..fbc423fe5cd5 100644 --- a/include/linux/keyslot-manager.h +++ b/include/linux/keyslot-manager.h @@ -3,96 +3,58 @@ * Copyright 2019 Google LLC */ -#include - -#ifdef CONFIG_BLOCK - #ifndef __LINUX_KEYSLOT_MANAGER_H #define __LINUX_KEYSLOT_MANAGER_H +#include + +struct keyslot_manager; + /** * struct keyslot_mgmt_ll_ops - functions to manage keyslots in hardware - * @keyslot_program: Program the specified key and algorithm into the - * specified slot in the inline encryption hardware. + * @keyslot_program: Program the specified key into the specified slot in the + * inline encryption hardware. * @keyslot_evict: Evict key from the specified keyslot in the hardware. - * The key, crypto_mode and data_unit_size are also passed - * down so that e.g. dm layers can evict keys from - * the devices that they map over. + * The key is provided so that e.g. dm layers can evict + * keys from the devices that they map over. * Returns 0 on success, -errno otherwise. - * @crypto_mode_supported: Check whether a crypto_mode and data_unit_size - * combo is supported. - * @keyslot_find: Returns the slot number that matches the key, - * or -ENOKEY if no match found, or -errno on - * error. * * This structure should be provided by storage device drivers when they set up * a keyslot manager - this structure holds the function ptrs that the keyslot * manager will use to manipulate keyslots in the hardware. */ struct keyslot_mgmt_ll_ops { - int (*keyslot_program)(void *ll_priv_data, const u8 *key, - enum blk_crypto_mode_num crypto_mode, - unsigned int data_unit_size, + int (*keyslot_program)(struct keyslot_manager *ksm, + const struct blk_crypto_key *key, unsigned int slot); - int (*keyslot_evict)(void *ll_priv_data, const u8 *key, - enum blk_crypto_mode_num crypto_mode, - unsigned int data_unit_size, + int (*keyslot_evict)(struct keyslot_manager *ksm, + const struct blk_crypto_key *key, unsigned int slot); - bool (*crypto_mode_supported)(void *ll_priv_data, - enum blk_crypto_mode_num crypto_mode, - unsigned int data_unit_size); - int (*keyslot_find)(void *ll_priv_data, const u8 *key, - enum blk_crypto_mode_num crypto_mode, - unsigned int data_unit_size); }; -#ifdef CONFIG_BLK_INLINE_ENCRYPTION -struct keyslot_manager; +struct keyslot_manager *keyslot_manager_create(unsigned int num_slots, + const struct keyslot_mgmt_ll_ops *ksm_ops, + const unsigned int crypto_mode_supported[BLK_ENCRYPTION_MODE_MAX], + void *ll_priv_data); -extern struct keyslot_manager *keyslot_manager_create(unsigned int num_slots, - const struct keyslot_mgmt_ll_ops *ksm_ops, - void *ll_priv_data); +int keyslot_manager_get_slot_for_key(struct keyslot_manager *ksm, + const struct blk_crypto_key *key); -extern int -keyslot_manager_get_slot_for_key(struct keyslot_manager *ksm, - const u8 *key, - enum blk_crypto_mode_num crypto_mode, - unsigned int data_unit_size); +void keyslot_manager_get_slot(struct keyslot_manager *ksm, unsigned int slot); -extern void keyslot_manager_get_slot(struct keyslot_manager *ksm, - unsigned int slot); +void keyslot_manager_put_slot(struct keyslot_manager *ksm, unsigned int slot); -extern void keyslot_manager_put_slot(struct keyslot_manager *ksm, - unsigned int slot); +bool keyslot_manager_crypto_mode_supported(struct keyslot_manager *ksm, + enum blk_crypto_mode_num crypto_mode, + unsigned int data_unit_size); -extern bool -keyslot_manager_crypto_mode_supported(struct keyslot_manager *ksm, - enum blk_crypto_mode_num crypto_mode, - unsigned int data_unit_size); +int keyslot_manager_evict_key(struct keyslot_manager *ksm, + const struct blk_crypto_key *key); -extern bool -keyslot_manager_rq_crypto_mode_supported(struct request_queue *q, - enum blk_crypto_mode_num crypto_mode, - unsigned int data_unit_size); +void keyslot_manager_reprogram_all_keys(struct keyslot_manager *ksm); -extern int keyslot_manager_evict_key(struct keyslot_manager *ksm, - const u8 *key, - enum blk_crypto_mode_num crypto_mode, - unsigned int data_unit_size); +void *keyslot_manager_private(struct keyslot_manager *ksm); -extern void keyslot_manager_destroy(struct keyslot_manager *ksm); - -#else /* CONFIG_BLK_INLINE_ENCRYPTION */ - -static inline bool -keyslot_manager_rq_crypto_mode_supported(struct request_queue *q, - enum blk_crypto_mode_num crypto_mode, - unsigned int data_unit_size) -{ - return false; -} -#endif /* CONFIG_BLK_INLINE_ENCRYPTION */ +void keyslot_manager_destroy(struct keyslot_manager *ksm); #endif /* __LINUX_KEYSLOT_MANAGER_H */ - -#endif /* CONFIG_BLOCK */