diff --git a/drivers/scsi/ufs/Kconfig b/drivers/scsi/ufs/Kconfig index f25d1491a4d4..1c3484c72b6b 100644 --- a/drivers/scsi/ufs/Kconfig +++ b/drivers/scsi/ufs/Kconfig @@ -196,3 +196,61 @@ config SCSI_UFS_CRYPTO_QTI Enable Vendor Crypto Engine Support in UFS Enabling this allows kernel to use UFS crypto operations defined and implemented by QTI. + +config UFSFEATURE + bool "UFS feature activate" + depends on SCSI_UFSHCD + ---help--- + UFS feature activate such as hpb, tw and etc. + +config UFSHPB + bool "UFS Host Performance Booster support" + depends on SCSI_UFSHCD && UFSFEATURE + ---help--- + UFS HPB Feature Enable + +config HPB_DEBUG + bool "HPB Debug Log Enable" + depends on SCSI_UFSHCD && UFSFEATURE && UFSHPB + default n + ---help--- + UFS HPB Debug Enable (dmesg log[=if set debug sysfs] & BLK_TRACE msg) + +config HPB_ERR_INJECTION + bool "HPB ERROR INJECTION" + depends on SCSI_UFSHCD && UFSFEATURE && UFSHPB + ---help--- + error injection for checking HPB entry integrity + +config UFSTW + bool "UFS Turbo Write support" + depends on SCSI_UFSHCD && UFSFEATURE + ---help--- + UFS TW Feature Enable + +config UFSTW_IGNORE_GUARANTEE_BIT + bool "Ignore UFS Turbo Write Life Time Guarantee bit for POC" + default n + depends on SCSI_UFSHCD && UFSFEATURE && UFSTW + ---help--- + ignore the guarantee bit[31] of dTurboWriteBufferLifeTimeEst for PoC + +config UFSTW_BOOT_ENABLED + bool "Turbo Write enabled at boot time" + default y + depends on SCSI_UFSHCD && UFSFEATURE && UFSTW + ---help--- + fTurboWriteEn and fTurboWriteBufferFlushDuringHibnerEnter flags + are enabled at boot time. + +config SCSI_SKHPB + bool "Activate HPB Host-aware Performance Booster" + depends on (SCSI_UFSHCD || SCSI_UFSHCD_QTI) + help + Activate or deactive SKHPB driver + +config MICRON_HPB + bool "Activate micron HPB Host-aware Performance Booster" + depends on SCSI_UFSHCD && UFSFEATURE && UFSHPB + help + Activate or deactive micron HPB driver diff --git a/drivers/scsi/ufs/Makefile b/drivers/scsi/ufs/Makefile index 8a0421a1b569..f0d24fac1ee8 100644 --- a/drivers/scsi/ufs/Makefile +++ b/drivers/scsi/ufs/Makefile @@ -5,8 +5,12 @@ obj-$(CONFIG_SCSI_UFS_DWC_TC_PLATFORM) += tc-dwc-g210-pltfrm.o ufshcd-dwc.o tc-d obj-$(CONFIG_SCSI_UFS_CDNS_PLATFORM) += cdns-pltfrm.o obj-$(CONFIG_SCSI_UFS_QCOM) += ufs-qcom.o obj-$(CONFIG_SCSI_UFSHCD) += ufshcd-core.o +obj-$(CONFIG_UFSFEATURE) += ufsfeature.o +obj-$(CONFIG_UFSHPB) += ufshpb.o +obj-$(CONFIG_UFSTW) += ufstw.o ufshcd-core-y += ufshcd.o ufs-sysfs.o ufshcd-core-$(CONFIG_SCSI_UFS_BSG) += ufs_bsg.o +obj-$(CONFIG_SCSI_SKHPB) += ufshpb_skh.o obj-$(CONFIG_SCSI_UFSHCD_PCI) += ufshcd-pci.o obj-$(CONFIG_SCSI_UFSHCD_PLATFORM) += ufshcd-pltfrm.o obj-$(CONFIG_SCSI_UFS_HISI) += ufs-hisi.o diff --git a/drivers/scsi/ufs/ufs-qcom.c b/drivers/scsi/ufs/ufs-qcom.c index 562ac279de39..8d976bc3e3e0 100644 --- a/drivers/scsi/ufs/ufs-qcom.c +++ b/drivers/scsi/ufs/ufs-qcom.c @@ -2257,6 +2257,14 @@ ufs_qcom_query_ioctl(struct ufs_hba *hba, u8 lun, void __user *buffer) goto out_release_mem; } +#if defined(CONFIG_UFSFEATURE) + if (ufsf_check_query(ioctl_data->opcode)) { + err = ufsf_query_ioctl(hba, lun, buffer, ioctl_data, + UFSFEATURE_SELECTOR); + goto out_release_mem; + } +#endif + /* verify legal parameters & send query */ switch (ioctl_data->opcode) { case UPIU_QUERY_OPCODE_READ_DESC: diff --git a/drivers/scsi/ufs/ufs.h b/drivers/scsi/ufs/ufs.h index 22f03719b419..39a2b164c661 100644 --- a/drivers/scsi/ufs/ufs.h +++ b/drivers/scsi/ufs/ufs.h @@ -152,6 +152,14 @@ enum flag_idn { QUERY_FLAG_IDN_WB_EN = 0x0E, QUERY_FLAG_IDN_WB_BUFF_FLUSH_EN = 0x0F, QUERY_FLAG_IDN_WB_BUFF_FLUSH_DURING_HIBERN8 = 0x10, +#if defined(CONFIG_UFSHPB) || defined(CONFIG_SCSI_SKHPB) + QUERY_FLAG_IDN_HPB_RESET = 0x11, +#endif +#if defined(CONFIG_UFSTW) + QUERY_FLAG_IDN_TW_EN = 0x0E, + QUERY_FLAG_IDN_TW_BUF_FLUSH_EN = 0x0F, + QUERY_FLAG_IDN_TW_FLUSH_DURING_HIBERN = 0x10, +#endif }; /* Attribute idn for Query requests */ @@ -184,8 +192,19 @@ enum attr_idn { QUERY_ATTR_IDN_AVAIL_WB_BUFF_SIZE = 0x1D, QUERY_ATTR_IDN_WB_BUFF_LIFE_TIME_EST = 0x1E, QUERY_ATTR_IDN_CURR_WB_BUFF_SIZE = 0x1F, +#if defined(CONFIG_UFSTW) + QUERY_ATTR_IDN_TW_FLUSH_STATUS = 0x1C, + QUERY_ATTR_IDN_TW_AVAIL_BUF_SIZE = 0x1D, + QUERY_ATTR_IDN_TW_BUF_LIFETIME_EST = 0x1E, + QUERY_ATTR_IDN_TW_CURR_BUF_SIZE = 0x1F, +#endif +#if defined(CONFIG_UFSFEATURE) + QUERY_ATTR_IDN_SUP_VENDOR_OPTIONS = 0xFF, +#endif }; +#define QUERY_ATTR_IDN_BOOT_LU_EN_MAX 0x02 + /* Descriptor idn for Query requests */ enum desc_idn { QUERY_DESC_IDN_DEVICE = 0x0, @@ -207,11 +226,11 @@ enum desc_header_offset { }; enum ufs_desc_def_size { - QUERY_DESC_DEVICE_DEF_SIZE = 0x59, - QUERY_DESC_CONFIGURATION_DEF_SIZE = 0x90, + QUERY_DESC_DEVICE_DEF_SIZE = 0x5F, + QUERY_DESC_CONFIGURATION_DEF_SIZE = 0xE6, QUERY_DESC_UNIT_DEF_SIZE = 0x2D, QUERY_DESC_INTERCONNECT_DEF_SIZE = 0x06, - QUERY_DESC_GEOMETRY_DEF_SIZE = 0x48, + QUERY_DESC_GEOMETRY_DEF_SIZE = 0x59, QUERY_DESC_POWER_DEF_SIZE = 0x62, QUERY_DESC_HEALTH_DEF_SIZE = 0x25, }; @@ -236,6 +255,15 @@ enum unit_desc_param { UNIT_DESC_PARAM_CTX_CAPABILITIES = 0x20, UNIT_DESC_PARAM_LARGE_UNIT_SIZE_M1 = 0x22, UNIT_DESC_PARAM_WB_BUF_ALLOC_UNITS = 0x29, +#if defined(CONFIG_UFSHPB) || defined(CONFIG_SCSI_SKHPB) + UNIT_DESC_HPB_LU_MAX_ACTIVE_REGIONS = 0x23, + UNIT_DESC_HPB_LU_PIN_REGION_START_OFFSET = 0x25, + UNIT_DESC_HPB_LU_NUM_PIN_REGIONS = 0x27, +#endif +#if defined(CONFIG_UFSTW) + UNIT_DESC_TW_LU_WRITE_BUFFER_ALLOC_UNIT = 0x29, + +#endif }; /* Device descriptor parameters offsets in bytes*/ @@ -279,6 +307,19 @@ enum device_desc_param { DEVICE_DESC_PARAM_WB_PRESRV_USRSPC_EN = 0x53, DEVICE_DESC_PARAM_WB_TYPE = 0x54, DEVICE_DESC_PARAM_WB_SHARED_ALLOC_UNITS = 0x55, +#if defined(CONFIG_UFSHPB) || defined(CONFIG_SCSI_SKHPB) + DEVICE_DESC_PARAM_HPB_VER = 0x40, + DEVICE_DESC_PARAM_HPB_CONTROL = 0x42, +#endif +#if defined(CONFIG_UFSFEATURE) + DEVICE_DESC_PARAM_EX_FEAT_SUP = 0x4F, +#endif +#if defined(CONFIG_UFSTW) + DEVICE_DESC_PARAM_TW_VER = 0x4D, + DEVICE_DESC_PARAM_TW_RETURN_TO_USER = 0x53, + DEVICE_DESC_PARAM_TW_BUF_TYPE = 0x54, + DEVICE_DESC_PARAM_TW_SHARED_BUF_ALLOC_UNITS = 0x55, +#endif }; /* Interconnect descriptor parameters offsets in bytes*/ @@ -328,6 +369,20 @@ enum geometry_desc_param { GEOMETRY_DESC_PARAM_WB_BUFF_CAP_ADJ = 0x54, GEOMETRY_DESC_PARAM_WB_SUP_RED_TYPE = 0x55, GEOMETRY_DESC_PARAM_WB_SUP_WB_TYPE = 0x56, +#if defined(CONFIG_UFSHPB) || defined(CONFIG_SCSI_SKHPB) + GEOMETRY_DESC_HPB_REGION_SIZE = 0x48, + GEOMETRY_DESC_HPB_NUMBER_LU = 0x49, + GEOMETRY_DESC_HPB_SUBREGION_SIZE = 0x4A, + GEOMETRY_DESC_HPB_DEVICE_MAX_ACTIVE_REGIONS = 0x4B, +#endif +#if defined(CONFIG_UFSTW) + GEOMETRY_DESC_TW_GROUP_NUM_CAP = 0x4E, + GEOMETRY_DESC_TW_MAX_SIZE = 0x4F, + GEOMETRY_DESC_TW_NUMBER_LU = 0x53, + GEOMETRY_DESC_TW_CAP_ADJ_FAC = 0x54, + GEOMETRY_DESC_TW_SUPPORT_USER_REDUCTION_TYPES = 0x55, + GEOMETRY_DESC_TW_SUPPORT_BUF_TYPE = 0x56, +#endif }; /* Health descriptor parameters offsets in bytes*/ @@ -391,6 +446,9 @@ enum power_desc_param_offset { enum { MASK_EE_STATUS = 0xFFFF, MASK_EE_URGENT_BKOPS = (1 << 2), +#if defined(CONFIG_UFSTW) + MASK_EE_TW = (1 << 5), +#endif }; /* Background operation status */ @@ -469,6 +527,9 @@ enum { MASK_RSP_EXCEPTION_EVENT = 0x10000, MASK_TM_SERVICE_RESP = 0xFF, MASK_TM_FUNC = 0xFF, +#if defined(CONFIG_SCSI_SKHPB) + MASK_RSP_UPIU_HPB_UPDATE_ALERT = 0x20000, +#endif }; /* Task management service response */ diff --git a/drivers/scsi/ufs/ufsfeature.c b/drivers/scsi/ufs/ufsfeature.c new file mode 100755 index 000000000000..7f65f74efb04 --- /dev/null +++ b/drivers/scsi/ufs/ufsfeature.c @@ -0,0 +1,703 @@ +/* + * Universal Flash Storage Feature Support + * + * Copyright (C) 2017-2018 Samsung Electronics Co., Ltd. + * + * Authors: + * Yongmyung Lee + * Jinyoung Choi + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * See the COPYING file in the top-level directory or visit + * + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * This program is provided "AS IS" and "WITH ALL FAULTS" and + * without warranty of any kind. You are solely responsible for + * determining the appropriateness of using and distributing + * the program and assume all risks associated with your exercise + * of rights with respect to the program, including but not limited + * to infringement of third party rights, the risks and costs of + * program errors, damage to or loss of data, programs or equipment, + * and unavailability or interruption of operations. Under no + * circumstances will the contributor of this Program be liable for + * any damages of any kind arising from your use or distribution of + * this program. + * + * The Linux Foundation chooses to take subject only to the GPLv2 + * license terms, and distributes only under these terms. + */ + +#include "ufsfeature.h" +#include "ufshcd.h" +#include "ufs-qcom.h" + +#define QUERY_REQ_TIMEOUT 1500 /* msec */ + +static inline void ufsf_init_query(struct ufs_hba *hba, + struct ufs_query_req **request, + struct ufs_query_res **response, + enum query_opcode opcode, u8 idn, + u8 index, u8 selector) +{ + *request = &hba->dev_cmd.query.request; + *response = &hba->dev_cmd.query.response; + memset(*request, 0, sizeof(struct ufs_query_req)); + memset(*response, 0, sizeof(struct ufs_query_res)); + (*request)->upiu_req.opcode = opcode; + (*request)->upiu_req.idn = idn; + (*request)->upiu_req.index = index; + (*request)->upiu_req.selector = selector; +} + +/* + * ufs feature common functions. + */ +static int ufsf_query_flag(struct ufs_hba *hba, enum query_opcode opcode, + enum flag_idn idn, u8 index, bool *flag_res) +{ + struct ufs_query_req *request = NULL; + struct ufs_query_res *response = NULL; + int err; + + BUG_ON(!hba); + + ufshcd_hold(hba, false); + mutex_lock(&hba->dev_cmd.lock); + + /* + * Init the query response and request parameters + */ + ufsf_init_query(hba, &request, &response, opcode, idn, index, + UFSFEATURE_SELECTOR); + + switch (opcode) { + case UPIU_QUERY_OPCODE_SET_FLAG: + case UPIU_QUERY_OPCODE_CLEAR_FLAG: + case UPIU_QUERY_OPCODE_TOGGLE_FLAG: + request->query_func = UPIU_QUERY_FUNC_STANDARD_WRITE_REQUEST; + break; + case UPIU_QUERY_OPCODE_READ_FLAG: + request->query_func = UPIU_QUERY_FUNC_STANDARD_READ_REQUEST; + if (!flag_res) { + /* No dummy reads */ + dev_err(hba->dev, "%s: Invalid argument for read request\n", + __func__); + err = -EINVAL; + goto out_unlock; + } + break; + default: + dev_err(hba->dev, + "%s: Expected query flag opcode but got = %d\n", + __func__, opcode); + err = -EINVAL; + goto out_unlock; + } + + /* Send query request */ + err = ufshcd_exec_dev_cmd(hba, DEV_CMD_TYPE_QUERY, QUERY_REQ_TIMEOUT); + if (err) { + dev_err(hba->dev, + "%s: Sending flag query for idn %d failed, err = %d\n", + __func__, idn, err); + goto out_unlock; + } + + if (flag_res) + *flag_res = (be32_to_cpu(response->upiu_res.value) & + MASK_QUERY_UPIU_FLAG_LOC) & 0x1; + +out_unlock: + mutex_unlock(&hba->dev_cmd.lock); + ufshcd_release(hba); + return err; +} + +int ufsf_query_flag_retry(struct ufs_hba *hba, enum query_opcode opcode, + enum flag_idn idn, u8 idx, bool *flag_res) +{ + int ret; + int retries; + + BUG_ON(idx > 7); + + for (retries = 0; retries < UFSF_QUERY_REQ_RETRIES; retries++) { + ret = ufsf_query_flag(hba, opcode, idn, idx, flag_res); + if (ret) + dev_dbg(hba->dev, + "%s: failed with error %d, retries %d\n", + __func__, ret, retries); + else + break; + } + if (ret) + dev_err(hba->dev, + "%s: query flag, opcode %d, idn %d, failed with error %d after %d retires\n", + __func__, opcode, idn, ret, retries); + return ret; +} + +int ufsf_query_attr_retry(struct ufs_hba *hba, enum query_opcode opcode, + enum attr_idn idn, u8 idx, u32 *attr_val) +{ + int ret; + int retries; + + for (retries = 0; retries < UFSF_QUERY_REQ_RETRIES; retries++) { + ret = ufshcd_query_attr(hba, opcode, idn, idx, + UFSFEATURE_SELECTOR, attr_val); + if (ret) + dev_dbg(hba->dev, + "%s: failed with error %d, retries %d\n", + __func__, ret, retries); + else + break; + } + if (ret) + dev_err(hba->dev, + "%s: query attr, opcode %d, idn %d, failed with error %d after %d retires\n", + __func__, opcode, idn, ret, retries); + return ret; +} + +static int ufsf_read_desc(struct ufs_hba *hba, u8 desc_id, u8 desc_index, + u8 selector, u8 *desc_buf, u32 size) +{ + int err = 0; + + pm_runtime_get_sync(hba->dev); + + err = ufshcd_query_descriptor_retry(hba, UPIU_QUERY_OPCODE_READ_DESC, + desc_id, desc_index, + selector, + desc_buf, &size); + if (err) + ERR_MSG("reading Device Desc failed. err = %d", err); + + pm_runtime_put_sync(hba->dev); + + return err; +} + +static int ufsf_read_dev_desc(struct ufsf_feature *ufsf, u8 selector) +{ + u8 desc_buf[UFSF_QUERY_DESC_DEVICE_MAX_SIZE]; + int ret; + + ret = ufsf_read_desc(ufsf->hba, QUERY_DESC_IDN_DEVICE, 0, selector, + desc_buf, UFSF_QUERY_DESC_DEVICE_MAX_SIZE); + if (ret) + return ret; + + ufsf->num_lu = desc_buf[DEVICE_DESC_PARAM_NUM_LU]; + INFO_MSG("device lu count %d", ufsf->num_lu); + + INFO_MSG("sel=%u length=%u(0x%x) bSupport=0x%.2x, extend=0x%.2x_%.2x", + selector, desc_buf[DEVICE_DESC_PARAM_LEN], + desc_buf[DEVICE_DESC_PARAM_LEN], + desc_buf[DEVICE_DESC_PARAM_UFS_FEAT], + desc_buf[DEVICE_DESC_PARAM_EX_FEAT_SUP+2], + desc_buf[DEVICE_DESC_PARAM_EX_FEAT_SUP+3]); + + INFO_MSG("Driver Feature Version : (%.6X%s)", UFSFEATURE_DD_VER, + UFSFEATURE_DD_VER_POST); + + +#if defined(CONFIG_UFSHPB) + ufshpb_get_dev_info(ufsf, desc_buf); +#endif +#if defined(CONFIG_UFSTW) + ufstw_get_dev_info(ufsf, desc_buf); +#endif + return 0; +} + +static int ufsf_read_geo_desc(struct ufsf_feature *ufsf, u8 selector) +{ + u8 geo_buf[UFSF_QUERY_DESC_GEOMETRY_MAX_SIZE]; + int ret; + + ret = ufsf_read_desc(ufsf->hba, QUERY_DESC_IDN_GEOMETRY, 0, selector, + geo_buf, UFSF_QUERY_DESC_GEOMETRY_MAX_SIZE); + if (ret) + return ret; + +#if defined(CONFIG_UFSHPB) + if (ufshpb_get_state(ufsf) == HPB_NEED_INIT) + ufshpb_get_geo_info(ufsf, geo_buf); +#endif + +#if defined(CONFIG_UFSTW) + if (ufstw_get_state(ufsf) == TW_NEED_INIT) + ufstw_get_geo_info(ufsf, geo_buf); +#endif + return 0; +} + +static void ufsf_read_unit_desc(struct ufsf_feature *ufsf, int lun, u8 selector) +{ + u8 unit_buf[UFSF_QUERY_DESC_UNIT_MAX_SIZE]; + int lu_enable, ret = 0; + + ret = ufsf_read_desc(ufsf->hba, QUERY_DESC_IDN_UNIT, lun, selector, + unit_buf, UFSF_QUERY_DESC_UNIT_MAX_SIZE); + if (ret) { + ERR_MSG("read unit desc failed. ret (%d)", ret); + goto out; + } + + lu_enable = unit_buf[UNIT_DESC_PARAM_LU_ENABLE]; + if (!lu_enable) + return; + +#if defined(CONFIG_UFSHPB) + if (ufshpb_get_state(ufsf) == HPB_NEED_INIT) + ufshpb_get_lu_info(ufsf, lun, unit_buf); +#endif + +#if defined(CONFIG_UFSTW) + if (ufstw_get_state(ufsf) == TW_NEED_INIT) + ufstw_alloc_lu(ufsf, lun, unit_buf); +#endif +out: + return; +} + +void ufsf_device_check(struct ufs_hba *hba) +{ + struct ufsf_feature *ufsf = &hba->ufsf; + int ret, lun; + u32 status; + + ufsf->hba = hba; + + ufshcd_query_attr(ufsf->hba, UPIU_QUERY_OPCODE_READ_ATTR, + QUERY_ATTR_IDN_SUP_VENDOR_OPTIONS, 0, 0, &status); + INFO_MSG("UFS FEATURE SELECTOR Dev %d - D/D %d", status, + UFSFEATURE_SELECTOR); + + ret = ufsf_read_dev_desc(ufsf, UFSFEATURE_SELECTOR); + if (ret) + return; + + ret = ufsf_read_geo_desc(ufsf, UFSFEATURE_SELECTOR); + if (ret) + return; + + seq_scan_lu(lun) + ufsf_read_unit_desc(ufsf, lun, UFSFEATURE_SELECTOR); +} + +static int ufsf_execute_dev_ctx_req(struct ufsf_feature *ufsf, + int lun, unsigned char *cdb, + void *buf, int len) +{ + struct scsi_sense_hdr sshdr; + struct scsi_device *sdev; + int ret = 0; + + sdev = ufsf->sdev_ufs_lu[lun]; + if (!sdev) { + WARN_MSG("cannot find scsi_device"); + return -ENODEV; + } + + ufsf->issue_ioctl = true; + ret = scsi_execute(sdev, cdb, DMA_FROM_DEVICE, buf, len, NULL, &sshdr, + msecs_to_jiffies(30000), 3, 0, 0, NULL); + ufsf->issue_ioctl = false; + + return ret; +} + +static inline void ufsf_set_read_dev_ctx(unsigned char *cdb, int lba, int len) +{ + cdb[0] = READ_10; + cdb[1] = 0x02; + cdb[2] = GET_BYTE_3(lba); + cdb[3] = GET_BYTE_2(lba); + cdb[4] = GET_BYTE_1(lba); + cdb[5] = GET_BYTE_0(lba); + cdb[6] = GET_BYTE_2(len); + cdb[7] = GET_BYTE_1(len); + cdb[8] = GET_BYTE_0(len); +} + +int ufsf_issue_req_dev_ctx(struct ufsf_feature *ufsf, int lun, + unsigned char *buf, int buf_len) +{ + unsigned char cdb[10] = { 0 }; + int cmd_len = buf_len >> OS_PAGE_SHIFT; + int ret = 0; + + ufsf_set_read_dev_ctx(cdb, READ10_DEBUG_LBA, cmd_len); + + ret = ufsf_execute_dev_ctx_req(ufsf, lun, cdb, buf, buf_len); + + if (ret < 0) + ERR_MSG("failed with err %d", ret); + + return ret; +} + +static void ufsf_print_query_buf(unsigned char *field, int size) +{ + unsigned char buf[255]; + int count = 0; + int i; + + count += snprintf(buf, 8, "(0x00):"); + + for (i = 0; i < size; i++) { + count += snprintf(buf + count, 4, " %.2X", field[i]); + + if ((i + 1) % 16 == 0) { + buf[count] = '\n'; + buf[count + 1] = '\0'; + printk(buf); + count = 0; + count += snprintf(buf, 8, "(0x%.2X):", i + 1); + } else if ((i + 1) % 4 == 0) + count += snprintf(buf + count, 3, " :"); + } + buf[count] = '\n'; + buf[count + 1] = '\0'; + printk(buf); +} + +inline int ufsf_check_query(__u32 opcode) +{ + return (opcode & 0xffff0000) >> 16 == UFSFEATURE_QUERY_OPCODE; +} + +static inline void ufsf_set_read10_debug_cmd(unsigned char *cdb, int lba, + int len) +{ + cdb[0] = READ_10; + cdb[1] = 0x02; + cdb[2] = GET_BYTE_3(lba); + cdb[3] = GET_BYTE_2(lba); + cdb[4] = GET_BYTE_1(lba); + cdb[5] = GET_BYTE_0(lba); + cdb[6] = GET_BYTE_2(len); + cdb[7] = GET_BYTE_1(len); + cdb[8] = GET_BYTE_0(len); +} + +int ufsf_query_ioctl(struct ufs_hba *hba, int lun, void __user *buffer, + struct ufs_ioctl_query_data *ioctl_data, u8 selector) +{ + unsigned char *kernel_buf; + int opcode; + int err = 0; + int index = 0; + int length = 0; + int buf_len = 0; + + opcode = ioctl_data->opcode & 0xffff; + + INFO_MSG("op %u idn %u sel %u size %u(0x%X)", opcode, ioctl_data->idn, + selector, ioctl_data->buf_size, ioctl_data->buf_size); + + buf_len = (ioctl_data->idn == QUERY_DESC_IDN_STRING) ? + IOCTL_DEV_CTX_MAX_SIZE : QUERY_DESC_MAX_SIZE; + + kernel_buf = kzalloc(buf_len, GFP_KERNEL); + if (!kernel_buf) { + err = -ENOMEM; + goto out; + } + + switch (opcode) { + case UPIU_QUERY_OPCODE_WRITE_DESC: + err = copy_from_user(kernel_buf, buffer + + sizeof(struct ufs_ioctl_query_data), + ioctl_data->buf_size); + INFO_MSG("buf size %d", ioctl_data->buf_size); + ufsf_print_query_buf(kernel_buf, ioctl_data->buf_size); + if (err) + goto out_release_mem; + break; + + case UPIU_QUERY_OPCODE_READ_DESC: + switch (ioctl_data->idn) { + case QUERY_DESC_IDN_UNIT: + if (!ufs_is_valid_unit_desc_lun(lun)) { + ERR_MSG("No unit descriptor for lun 0x%x", lun); + err = -EINVAL; + goto out_release_mem; + } + index = lun; + INFO_MSG("read lu desc lun: %d", index); + break; +#if 0 + case QUERY_DESC_IDN_STRING: + if (!ufs_is_valid_unit_desc_lun(lun)) { + ERR_MSG("No unit descriptor for lun 0x%x", lun); + err = -EINVAL; + goto out_release_mem; + } + err = ufsf_issue_req_dev_ctx(ufsf, lun, kernel_buf, + ioctl_data->buf_size); + if (err < 0) + goto out_release_mem; + + goto copy_buffer; +#endif + case QUERY_DESC_IDN_DEVICE: + case QUERY_DESC_IDN_GEOMETRY: + case QUERY_DESC_IDN_CONFIGURATION: + break; + + default: + ERR_MSG("invalid idn %d", ioctl_data->idn); + err = -EINVAL; + goto out_release_mem; + } + break; + default: + ERR_MSG("invalid opcode %d", opcode); + err = -EINVAL; + goto out_release_mem; + } + + length = ioctl_data->buf_size; + + err = ufshcd_query_descriptor_retry(hba, opcode, ioctl_data->idn, + index, selector, kernel_buf, + &length); + if (err) + goto out_release_mem; +#if 0 +copy_buffer: +#endif + + if (opcode == UPIU_QUERY_OPCODE_READ_DESC) { + err = copy_to_user(buffer, ioctl_data, + sizeof(struct ufs_ioctl_query_data)); + if (err) + ERR_MSG("Failed copying back to user."); + + err = copy_to_user(buffer + sizeof(struct ufs_ioctl_query_data), + kernel_buf, ioctl_data->buf_size); + if (err) + ERR_MSG("Fail: copy rsp_buffer to user space."); + } +out_release_mem: + kfree(kernel_buf); +out: + return err; +} + +inline int ufsf_get_scsi_device(struct ufs_hba *hba, struct scsi_device *sdev) +{ + unsigned long flags; + int ret; + + spin_lock_irqsave(hba->host->host_lock, flags); + ret = scsi_device_get(sdev); + if (!ret && !scsi_device_online(sdev)) { + spin_unlock_irqrestore(hba->host->host_lock, flags); + scsi_device_put(sdev); + ERR_MSG("scsi_device_get failed.(%d)", ret); + return -ENODEV; + } + spin_unlock_irqrestore(hba->host->host_lock, flags); + + return ret; +} + +inline bool ufsf_is_valid_lun(int lun) +{ + return lun < UFS_UPIU_MAX_GENERAL_LUN; +} + +inline void ufsf_slave_configure(struct ufsf_feature *ufsf, + struct scsi_device *sdev) +{ + if (ufsf_is_valid_lun(sdev->lun)) { + ufsf->sdev_ufs_lu[sdev->lun] = sdev; + ufsf->slave_conf_cnt++; + INFO_MSG("lun[%d] sdev(%p) q(%p) slave_conf_cnt(%d/%d)", + (int)sdev->lun, sdev, sdev->request_queue, + ufsf->slave_conf_cnt, ufsf->num_lu); + +#if defined(CONFIG_UFSHPB) + if (ufsf->num_lu == ufsf->slave_conf_cnt) { + if (ufshpb_get_state(ufsf) == HPB_NEED_INIT) { + INFO_MSG("wakeup ufshpb_init_handler"); + wake_up(&ufsf->hpb_wait); + } + } +#endif + } +} + +inline void ufsf_prep_fn(struct ufsf_feature *ufsf, + struct ufshcd_lrb *lrbp) +{ +#if defined(CONFIG_UFSHPB) + if (ufshpb_get_state(ufsf) == HPB_PRESENT && + ufsf->issue_ioctl == false) + ufshpb_prep_fn(ufsf, lrbp); +#endif + +#if defined(CONFIG_UFSTW) + if (ufstw_get_state(ufsf) == TW_PRESENT) + ufstw_prep_fn(ufsf, lrbp); +#endif +} + +inline void ufsf_reset_lu(struct ufsf_feature *ufsf) +{ +#if defined(CONFIG_UFSTW) + INFO_MSG("run reset_lu.. tw_state(%d) -> TW_RESET", + ufstw_get_state(ufsf)); + ufstw_set_state(ufsf, TW_RESET); + ufstw_reset(ufsf, false); +#endif +} + +inline void ufsf_reset_host(struct ufsf_feature *ufsf) +{ +#if defined(CONFIG_UFSHPB) + INFO_MSG("run reset_host.. hpb_state(%d) -> HPB_RESET", + ufshpb_get_state(ufsf)); + if (ufshpb_get_state(ufsf) == HPB_PRESENT) + ufshpb_reset_host(ufsf); +#endif + +#if defined(CONFIG_UFSTW) + INFO_MSG("run reset_host.. tw_state(%d) -> TW_RESET", + ufstw_get_state(ufsf)); + if (ufstw_get_state(ufsf) == TW_PRESENT) + ufstw_reset_host(ufsf); +#endif +} + +inline void ufsf_init(struct ufsf_feature *ufsf) +{ +#if defined(CONFIG_UFSHPB) + if (ufshpb_get_state(ufsf) == HPB_NEED_INIT) { + INFO_MSG("init start.. hpb_state (%d)", HPB_NEED_INIT); + schedule_work(&ufsf->hpb_init_work); + } +#endif + +#if defined(CONFIG_UFSTW) + if (ufstw_get_state(ufsf) == TW_NEED_INIT) + ufstw_init(ufsf); +#endif +} + +inline void ufsf_reset(struct ufsf_feature *ufsf) +{ +#if defined(CONFIG_UFSHPB) + if (ufshpb_get_state(ufsf) == HPB_RESET) { + INFO_MSG("reset start.. hpb_state %d", HPB_RESET); + ufshpb_reset(ufsf); + } +#endif + +#if defined(CONFIG_UFSTW) + if (ufstw_get_state(ufsf) == TW_RESET && + !ufsf->hba->pm_op_in_progress) { + INFO_MSG("reset start.. tw_state %d", + ufstw_get_state(ufsf)); + ufstw_reset(ufsf, false); + } +#endif +} + +inline void ufsf_remove(struct ufsf_feature *ufsf) +{ +#if defined(CONFIG_UFSHPB) + if (ufshpb_get_state(ufsf) == HPB_PRESENT) + ufshpb_remove(ufsf, HPB_NEED_INIT); +#endif + +#if defined(CONFIG_UFSTW) + if (ufstw_get_state(ufsf) == TW_PRESENT) + ufstw_remove(ufsf); +#endif +} + +inline void ufsf_set_init_state(struct ufsf_feature *ufsf) +{ + ufsf->slave_conf_cnt = 0; + ufsf->issue_ioctl = false; +#if defined(CONFIG_UFSHPB) + ufshpb_set_state(ufsf, HPB_NEED_INIT); + INIT_WORK(&ufsf->hpb_init_work, ufshpb_init_handler); + init_waitqueue_head(&ufsf->hpb_wait); +#endif +#if defined(CONFIG_UFSTW) + ufstw_set_state(ufsf, TW_NEED_INIT); +#endif +} + +inline void ufsf_suspend(struct ufsf_feature *ufsf) +{ +#if defined(CONFIG_UFSHPB) + /* + * if suspend failed, pm could call the suspend function again, + * in this case, ufshpb state already had been changed to SUSPEND state. + * so, we will not call ufshpb_suspend. + */ + if (ufshpb_get_state(ufsf) == HPB_PRESENT) + ufshpb_suspend(ufsf); +#endif +} + +inline void ufsf_resume(struct ufsf_feature *ufsf) +{ +#if defined(CONFIG_UFSHPB) + if (ufshpb_get_state(ufsf) == HPB_SUSPEND || + ufshpb_get_state(ufsf) == HPB_PRESENT) { + if (ufshpb_get_state(ufsf) == HPB_PRESENT) + WARN_MSG("warning.. hpb state PRESENT in resuming"); + ufshpb_resume(ufsf); + } +#endif + +#if defined(CONFIG_UFSTW) + if (ufstw_get_state(ufsf) == TW_RESET) + ufstw_reset(ufsf, true); +#endif +} + + +inline void ufsf_change_lun(struct ufsf_feature *ufsf, + struct ufshcd_lrb *lrbp) +{ + int ctx_lba = LI_EN_32(lrbp->cmd->cmnd + 2); + + if (unlikely(ufsf->issue_ioctl == true && + ctx_lba == READ10_DEBUG_LBA)) { + lrbp->lun = READ10_DEBUG_LUN; + INFO_MSG("lun 0x%X lba 0x%X", lrbp->lun, ctx_lba); + } +} + +/* + * Wrapper functions for ufshpb. + */ +#if defined(CONFIG_UFSHPB) +inline void ufsf_hpb_noti_rb(struct ufsf_feature *ufsf, struct ufshcd_lrb *lrbp) +{ + if (ufshpb_get_state(ufsf) == HPB_PRESENT) + ufshpb_rsp_upiu(ufsf, lrbp); +} + +#else +inline void ufsf_hpb_noti_rb(struct ufsf_feature *ufsf, + struct ufshcd_lrb *lrbp) {} +#endif diff --git a/drivers/scsi/ufs/ufsfeature.h b/drivers/scsi/ufs/ufsfeature.h new file mode 100755 index 000000000000..4b5b7b56adb2 --- /dev/null +++ b/drivers/scsi/ufs/ufsfeature.h @@ -0,0 +1,192 @@ +/* + * Universal Flash Storage Feature Support + * + * Copyright (C) 2017-2018 Samsung Electronics Co., Ltd. + * + * Authors: + * Yongmyung Lee + * Jinyoung Choi + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * See the COPYING file in the top-level directory or visit + * + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * This program is provided "AS IS" and "WITH ALL FAULTS" and + * without warranty of any kind. You are solely responsible for + * determining the appropriateness of using and distributing + * the program and assume all risks associated with your exercise + * of rights with respect to the program, including but not limited + * to infringement of third party rights, the risks and costs of + * program errors, damage to or loss of data, programs or equipment, + * and unavailability or interruption of operations. Under no + * circumstances will the contributor of this Program be liable for + * any damages of any kind arising from your use or distribution of + * this program. + * + * The Linux Foundation chooses to take subject only to the GPLv2 + * license terms, and distributes only under these terms. + */ + +#ifndef _UFSFEATURE_H_ +#define _UFSFEATURE_H_ + +#include + +#include "ufs.h" + +#include "ufshpb.h" +#include "ufstw.h" + +/* Version info */ +#define UFSFEATURE_DD_VER 0x010100 +#define UFSFEATURE_DD_VER_POST "" + +#define UFSFEATURE_QUERY_OPCODE 0x5500 + +/* For read10 debug */ +#define READ10_DEBUG_LUN 0x7F +#define READ10_DEBUG_LBA 0x48504230 + +/* Constant value*/ +#define SECTOR 512 +#define BLOCK 4096 +#define SECTORS_PER_BLOCK (BLOCK / SECTOR) +#define BITS_PER_DWORD 32 +#define sects_per_blk_shift 3 +#define bits_per_dword_shift 5 +#define bits_per_dword_mask 0x1F +#define bits_per_byte_shift 3 + +#define IOCTL_DEV_CTX_MAX_SIZE OS_PAGE_SIZE +#define OS_PAGE_SIZE 4096 +#define OS_PAGE_SHIFT 12 + +#define UFSF_QUERY_REQ_RETRIES 1 + +/* Description */ +#define UFSF_QUERY_DESC_DEVICE_MAX_SIZE 0x5F +#define UFSF_QUERY_DESC_CONFIGURAION_MAX_SIZE 0xE6 +#define UFSF_QUERY_DESC_UNIT_MAX_SIZE 0x2D +#define UFSF_QUERY_DESC_GEOMETRY_MAX_SIZE 0x59 + +#define UFSFEATURE_SELECTOR 0x01 + +/* query_flag */ +#define MASK_QUERY_UPIU_FLAG_LOC 0xFF + +/* For read10 debug */ +#define READ10_DEBUG_LUN 0x7F +#define READ10_DEBUG_LBA 0x48504230 + +/* BIG -> LI */ +#define LI_EN_16(x) be16_to_cpu(*(__be16 *)(x)) +#define LI_EN_32(x) be32_to_cpu(*(__be32 *)(x)) +#define LI_EN_64(x) be64_to_cpu(*(__be64 *)(x)) + +/* LI -> BIG */ +#define GET_BYTE_0(num) (((num) >> 0) & 0xff) +#define GET_BYTE_1(num) (((num) >> 8) & 0xff) +#define GET_BYTE_2(num) (((num) >> 16) & 0xff) +#define GET_BYTE_3(num) (((num) >> 24) & 0xff) +#define GET_BYTE_4(num) (((num) >> 32) & 0xff) +#define GET_BYTE_5(num) (((num) >> 40) & 0xff) +#define GET_BYTE_6(num) (((num) >> 48) & 0xff) +#define GET_BYTE_7(num) (((num) >> 56) & 0xff) + +#define INFO_MSG(msg, args...) pr_info("%s:%d info: " msg "\n", \ + __func__, __LINE__, ##args) +#define ERR_MSG(msg, args...) pr_err("%s:%d err: " msg "\n", \ + __func__, __LINE__, ##args) +#define WARN_MSG(msg, args...) pr_warn("%s:%d warn: " msg "\n", \ + __func__, __LINE__, ##args) + +#define seq_scan_lu(lun) for (lun = 0; lun < UFS_UPIU_MAX_GENERAL_LUN; lun++) + +#define TMSG(ufsf, lun, msg, args...) \ + do { if (ufsf->sdev_ufs_lu[lun] && \ + ufsf->sdev_ufs_lu[lun]->request_queue) \ + blk_add_trace_msg( \ + ufsf->sdev_ufs_lu[lun]->request_queue, \ + msg, ##args); \ + } while (0) \ + +struct ufsf_lu_desc { + /* Common info */ + int lu_enable; /* 03h bLUEnable */ + int lu_queue_depth; /* 06h lu queue depth info*/ + int lu_logblk_size; /* 0Ah bLogicalBlockSize. default 0x0C = 4KB */ + u64 lu_logblk_cnt; /* 0Bh qLogicalBlockCount. */ + +#if defined(CONFIG_UFSHPB) + u16 lu_max_active_hpb_rgns; /* 23h:24h wLUMaxActiveHPBRegions */ + u16 lu_hpb_pinned_rgn_startidx; /* 25h:26h wHPBPinnedRegionStartIdx */ + u16 lu_num_hpb_pinned_rgns; /* 27h:28h wNumHPBPinnedRegions */ + int lu_hpb_pinned_end_offset; +#endif +#if defined(CONFIG_UFSTW) + unsigned int tw_lu_buf_size; +#endif +}; + +struct ufsf_feature { + struct ufs_hba *hba; + int num_lu; + int slave_conf_cnt; + struct scsi_device *sdev_ufs_lu[UFS_UPIU_MAX_GENERAL_LUN]; + bool issue_ioctl; + +#if defined(CONFIG_UFSHPB) + struct ufshpb_dev_info hpb_dev_info; + struct ufshpb_lu *hpb_lup[UFS_UPIU_MAX_GENERAL_LUN]; + struct work_struct hpb_init_work; + struct work_struct hpb_eh_work; + wait_queue_head_t hpb_wait; + atomic_t hpb_state; + struct kref hpb_kref; +#endif +#if defined(CONFIG_UFSTW) + struct ufstw_dev_info tw_dev_info; + struct ufstw_lu *tw_lup[UFS_UPIU_MAX_GENERAL_LUN]; + atomic_t tw_state; +#endif +}; + +struct ufs_hba; +struct ufshcd_lrb; +struct ufs_ioctl_query_data; + +void ufsf_device_check(struct ufs_hba *hba); +int ufsf_check_query(__u32 opcode); +int ufsf_query_ioctl(struct ufs_hba *hba, int lun, void __user *buffer, + struct ufs_ioctl_query_data *ioctl_data, + u8 selector); +int ufsf_query_flag_retry(struct ufs_hba *hba, enum query_opcode opcode, + enum flag_idn idn, u8 idx, bool *flag_res); +int ufsf_query_attr_retry(struct ufs_hba *hba, enum query_opcode opcode, + enum attr_idn idn, u8 idx, u32 *attr_val); +int ufsf_get_scsi_device(struct ufs_hba *hba, struct scsi_device *sdev); +bool ufsf_is_valid_lun(int lun); +void ufsf_slave_configure(struct ufsf_feature *ufsf, struct scsi_device *sdev); +void ufsf_change_lun(struct ufsf_feature *ufsf, struct ufshcd_lrb *lrbp); + +void ufsf_prep_fn(struct ufsf_feature *ufsf, struct ufshcd_lrb *lrbp); +void ufsf_reset_lu(struct ufsf_feature *ufsf); +void ufsf_reset_host(struct ufsf_feature *ufsf); +void ufsf_init(struct ufsf_feature *ufsf); +void ufsf_reset(struct ufsf_feature *ufsf); +void ufsf_remove(struct ufsf_feature *ufsf); +void ufsf_set_init_state(struct ufsf_feature *ufsf); +void ufsf_suspend(struct ufsf_feature *ufsf); +void ufsf_resume(struct ufsf_feature *ufsf); + +/* for hpb */ +void ufsf_hpb_noti_rb(struct ufsf_feature *ufsf, struct ufshcd_lrb *lrbp); +#endif /* End of Header */ diff --git a/drivers/scsi/ufs/ufshcd.c b/drivers/scsi/ufs/ufshcd.c index 569d2efbf783..c66f206410bd 100644 --- a/drivers/scsi/ufs/ufshcd.c +++ b/drivers/scsi/ufs/ufshcd.c @@ -66,6 +66,7 @@ #define UIC_CMD_TIMEOUT 500 #endif +unsigned int storage_mfrid; /* NOP OUT retries waiting for NOP IN response */ #define NOP_OUT_RETRIES 10 /* Timeout after 30 msecs if NOP OUT hangs without response */ @@ -200,7 +201,21 @@ struct ufs_pm_lvl_states ufs_pm_lvl_states[] = { {UFS_POWERDOWN_PWR_MODE, UIC_LINK_HIBERN8_STATE}, {UFS_POWERDOWN_PWR_MODE, UIC_LINK_OFF_STATE}, }; - +#if defined(CONFIG_SCSI_SKHPB) +static inline int is_support_hpb_100_device(unsigned int mfrid){ + return IS_SKHYNIX_DEVICE(mfrid); +} +#endif +#if defined(CONFIG_UFSFEATURE) +static inline int is_support_hpb_200_device(unsigned int mfrid){ +#if defined(CONFIG_MICRON_HPB) + return (IS_SAMSUNG_DEVICE(mfrid) || IS_MICRON_DEVICE(mfrid)); +#else + return IS_SAMSUNG_DEVICE(mfrid); +#endif + return 0; +} +#endif static inline enum ufs_dev_pwr_mode ufs_get_pm_lvl_to_dev_pwr_mode(enum ufs_pm_level lvl) { @@ -255,7 +270,36 @@ static struct ufs_dev_fix ufs_fixups[] = { UFS_FIX(UFS_VENDOR_SAMSUNG, "KLUDG4UHDB-B2D1", UFS_DEVICE_QUIRK_PA_HIBER8TIME), #endif +#if defined(CONFIG_SCSI_SKHPB) + UFS_FIX(UFS_VENDOR_SKHYNIX, "H28S", + SKHPB_QUIRK_PURGE_HINT_INFO_WHEN_SLEEP), + UFS_FIX(UFS_VENDOR_SKHYNIX, "H9HQ15ACPMA", + SKHPB_QUIRK_PURGE_HINT_INFO_WHEN_SLEEP), + UFS_FIX(UFS_VENDOR_SKHYNIX, "H9HQ15AECMA", + SKHPB_QUIRK_PURGE_HINT_INFO_WHEN_SLEEP), + UFS_FIX(UFS_VENDOR_SKHYNIX, "H9HQ15AECMM", + SKHPB_QUIRK_PURGE_HINT_INFO_WHEN_SLEEP), + UFS_FIX(UFS_VENDOR_SKHYNIX, "H9HQ15AFAMA", + SKHPB_QUIRK_PURGE_HINT_INFO_WHEN_SLEEP), + UFS_FIX(UFS_VENDOR_SKHYNIX, "H9HQ15AFAMM", + SKHPB_QUIRK_PURGE_HINT_INFO_WHEN_SLEEP), + UFS_FIX(UFS_VENDOR_SKHYNIX, "H9HQ15AJAMM", + SKHPB_QUIRK_PURGE_HINT_INFO_WHEN_SLEEP), + + UFS_FIX(UFS_VENDOR_SKHYNIX, "H9HQ21AECMM", + SKHPB_QUIRK_PURGE_HINT_INFO_WHEN_SLEEP), + UFS_FIX(UFS_VENDOR_SKHYNIX, "H9HQ21AECMZ", + SKHPB_QUIRK_PURGE_HINT_INFO_WHEN_SLEEP), + UFS_FIX(UFS_VENDOR_SKHYNIX, "H9HQ21AFAMM", + SKHPB_QUIRK_PURGE_HINT_INFO_WHEN_SLEEP), + UFS_FIX(UFS_VENDOR_SKHYNIX, "H9HQ21AFAMZ", + SKHPB_QUIRK_PURGE_HINT_INFO_WHEN_SLEEP), + UFS_FIX(UFS_VENDOR_SKHYNIX, "H9HQ21AJAMM", + SKHPB_QUIRK_PURGE_HINT_INFO_WHEN_SLEEP), + UFS_FIX(UFS_VENDOR_SKHYNIX, "H9HQ21AHDMM", + SKHPB_QUIRK_PURGE_HINT_INFO_WHEN_SLEEP), +#endif END_FIX }; @@ -2563,9 +2607,23 @@ static int ufshcd_comp_scsi_upiu(struct ufs_hba *hba, struct ufshcd_lrb *lrbp) lrbp->command_type = UTP_CMD_TYPE_UFS_STORAGE; if (likely(lrbp->cmd)) { +#if defined(CONFIG_UFSFEATURE) + if (is_support_hpb_200_device(storage_mfrid)){ + ufsf_change_lun(&hba->ufsf, lrbp); + ufsf_prep_fn(&hba->ufsf, lrbp); + } +#endif ufshcd_prepare_req_desc_hdr(lrbp, &upiu_flags, lrbp->cmd->sc_data_direction); ufshcd_prepare_utp_scsi_cmd_upiu(lrbp, upiu_flags); +#if defined(CONFIG_SCSI_SKHPB) + if (is_support_hpb_100_device(storage_mfrid)){ + if (hba->skhpb_state == SKHPB_PRESENT && + hba->issue_ioctl == false) { + skhpb_prep_fn(hba, lrbp); + } + } +#endif } else { ret = -EINVAL; } @@ -2655,6 +2713,22 @@ static int ufshcd_queuecommand(struct Scsi_Host *host, struct scsi_cmnd *cmd) lrbp->req_abort_skip = false; ufshcd_comp_scsi_upiu(hba, lrbp); +#if defined(CONFIG_UFSFEATURE) && defined(CONFIG_UFSHPB) + if (is_support_hpb_200_device(storage_mfrid)){ + if (cmd->cmnd[0] != 0x28) + BUG_ON(cmd->requeue_cnt); + + if (cmd->requeue_cnt) + err = -EAGAIN; + + if (err) { + ufshcd_release(hba); + lrbp->cmd = NULL; + clear_bit_unlock(tag, &hba->lrb_in_use); + goto out; + } + } +#endif err = ufshcd_map_sg(hba, lrbp); if (err) { @@ -2897,8 +2971,13 @@ static inline void ufshcd_put_dev_cmd_tag(struct ufs_hba *hba, int tag) * NOTE: Since there is only one available tag for device management commands, * it is expected you hold the hba->dev_cmd.lock mutex. */ +#if defined(CONFIG_UFSFEATURE) +int ufshcd_exec_dev_cmd(struct ufs_hba *hba, + enum dev_cmd_type cmd_type, int timeout) +#else static int ufshcd_exec_dev_cmd(struct ufs_hba *hba, enum dev_cmd_type cmd_type, int timeout) +#endif { struct ufshcd_lrb *lrbp; int err; @@ -2967,8 +3046,13 @@ static inline void ufshcd_init_query(struct ufs_hba *hba, (*request)->upiu_req.selector = selector; } +#if defined(CONFIG_SCSI_SKHPB) +int ufshcd_query_flag_retry(struct ufs_hba *hba, + enum query_opcode opcode, enum flag_idn idn, u8 index, bool *flag_res) +#else static int ufshcd_query_flag_retry(struct ufs_hba *hba, enum query_opcode opcode, enum flag_idn idn, u8 index, bool *flag_res) +#endif { int ret; int retries; @@ -4952,13 +5036,23 @@ static int ufshcd_slave_configure(struct scsi_device *sdev) struct request_queue *q = sdev->request_queue; struct ufs_hba *hba = shost_priv(sdev->host); +#if defined(CONFIG_UFSFEATURE) + if (is_support_hpb_200_device(storage_mfrid)){ + ufsf_slave_configure(&hba->ufsf, sdev); + } +#endif blk_queue_update_dma_pad(q, PRDT_DATA_BYTE_COUNT_PAD - 1); ufshcd_crypto_setup_rq_keyslot_manager(hba, q); if (ufshcd_is_rpm_autosuspend_allowed(hba)) sdev->rpm_autosuspend = 1; - +#if defined(CONFIG_SCSI_SKHPB) + if (is_support_hpb_100_device(storage_mfrid)){ + if (sdev->lun < UFS_UPIU_MAX_GENERAL_LUN) + hba->sdev_ufs_lu[sdev->lun] = sdev; + } +#endif return 0; } @@ -5087,6 +5181,20 @@ ufshcd_transfer_rsp_status(struct ufs_hba *hba, struct ufshcd_lrb *lrbp) if (schedule_work(&hba->eeh_work)) pm_runtime_get_noresume(hba->dev); } + +#if defined(CONFIG_UFSFEATURE) + if (is_support_hpb_200_device(storage_mfrid)){ + if (scsi_status == SAM_STAT_GOOD) + ufsf_hpb_noti_rb(&hba->ufsf, lrbp); + } +#endif +#if defined(CONFIG_SCSI_SKHPB) + if (is_support_hpb_100_device(storage_mfrid)){ + if (hba->skhpb_state == SKHPB_PRESENT && + scsi_status == SAM_STAT_GOOD) + skhpb_rsp_upiu(hba, lrbp); + } +#endif break; case UPIU_TRANSACTION_REJECT_UPIU: /* TODO: handle Reject UPIU Response */ @@ -5188,6 +5296,13 @@ static void __ufshcd_transfer_req_compl(struct ufs_hba *hba, cmd = lrbp->cmd; ufshcd_vops_compl_xfer_req(hba, index, (cmd) ? true : false); if (cmd) { +#if defined(CONFIG_UFSFEATURE) && defined(CONFIG_UFSHPB) && defined(CONFIG_HPB_DEBUG) + if (is_support_hpb_200_device(storage_mfrid)) + trace_printk("%llu + %u cmd 0x%X comp tag[%d] out %lX\n", + (unsigned long long) blk_rq_pos(cmd->request), + (unsigned int) blk_rq_sectors(cmd->request), + cmd->cmnd[0], index, hba->outstanding_reqs); +#endif ufshcd_add_command_trace(hba, index, "complete"); result = ufshcd_transfer_rsp_status(hba, lrbp); scsi_dma_unmap(cmd); @@ -6732,6 +6847,18 @@ out: hba->req_abort_count = 0; ufshcd_update_reg_hist(&hba->ufs_stats.dev_reset, (u32)err); if (!err) { +#if defined(CONFIG_UFSFEATURE) + if (is_support_hpb_200_device(storage_mfrid)) + ufsf_reset_lu(&hba->ufsf); +#endif +#if defined(CONFIG_SCSI_SKHPB) + if (is_support_hpb_100_device(storage_mfrid)){ + if (hba->skhpb_state == SKHPB_PRESENT) + hba->skhpb_state = SKHPB_RESET; + schedule_delayed_work(&hba->skhpb_init_work, + msecs_to_jiffies(10)); + } +#endif err = SUCCESS; } else { dev_err(hba->dev, "%s: failed with err %d\n", __func__, err); @@ -6942,6 +7069,12 @@ static int ufshcd_host_reset_and_restore(struct ufs_hba *hba) int err; unsigned long flags; +#if defined(CONFIG_UFSFEATURE) + if (is_support_hpb_200_device(storage_mfrid)){ + ufsf_reset_host(&hba->ufsf); + } +#endif + /* * Stop the host controller and complete the requests * cleared by h/w @@ -7300,18 +7433,25 @@ static void ufshcd_wb_probe(struct ufs_hba *hba, u8 *desc_buf) */ if (!(dev_info->wspecversion >= 0x310 || dev_info->wspecversion == 0x220 || - (hba->dev_quirks & UFS_DEVICE_QUIRK_SUPPORT_EXTENDED_FEATURES))) + (hba->dev_quirks & UFS_DEVICE_QUIRK_SUPPORT_EXTENDED_FEATURES))) { + dev_err(hba->dev, "%s: vendor quirks don't support wb, wb_disable\n",__func__); goto wb_disabled; + } - if (hba->desc_size.dev_desc < DEVICE_DESC_PARAM_EXT_UFS_FEATURE_SUP + 4) + if (hba->desc_size.dev_desc < DEVICE_DESC_PARAM_EXT_UFS_FEATURE_SUP + 4) { + dev_err(hba->dev, "%s: ufs device don't support ext ufs feature, wb_disable\n", + __func__); goto wb_disabled; + } dev_info->d_ext_ufs_feature_sup = get_unaligned_be32(desc_buf + DEVICE_DESC_PARAM_EXT_UFS_FEATURE_SUP); - if (!(dev_info->d_ext_ufs_feature_sup & UFS_DEV_WRITE_BOOSTER_SUP)) + if (!(dev_info->d_ext_ufs_feature_sup & UFS_DEV_WRITE_BOOSTER_SUP)) { + dev_err(hba->dev, "%s: ufs device don't support write booster feature, wb_disable\n",__func__); goto wb_disabled; + } /* * WB may be supported but not configured while provisioning. @@ -7329,8 +7469,10 @@ static void ufshcd_wb_probe(struct ufs_hba *hba, u8 *desc_buf) dev_info->d_wb_alloc_units = get_unaligned_be32(desc_buf + DEVICE_DESC_PARAM_WB_SHARED_ALLOC_UNITS); - if (!dev_info->d_wb_alloc_units) + if (!dev_info->d_wb_alloc_units) { + dev_err(hba->dev, "%s: read wb_share_alloc_units is 0 , wb_disable\n",__func__); goto wb_disabled; + } } else { for (lun = 0; lun < UFS_UPIU_MAX_WB_LUN_ID; lun++) { d_lu_wb_buf_alloc = 0; @@ -7345,8 +7487,10 @@ static void ufshcd_wb_probe(struct ufs_hba *hba, u8 *desc_buf) } } - if (!d_lu_wb_buf_alloc) + if (!d_lu_wb_buf_alloc) { + dev_err(hba->dev, "%s: read wb_share_alloc_units is 0 , wb_disable\n",__func__); goto wb_disabled; + } } return; @@ -7799,6 +7943,13 @@ static int ufshcd_add_lus(struct ufs_hba *hba) ufs_bsg_probe(hba); scsi_scan_host(hba->host); +#if defined(CONFIG_UFSFEATURE) + if (is_support_hpb_200_device(storage_mfrid)){ + ufsf_device_check(hba); + ufsf_init(&hba->ufsf); + } +#endif + out: return ret; } @@ -7918,10 +8069,20 @@ reinit: ufshcd_set_active_icc_lvl(hba); ufshcd_wb_config(hba); + +#if defined(CONFIG_SCSI_SKHPB) + if (is_support_hpb_100_device(storage_mfrid)) + schedule_delayed_work(&hba->skhpb_init_work, 0); +#endif /* Enable Auto-Hibernate if configured */ ufshcd_auto_hibern8_enable(hba); out: +#if defined(CONFIG_UFSFEATURE) + if (is_support_hpb_200_device(storage_mfrid)) + ufsf_reset(&hba->ufsf); +#endif + spin_lock_irqsave(hba->host->host_lock, flags); if (ret) hba->ufshcd_state = UFSHCD_STATE_ERROR; @@ -8783,6 +8944,11 @@ static int ufshcd_suspend(struct ufs_hba *hba, enum ufs_pm_op pm_op) req_link_state = UIC_LINK_OFF_STATE; } +#if defined(CONFIG_UFSFEATURE) + if (is_support_hpb_200_device(storage_mfrid)) + ufsf_suspend(&hba->ufsf); +#endif + ret = ufshcd_crypto_suspend(hba, pm_op); if (ret) goto out; @@ -8793,6 +8959,10 @@ static int ufshcd_suspend(struct ufs_hba *hba, enum ufs_pm_op pm_op) */ ufshcd_hold(hba, false); hba->clk_gating.is_suspended = true; +#if defined(CONFIG_SCSI_SKHPB) + if(is_support_hpb_100_device(storage_mfrid)) + skhpb_suspend(hba); +#endif if (hba->clk_scaling.is_allowed) { cancel_work_sync(&hba->clk_scaling.suspend_work); @@ -8917,6 +9087,11 @@ enable_gating: hba->dev_info.b_rpm_dev_flush_capable = false; ufshcd_release(hba); ufshcd_crypto_resume(hba, pm_op); +#if defined(CONFIG_UFSFEATURE) + if (is_support_hpb_200_device(storage_mfrid)){ + ufsf_resume(&hba->ufsf); + } +#endif out: if (hba->dev_info.b_rpm_dev_flush_capable) { schedule_delayed_work(&hba->rpm_dev_flush_recheck_work, @@ -9059,6 +9234,15 @@ static int ufshcd_resume(struct ufs_hba *hba, enum ufs_pm_op pm_op) cancel_delayed_work(&hba->rpm_dev_flush_recheck_work); } +#if defined(CONFIG_UFSFEATURE) + if (is_support_hpb_200_device(storage_mfrid)) + ufsf_resume(&hba->ufsf); +#endif +#if defined(CONFIG_SCSI_SKHPB) + if (is_support_hpb_100_device(storage_mfrid)) + skhpb_resume(hba); +#endif + /* Schedule clock gating in case of no access to UFS device yet */ ufshcd_release(hba); @@ -9313,8 +9497,20 @@ EXPORT_SYMBOL(ufshcd_shutdown); */ void ufshcd_remove(struct ufs_hba *hba) { +#if defined(CONFIG_UFSFEATURE) + if (is_support_hpb_200_device(storage_mfrid)){ + ufsf_remove(&hba->ufsf); + } +#endif ufs_bsg_remove(hba); ufs_sysfs_remove_nodes(hba->dev); + +#if defined(CONFIG_SCSI_SKHPB) + if (is_support_hpb_100_device(storage_mfrid)){ + skhpb_release(hba, SKHPB_NEED_INIT); + } +#endif + scsi_remove_host(hba->host); destroy_workqueue(hba->eh_wq); /* disable interrupts */ @@ -9586,6 +9782,17 @@ int ufshcd_init(struct ufs_hba *hba, void __iomem *mmio_base, unsigned int irq) */ ufshcd_set_ufs_dev_active(hba); +#if defined(CONFIG_UFSFEATURE) + if (is_support_hpb_200_device(storage_mfrid)){ + ufsf_set_init_state(&hba->ufsf); + } +#endif +#if defined(CONFIG_SCSI_SKHPB) + if (is_support_hpb_100_device(storage_mfrid)){ + /* initialize hpb structures */ + ufshcd_init_hpb(hba); + } +#endif async_schedule(ufshcd_async_scan, hba); ufs_sysfs_add_nodes(hba->dev); @@ -9603,6 +9810,12 @@ out_disable: out_error: return err; } +static int __init storage_mfrid_setup(char *str) +{ + storage_mfrid = simple_strtol(str, NULL, 16); + return 1; +} +__setup("storage_mfrid=",storage_mfrid_setup); EXPORT_SYMBOL_GPL(ufshcd_init); MODULE_AUTHOR("Santosh Yaragnavi "); diff --git a/drivers/scsi/ufs/ufshcd.h b/drivers/scsi/ufs/ufshcd.h index d9003a0b12a4..ce44b1db6e5b 100644 --- a/drivers/scsi/ufs/ufshcd.h +++ b/drivers/scsi/ufs/ufshcd.h @@ -73,10 +73,22 @@ #include "ufs_quirks.h" #include "ufshci.h" +#if defined(CONFIG_UFSFEATURE) +#include "ufsfeature.h" +#endif + +#if defined(CONFIG_SCSI_SKHPB) +#include "ufshpb_skh.h" +#endif + #define UFSHCD "ufshcd" #define UFSHCD_DRIVER_VERSION "0.2" #define UFSHCD_QUIRK_BROKEN_AUTO_HIBERN8 0x40000 +extern unsigned int storage_mfrid; +#define IS_SAMSUNG_DEVICE(mfrid) (UFS_VENDOR_SAMSUNG == (mfrid)) +#define IS_SKHYNIX_DEVICE(mfrid) (UFS_VENDOR_SKHYNIX == (mfrid)) +#define IS_MICRON_DEVICE(mfrid) (UFS_VENDOR_MICRON == (mfrid)) struct ufs_hba; @@ -1016,6 +1028,24 @@ struct ufs_hba { struct device bsg_dev; struct request_queue *bsg_queue; +#if defined(CONFIG_SCSI_SKHPB) + /* HPB support */ + u32 skhpb_feat; + int skhpb_state; + int skhpb_max_regions; + struct delayed_work skhpb_init_work; + bool issue_ioctl; + struct skhpb_lu *skhpb_lup[UFS_UPIU_MAX_GENERAL_LUN]; + struct work_struct skhpb_eh_work; + u32 skhpb_quirk; + u8 hpb_control_mode; +#define SKHPB_U8_MAX 0xFF + u8 skhpb_quicklist_lu_enable[UFS_UPIU_MAX_GENERAL_LUN]; +#endif + +#if defined(CONFIG_SCSI_SKHPB) + struct scsi_device *sdev_ufs_lu[UFS_UPIU_MAX_GENERAL_LUN]; +#endif #ifdef CONFIG_SCSI_UFS_CRYPTO /* crypto */ union ufs_crypto_capabilities crypto_capabilities; @@ -1028,6 +1058,10 @@ struct ufs_hba { bool wb_buf_flush_enabled; bool wb_enabled; struct delayed_work rpm_dev_flush_recheck_work; + +#if defined(CONFIG_UFSFEATURE) + struct ufsf_feature ufsf; +#endif ANDROID_KABI_RESERVE(1); ANDROID_KABI_RESERVE(2); ANDROID_KABI_RESERVE(3); @@ -1123,7 +1157,11 @@ static inline bool ufshcd_is_auto_hibern8_enabled(struct ufs_hba *hba) static inline bool ufshcd_is_wb_allowed(struct ufs_hba *hba) { +#if defined(CONFIG_UFSTW) + return false; +#else return hba->caps & UFSHCD_CAP_WB_EN; +#endif } #define ufshcd_writel(hba, val, reg) \ @@ -1297,6 +1335,10 @@ void ufshcd_fixup_dev_quirks(struct ufs_hba *hba, struct ufs_dev_fix *fixups); #define SD_ASCII_STD true #define SD_RAW false +#if defined(CONFIG_UFSFEATURE) +int ufshcd_exec_dev_cmd(struct ufs_hba *hba, + enum dev_cmd_type cmd_type, int timeout); +#endif int ufshcd_hold(struct ufs_hba *hba, bool async); void ufshcd_release(struct ufs_hba *hba); @@ -1500,6 +1542,10 @@ static inline u8 ufshcd_scsi_to_upiu_lun(unsigned int scsi_lun) return scsi_lun & UFS_UPIU_MAX_UNIT_NUM_ID; } +#if defined(CONFIG_SCSI_SKHPB) +int ufshcd_query_flag_retry(struct ufs_hba *hba, + enum query_opcode opcode, enum flag_idn idn, u8 index, bool *flag_res); +#endif int ufshcd_dump_regs(struct ufs_hba *hba, size_t offset, size_t len, const char *prefix); diff --git a/drivers/scsi/ufs/ufshpb.c b/drivers/scsi/ufs/ufshpb.c new file mode 100755 index 000000000000..1659a648a846 --- /dev/null +++ b/drivers/scsi/ufs/ufshpb.c @@ -0,0 +1,4099 @@ +/* + * Universal Flash Storage Host Performance Booster + * + * Copyright (C) 2017-2018 Samsung Electronics Co., Ltd. + * + * Authors: + * Yongmyung Lee + * Jinyoung Choi + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * See the COPYING file in the top-level directory or visit + * + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * This program is provided "AS IS" and "WITH ALL FAULTS" and + * without warranty of any kind. You are solely responsible for + * determining the appropriateness of using and distributing + * the program and assume all risks associated with your exercise + * of rights with respect to the program, including but not limited + * to infringement of third party rights, the risks and costs of + * program errors, damage to or loss of data, programs or equipment, + * and unavailability or interruption of operations. Under no + * circumstances will the contributor of this Program be liable for + * any damages of any kind arising from your use or distribution of + * this program. + * + * The Linux Foundation chooses to take subject only to the GPLv2 + * license terms, and distributes only under these terms. + */ + +#include "ufshcd.h" +#include "ufshpb.h" + +#define UFSHCD_REQ_SENSE_SIZE 18 + +static int ufshpb_create_sysfs(struct ufsf_feature *ufsf, + struct ufshpb_lu *hpb); +static void ufshpb_remove_sysfs(struct ufshpb_lu *hpb); + + +static inline void +ufshpb_get_pos_from_lpn(struct ufshpb_lu *hpb, unsigned long lpn, int *rgn_idx, + int *srgn_idx, int *offset) +{ + int rgn_offset; + + *rgn_idx = lpn >> hpb->entries_per_rgn_shift; + rgn_offset = lpn & hpb->entries_per_rgn_mask; + *srgn_idx = rgn_offset >> hpb->entries_per_srgn_shift; + *offset = rgn_offset & hpb->entries_per_srgn_mask; +} + +inline int ufshpb_valid_srgn(struct ufshpb_region *rgn, + struct ufshpb_subregion *srgn) +{ + return rgn->rgn_state != HPB_RGN_INACTIVE && + srgn->srgn_state == HPB_SRGN_CLEAN; +} + +/* Must be held hpb_lock */ +static bool ufshpb_ppn_dirty_check(struct ufshpb_lu *hpb, unsigned long lpn, + int transfer_len) +{ + struct ufshpb_region *rgn; + struct ufshpb_subregion *srgn; + unsigned long cur_lpn = lpn; + int rgn_idx, srgn_idx, srgn_offset, find_size; + int scan_cnt = transfer_len; + + do { + ufshpb_get_pos_from_lpn(hpb, cur_lpn, &rgn_idx, &srgn_idx, + &srgn_offset); + rgn = hpb->rgn_tbl + rgn_idx; + srgn = rgn->srgn_tbl + srgn_idx; + + if (!ufshpb_valid_srgn(rgn, srgn)) + return true; + + if (unlikely(!srgn->mctx || !srgn->mctx->ppn_dirty)) + return true; + + if (hpb->entries_per_srgn < srgn_offset + scan_cnt) { + int cnt = hpb->entries_per_srgn - srgn_offset; + + find_size = hpb->entries_per_srgn; + scan_cnt -= cnt; + cur_lpn += cnt; + } else { + find_size = srgn_offset + scan_cnt; + scan_cnt = 0; + } + + srgn_offset = + find_next_bit((unsigned long *)srgn->mctx->ppn_dirty, + hpb->entries_per_srgn, srgn_offset); + + if (srgn_offset < hpb->entries_per_srgn) + return srgn_offset < find_size; + + } while (scan_cnt); + + return false; +} + +static void ufshpb_set_read16_cmd(struct ufshpb_lu *hpb, struct scsi_cmnd *cmd, + u64 ppn, unsigned int transfer_len, + int hpb_ctx_id) +{ + unsigned char *cdb = cmd->cmnd; + + cdb[0] = READ_16; + cdb[2] = cmd->cmnd[2]; + cdb[3] = cmd->cmnd[3]; + cdb[4] = cmd->cmnd[4]; + cdb[5] = cmd->cmnd[5]; + cdb[6] = GET_BYTE_7(ppn); + cdb[7] = GET_BYTE_6(ppn); + cdb[8] = GET_BYTE_5(ppn); + cdb[9] = GET_BYTE_4(ppn); + cdb[10] = GET_BYTE_3(ppn); + cdb[11] = GET_BYTE_2(ppn); + cdb[12] = GET_BYTE_1(ppn); + cdb[13] = GET_BYTE_0(ppn); + + if (hpb_ctx_id < MAX_HPB_CONTEXT_ID) + cdb[14] = (1 << 7) | hpb_ctx_id; + else + cdb[14] = UFSHPB_GROUP_NUMBER; + + cdb[15] = transfer_len; + + cmd->cmd_len = UFS_CDB_SIZE; +} + +/* called with hpb_lock (irq) */ +static inline void +ufshpb_set_dirty_bits(struct ufshpb_lu *hpb, struct ufshpb_region *rgn, + struct ufshpb_subregion *srgn, int dword, int offset, + unsigned int cnt) +{ + const unsigned long mask = ((1UL << cnt) - 1) & 0xffffffff; + + if (rgn->rgn_state == HPB_RGN_INACTIVE) + return; + + BUG_ON(!srgn->mctx); + srgn->mctx->ppn_dirty[dword] |= (mask << offset); +} + +static inline void ufshpb_get_bit_offset(struct ufshpb_lu *hpb, int srgn_offset, + int *dword, int *offset) +{ + *dword = srgn_offset >> bits_per_dword_shift; + *offset = srgn_offset & bits_per_dword_mask; +} + +static void ufshpb_set_dirty(struct ufshpb_lu *hpb, struct ufshcd_lrb *lrbp, + int rgn_idx, int srgn_idx, int srgn_offset) +{ + struct ufshpb_region *rgn; + struct ufshpb_subregion *srgn; + int cnt, bit_cnt, bit_dword, bit_offset; + + cnt = blk_rq_sectors(lrbp->cmd->request) >> sects_per_blk_shift; + ufshpb_get_bit_offset(hpb, srgn_offset, &bit_dword, &bit_offset); + + do { + bit_cnt = min(cnt, BITS_PER_DWORD - bit_offset); + + rgn = hpb->rgn_tbl + rgn_idx; + srgn = rgn->srgn_tbl + srgn_idx; + + ufshpb_set_dirty_bits(hpb, rgn, srgn, bit_dword, bit_offset, + bit_cnt); + + bit_offset = 0; + bit_dword++; + + if (bit_dword == hpb->dwords_per_srgn) { + bit_dword = 0; + srgn_idx++; + + if (srgn_idx == hpb->srgns_per_rgn) { + srgn_idx = 0; + rgn_idx++; + } + } + cnt -= bit_cnt; + } while (cnt); + + BUG_ON(cnt < 0); +} + +static inline bool ufshpb_is_read_cmd(struct scsi_cmnd *cmd) +{ + return (cmd->cmnd[0] == READ_10 || cmd->cmnd[0] == READ_16); +} + +static inline bool ufshpb_is_write_cmd(struct scsi_cmnd *cmd) +{ + return (cmd->cmnd[0] == WRITE_10 || cmd->cmnd[0] == WRITE_16); +} + +static inline bool ufshpb_is_discard_cmd(struct scsi_cmnd *cmd) +{ + return cmd->cmnd[0] == UNMAP; +} + +static u64 ufshpb_get_ppn(struct ufshpb_map_ctx *mctx, int pos, int *error) +{ + u64 *ppn_table; + struct page *page = NULL; + int index, offset; + + index = pos / HPB_ENTREIS_PER_OS_PAGE; + offset = pos % HPB_ENTREIS_PER_OS_PAGE; + + page = mctx->m_page[index]; + if (unlikely(!page)) { + *error = -ENOMEM; + ERR_MSG("mctx %p cannot get m_page", mctx); + return 0; + } + + ppn_table = page_address(page); + if (unlikely(!ppn_table)) { + *error = -ENOMEM; + ERR_MSG("mctx %p cannot get ppn_table vm", mctx); + return 0; + } + + return ppn_table[offset]; +} + +#if defined(CONFIG_HPB_ERR_INJECTION) +static u64 ufshpb_get_bitflip_ppn(struct ufshpb_lu *hpb, u64 ppn) +{ + return (ppn ^ hpb->err_injection_bitflip); +} + +static u64 ufshpb_get_offset_ppn(struct ufshpb_lu *hpb, u64 ppn) +{ + return (ppn + hpb->err_injection_offset); +} + +static u64 ufshpb_get_random_ppn(struct ufshpb_lu *hpb, u64 ppn) +{ + u64 random_ppn = 0; + + if (!hpb->err_injection_random) + return ppn; + + get_random_bytes(&random_ppn, sizeof(random_ppn)); + + return random_ppn; +} + +static u64 ufshpb_get_err_injection_ppn(struct ufshpb_lu *hpb, u64 ppn) +{ + u64 new_ppn = 0; + + if (hpb->err_injection_select == HPB_ERR_INJECTION_BITFLIP) + new_ppn = ufshpb_get_bitflip_ppn(hpb, ppn); + else if (hpb->err_injection_select == HPB_ERR_INJECTION_OFFSET) + new_ppn = ufshpb_get_offset_ppn(hpb, ppn); + else if (hpb->err_injection_select == HPB_ERR_INJECTION_RANDOM) + new_ppn = ufshpb_get_random_ppn(hpb, ppn); + + return new_ppn; +} +#endif + +inline int ufshpb_get_state(struct ufsf_feature *ufsf) +{ + return atomic_read(&ufsf->hpb_state); +} + +inline void ufshpb_set_state(struct ufsf_feature *ufsf, int state) +{ + atomic_set(&ufsf->hpb_state, state); +} + +static inline int ufshpb_lu_get(struct ufshpb_lu *hpb) +{ + if (!hpb) + return -ENODEV; + + if (ufshpb_get_state(hpb->ufsf) != HPB_PRESENT) + return -EBUSY; + + kref_get(&hpb->ufsf->hpb_kref); + return 0; +} + +static inline void ufshpb_schedule_error_handler(struct kref *kref) +{ + struct ufsf_feature *ufsf; + + ufsf = container_of(kref, struct ufsf_feature, hpb_kref); + schedule_work(&ufsf->hpb_eh_work); +} + +static inline void ufshpb_lu_put(struct ufshpb_lu *hpb) +{ + kref_put(&hpb->ufsf->hpb_kref, ufshpb_schedule_error_handler); +} + +static void ufshpb_failed(struct ufshpb_lu *hpb, const char *f) +{ + ERR_MSG("ufshpb_driver failed. function (%s)", f); + ufshpb_set_state(hpb->ufsf, HPB_FAILED); + ufshpb_lu_put(hpb); +} + +/* + * end_io() is called with held the queue lock. + * so, blk_put_request is not available directly. + */ +static void ufshpb_put_blk_request(struct request *req) +{ + struct request_queue *q = req->q; + + if (q->mq_ops) { + blk_mq_free_request(req); + } else { + lockdep_assert_held(&q->queue_lock); + blk_put_request(req); + } +} + +static inline void ufshpb_put_pre_req(struct ufshpb_lu *hpb, + struct ufshpb_req *pre_req) +{ + pre_req->req = NULL; + pre_req->bio = NULL; + list_add_tail(&pre_req->list_req, &hpb->lh_pre_req_free); + hpb->num_inflight_pre_req--; +} + +static struct ufshpb_req *ufshpb_get_pre_req(struct ufshpb_lu *hpb) +{ + struct ufshpb_req *pre_req; + + if (hpb->num_inflight_pre_req >= hpb->throttle_pre_req) { + HPB_FTRACE(hpb, "inflight %d throttle %d", + hpb->num_inflight_pre_req, hpb->throttle_pre_req); + return NULL; + } + + pre_req = list_first_entry_or_null(&hpb->lh_pre_req_free, + struct ufshpb_req, list_req); + if (unlikely(!pre_req)) { + HPB_DEBUG(hpb, "no pre_req"); + return NULL; + } + + list_del_init(&pre_req->list_req); + hpb->num_inflight_pre_req++; + + return pre_req; +} + +static void ufshpb_pre_req_compl_fn(struct request *req, blk_status_t error) +{ + struct ufshpb_req *pre_req = (struct ufshpb_req *)req->end_io_data; + struct ufshpb_lu *hpb = pre_req->hpb; + unsigned long flags; + struct scsi_sense_hdr sshdr; + + if (error) { + ERR_MSG("block status (%d)", error); + scsi_normalize_sense(pre_req->sense, SCSI_SENSE_BUFFERSIZE, + &sshdr); + ERR_MSG("code %x sense_key %x asc %x ascq %x", + sshdr.response_code, + sshdr.sense_key, sshdr.asc, sshdr.ascq); + ERR_MSG("byte4 %x byte5 %x byte6 %x additional_len %x", + sshdr.byte4, sshdr.byte5, + sshdr.byte6, sshdr.additional_length); + } + + bio_put(pre_req->bio); + ufshpb_put_blk_request(pre_req->req); + spin_lock_irqsave(&hpb->hpb_lock, flags); + ufshpb_put_pre_req(pre_req->hpb, pre_req); + spin_unlock_irqrestore(&hpb->hpb_lock, flags); + + ufshpb_lu_put(pre_req->hpb); +} + +static int ufshpb_prep_entry(struct ufshpb_req *pre_req, struct page *page) +{ + struct ufshpb_lu *hpb = pre_req->hpb; + struct ufshpb_region *rgn; + struct ufshpb_subregion *srgn; + u64 *addr; + u64 entry_ppn = 0; + unsigned long lpn = pre_req->wb.lpn; + int rgn_idx, srgn_idx, srgn_offset; + int i, error = 0; + unsigned long flags; + + addr = page_address(page); + + spin_lock_irqsave(&hpb->hpb_lock, flags); + for (i = 0; i < pre_req->wb.len; i++, lpn++) { + ufshpb_get_pos_from_lpn(hpb, lpn, &rgn_idx, &srgn_idx, + &srgn_offset); + + rgn = hpb->rgn_tbl + rgn_idx; + srgn = rgn->srgn_tbl + srgn_idx; + + if (!ufshpb_valid_srgn(rgn, srgn)) + goto mctx_error; + + BUG_ON(!srgn->mctx); + + entry_ppn = ufshpb_get_ppn(srgn->mctx, srgn_offset, &error); + if (error) + goto mctx_error; + +#if defined(CONFIG_HPB_ERR_INJECTION) + if (hpb->err_injection_select != HPB_ERR_INJECTION_DISABLE) + entry_ppn = + ufshpb_get_err_injection_ppn(hpb, entry_ppn); +#endif + + addr[i] = entry_ppn; + } + spin_unlock_irqrestore(&hpb->hpb_lock, flags); + return 0; +mctx_error: + spin_unlock_irqrestore(&hpb->hpb_lock, flags); + return -ENOMEM; +} + +static int ufshpb_pre_req_add_bio_page(struct request_queue *q, + struct ufshpb_req *pre_req) +{ + struct page *page = pre_req->wb.m_page; + struct bio *bio = pre_req->bio; + int entries_bytes, ret; + + BUG_ON(!page); + + ret = ufshpb_prep_entry(pre_req, page); + if (ret) + return ret; + + entries_bytes = pre_req->wb.len * sizeof(u64); + + ret = bio_add_pc_page(q, bio, page, entries_bytes, 0); + if (ret != entries_bytes) { + ERR_MSG("bio_add_pc_page fail (%d)", ret); + return -ENOMEM; + } + + return 0; +} + +static inline unsigned long ufshpb_get_lpn(struct request *rq) +{ + return blk_rq_pos(rq) / SECTORS_PER_BLOCK; +} + +static inline unsigned int ufshpb_get_len(struct request *rq) +{ + return blk_rq_sectors(rq) / SECTORS_PER_BLOCK; +} + +static inline unsigned int ufshpb_is_unaligned(struct request *rq) +{ + return blk_rq_sectors(rq) % SECTORS_PER_BLOCK; +} + +static inline int ufshpb_issue_ctx_id_ticket(struct ufshpb_lu *hpb) +{ + int hpb_ctx_id; + + hpb->ctx_id_ticket++; + if (hpb->ctx_id_ticket >= MAX_HPB_CONTEXT_ID) + hpb->ctx_id_ticket = 0; + hpb_ctx_id = hpb->ctx_id_ticket; + + return hpb_ctx_id; +} + +static inline void ufshpb_set_prefetch_cmd(unsigned char *cdb, + unsigned long lpn, unsigned int len, + int hpb_ctx_id) +{ + int len_byte = len * HPB_ENTRY_SIZE; + + cdb[0] = UFSHPB_WRITE_BUFFER; + cdb[1] = UFSHPB_WB_ID_PREFETCH; + cdb[2] = GET_BYTE_3(lpn); + cdb[3] = GET_BYTE_2(lpn); + cdb[4] = GET_BYTE_1(lpn); + cdb[5] = GET_BYTE_0(lpn); + cdb[6] = (1 << 7) | hpb_ctx_id; + cdb[7] = GET_BYTE_1(len_byte); + cdb[8] = GET_BYTE_0(len_byte); + cdb[9] = 0x00; /* Control = 0x00 */ +} + +static void ufshpb_hctx_unlock_mimic(struct blk_mq_hw_ctx *hctx, int srcu_idx) +__releases(hctx->srcu) +{ + if (!(hctx->flags & BLK_MQ_F_BLOCKING)) + rcu_read_unlock(); + else + srcu_read_unlock(hctx->srcu, srcu_idx); +} + +static void ufshpb_hctx_lock_mimic(struct blk_mq_hw_ctx *hctx, int *srcu_idx) +__acquires(hctx->srcu) +{ + if (!(hctx->flags & BLK_MQ_F_BLOCKING)) { + /* shut up gcc false positive */ + *srcu_idx = 0; + rcu_read_lock(); + } else + *srcu_idx = srcu_read_lock(hctx->srcu); +} + +static int ufshpb_blk_mq_dispatch_rq_list_mimic(struct blk_mq_hw_ctx *hctx, + struct request *rq) +{ + struct request_queue *q = rq->q; + struct blk_mq_queue_data bd; + blk_status_t ret = 0; + + if (!blk_mq_get_dispatch_budget(hctx)) + return -EAGAIN; + + if (!blk_mq_get_driver_tag(rq)) { + blk_mq_put_dispatch_budget(hctx); + return -EAGAIN; + } + + bd.rq = rq; + + ret = q->mq_ops->queue_rq(hctx, &bd); + if (ret == BLK_STS_RESOURCE || ret == BLK_STS_DEV_RESOURCE) { + /* + * If an I/O scheduler has been configured and we got a + * driver tag for the next request already, free it + * again. + */ + return -EIO; + } + + if (unlikely(ret != BLK_STS_OK)) + return -EIO; + + return (int)ret; +} + +static int ufshpb_blk_mq_sched_dispatch_requests(struct blk_mq_hw_ctx *hctx, + struct request *rq) +{ + int ret; + + /* RCU or SRCU read lock is needed before checking quiesced flag */ + if (unlikely(blk_mq_hctx_stopped(hctx) || blk_queue_quiesced(rq->q))) + return -EPERM; + + hctx->run++; + + ret = ufshpb_blk_mq_dispatch_rq_list_mimic(hctx, rq); + + return ret; +} + +static int ufshpb_blk_mq_run_hw_queue_mimic(struct blk_mq_hw_ctx *hctx, + struct request *rq) +{ + int srcu_idx; + int ret = 0; + + /* + * We should be running this queue from one of the CPUs that + * are mapped to it. + * + * There are at least two related races now between setting + * hctx->next_cpu from blk_mq_hctx_next_cpu() and running + * __blk_mq_run_hw_queue(): + * + * - hctx->next_cpu is found offline in blk_mq_hctx_next_cpu(), + * but later it becomes online, then this warning is harmless + * at all + * + * - hctx->next_cpu is found online in blk_mq_hctx_next_cpu(), + * but later it becomes offline, then the warning can't be + * triggered, and we depend on blk-mq timeout handler to + * handle dispatched requests to this hctx + */ + if (!cpumask_test_cpu(raw_smp_processor_id(), hctx->cpumask) && + cpu_online(hctx->next_cpu)) { + printk(KERN_WARNING "run queue from wrong CPU %d, hctx %s\n", + raw_smp_processor_id(), + cpumask_empty(hctx->cpumask) ? "inactive" : "active"); + dump_stack(); + } + + /* + * We can't run the queue inline with ints disabled. Ensure that + * we catch bad users of this early. + */ + WARN_ON_ONCE(in_interrupt()); + + if (hctx->flags & BLK_MQ_F_BLOCKING) + return -EPERM; + + ufshpb_hctx_lock_mimic(hctx, &srcu_idx); + ret = ufshpb_blk_mq_sched_dispatch_requests(hctx, rq); + ufshpb_hctx_unlock_mimic(hctx, srcu_idx); + + return ret; +} + +static int ufshpb_insert_pre_req_directly(struct request *rq) +{ + struct blk_mq_hw_ctx *hctx = rq->mq_hctx; + int cpu = get_cpu(); + int ret = 0; + + if (likely(cpumask_test_cpu(cpu, hctx->cpumask))) { + ret = ufshpb_blk_mq_run_hw_queue_mimic(hctx, rq); + put_cpu(); + return ret; + } + put_cpu(); + return -EPERM; +} + +static int ufshpb_blk_execute_rq_nowait_mimic(struct request_queue *q, + struct request *rq, + rq_end_io_fn *done) +{ + int ret = 0; + + WARN_ON(irqs_disabled()); + WARN_ON(!blk_rq_is_passthrough(rq)); + + rq->rq_disk = NULL; + rq->end_io = done; + + ret = ufshpb_insert_pre_req_directly(rq); + return ret; +} + +static int ufshpb_execute_pre_req(struct ufshpb_lu *hpb, struct scsi_cmnd *cmd, + struct ufshpb_req *pre_req, int hpb_ctx_id) +{ + struct scsi_device *sdev = cmd->device; + struct request_queue *q = sdev->request_queue; + struct request *req; + struct scsi_request *rq; + struct bio *bio = pre_req->bio; + int ret = 0; + + pre_req->hpb = hpb; + pre_req->wb.lpn = ufshpb_get_lpn(cmd->request); + pre_req->wb.len = ufshpb_get_len(cmd->request); + ret = ufshpb_pre_req_add_bio_page(q, pre_req); + if (ret) + return ret; + + req = pre_req->req; + + /* 1. request setup */ + blk_rq_append_bio(req, &bio); + req->rq_flags |= RQF_QUIET; + req->timeout = 30 * HZ; + req->end_io_data = (void *)pre_req; + + /* 2. scsi_request setup */ + rq = scsi_req(req); + rq->retries = 1; + ufshpb_set_prefetch_cmd(rq->cmd, pre_req->wb.lpn, pre_req->wb.len, + hpb_ctx_id); + rq->cmd_len = scsi_command_size(rq->cmd); + + ret = ufshpb_blk_execute_rq_nowait_mimic(q, req, + ufshpb_pre_req_compl_fn); + + if (!ret) + atomic64_inc(&hpb->pre_req_cnt); + + return ret; +} + +static int ufshpb_issue_pre_req(struct ufshpb_lu *hpb, struct scsi_cmnd *cmd, + int *hpb_ctx_id) +{ + struct ufshpb_req *pre_req; + struct request *req = NULL; + struct bio *bio = NULL; + unsigned long flags; + int ctx_id; + int ret = 0; + + req = blk_get_request(cmd->device->request_queue, + REQ_OP_SCSI_OUT | REQ_SYNC, + BLK_MQ_REQ_PREEMPT | BLK_MQ_REQ_NOWAIT); + if (IS_ERR(req)) + return -EAGAIN; + + bio = bio_alloc(GFP_ATOMIC, 1); + if (!bio) { + blk_put_request(req); + return -EAGAIN; + } + + spin_lock_irqsave(&hpb->hpb_lock, flags); + pre_req = ufshpb_get_pre_req(hpb); + if (!pre_req) { + ret = -EAGAIN; + goto unlock_out; + } + ctx_id = ufshpb_issue_ctx_id_ticket(hpb); + spin_unlock_irqrestore(&hpb->hpb_lock, flags); + + pre_req->req = req; + pre_req->bio = bio; + +#if defined(CONFIG_HPB_DEBUG) + { + unsigned int start_lpn, end_lpn; + int s_rgn_idx, s_srgn_idx, s_offset; + int e_rgn_idx, e_srgn_idx, e_offset; + + start_lpn = ufshpb_get_lpn(cmd->request); + end_lpn = start_lpn + ufshpb_get_len(cmd->request) - 1; + + ufshpb_get_pos_from_lpn(hpb, start_lpn, &s_rgn_idx, &s_srgn_idx, + &s_offset); + ufshpb_get_pos_from_lpn(hpb, end_lpn, &e_rgn_idx, &e_srgn_idx, + &e_offset); + HPB_FTRACE(hpb, "R[%05d](%d) - R[%05d](%d)", + s_rgn_idx, hpb->rgn_tbl[s_rgn_idx].rgn_state, + e_rgn_idx, hpb->rgn_tbl[e_rgn_idx].rgn_state); + } +#endif + + ret = ufshpb_lu_get(hpb); + if (unlikely(ret)) { + if (ret == -ENODEV) + ERR_MSG("lu_get failed (%d)", ret); + HPB_DEBUG(hpb, "lu_get failed (%d)", ret); + goto free_pre_req; + } + + ret = ufshpb_execute_pre_req(hpb, cmd, pre_req, ctx_id); + if (ret) { + ufshpb_lu_put(hpb); + goto free_pre_req; + } + + *hpb_ctx_id = ctx_id; + + return ret; +free_pre_req: + spin_lock_irqsave(&hpb->hpb_lock, flags); + ufshpb_put_pre_req(hpb, pre_req); +unlock_out: + spin_unlock_irqrestore(&hpb->hpb_lock, flags); + bio_put(bio); + blk_put_request(req); + return ret; +} + +static inline bool ufshpb_is_support_chunk(int transfer_len) +{ + return transfer_len <= HPB_MULTI_CHUNK_HIGH; +} + +static int ufshpb_fill_passthrough_rq(struct scsi_cmnd *cmd) +{ + struct request *rq = cmd->request; + struct scsi_device *sdp = cmd->device; + struct bio *bio = rq->bio; + sector_t sector = 0; + unsigned int data_len = 0; + u8 *p = NULL; + + /* + * UFS is 4K block device. + */ + if (unlikely(sdp->sector_size != BLOCK)) { + ERR_MSG("this device is not 4K block device"); + return -EACCES; + } + + switch (cmd->cmnd[0]) { + case UNMAP: + /* + * UNMAP command has just a block descriptor(=a phys_segment). + */ + if (unlikely(!bio) || + unlikely(bio->bi_vcnt != 1)) { + ERR_MSG("UNMAP bio info is incorrect"); + return -EINVAL; + } + + p = page_address(bio->bi_io_vec->bv_page); + if (unlikely(!p)) { + ERR_MSG("can't get bio page address"); + return -ENOMEM; + } + + sector = LI_EN_64(p + 8); + data_len = LI_EN_32(p + 16); + break; + case READ_10: + case WRITE_10: + sector = LI_EN_32(cmd->cmnd + 2); + data_len = LI_EN_16(cmd->cmnd + 7); + break; + case READ_16: + case WRITE_16: + sector = LI_EN_64(cmd->cmnd + 2); + data_len = LI_EN_32(cmd->cmnd + 10); + break; + default: + return 0; + }; + + rq->__sector = sector << sects_per_blk_shift; + rq->__data_len = data_len << sects_per_blk_shift << SECTOR_SHIFT; + + return 0; +} + +/* routine : READ10 -> HPB_READ */ +void ufshpb_prep_fn(struct ufsf_feature *ufsf, struct ufshcd_lrb *lrbp) +{ + struct ufshpb_lu *hpb; + struct ufshpb_region *rgn; + struct ufshpb_subregion *srgn; + struct request *rq; + struct scsi_cmnd *cmd = lrbp->cmd; + u64 ppn = 0; + unsigned long lpn, flags; + int hpb_ctx_id = MAX_HPB_CONTEXT_ID; + int transfer_len = TRANSFER_LEN; + int rgn_idx, srgn_idx, srgn_offset, ret, error = 0; + + /* WKLU could not be HPB-LU */ + if (!lrbp || !ufsf_is_valid_lun(lrbp->lun)) { + cmd->requeue_cnt = 0; + return; + } + + cmd = lrbp->cmd; + + if (!ufshpb_is_read_cmd(cmd)) + cmd->requeue_cnt = 0; + + if (!ufshpb_is_read_cmd(cmd) && !ufshpb_is_write_cmd(cmd) && + !ufshpb_is_discard_cmd(cmd)) + return; + + rq = cmd->request; + if (unlikely(blk_rq_is_passthrough(rq) && + rq->__sector == (sector_t)-1)) { + ret = ufshpb_fill_passthrough_rq(cmd); + if (ret) + return; + } + + hpb = ufsf->hpb_lup[lrbp->lun]; + ret = ufshpb_lu_get(hpb); + if (unlikely(ret)) { + HPB_DEBUG(hpb, "lu_get failed (%d)", ret); + cmd->requeue_cnt = 0; + return; + } + + if (hpb->force_disable) { + if (ufshpb_is_read_cmd(cmd)) + HPB_FTRACE(hpb, "%llu + %u READ_10 (force_disable)", + (unsigned long long) blk_rq_pos(rq), + (unsigned int) blk_rq_sectors(rq)); + + goto put_hpb; + } + + lpn = ufshpb_get_lpn(rq); + ufshpb_get_pos_from_lpn(hpb, lpn, &rgn_idx, &srgn_idx, &srgn_offset); + rgn = hpb->rgn_tbl + rgn_idx; + srgn = rgn->srgn_tbl + srgn_idx; + + /* + * If cmd type is WRITE, bitmap set to dirty. + */ + if (ufshpb_is_write_cmd(cmd) || ufshpb_is_discard_cmd(cmd)) { + spin_lock_irqsave(&hpb->hpb_lock, flags); + if (rgn->rgn_state != HPB_RGN_INACTIVE) { + ufshpb_set_dirty(hpb, lrbp, rgn_idx, srgn_idx, + srgn_offset); + } + spin_unlock_irqrestore(&hpb->hpb_lock, flags); + } + + if (!ufshpb_is_read_cmd(cmd)) + goto put_hpb; + + if (unlikely(ufshpb_is_unaligned(rq))) { + HPB_FTRACE(hpb, "%llu + %u READ_10 R[%05d](%d) (not aligned)", + (unsigned long long) blk_rq_pos(rq), + (unsigned int) blk_rq_sectors(rq), + rgn_idx, rgn->rgn_state); + + goto put_hpb; + } + + transfer_len = ufshpb_get_len(rq); + if (unlikely(!transfer_len)) + goto put_hpb; + + if (!ufshpb_is_support_chunk(transfer_len)) { + HPB_FTRACE(hpb, "%llu + %u READ_10 R[%05d](%d) (not support chunk)", + (unsigned long long) blk_rq_pos(rq), + (unsigned int) blk_rq_sectors(rq), + rgn_idx, rgn->rgn_state); + + goto put_hpb; + } +#if defined(CONFIG_MICRON_HPB) + if (IS_MICRON_DEVICE(storage_mfrid) && transfer_len >= hpb->pre_req_min_tr_len && + transfer_len <= hpb->pre_req_max_tr_len) { + goto put_hpb; + } +#endif + spin_lock_irqsave(&hpb->hpb_lock, flags); + if (ufshpb_ppn_dirty_check(hpb, lpn, transfer_len)) { + atomic64_inc(&hpb->miss); + spin_unlock_irqrestore(&hpb->hpb_lock, flags); + HPB_FTRACE(hpb, "%llu + %u READ_10 R[%05d](%d) (hpb map invalid)", + (unsigned long long) blk_rq_pos(rq), + (unsigned int) blk_rq_sectors(rq), + rgn_idx, rgn->rgn_state); + + goto put_hpb; + } + + ppn = ufshpb_get_ppn(srgn->mctx, srgn_offset, &error); +#if defined(CONFIG_HPB_ERR_INJECTION) + if (hpb->err_injection_select != HPB_ERR_INJECTION_DISABLE) + ppn = ufshpb_get_err_injection_ppn(hpb, ppn); +#endif + spin_unlock_irqrestore(&hpb->hpb_lock, flags); + if (unlikely(error)) { + ERR_MSG("ppn failed (%d) R[%05d](%d) S[%d](%d)", + error, rgn_idx, rgn->rgn_state, + srgn_idx, srgn->srgn_state); + ufshpb_lu_put(hpb); + goto wakeup_ee_worker; + } + + /* + * WRITE_BUFFER CMD support 36K (len=9) ~ 512K (len=128) default. + * it is possible to change range of transfer_len through sysfs. + */ + if (transfer_len >= hpb->pre_req_min_tr_len && + transfer_len <= hpb->pre_req_max_tr_len) { + ret = ufshpb_issue_pre_req(hpb, cmd, &hpb_ctx_id); + if (ret) { + if (ret == -EAGAIN) { + if (cmd->requeue_cnt < + hpb->pre_req_requeue_cnt) { + cmd->requeue_cnt++; + goto requeue; + } + } + atomic64_inc(&hpb->miss); + goto put_hpb; + } + } + + ufshpb_set_read16_cmd(hpb, cmd, ppn, transfer_len, hpb_ctx_id); + HPB_FTRACE(hpb, "%llu + %u READ_HPB R[%05d](%d) - context_id %d", + (unsigned long long) blk_rq_pos(rq), + (unsigned int) blk_rq_sectors(rq), + rgn_idx, rgn->rgn_state, hpb_ctx_id); + + atomic64_inc(&hpb->hit); +put_hpb: + cmd->requeue_cnt = 0; +requeue: + ufshpb_lu_put(hpb); + return; +wakeup_ee_worker: + ufshpb_failed(hpb, __func__); +} + +static inline void ufshpb_put_map_req(struct ufshpb_lu *hpb, + struct ufshpb_req *map_req) +{ + map_req->req = NULL; + map_req->bio = NULL; + list_add_tail(&map_req->list_req, &hpb->lh_map_req_free); + hpb->num_inflight_map_req--; +} + +static struct ufshpb_req *ufshpb_get_map_req(struct ufshpb_lu *hpb) +{ + struct ufshpb_req *map_req; + + if (hpb->num_inflight_map_req >= hpb->throttle_map_req) { + HPB_FTRACE(hpb, "inflight %d throttle %d", + hpb->num_inflight_map_req, hpb->throttle_map_req); + return NULL; + } + + map_req = list_first_entry_or_null(&hpb->lh_map_req_free, + struct ufshpb_req, list_req); + if (!map_req) { + HPB_DEBUG(hpb, "no map_req"); + return NULL; + } + + list_del_init(&map_req->list_req); + hpb->num_inflight_map_req++; + + return map_req; +} + +static int ufshpb_clean_dirty_bitmap(struct ufshpb_lu *hpb, + struct ufshpb_subregion *srgn) +{ + struct ufshpb_region *rgn; + + BUG_ON(!srgn->mctx); + + rgn = hpb->rgn_tbl + srgn->rgn_idx; + + if (rgn->rgn_state == HPB_RGN_INACTIVE) { + HPB_DEBUG(hpb, "evicted R[%05d](%d) S[%d](%d)", + srgn->rgn_idx, rgn->rgn_state, + srgn->srgn_idx, srgn->srgn_state); + return -EINVAL; + } + + memset(srgn->mctx->ppn_dirty, 0x00, + hpb->entries_per_srgn >> bits_per_byte_shift); + + return 0; +} + +static void ufshpb_clean_active_subregion(struct ufshpb_lu *hpb, + struct ufshpb_subregion *srgn) +{ + struct ufshpb_region *rgn; + + BUG_ON(!srgn->mctx); + + rgn = hpb->rgn_tbl + srgn->rgn_idx; + + if (rgn->rgn_state == HPB_RGN_INACTIVE) { + HPB_DEBUG(hpb, "evicted R[%05d](%d) S[%d](%d)", + srgn->rgn_idx, rgn->rgn_state, + srgn->srgn_idx, srgn->srgn_state); + return; + } + srgn->srgn_state = HPB_SRGN_CLEAN; +} + +static void ufshpb_error_active_subregion(struct ufshpb_lu *hpb, + struct ufshpb_subregion *srgn) +{ + struct ufshpb_region *rgn; + + BUG_ON(!srgn->mctx); + + rgn = hpb->rgn_tbl + srgn->rgn_idx; + + if (rgn->rgn_state == HPB_RGN_INACTIVE) { + ERR_MSG("evicted R[%05d](%d) S[%d](%d)", + srgn->rgn_idx, rgn->rgn_state, + srgn->srgn_idx, srgn->srgn_state); + return; + } + srgn->srgn_state = HPB_SRGN_DIRTY; +} + +#if defined(CONFIG_HPB_DEBUG) +static void ufshpb_check_ppn(struct ufshpb_lu *hpb, int rgn_idx, int srgn_idx, + struct ufshpb_map_ctx *mctx, const char *str) +{ + int error = 0; + u64 val[2]; + + BUG_ON(!mctx); + + val[0] = ufshpb_get_ppn(mctx, 0, &error); + if (!error) + val[1] = ufshpb_get_ppn(mctx, hpb->entries_per_srgn - 1, + &error); + if (error) + val[0] = val[1] = 0; + + HPB_DEBUG(hpb, "%s RB R[%05d](%d) S[%d](%d) (%llx ~ %llx)", + str, rgn_idx, hpb->rgn_tbl[rgn_idx].rgn_state, + srgn_idx, hpb->rgn_tbl[rgn_idx].srgn_tbl[srgn_idx].srgn_state, + val[0], val[1]); +} +#endif + +static void ufshpb_map_compl_process(struct ufshpb_req *map_req) +{ + struct ufshpb_lu *hpb = map_req->hpb; + struct ufshpb_region *rgn = hpb->rgn_tbl + map_req->rb.rgn_idx; + struct ufshpb_subregion *srgn = rgn->srgn_tbl + map_req->rb.srgn_idx; + unsigned long flags; + + atomic64_inc(&hpb->map_compl_cnt); + +#if defined(CONFIG_HPB_DEBUG) + if (hpb->debug) + ufshpb_check_ppn(hpb, srgn->rgn_idx, srgn->srgn_idx, srgn->mctx, + "COMPL"); +#endif + HPB_FTRACE(hpb, "R[%05d](%d)", + map_req->rb.rgn_idx, rgn->rgn_state); + + spin_lock_irqsave(&hpb->hpb_lock, flags); + ufshpb_clean_active_subregion(hpb, srgn); + spin_unlock_irqrestore(&hpb->hpb_lock, flags); +} + +static inline void ufshpb_act_rsp_list_add(struct ufshpb_lu *hpb, + struct ufshpb_subregion *srgn) +{ + atomic64_inc(&hpb->act_rsp_list_cnt); + list_add_tail(&srgn->list_act_srgn, &hpb->lh_act_srgn); + HPB_FTRACE(hpb, "R[%05d](%d) S[%d](%d)", + srgn->rgn_idx, hpb->rgn_tbl[srgn->rgn_idx].rgn_state, + srgn->srgn_idx, srgn->srgn_state); +} + +static inline void ufshpb_act_rsp_list_del(struct ufshpb_lu *hpb, + struct ufshpb_subregion *srgn) +{ + atomic64_dec(&hpb->act_rsp_list_cnt); + list_del_init(&srgn->list_act_srgn); + HPB_FTRACE(hpb, "R[%05d](%d) S[%d](%d)", + srgn->rgn_idx, hpb->rgn_tbl[srgn->rgn_idx].rgn_state, + srgn->srgn_idx, srgn->srgn_state); +} + +static inline void ufshpb_inact_rsp_list_add(struct ufshpb_lu *hpb, + struct ufshpb_region *rgn) +{ + atomic64_inc(&hpb->inact_rsp_list_cnt); + list_add_tail(&rgn->list_inact_rgn, &hpb->lh_inact_rgn); + HPB_FTRACE(hpb, "R[%05d](%d)", + rgn->rgn_idx, hpb->rgn_tbl[rgn->rgn_idx].rgn_state); +} + +static inline void ufshpb_inact_rsp_list_del(struct ufshpb_lu *hpb, + struct ufshpb_region *rgn) +{ + atomic64_dec(&hpb->inact_rsp_list_cnt); + list_del_init(&rgn->list_inact_rgn); + HPB_FTRACE(hpb, "R[%05d](%d)", + rgn->rgn_idx, hpb->rgn_tbl[rgn->rgn_idx].rgn_state); +} + +static void ufshpb_update_active_info(struct ufshpb_lu *hpb, int rgn_idx, + int srgn_idx) +{ + struct ufshpb_region *rgn; + struct ufshpb_subregion *srgn; + + rgn = hpb->rgn_tbl + rgn_idx; + srgn = rgn->srgn_tbl + srgn_idx; + + if (!list_empty(&rgn->list_inact_rgn)) + ufshpb_inact_rsp_list_del(hpb, rgn); + + if (list_empty(&srgn->list_act_srgn)) + ufshpb_act_rsp_list_add(hpb, srgn); +} + +static void ufshpb_update_inactive_info(struct ufshpb_lu *hpb, int rgn_idx) +{ + struct ufshpb_region *rgn; + struct ufshpb_subregion *srgn; + int srgn_idx; + + rgn = hpb->rgn_tbl + rgn_idx; + + for (srgn_idx = 0; srgn_idx < rgn->srgn_cnt; srgn_idx++) { + srgn = rgn->srgn_tbl + srgn_idx; + + if (!list_empty(&srgn->list_act_srgn)) + ufshpb_act_rsp_list_del(hpb, srgn); + } + + if (list_empty(&rgn->list_inact_rgn)) + ufshpb_inact_rsp_list_add(hpb, rgn); +} + +static int ufshpb_map_req_error(struct ufshpb_req *map_req) +{ + struct ufshpb_lu *hpb = map_req->hpb; + struct ufshpb_region *rgn; + struct ufshpb_subregion *srgn; + struct scsi_sense_hdr sshdr; + unsigned long flags; + + rgn = hpb->rgn_tbl + map_req->rb.rgn_idx; + srgn = rgn->srgn_tbl + map_req->rb.srgn_idx; + + scsi_normalize_sense(map_req->sense, SCSI_SENSE_BUFFERSIZE, &sshdr); + + ERR_MSG("code %x sense_key %x asc %x ascq %x", + sshdr.response_code, sshdr.sense_key, sshdr.asc, sshdr.ascq); + ERR_MSG("byte4 %x byte5 %x byte6 %x additional_len %x", + sshdr.byte4, sshdr.byte5, sshdr.byte6, sshdr.additional_length); + + if (sshdr.sense_key != ILLEGAL_REQUEST) + return 0; + + spin_lock_irqsave(&hpb->hpb_lock, flags); + if (rgn->rgn_state == HPB_RGN_PINNED) { + if (sshdr.asc == 0x06 && sshdr.ascq == 0x01) { + HPB_DEBUG(hpb, "retry R[%05d] S[%d]", + map_req->rb.rgn_idx, map_req->rb.srgn_idx); + spin_unlock_irqrestore(&hpb->hpb_lock, flags); + spin_lock(&hpb->retry_list_lock); + list_add_tail(&map_req->list_req, + &hpb->lh_map_req_retry); + spin_unlock(&hpb->retry_list_lock); + + spin_lock_irqsave(&hpb->hpb_lock, flags); + ufshpb_error_active_subregion(hpb, srgn); + spin_unlock_irqrestore(&hpb->hpb_lock, flags); + + schedule_delayed_work(&hpb->retry_work, + msecs_to_jiffies(RETRY_DELAY_MS)); + return -EAGAIN; + } + HPB_DEBUG(hpb, "dirty R[%05d] S[%d]", + map_req->rb.rgn_idx, map_req->rb.srgn_idx); + ufshpb_error_active_subregion(hpb, srgn); + spin_unlock_irqrestore(&hpb->hpb_lock, flags); + } else { + ufshpb_error_active_subregion(hpb, srgn); + + spin_unlock_irqrestore(&hpb->hpb_lock, flags); + + spin_lock_irqsave(&hpb->rsp_list_lock, flags); + ufshpb_update_inactive_info(hpb, map_req->rb.rgn_idx); + spin_unlock_irqrestore(&hpb->rsp_list_lock, flags); + HPB_DEBUG(hpb, "inact R[%05d]", map_req->rb.rgn_idx); + schedule_work(&hpb->task_work); + } + + return 0; +} + +/* routine : map_req compl */ +static void ufshpb_map_req_compl_fn(struct request *req, blk_status_t error) +{ + struct ufshpb_req *map_req = (struct ufshpb_req *) req->end_io_data; + struct ufshpb_lu *hpb = map_req->hpb; + unsigned long flags; + int ret; + + if (ufshpb_get_state(hpb->ufsf) == HPB_FAILED) + goto free_map_req; + + if (error) { + ERR_MSG("ERR %d R[%05d](%d) S[%d](%d)", error, + map_req->rb.rgn_idx, + hpb->rgn_tbl[map_req->rb.rgn_idx].rgn_state, + map_req->rb.srgn_idx, + hpb->rgn_tbl[map_req->rb.rgn_idx].srgn_tbl[map_req->rb.srgn_idx].srgn_state); + ret = ufshpb_map_req_error(map_req); + if (ret) + goto retry_map_req; + } else + ufshpb_map_compl_process(map_req); + +free_map_req: + bio_put(map_req->bio); + ufshpb_put_blk_request(map_req->req); + spin_lock_irqsave(&hpb->hpb_lock, flags); + ufshpb_put_map_req(map_req->hpb, map_req); + spin_unlock_irqrestore(&hpb->hpb_lock, flags); +retry_map_req: + scsi_device_put(hpb->ufsf->sdev_ufs_lu[hpb->lun]); + ufshpb_lu_put(hpb); +} + +static inline void ufshpb_set_read_buf_cmd(unsigned char *cdb, int rgn_idx, + int srgn_idx, int srgn_mem_size) +{ + cdb[0] = UFSHPB_READ_BUFFER; + cdb[1] = UFSHPB_RB_ID_READ; + cdb[2] = GET_BYTE_1(rgn_idx); + cdb[3] = GET_BYTE_0(rgn_idx); + cdb[4] = GET_BYTE_1(srgn_idx); + cdb[5] = GET_BYTE_0(srgn_idx); + cdb[6] = GET_BYTE_2(srgn_mem_size); + cdb[7] = GET_BYTE_1(srgn_mem_size); + cdb[8] = GET_BYTE_0(srgn_mem_size); + cdb[9] = 0x00; +} + +static int ufshpb_map_req_add_bio_page(struct ufshpb_lu *hpb, + struct request_queue *q, struct bio *bio, + struct ufshpb_map_ctx *mctx) +{ + struct page *page = NULL; + int i, ret = 0; + + bio_reset(bio); + + for (i = 0; i < hpb->mpages_per_srgn; i++) { + /* virt_to_page(p + (OS_PAGE_SIZE * i)); */ + page = mctx->m_page[i]; + if (!page) + return -ENOMEM; + + ret = bio_add_pc_page(q, bio, page, hpb->mpage_bytes, 0); + + if (ret != hpb->mpage_bytes) { + ERR_MSG("bio_add_pc_page fail (%d)", ret); + return -ENOMEM; + } + } + + return 0; +} + +static int ufshpb_execute_map_req(struct ufshpb_lu *hpb, + struct ufshpb_req *map_req) +{ + struct scsi_device *sdev; + struct request_queue *q; + struct request *req; + struct bio *bio = map_req->bio; + struct scsi_request *rq; + int ret = 0; + + sdev = hpb->ufsf->sdev_ufs_lu[hpb->lun]; + if (!sdev) { + ERR_MSG("cannot find scsi_device"); + return -ENODEV; + } + + ret = ufsf_get_scsi_device(hpb->ufsf->hba, sdev); + if (ret) + return ret; + + q = sdev->request_queue; + + ret = ufshpb_map_req_add_bio_page(hpb, q, bio, map_req->rb.mctx); + if (ret) { + scsi_device_put(sdev); + return ret; + } + + req = map_req->req; + + /* 1. request setup */ + blk_rq_append_bio(req, &bio); /* req->__data_len is setted */ + req->rq_flags |= RQF_QUIET; + req->timeout = 30 * HZ; + req->end_io_data = (void *)map_req; + + /* 2. scsi_request setup */ + rq = scsi_req(req); + rq->retries = 3; + + ufshpb_set_read_buf_cmd(rq->cmd, map_req->rb.rgn_idx, + map_req->rb.srgn_idx, hpb->srgn_mem_size); + rq->cmd_len = scsi_command_size(rq->cmd); + +#if defined(CONFIG_HPB_DEBUG) + if (hpb->debug) + ufshpb_check_ppn(hpb, map_req->rb.rgn_idx, map_req->rb.srgn_idx, + map_req->rb.mctx, "ISSUE"); +#endif + ret = ufshpb_blk_execute_rq_nowait_mimic(q, req, ufshpb_map_req_compl_fn); + if (ret) { + scsi_device_put(sdev); + return ret; + } + + atomic64_inc(&hpb->map_req_cnt); + HPB_FTRACE(hpb, "issue R[%05d](%d) S[%d](%d)", + map_req->rb.rgn_idx, + hpb->rgn_tbl[map_req->rb.rgn_idx].rgn_state, + map_req->rb.srgn_idx, + hpb->rgn_tbl[map_req->rb.rgn_idx].srgn_tbl[map_req->rb.srgn_idx].srgn_state); + + return 0; +} + +static struct ufshpb_map_ctx *ufshpb_get_map_ctx(struct ufshpb_lu *hpb, + int *err) +{ + struct ufshpb_map_ctx *mctx; + + mctx = list_first_entry_or_null(&hpb->lh_map_ctx_free, + struct ufshpb_map_ctx, list_table); + if (mctx) { + list_del_init(&mctx->list_table); + hpb->debug_free_table--; + return mctx; + } + *err = -ENOMEM; + return NULL; +} + +static inline void ufshpb_add_lru_info(struct victim_select_info *lru_info, + struct ufshpb_region *rgn) +{ + rgn->rgn_state = HPB_RGN_ACTIVE; + list_add_tail(&rgn->list_lru_rgn, &lru_info->lh_lru_rgn); + atomic64_inc(&lru_info->active_cnt); +} + +static inline int ufshpb_add_region(struct ufshpb_lu *hpb, + struct ufshpb_region *rgn) +{ + struct victim_select_info *lru_info; + int srgn_idx; + int err = 0; + + lru_info = &hpb->lru_info; + + for (srgn_idx = 0; srgn_idx < rgn->srgn_cnt; srgn_idx++) { + struct ufshpb_subregion *srgn; + + srgn = rgn->srgn_tbl + srgn_idx; + + srgn->mctx = ufshpb_get_map_ctx(hpb, &err); + if (!srgn->mctx) { + ERR_MSG("mctx ERR %d R[%05d](%d) S[%d](%d) free_table %d", + err, srgn->rgn_idx, rgn->rgn_state, + srgn_idx, srgn->srgn_state, + hpb->debug_free_table); + goto out; + } + + srgn->srgn_state = HPB_SRGN_DIRTY; + } + + HPB_FTRACE(hpb, "add R[%05d](%d)", + rgn->rgn_idx, rgn->rgn_state); + + ufshpb_add_lru_info(lru_info, rgn); +out: + return err; +} + +static inline void ufshpb_put_map_ctx(struct ufshpb_lu *hpb, + struct ufshpb_map_ctx *mctx) +{ + list_add(&mctx->list_table, &hpb->lh_map_ctx_free); + hpb->debug_free_table++; +} + +static inline void ufshpb_purge_active_subregion(struct ufshpb_lu *hpb, + struct ufshpb_subregion *srgn, + int state) +{ + if (unlikely(state == HPB_SRGN_UNUSED)) { + ufshpb_put_map_ctx(hpb, srgn->mctx); + srgn->mctx = NULL; + } + + srgn->srgn_state = state; +} + +static inline void ufshpb_cleanup_lru_info(struct victim_select_info *lru_info, + struct ufshpb_region *rgn) +{ + list_del_init(&rgn->list_lru_rgn); + rgn->rgn_state = HPB_RGN_INACTIVE; + atomic64_dec(&lru_info->active_cnt); +} + +static void __ufshpb_evict_region(struct ufshpb_lu *hpb, + struct ufshpb_region *rgn) +{ + struct victim_select_info *lru_info; + struct ufshpb_subregion *srgn; + int srgn_idx; + + lru_info = &hpb->lru_info; + + HPB_FTRACE(hpb, "evict R[%05d](%d)", + rgn->rgn_idx, rgn->rgn_state); + + ufshpb_cleanup_lru_info(lru_info, rgn); + + for (srgn_idx = 0; srgn_idx < rgn->srgn_cnt; srgn_idx++) { + srgn = rgn->srgn_tbl + srgn_idx; + + ufshpb_purge_active_subregion(hpb, srgn, HPB_SRGN_UNUSED); + } +} + +static void ufshpb_hit_lru_info(struct victim_select_info *lru_info, + struct ufshpb_region *rgn) +{ + switch (lru_info->selection_type) { + case LRU: + list_move_tail(&rgn->list_lru_rgn, &lru_info->lh_lru_rgn); + break; + default: + break; + } +} + +/* + * Must be held hpb_lock before call this func. + */ +static int ufshpb_check_issue_state_srgns(struct ufshpb_lu *hpb, + struct ufshpb_region *rgn) +{ + struct ufshpb_subregion *srgn; + int srgn_idx; + + for (srgn_idx = 0; srgn_idx < rgn->srgn_cnt; srgn_idx++) { + srgn = rgn->srgn_tbl + srgn_idx; + + if (srgn->srgn_state == HPB_SRGN_ISSUED) { + HPB_FTRACE(hpb, "R[%05d](%d)", + rgn->rgn_idx, rgn->rgn_state); + + return -EPERM; + } + } + return 0; +} + +static struct ufshpb_region *ufshpb_victim_lru_info(struct ufshpb_lu *hpb) +{ + struct victim_select_info *lru_info = &hpb->lru_info; + struct ufshpb_region *rgn; + struct ufshpb_region *victim_rgn = NULL; + + switch (lru_info->selection_type) { + case LRU: + list_for_each_entry(rgn, &lru_info->lh_lru_rgn, list_lru_rgn) { + if (!rgn) + break; + + if (ufshpb_check_issue_state_srgns(hpb, rgn)) + continue; + + victim_rgn = rgn; + break; + } + break; + default: + break; + } + + return victim_rgn; +} + +static int ufshpb_evict_region(struct ufshpb_lu *hpb, struct ufshpb_region *rgn) +{ + unsigned long flags; + + spin_lock_irqsave(&hpb->hpb_lock, flags); + if (rgn->rgn_state == HPB_RGN_PINNED) { + /* Pinned region should not drop-out.*/ + ERR_MSG("drop-out ERR R[%05d](%d)", + rgn->rgn_idx, rgn->rgn_state); + goto out; + } + + if (!list_empty(&rgn->list_lru_rgn)) { + if (ufshpb_check_issue_state_srgns(hpb, rgn)) + goto evict_fail; + + __ufshpb_evict_region(hpb, rgn); + } +out: + spin_unlock_irqrestore(&hpb->hpb_lock, flags); + return 0; +evict_fail: + spin_unlock_irqrestore(&hpb->hpb_lock, flags); + return -EPERM; +} + +static inline struct +ufshpb_rsp_field *ufshpb_get_hpb_rsp(struct ufshcd_lrb *lrbp) +{ + return (struct ufshpb_rsp_field *)&lrbp->ucd_rsp_ptr->sr.sense_data_len; +} + +static int ufshpb_issue_map_req(struct ufshpb_lu *hpb, + struct ufshpb_subregion *srgn) +{ + struct scsi_device *sdev = hpb->ufsf->sdev_ufs_lu[hpb->lun]; + struct request *req = NULL; + struct bio *bio = NULL; + struct ufshpb_req *map_req = NULL; + unsigned long flags; + int ret = 0; + + req = blk_get_request(sdev->request_queue, REQ_OP_SCSI_IN, + BLK_MQ_REQ_PREEMPT); + if (IS_ERR(req)) + return -EAGAIN; + + bio = bio_alloc(GFP_KERNEL, hpb->mpages_per_srgn); + if (!bio) { + blk_put_request(req); + return -EAGAIN; + } + + spin_lock_irqsave(&hpb->hpb_lock, flags); + + if (srgn->srgn_state == HPB_SRGN_ISSUED) { + ret = -EAGAIN; + goto unlock_out; + } + + ret = ufshpb_clean_dirty_bitmap(hpb, srgn); + if (ret) { + ret = -EAGAIN; + goto unlock_out; + } + + map_req = ufshpb_get_map_req(hpb); + if (!map_req) { + ret = -ENOMEM; + goto unlock_out; + } + + srgn->srgn_state = HPB_SRGN_ISSUED; + + spin_unlock_irqrestore(&hpb->hpb_lock, flags); + + map_req->req = req; + map_req->bio = bio; + map_req->hpb = hpb; + map_req->rb.rgn_idx = srgn->rgn_idx; + map_req->rb.srgn_idx = srgn->srgn_idx; + map_req->rb.mctx = srgn->mctx; + map_req->rb.lun = hpb->lun; + + ret = ufshpb_lu_get(hpb); + if (unlikely(ret)) { + if (ret == -ENODEV) + ERR_MSG("lu_get failed (%d)", ret); + HPB_DEBUG(hpb, "lu_get failed (%d)", ret); + + goto free_map_req; + } + + ret = ufshpb_execute_map_req(hpb, map_req); + if (ret) + goto map_req_failed; + + return ret; +map_req_failed: + ufshpb_lu_put(hpb); +free_map_req: + spin_lock_irqsave(&hpb->hpb_lock, flags); + srgn->srgn_state = HPB_SRGN_DIRTY; + ufshpb_put_map_req(hpb, map_req); +unlock_out: + spin_unlock_irqrestore(&hpb->hpb_lock, flags); + bio_put(bio); + blk_put_request(req); + return ret; +} + +static int ufshpb_load_region(struct ufshpb_lu *hpb, struct ufshpb_region *rgn) +{ + struct ufshpb_region *victim_rgn; + struct victim_select_info *lru_info = &hpb->lru_info; + unsigned long flags; + int ret = 0; + + /* + * if already region is added to lru_list, + * just initiate the information of lru. + * because the region already has the map ctx. + * (!list_empty(&rgn->list_region) == region->state=active...) + */ + spin_lock_irqsave(&hpb->hpb_lock, flags); + if (!list_empty(&rgn->list_lru_rgn)) { + ufshpb_hit_lru_info(lru_info, rgn); + goto out; + } + + if (rgn->rgn_state == HPB_RGN_INACTIVE) { + if (atomic64_read(&lru_info->active_cnt) + == lru_info->max_lru_active_cnt) { + victim_rgn = ufshpb_victim_lru_info(hpb); + if (!victim_rgn) { + HPB_DEBUG(hpb, "victim_rgn is NULL"); + ret = -ENOMEM; + goto out; + } + HPB_FTRACE(hpb, "lru_max %lld R[%05d](%d)", + atomic64_read(&lru_info->active_cnt), + victim_rgn->rgn_idx, victim_rgn->rgn_state); + + __ufshpb_evict_region(hpb, victim_rgn); + } + + ret = ufshpb_add_region(hpb, rgn); + if (ret) { + ERR_MSG("add_region failed (%d)", ret); + spin_unlock_irqrestore(&hpb->hpb_lock, flags); + goto wakeup_ee_worker; + } + } +out: + spin_unlock_irqrestore(&hpb->hpb_lock, flags); + return ret; +wakeup_ee_worker: + ufshpb_failed(hpb, __func__); + return ret; +} + +static void ufshpb_rsp_req_region_update(struct ufshpb_lu *hpb, + struct ufshpb_rsp_field *rsp_field) +{ + struct ufshpb_region *rgn; + struct ufshpb_subregion *srgn; + int num, rgn_idx, srgn_idx; + + HPB_FTRACE(hpb, "ACT %d INACT %d", + rsp_field->active_rgn_cnt, rsp_field->inactive_rgn_cnt); + + /* + * If active rgn = inactive rgn, choose inactive rgn. + * So, active process -> inactive process + */ + for (num = 0; num < rsp_field->active_rgn_cnt; num++) { + rgn_idx = be16_to_cpu(rsp_field->hpb_active_field[num].active_rgn); + srgn_idx = be16_to_cpu(rsp_field->hpb_active_field[num].active_srgn); + + if (unlikely(rgn_idx >= hpb->rgns_per_lu)) { + ERR_MSG("R[%05d] is wrong. (Max %d)", + rgn_idx, hpb->rgns_per_lu); + continue; + } + + rgn = hpb->rgn_tbl + rgn_idx; + + if (unlikely(srgn_idx >= rgn->srgn_cnt)) { + ERR_MSG("R[%05d] S[%d] is wrong. (Max %d)", + rgn_idx, srgn_idx, rgn->srgn_cnt); + continue; + } + + srgn = rgn->srgn_tbl + srgn_idx; + + HPB_FTRACE(hpb, "ACT %d R[%05d](%d) S[%d](%d)", + num + 1, rgn_idx, rgn->rgn_state, + srgn_idx, srgn->srgn_idx); + spin_lock(&hpb->rsp_list_lock); + ufshpb_update_active_info(hpb, rgn_idx, srgn_idx); + spin_unlock(&hpb->rsp_list_lock); + + /* It is just blocking HPB_READ */ + spin_lock(&hpb->hpb_lock); + if (srgn->srgn_state == HPB_SRGN_CLEAN) + srgn->srgn_state = HPB_SRGN_DIRTY; + spin_unlock(&hpb->hpb_lock); + atomic64_inc(&hpb->rb_active_cnt); + } + + for (num = 0; num < rsp_field->inactive_rgn_cnt; num++) { + rgn_idx = be16_to_cpu(rsp_field->hpb_inactive_field[num]); + + if (unlikely(rgn_idx >= hpb->rgns_per_lu)) { + ERR_MSG("R[%05d] is wrong. (Max %d)", + rgn_idx, hpb->rgns_per_lu); + continue; + } + + rgn = hpb->rgn_tbl + rgn_idx; + + HPB_FTRACE(hpb, "INACT %d R[%05d](%d)", + num + 1, rgn_idx, rgn->rgn_state); + spin_lock(&hpb->rsp_list_lock); + ufshpb_update_inactive_info(hpb, rgn_idx); + spin_unlock(&hpb->rsp_list_lock); + + /* + * to block HPB_READ + */ + spin_lock(&hpb->hpb_lock); + if (rgn->rgn_state != HPB_RGN_INACTIVE) { + for (srgn_idx = 0; srgn_idx < rgn->srgn_cnt; srgn_idx++) { + srgn = rgn->srgn_tbl + srgn_idx; + if (srgn->srgn_state == HPB_SRGN_CLEAN) + srgn->srgn_state = HPB_SRGN_DIRTY; + } + } + spin_unlock(&hpb->hpb_lock); + + atomic64_inc(&hpb->rb_inactive_cnt); + } + + schedule_work(&hpb->task_work); +} + +static inline int ufshpb_may_field_valid(struct ufshcd_lrb *lrbp, + struct ufshpb_rsp_field *rsp_field) +{ + if (be16_to_cpu(rsp_field->sense_data_len) != DEV_SENSE_SEG_LEN || + rsp_field->desc_type != DEV_DES_TYPE || + rsp_field->additional_len != DEV_ADDITIONAL_LEN || + rsp_field->hpb_type == HPB_RSP_NONE || + rsp_field->active_rgn_cnt > MAX_ACTIVE_NUM || + rsp_field->inactive_rgn_cnt > MAX_INACTIVE_NUM || + (!rsp_field->active_rgn_cnt && !rsp_field->inactive_rgn_cnt)) + return -EINVAL; + + if (!ufsf_is_valid_lun(lrbp->lun)) { + ERR_MSG("LU(%d) is not supported", lrbp->lun); + return -EINVAL; + } + + return 0; +} + +static bool ufshpb_is_empty_rsp_lists(struct ufshpb_lu *hpb) +{ + bool ret = true; + unsigned long flags; + + spin_lock_irqsave(&hpb->rsp_list_lock, flags); + if (!list_empty(&hpb->lh_inact_rgn) || !list_empty(&hpb->lh_act_srgn)) + ret = false; + spin_unlock_irqrestore(&hpb->rsp_list_lock, flags); + + return ret; +} + +/* routine : isr (ufs) */ +void ufshpb_rsp_upiu(struct ufsf_feature *ufsf, struct ufshcd_lrb *lrbp) +{ + struct ufshpb_lu *hpb; + struct ufshpb_rsp_field *rsp_field; + int data_seg_len, ret; + + data_seg_len = be32_to_cpu(lrbp->ucd_rsp_ptr->header.dword_2) + & MASK_RSP_UPIU_DATA_SEG_LEN; + + if (!data_seg_len) { + bool do_task_work = false; + + if (!ufsf_is_valid_lun(lrbp->lun)) + return; + + hpb = ufsf->hpb_lup[lrbp->lun]; + ret = ufshpb_lu_get(hpb); + if (unlikely(ret)) { + HPB_DEBUG(hpb, "lu_get failed (%d)", ret); + return; + } + + do_task_work = !ufshpb_is_empty_rsp_lists(hpb); + if (do_task_work) + schedule_work(&hpb->task_work); + + goto put_hpb; + } + + rsp_field = ufshpb_get_hpb_rsp(lrbp); + + if (ufshpb_may_field_valid(lrbp, rsp_field)) { + WARN_ON(rsp_field->additional_len != DEV_ADDITIONAL_LEN); + return; + } + + hpb = ufsf->hpb_lup[lrbp->lun]; + ret = ufshpb_lu_get(hpb); + if (unlikely(ret)) { + HPB_DEBUG(hpb, "lu_get failed (%d)", ret); + return; + } + + if (hpb->force_map_req_disable) + goto put_hpb; + + atomic64_inc(&hpb->rb_noti_cnt); + + switch (rsp_field->hpb_type) { + case HPB_RSP_REQ_REGION_UPDATE: + WARN_ON(data_seg_len != DEV_DATA_SEG_LEN); + ufshpb_rsp_req_region_update(hpb, rsp_field); + goto put_hpb; + default: + HPB_DEBUG(hpb, "HPB_TYPE %d is not available", + rsp_field->hpb_type); + goto put_hpb; + } + +put_hpb: + ufshpb_lu_put(hpb); +} + +static int ufshpb_execute_map_req_wait(struct ufshpb_lu *hpb, + unsigned char *cmd, + struct ufshpb_subregion *srgn) +{ + struct ufsf_feature *ufsf = hpb->ufsf; + struct scsi_device *sdev; + struct request_queue *q; + struct request *req; + struct scsi_request *rq; + struct bio *bio; + struct scsi_sense_hdr sshdr; + unsigned long flags; + int ret = 0; + + sdev = ufsf->sdev_ufs_lu[hpb->lun]; + if (!sdev) { + ERR_MSG("cannot find scsi_device"); + return -ENODEV; + } + + q = sdev->request_queue; + + ret = ufsf_get_scsi_device(ufsf->hba, sdev); + if (ret) + return ret; + + req = blk_get_request(q, REQ_OP_SCSI_IN, BLK_MQ_REQ_PREEMPT); + if (IS_ERR(req)) { + ERR_MSG("cannot get request"); + ret = -EIO; + goto sdev_put_out; + } + + bio = bio_alloc(GFP_KERNEL, hpb->mpages_per_srgn); + if (!bio) { + ret = -ENOMEM; + goto req_put_out; + } + + ret = ufshpb_map_req_add_bio_page(hpb, q, bio, srgn->mctx); + if (ret) + goto mem_free_out; + + /* 1. request setup*/ + blk_rq_append_bio(req, &bio); /* req->__data_len */ + req->rq_flags |= RQF_QUIET; + req->timeout = 30 * HZ; + + /* 2. scsi_request setup */ + rq = scsi_req(req); + rq->retries = 3; + rq->cmd_len = scsi_command_size(cmd); + memcpy(rq->cmd, cmd, rq->cmd_len); + + blk_execute_rq(q, NULL, req, 1); + if (rq->result) { + ret = -EIO; + scsi_normalize_sense(rq->sense, SCSI_SENSE_BUFFERSIZE, &sshdr); + ERR_MSG("code %x sense_key %x asc %x ascq %x", + sshdr.response_code, sshdr.sense_key, sshdr.asc, + sshdr.ascq); + ERR_MSG("byte4 %x byte5 %x byte6 %x additional_len %x", + sshdr.byte4, sshdr.byte5, sshdr.byte6, + sshdr.additional_length); + spin_lock_irqsave(&hpb->hpb_lock, flags); + ufshpb_error_active_subregion(hpb, srgn); + spin_unlock_irqrestore(&hpb->hpb_lock, flags); + } +mem_free_out: + bio_put(bio); +req_put_out: + blk_put_request(req); +sdev_put_out: + scsi_device_put(sdev); + return ret; +} + +static void ufshpb_issue_map_req_from_list(struct ufshpb_lu *hpb) +{ + struct ufshpb_subregion *srgn; + unsigned long flags; + int ret; + + spin_lock_irqsave(&hpb->rsp_list_lock, flags); + + while ((srgn = list_first_entry_or_null(&hpb->lh_pinned_srgn, + struct ufshpb_subregion, + list_act_srgn))) { + unsigned char cmd[10] = { 0 }; + + list_del_init(&srgn->list_act_srgn); + spin_unlock_irqrestore(&hpb->rsp_list_lock, flags); + + ufshpb_set_read_buf_cmd(cmd, srgn->rgn_idx, srgn->srgn_idx, + hpb->srgn_mem_size); + +#if defined(CONFIG_HPB_DEBUG) + if (hpb->debug) + ufshpb_check_ppn(hpb, srgn->rgn_idx, srgn->srgn_idx, + srgn->mctx, "ISSUE"); +#endif + HPB_FTRACE(hpb, "issue R[%05d](%d)", + srgn->rgn_idx, hpb->rgn_tbl[srgn->rgn_idx].rgn_state); + + ret = ufshpb_execute_map_req_wait(hpb, cmd, srgn); + if (ret < 0) { + ERR_MSG("issue ERR %d R[%05d](%d) S%0d(%d)", + ret, + srgn->rgn_idx, hpb->rgn_tbl[srgn->rgn_idx].rgn_state, + srgn->srgn_idx, srgn->srgn_state); + spin_lock_irqsave(&hpb->rsp_list_lock, flags); + if (list_empty(&srgn->list_act_srgn)) + list_add(&srgn->list_act_srgn, + &hpb->lh_pinned_srgn); + continue; + } + +#if defined(CONFIG_HPB_DEBUG) + if (hpb->debug) + ufshpb_check_ppn(hpb, srgn->rgn_idx, srgn->srgn_idx, + srgn->mctx, "COMPL"); +#endif + HPB_FTRACE(hpb, "compl R[%05d](%d)", + srgn->rgn_idx, hpb->rgn_tbl[srgn->rgn_idx].rgn_state); + + spin_lock_irqsave(&hpb->hpb_lock, flags); + ufshpb_clean_active_subregion(hpb, srgn); + spin_unlock_irqrestore(&hpb->hpb_lock, flags); + + spin_lock_irqsave(&hpb->rsp_list_lock, flags); + } + + spin_unlock_irqrestore(&hpb->rsp_list_lock, flags); +} + +static void ufshpb_pinned_work_handler(struct work_struct *work) +{ + struct ufshpb_lu *hpb; + int ret; + + hpb = container_of(work, struct ufshpb_lu, pinned_work); + HPB_DEBUG(hpb, "worker start"); + + pm_runtime_get_sync(hpb->ufsf->hba->dev); + ret = ufshpb_lu_get(hpb); + if (unlikely(ret)) { + if (ret == -ENODEV) + ERR_MSG("lu_get failed (%d)", ret); + HPB_DEBUG(hpb, "lu_get failed (%d)", ret); + return; + } + + if (!list_empty(&hpb->lh_pinned_srgn)) + ufshpb_issue_map_req_from_list(hpb); + /* + * if its function failed at init time, + * ufshpb-device will request map-req, + * so it is not critical-error, and just finish work-handler + */ + + ufshpb_lu_put(hpb); + pm_runtime_put_sync(hpb->ufsf->hba->dev); + HPB_DEBUG(hpb, "worker end"); +} + +static int ufshpb_check_pm(struct ufshpb_lu *hpb) +{ + struct ufs_hba *hba = hpb->ufsf->hba; + + if (hba->pm_op_in_progress || + hba->curr_dev_pwr_mode != UFS_ACTIVE_PWR_MODE) { + INFO_MSG("hba current power state %d pm_progress %d", + hba->curr_dev_pwr_mode, + hba->pm_op_in_progress); + return -ENODEV; + } + return 0; +} + +static void ufshpb_retry_work_handler(struct work_struct *work) +{ + struct ufshpb_lu *hpb; + struct delayed_work *dwork = to_delayed_work(work); + struct ufshpb_req *map_req, *next; + struct ufshpb_region *rgn; + struct ufshpb_subregion *srgn; + unsigned long flags; + bool do_retry = false; + int ret = 0; + + LIST_HEAD(retry_list); + + hpb = container_of(dwork, struct ufshpb_lu, retry_work); + + if (ufshpb_check_pm(hpb)) + return; + + ret = ufshpb_lu_get(hpb); + if (unlikely(ret)) { + if (ret == -ENODEV) + ERR_MSG("lu_get failed (%d)", ret); + HPB_DEBUG(hpb, "lu_get failed (%d)", ret); + return; + } + + HPB_DEBUG(hpb, "worker start"); + + spin_lock_bh(&hpb->retry_list_lock); + list_splice_init(&hpb->lh_map_req_retry, &retry_list); + spin_unlock_bh(&hpb->retry_list_lock); + + list_for_each_entry_safe(map_req, next, &retry_list, list_req) { + if (ufshpb_get_state(hpb->ufsf) == HPB_SUSPEND) { + INFO_MSG("suspend state. Issue READ_BUFFER in the next turn"); + spin_lock_bh(&hpb->retry_list_lock); + list_splice_init(&retry_list, &hpb->lh_map_req_retry); + spin_unlock_bh(&hpb->retry_list_lock); + do_retry = true; + break; + } + + list_del_init(&map_req->list_req); + + rgn = hpb->rgn_tbl + map_req->rb.rgn_idx; + srgn = rgn->srgn_tbl + map_req->rb.srgn_idx; + + bio_put(map_req->bio); + blk_put_request(map_req->req); + spin_lock_irqsave(&hpb->hpb_lock, flags); + ufshpb_put_map_req(hpb, map_req); + spin_unlock_irqrestore(&hpb->hpb_lock, flags); + + ret = ufshpb_lu_get(hpb); + if (unlikely(ret)) { + if (ret == -ENODEV) + ERR_MSG("lu_get failed (%d)", ret); + HPB_DEBUG(hpb, "lu_get failed (%d)", ret); + continue; + } + + ret = ufshpb_issue_map_req(hpb, srgn); + if (ret) { + ERR_MSG("issue ERR %d R[%05d](%d) S%0d(%d)", + ret, rgn->rgn_idx, rgn->rgn_state, + srgn->srgn_idx, srgn->srgn_state); + spin_lock_bh(&hpb->retry_list_lock); + list_splice_init(&retry_list, &hpb->lh_map_req_retry); + spin_unlock_bh(&hpb->retry_list_lock); + do_retry = true; + break; + } + } + HPB_DEBUG(hpb, "worker end"); + ufshpb_lu_put(hpb); + + if (do_retry) + schedule_delayed_work(&hpb->retry_work, + msecs_to_jiffies(RETRY_DELAY_MS)); +} + +static void ufshpb_add_starved_list(struct ufshpb_lu *hpb, + struct ufshpb_region *rgn, + struct list_head *starved_list) +{ + struct ufshpb_subregion *srgn; + int srgn_idx; + + if (!list_empty(&rgn->list_inact_rgn)) + return; + + for (srgn_idx = 0; srgn_idx < rgn->srgn_cnt; srgn_idx++) { + srgn = rgn->srgn_tbl + srgn_idx; + + if (!list_empty(&srgn->list_act_srgn)) + return; + } + /* It will be added to hpb->lh_inact_rgn */ + atomic64_inc(&hpb->inact_rsp_list_cnt); + list_add_tail(&rgn->list_inact_rgn, starved_list); + + HPB_FTRACE(hpb, "R[%05d](%d)", + rgn->rgn_idx, rgn->rgn_state); + +} + +static void ufshpb_run_inactive_region_list(struct ufshpb_lu *hpb) +{ + struct ufshpb_region *rgn; + unsigned long flags; + int ret; + LIST_HEAD(starved_list); + + spin_lock_irqsave(&hpb->rsp_list_lock, flags); + while ((rgn = list_first_entry_or_null(&hpb->lh_inact_rgn, + struct ufshpb_region, + list_inact_rgn))) { + if (ufshpb_get_state(hpb->ufsf) == HPB_SUSPEND) { + INFO_MSG("suspend state. Inactivate rgn the next turn"); + break; + } + + HPB_FTRACE(hpb, "inact R[%05d](%d)", + rgn->rgn_idx, rgn->rgn_state); + + ufshpb_inact_rsp_list_del(hpb, rgn); + spin_unlock_irqrestore(&hpb->rsp_list_lock, flags); + + ret = ufshpb_evict_region(hpb, rgn); + if (ret) { + spin_lock_irqsave(&hpb->rsp_list_lock, flags); + ufshpb_add_starved_list(hpb, rgn, &starved_list); + spin_unlock_irqrestore(&hpb->rsp_list_lock, flags); + } + + spin_lock_irqsave(&hpb->rsp_list_lock, flags); + } + + list_splice(&starved_list, &hpb->lh_inact_rgn); + spin_unlock_irqrestore(&hpb->rsp_list_lock, flags); +} + +static void ufshpb_add_active_list(struct ufshpb_lu *hpb, + struct ufshpb_region *rgn, + struct ufshpb_subregion *srgn) +{ + if (!list_empty(&rgn->list_inact_rgn)) + return; + + if (!list_empty(&srgn->list_act_srgn)) { + list_move(&srgn->list_act_srgn, &hpb->lh_act_srgn); + return; + } + + HPB_FTRACE(hpb, "R[%05d](%d)", + rgn->rgn_idx, rgn->rgn_state); + + ufshpb_act_rsp_list_add(hpb, srgn); +} + +static void ufshpb_run_active_subregion_list(struct ufshpb_lu *hpb) +{ + struct ufshpb_region *rgn; + struct ufshpb_subregion *srgn; + unsigned long flags; + int ret = 0; + + spin_lock_irqsave(&hpb->rsp_list_lock, flags); + while ((srgn = list_first_entry_or_null(&hpb->lh_act_srgn, + struct ufshpb_subregion, + list_act_srgn))) { + if (ufshpb_get_state(hpb->ufsf) == HPB_SUSPEND) { + INFO_MSG("suspend state. Issuing READ_BUFFER the next turn"); + break; + } + + ufshpb_act_rsp_list_del(hpb, srgn); + + if (hpb->force_map_req_disable) { + HPB_DEBUG(hpb, "map_req disabled"); + continue; + } + + spin_unlock_irqrestore(&hpb->rsp_list_lock, flags); + + rgn = hpb->rgn_tbl + srgn->rgn_idx; + + HPB_FTRACE(hpb, "R[%05d](%d)", + rgn->rgn_idx, rgn->rgn_state); + + ret = ufshpb_load_region(hpb, rgn); + if (ret) + break; + + ret = ufshpb_issue_map_req(hpb, srgn); + if (ret) + break; + + spin_lock_irqsave(&hpb->rsp_list_lock, flags); + } + + if (ret) { + spin_lock_irqsave(&hpb->rsp_list_lock, flags); + ufshpb_add_active_list(hpb, rgn, srgn); + } + spin_unlock_irqrestore(&hpb->rsp_list_lock, flags); +} + +static void ufshpb_task_work_handler(struct work_struct *work) +{ + struct ufshpb_lu *hpb; + int ret; + + hpb = container_of(work, struct ufshpb_lu, task_work); + ret = ufshpb_lu_get(hpb); + if (unlikely(ret)) { + if (ret == -ENODEV) + ERR_MSG("lu_get failed (%d)", ret); + HPB_DEBUG(hpb, "lu_get failed (%d)", ret); + return; + } + + ufshpb_run_inactive_region_list(hpb); + ufshpb_run_active_subregion_list(hpb); + + ufshpb_lu_put(hpb); +} + +static inline void ufshpb_map_req_mempool_remove(struct ufshpb_lu *hpb) +{ + struct ufshpb_req *map_req; + int i; + + for (i = 0; i < hpb->qd; i++) { + map_req = hpb->map_req + i; + if (map_req->req) + blk_put_request(map_req->req); + if (map_req->bio) + bio_put(map_req->bio); + } + + kfree(hpb->map_req); +} + +static inline void ufshpb_pre_req_mempool_remove(struct ufshpb_lu *hpb) +{ + struct ufshpb_req *pre_req; + int i; + + for (i = 0; i < hpb->qd; i++) { + pre_req = hpb->pre_req + i; + if (pre_req->req) + blk_put_request(pre_req->req); + if (pre_req->bio) + bio_put(pre_req->bio); + __free_page(pre_req->wb.m_page); + } + + kfree(hpb->pre_req); +} + +static void ufshpb_table_mempool_remove(struct ufshpb_lu *hpb) +{ + struct ufshpb_map_ctx *mctx, *next; + int i; + + /* + * the mctx in the lh_map_ctx_free has been allocated completely. + */ + list_for_each_entry_safe(mctx, next, &hpb->lh_map_ctx_free, + list_table) { + for (i = 0; i < hpb->mpages_per_srgn; i++) + __free_page(mctx->m_page[i]); + + vfree(mctx->ppn_dirty); + kfree(mctx->m_page); + kfree(mctx); + hpb->alloc_mctx--; + } +} + +/* + * this function doesn't need to hold lock due to be called in init. + * (hpb_lock, rsp_list_lock, etc..) + */ +static int ufshpb_init_pinned_active_region(struct ufshpb_lu *hpb, + struct ufshpb_region *rgn) +{ + struct ufshpb_subregion *srgn; + int srgn_idx, j; + int err = 0; + + for (srgn_idx = 0; srgn_idx < rgn->srgn_cnt; srgn_idx++) { + srgn = rgn->srgn_tbl + srgn_idx; + + srgn->mctx = ufshpb_get_map_ctx(hpb, &err); + if (err) { + ERR_MSG("mctx ERR %d R[%05d](%d) S[%d](%d) free_table %d", + err, srgn->rgn_idx, rgn->rgn_state, + srgn_idx, srgn->srgn_state, + hpb->debug_free_table); + goto release; + } + + srgn->srgn_state = HPB_SRGN_ISSUED; + /* + * no need to clean ppn_dirty bitmap + * because it is vzalloc-ed @ufshpb_table_mempool_init() + */ + list_add_tail(&srgn->list_act_srgn, &hpb->lh_pinned_srgn); + } + + rgn->rgn_state = HPB_RGN_PINNED; + + return 0; + +release: + for (j = 0; j < srgn_idx; j++) { + srgn = rgn->srgn_tbl + j; + ufshpb_put_map_ctx(hpb, srgn->mctx); + } + + return err; +} + +static inline bool ufshpb_is_pinned_region(struct ufshpb_lu *hpb, int rgn_idx) +{ + if (hpb->lu_pinned_end_offset != PINNED_NOT_SET && + rgn_idx >= hpb->lu_pinned_rgn_startidx && + rgn_idx <= hpb->lu_pinned_end_offset) + return true; + + return false; +} + +static inline void ufshpb_init_jobs(struct ufshpb_lu *hpb) +{ + INIT_WORK(&hpb->pinned_work, ufshpb_pinned_work_handler); + INIT_DELAYED_WORK(&hpb->retry_work, ufshpb_retry_work_handler); + INIT_WORK(&hpb->task_work, ufshpb_task_work_handler); +} + +static inline void ufshpb_cancel_jobs(struct ufshpb_lu *hpb) +{ + cancel_work_sync(&hpb->pinned_work); + cancel_delayed_work_sync(&hpb->retry_work); + cancel_work_sync(&hpb->task_work); +} + +static void ufshpb_init_subregion_tbl(struct ufshpb_lu *hpb, + struct ufshpb_region *rgn) +{ + int srgn_idx; + + for (srgn_idx = 0; srgn_idx < rgn->srgn_cnt; srgn_idx++) { + struct ufshpb_subregion *srgn = rgn->srgn_tbl + srgn_idx; + + INIT_LIST_HEAD(&srgn->list_act_srgn); + + srgn->rgn_idx = rgn->rgn_idx; + srgn->srgn_idx = srgn_idx; + srgn->srgn_state = HPB_SRGN_UNUSED; + } +} + +static inline int ufshpb_alloc_subregion_tbl(struct ufshpb_lu *hpb, + struct ufshpb_region *rgn, + int srgn_cnt) +{ + rgn->srgn_tbl = + kzalloc(sizeof(struct ufshpb_subregion) * srgn_cnt, GFP_KERNEL); + if (!rgn->srgn_tbl) + return -ENOMEM; + + rgn->srgn_cnt = srgn_cnt; + return 0; +} + +static int ufshpb_table_mempool_init(struct ufshpb_lu *hpb) +{ + struct ufshpb_map_ctx *mctx = NULL; + int i, j, k; + + INIT_LIST_HEAD(&hpb->lh_map_ctx_free); + + hpb->alloc_mctx = hpb->lu_max_active_rgns * hpb->srgns_per_rgn; + + for (i = 0; i < hpb->alloc_mctx; i++) { + mctx = kmalloc(sizeof(struct ufshpb_map_ctx), GFP_KERNEL); + if (!mctx) + goto release_mem; + + mctx->m_page = + kzalloc(sizeof(struct page *) * hpb->mpages_per_srgn, + GFP_KERNEL); + if (!mctx->m_page) + goto release_mem; + + mctx->ppn_dirty = vzalloc(hpb->entries_per_srgn >> + bits_per_byte_shift); + if (!mctx->ppn_dirty) + goto release_mem; + + for (j = 0; j < hpb->mpages_per_srgn; j++) { + mctx->m_page[j] = alloc_page(GFP_KERNEL | __GFP_ZERO); + if (!mctx->m_page[j]) { + for (k = 0; k < j; k++) + __free_page(mctx->m_page[k]); + goto release_mem; + } + } + + INIT_LIST_HEAD(&mctx->list_table); + list_add(&mctx->list_table, &hpb->lh_map_ctx_free); + + hpb->debug_free_table++; + } + + INFO_MSG("The number of mctx = (%d). debug_free_table (%d)", + hpb->alloc_mctx, hpb->debug_free_table); + return 0; +release_mem: + /* + * mctxs already added in lh_map_ctx_free will be removed + * in the caller function. + */ + if (mctx) { + kfree(mctx->m_page); + vfree(mctx->ppn_dirty); + kfree(mctx); + } + return -ENOMEM; +} + +static int ufshpb_map_req_mempool_init(struct ufshpb_lu *hpb) +{ + struct scsi_device *sdev; + struct request_queue *q; + struct ufshpb_req *map_req = NULL; + int qd = hpb->qd; + int i; + + sdev = hpb->ufsf->sdev_ufs_lu[hpb->lun]; + q = sdev->request_queue; + + INIT_LIST_HEAD(&hpb->lh_map_req_free); + INIT_LIST_HEAD(&hpb->lh_map_req_retry); + + hpb->map_req = kzalloc(sizeof(struct ufshpb_req) * qd, GFP_KERNEL); + if (!hpb->map_req) + goto release_mem; + + for (i = 0; i < qd; i++) { + map_req = hpb->map_req + i; + INIT_LIST_HEAD(&map_req->list_req); + map_req->req = NULL; + map_req->bio = NULL; + + list_add_tail(&map_req->list_req, &hpb->lh_map_req_free); + } + + return 0; +release_mem: + kfree(hpb->map_req); + return -ENOMEM; +} + +static int ufshpb_pre_req_mempool_init(struct ufshpb_lu *hpb) +{ + struct scsi_device *sdev; + struct request_queue *q; + struct ufshpb_req *pre_req = NULL; + int qd = hpb->qd; + int i, j; + + INIT_LIST_HEAD(&hpb->lh_pre_req_free); + + sdev = hpb->ufsf->sdev_ufs_lu[hpb->lun]; + q = sdev->request_queue; + + hpb->pre_req = kzalloc(sizeof(struct ufshpb_req) * qd, GFP_KERNEL); + if (!hpb->pre_req) + goto release_mem; + + for (i = 0; i < qd; i++) { + pre_req = hpb->pre_req + i; + INIT_LIST_HEAD(&pre_req->list_req); + pre_req->req = NULL; + pre_req->bio = NULL; + + pre_req->wb.m_page = alloc_page(GFP_KERNEL | __GFP_ZERO); + if (!pre_req->wb.m_page) { + for (j = 0; j < i; j++) + __free_page(hpb->pre_req[j].wb.m_page); + + goto release_mem; + } + list_add_tail(&pre_req->list_req, &hpb->lh_pre_req_free); + } + + return 0; +release_mem: + kfree(hpb->pre_req); + return -ENOMEM; +} + +static void ufshpb_find_lu_qd(struct ufshpb_lu *hpb) +{ + struct scsi_device *sdev; + struct ufs_hba *hba; + + sdev = hpb->ufsf->sdev_ufs_lu[hpb->lun]; + hba = hpb->ufsf->hba; + + /* + * ufshcd_slave_alloc(sdev) -> ufshcd_set_queue_depth(sdev) + * a lu-queue-depth compared with lu_info and hba->nutrs + * is selected in ufshcd_set_queue_depth() + */ + hpb->qd = sdev->queue_depth; + INFO_MSG("lu (%d) queue_depth (%d)", hpb->lun, hpb->qd); + if (!hpb->qd) { + hpb->qd = hba->nutrs; + INFO_MSG("lu_queue_depth is 0. we use device's queue info."); + INFO_MSG("hba->nutrs = (%d)", hba->nutrs); + } + + hpb->throttle_map_req = hpb->qd; + hpb->throttle_pre_req = hpb->qd; + hpb->num_inflight_map_req = 0; + hpb->num_inflight_pre_req = 0; +} + +static void ufshpb_init_lu_constant(struct ufshpb_dev_info *hpb_dev_info, + struct ufshpb_lu *hpb) +{ + unsigned long long rgn_unit_size, rgn_mem_size; + int entries_per_rgn; + + hpb->debug = false; + + ufshpb_find_lu_qd(hpb); + + /* for pre_req */ + hpb->pre_req_min_tr_len = HPB_MULTI_CHUNK_LOW; + hpb->pre_req_max_tr_len = HPB_MULTI_CHUNK_HIGH; + hpb->ctx_id_ticket = 0; + hpb->pre_req_requeue_cnt = DEFAULT_WB_REQUEUE_CNT; + + /* From descriptors */ + rgn_unit_size = (unsigned long long) + SECTOR * (0x01 << hpb_dev_info->rgn_size); + rgn_mem_size = rgn_unit_size / BLOCK * HPB_ENTRY_SIZE; + + hpb->srgn_unit_size = (unsigned long long) + SECTOR * (0x01 << hpb_dev_info->srgn_size); + hpb->srgn_mem_size = hpb->srgn_unit_size / BLOCK * HPB_ENTRY_SIZE; + + /* relation : lu <-> region <-> sub region <-> entry */ + entries_per_rgn = rgn_mem_size / HPB_ENTRY_SIZE; + hpb->entries_per_srgn = hpb->srgn_mem_size / HPB_ENTRY_SIZE; + hpb->srgns_per_rgn = rgn_mem_size / hpb->srgn_mem_size; + + /* + * regions_per_lu = (lu_num_blocks * 4096) / region_unit_size + * = (lu_num_blocks * HPB_ENTRY_SIZE) / region_mem_size + */ + hpb->rgns_per_lu = + ((unsigned long long)hpb->lu_num_blocks + + (rgn_mem_size / HPB_ENTRY_SIZE) - 1) + / (rgn_mem_size / HPB_ENTRY_SIZE); + hpb->srgns_per_lu = + ((unsigned long long)hpb->lu_num_blocks + + (hpb->srgn_mem_size / HPB_ENTRY_SIZE) - 1) + / (hpb->srgn_mem_size / HPB_ENTRY_SIZE); + + /* mempool info */ + hpb->mpage_bytes = OS_PAGE_SIZE; + hpb->mpages_per_srgn = hpb->srgn_mem_size / hpb->mpage_bytes; + + /* Bitmask Info. */ + hpb->dwords_per_srgn = hpb->entries_per_srgn / BITS_PER_DWORD; + hpb->entries_per_rgn_shift = ffs(entries_per_rgn) - 1; + hpb->entries_per_rgn_mask = entries_per_rgn - 1; + hpb->entries_per_srgn_shift = ffs(hpb->entries_per_srgn) - 1; + hpb->entries_per_srgn_mask = hpb->entries_per_srgn - 1; + + INFO_MSG("===== From Device Descriptor! ====="); + INFO_MSG("hpb_region_size = (%d), hpb_subregion_size = (%d)", + hpb_dev_info->rgn_size, hpb_dev_info->srgn_size); + INFO_MSG("===== Constant Values(LU) ====="); + INFO_MSG("region_unit_size = (%lld), region_mem_size (%lld)", + rgn_unit_size, rgn_mem_size); + INFO_MSG("subregion_unit_size = (%lld), subregion_mem_size (%d)", + hpb->srgn_unit_size, hpb->srgn_mem_size); + + INFO_MSG("lu_num_blocks = (%d)", hpb->lu_num_blocks); + INFO_MSG("regions_per_lu = (%d), subregions_per_lu = (%d)", + hpb->rgns_per_lu, hpb->srgns_per_lu); + + INFO_MSG("subregions_per_region = %d", hpb->srgns_per_rgn); + INFO_MSG("entries_per_region (%u) shift (%u) mask (0x%X)", + entries_per_rgn, hpb->entries_per_rgn_shift, + hpb->entries_per_rgn_mask); + INFO_MSG("entries_per_subregion (%u) shift (%u) mask (0x%X)", + hpb->entries_per_srgn, hpb->entries_per_srgn_shift, + hpb->entries_per_srgn_mask); + INFO_MSG("mpages_per_subregion : (%d)", hpb->mpages_per_srgn); + INFO_MSG("==================================="); +} + +static int ufshpb_lu_hpb_init(struct ufsf_feature *ufsf, int lun) +{ + struct ufshpb_lu *hpb = ufsf->hpb_lup[lun]; + struct ufshpb_region *rgn_table, *rgn; + struct ufshpb_subregion *srgn; + int rgn_idx, srgn_idx, total_srgn_cnt, srgn_cnt, i, ret = 0; + + ufshpb_init_lu_constant(&ufsf->hpb_dev_info, hpb); + + rgn_table = kzalloc(sizeof(struct ufshpb_region) * hpb->rgns_per_lu, + GFP_KERNEL); + if (!rgn_table) { + ret = -ENOMEM; + goto out; + } + + INFO_MSG("active_region_table bytes: (%lu)", + (sizeof(struct ufshpb_region) * hpb->rgns_per_lu)); + + hpb->rgn_tbl = rgn_table; + + spin_lock_init(&hpb->hpb_lock); + spin_lock_init(&hpb->retry_list_lock); + spin_lock_init(&hpb->rsp_list_lock); + + /* init lru information */ + INIT_LIST_HEAD(&hpb->lru_info.lh_lru_rgn); + hpb->lru_info.selection_type = LRU; + + INIT_LIST_HEAD(&hpb->lh_pinned_srgn); + INIT_LIST_HEAD(&hpb->lh_act_srgn); + INIT_LIST_HEAD(&hpb->lh_inact_rgn); + + INIT_LIST_HEAD(&hpb->lh_map_ctx_free); + + ufshpb_init_jobs(hpb); + + ret = ufshpb_map_req_mempool_init(hpb); + if (ret) { + ERR_MSG("map_req_mempool init fail!"); + goto release_rgn_table; + } + + ret = ufshpb_pre_req_mempool_init(hpb); + if (ret) { + ERR_MSG("pre_req_mempool init fail!"); + goto release_map_req_mempool; + } + + ret = ufshpb_table_mempool_init(hpb); + if (ret) { + ERR_MSG("ppn table mempool init fail!"); + ufshpb_table_mempool_remove(hpb); + goto release_pre_req_mempool; + } + + total_srgn_cnt = hpb->srgns_per_lu; + INFO_MSG("total_subregion_count: (%d)", total_srgn_cnt); + for (rgn_idx = 0, srgn_cnt = 0; rgn_idx < hpb->rgns_per_lu; + rgn_idx++, total_srgn_cnt -= srgn_cnt) { + rgn = rgn_table + rgn_idx; + rgn->rgn_idx = rgn_idx; + + INIT_LIST_HEAD(&rgn->list_inact_rgn); + + /* init lru region information*/ + INIT_LIST_HEAD(&rgn->list_lru_rgn); + + srgn_cnt = min(total_srgn_cnt, hpb->srgns_per_rgn); + + ret = ufshpb_alloc_subregion_tbl(hpb, rgn, srgn_cnt); + if (ret) + goto release_srgns; + ufshpb_init_subregion_tbl(hpb, rgn); + + if (ufshpb_is_pinned_region(hpb, rgn_idx)) { + ret = ufshpb_init_pinned_active_region(hpb, rgn); + if (ret) + goto release_srgns; + } else { + rgn->rgn_state = HPB_RGN_INACTIVE; + } + } + + if (total_srgn_cnt != 0) { + ERR_MSG("error total_subregion_count: %d", + total_srgn_cnt); + goto release_srgns; + } + + atomic64_set(&hpb->lru_info.active_cnt, 0); + /* + * even if creating sysfs failed, ufshpb could run normally. + * so we don't deal with error handling + */ + ufshpb_create_sysfs(ufsf, hpb); + + return 0; +release_srgns: + for (i = 0; i < rgn_idx; i++) { + rgn = rgn_table + i; + if (rgn->srgn_tbl) { + for (srgn_idx = 0; srgn_idx < rgn->srgn_cnt; + srgn_idx++) { + srgn = rgn->srgn_tbl + srgn_idx; + if (srgn->mctx) + ufshpb_put_map_ctx(hpb, srgn->mctx); + } + kfree(rgn->srgn_tbl); + } + } + + ufshpb_table_mempool_remove(hpb); +release_pre_req_mempool: + ufshpb_pre_req_mempool_remove(hpb); +release_map_req_mempool: + ufshpb_map_req_mempool_remove(hpb); +release_rgn_table: + kfree(rgn_table); +out: + return ret; +} + +static inline int ufshpb_version_check(struct ufshpb_dev_info *hpb_dev_info, int oem_id) +{ + INFO_MSG("Support HPB Spec : Driver = (%.4X) Device = (%.4X) Vendor = (%.2x)", + UFSHPB_VER, hpb_dev_info->version,oem_id); + + INFO_MSG("HPB Driver Version : (%.6X%s)", + UFSHPB_DD_VER, UFSHPB_DD_VER_POST); + + if (hpb_dev_info->version != UFSHPB_VER && oem_id == 0x1ce) { + ERR_MSG("ERROR: HPB Spec Version mismatch. So HPB disabled."); + return -ENODEV; + } + if (hpb_dev_info->version != 0x200 && oem_id == 0x12c){ + ERR_MSG("ERROR: micron devices HPB Spec Version mismatch. So HPB disabled."); + return -ENODEV; + + } + return 0; +} + +void ufshpb_get_dev_info(struct ufsf_feature *ufsf, u8 *desc_buf) +{ + struct ufshpb_dev_info *hpb_dev_info = &ufsf->hpb_dev_info; + int ret; + int manufacturer = 0; + + if (desc_buf[DEVICE_DESC_PARAM_UFS_FEAT] & UFS_FEATURE_SUPPORT_HPB_BIT) + INFO_MSG("bUFSFeaturesSupport: HPB is set"); + else { + INFO_MSG("bUFSFeaturesSupport: HPB not support"); + ufshpb_set_state(ufsf, HPB_FAILED); + return; + } + + hpb_dev_info->version = LI_EN_16(desc_buf + DEVICE_DESC_PARAM_HPB_VER); + + manufacturer = LI_EN_16(desc_buf + DEVICE_DESC_PARAM_OEM_ID); + + ret = ufshpb_version_check(hpb_dev_info,manufacturer); + if (ret) + ufshpb_set_state(ufsf, HPB_FAILED); +} + +void ufshpb_get_geo_info(struct ufsf_feature *ufsf, u8 *geo_buf) +{ + struct ufshpb_dev_info *hpb_dev_info = &ufsf->hpb_dev_info; + int hpb_device_max_active_rgns = 0; + + hpb_dev_info->num_lu = geo_buf[GEOMETRY_DESC_HPB_NUMBER_LU]; + if (hpb_dev_info->num_lu == 0) { + ERR_MSG("Don't have a lu for hpb."); + ufshpb_set_state(ufsf, HPB_FAILED); + return; + } + + hpb_dev_info->rgn_size = geo_buf[GEOMETRY_DESC_HPB_REGION_SIZE]; + hpb_dev_info->srgn_size = geo_buf[GEOMETRY_DESC_HPB_SUBREGION_SIZE]; + hpb_device_max_active_rgns = + LI_EN_16(geo_buf + GEOMETRY_DESC_HPB_DEVICE_MAX_ACTIVE_REGIONS); + + INFO_MSG("[48] bHPBRegionSize (%u)", hpb_dev_info->rgn_size); + INFO_MSG("[49] bHPBNumberLU (%u)", hpb_dev_info->num_lu); + INFO_MSG("[4A] bHPBSubRegionSize (%u)", hpb_dev_info->srgn_size); + INFO_MSG("[4B:4C] wDeviceMaxActiveHPBRegions (%u)", + hpb_device_max_active_rgns); + + if (hpb_dev_info->rgn_size == 0 || hpb_dev_info->srgn_size == 0 || + hpb_device_max_active_rgns == 0) { + ERR_MSG("HPB NOT normally supported by device"); + ufshpb_set_state(ufsf, HPB_FAILED); + } +} + +void ufshpb_get_lu_info(struct ufsf_feature *ufsf, int lun, u8 *unit_buf) +{ + struct ufsf_lu_desc lu_desc; + struct ufshpb_lu *hpb; + + lu_desc.lu_enable = unit_buf[UNIT_DESC_PARAM_LU_ENABLE]; + lu_desc.lu_queue_depth = unit_buf[UNIT_DESC_PARAM_LU_Q_DEPTH]; + lu_desc.lu_logblk_size = unit_buf[UNIT_DESC_PARAM_LOGICAL_BLK_SIZE]; + lu_desc.lu_logblk_cnt = + LI_EN_64(unit_buf + UNIT_DESC_PARAM_LOGICAL_BLK_COUNT); + lu_desc.lu_max_active_hpb_rgns = + LI_EN_16(unit_buf + UNIT_DESC_HPB_LU_MAX_ACTIVE_REGIONS); + lu_desc.lu_hpb_pinned_rgn_startidx = + LI_EN_16(unit_buf + UNIT_DESC_HPB_LU_PIN_REGION_START_OFFSET); + lu_desc.lu_num_hpb_pinned_rgns = + LI_EN_16(unit_buf + UNIT_DESC_HPB_LU_NUM_PIN_REGIONS); + + if (lu_desc.lu_num_hpb_pinned_rgns > 0) { + lu_desc.lu_hpb_pinned_end_offset = + lu_desc.lu_hpb_pinned_rgn_startidx + + lu_desc.lu_num_hpb_pinned_rgns - 1; + } else + lu_desc.lu_hpb_pinned_end_offset = PINNED_NOT_SET; + + INFO_MSG("LUN(%d) [0A] bLogicalBlockSize (%d)", + lun, lu_desc.lu_logblk_size); + INFO_MSG("LUN(%d) [0B] qLogicalBlockCount (%llu)", + lun, lu_desc.lu_logblk_cnt); + INFO_MSG("LUN(%d) [03] bLuEnable (%d)", lun, lu_desc.lu_enable); + INFO_MSG("LUN(%d) [06] bLuQueueDepth (%d)", lun, + lu_desc.lu_queue_depth); + INFO_MSG("LUN(%d) [23:24] wLUMaxActiveHPBRegions (%d)", + lun, lu_desc.lu_max_active_hpb_rgns); + INFO_MSG("LUN(%d) [25:26] wHPBPinnedRegionStartIdx (%d)", + lun, lu_desc.lu_hpb_pinned_rgn_startidx); + INFO_MSG("LUN(%d) [27:28] wNumHPBPinnedRegions (%d)", + lun, lu_desc.lu_num_hpb_pinned_rgns); + INFO_MSG("LUN(%d) PINNED Start (%d) End (%d)", + lun, lu_desc.lu_hpb_pinned_rgn_startidx, + lu_desc.lu_hpb_pinned_end_offset); + + ufsf->hpb_lup[lun] = NULL; + + if (lu_desc.lu_enable == 0x02 && lu_desc.lu_max_active_hpb_rgns) { + ufsf->hpb_lup[lun] = kzalloc(sizeof(struct ufshpb_lu), + GFP_KERNEL); + if (!ufsf->hpb_lup[lun]) { + ERR_MSG("hpb_lup[%d] alloc failed", lun); + return; + } + + hpb = ufsf->hpb_lup[lun]; + hpb->ufsf = ufsf; + hpb->lun = lun; + hpb->lu_num_blocks = lu_desc.lu_logblk_cnt; + hpb->lu_max_active_rgns = lu_desc.lu_max_active_hpb_rgns; + hpb->lru_info.max_lru_active_cnt = + lu_desc.lu_max_active_hpb_rgns - + lu_desc.lu_num_hpb_pinned_rgns; + hpb->lu_pinned_rgn_startidx = + lu_desc.lu_hpb_pinned_rgn_startidx; + hpb->lu_pinned_end_offset = lu_desc.lu_hpb_pinned_end_offset; + } else { + INFO_MSG("===== LU (%d) is hpb-disabled.", lun); + } +} + +static void ufshpb_error_handler(struct work_struct *work) +{ + struct ufsf_feature *ufsf; + + ufsf = container_of(work, struct ufsf_feature, hpb_eh_work); + + WARN_MSG("driver has failed. but UFSHCD can run without UFSHPB"); + WARN_MSG("UFSHPB will be removed from the kernel"); + + ufshpb_remove(ufsf, HPB_FAILED); +} + +static int ufshpb_init(struct ufsf_feature *ufsf) +{ + int lun, ret; + struct ufshpb_lu *hpb; + int hpb_enabled_lun = 0; + + seq_scan_lu(lun) { + if (!ufsf->hpb_lup[lun]) + continue; + + /* + * HPB need info about request queue in order to issue + * RB-CMD for pinned region. + */ + if (!ufsf->sdev_ufs_lu[lun]) { + WARN_MSG("lun (%d) don't have scsi_device", lun); + continue; + } + + ret = ufshpb_lu_hpb_init(ufsf, lun); + if (ret) { + kfree(ufsf->hpb_lup[lun]); + continue; + } + hpb_enabled_lun++; + } + + if (hpb_enabled_lun == 0) { + ERR_MSG("No UFSHPB LU to init"); + ret = -ENODEV; + goto hpb_failed; + } + + INIT_WORK(&ufsf->hpb_eh_work, ufshpb_error_handler); + + kref_init(&ufsf->hpb_kref); + ufshpb_set_state(ufsf, HPB_PRESENT); + + seq_scan_lu(lun) + if (ufsf->hpb_lup[lun]) { + hpb = ufsf->hpb_lup[lun]; + + INFO_MSG("UFSHPB LU %d working", lun); + if (hpb->lu_pinned_end_offset != PINNED_NOT_SET) + schedule_work(&hpb->pinned_work); + } + + return 0; +hpb_failed: + ufshpb_set_state(ufsf, HPB_FAILED); + return ret; +} + +static void ufshpb_drop_retry_list(struct ufshpb_lu *hpb) +{ + struct ufshpb_req *map_req, *next; + struct request *req; + unsigned long flags; + LIST_HEAD(lh_reqs); + + if (list_empty(&hpb->lh_map_req_retry)) + return; + + spin_lock_irqsave(&hpb->hpb_lock, flags); + list_for_each_entry_safe(map_req, next, &hpb->lh_map_req_retry, + list_req) { + INFO_MSG("drop map_req %p ( %d - %d )", map_req, + map_req->rb.rgn_idx, map_req->rb.srgn_idx); + + list_del_init(&map_req->list_req); + + /* + * blk_put_request() will grap the queue_lock. + * so requests add to a list. + * lock seq : queue_lock -> hpb_lock. + */ + list_add_tail(&map_req->req->queuelist, &lh_reqs); + + bio_put(map_req->bio); + + ufshpb_put_map_req(hpb, map_req); + } + spin_unlock_irqrestore(&hpb->hpb_lock, flags); + + list_for_each_entry(req, &lh_reqs, queuelist) + blk_put_request(req); +} + +static void ufshpb_drop_rsp_lists(struct ufshpb_lu *hpb) +{ + struct ufshpb_region *rgn, *next_rgn; + struct ufshpb_subregion *srgn, *next_srgn; + unsigned long flags; + + spin_lock_irqsave(&hpb->rsp_list_lock, flags); + list_for_each_entry_safe(rgn, next_rgn, &hpb->lh_inact_rgn, + list_inact_rgn) { + list_del_init(&rgn->list_inact_rgn); + } + + list_for_each_entry_safe(srgn, next_srgn, &hpb->lh_act_srgn, + list_act_srgn) { + list_del_init(&srgn->list_act_srgn); + } + spin_unlock_irqrestore(&hpb->rsp_list_lock, flags); +} + +static void ufshpb_destroy_subregion_tbl(struct ufshpb_lu *hpb, + struct ufshpb_region *rgn) +{ + int srgn_idx; + + for (srgn_idx = 0; srgn_idx < rgn->srgn_cnt; srgn_idx++) { + struct ufshpb_subregion *srgn; + + srgn = rgn->srgn_tbl + srgn_idx; + srgn->srgn_state = HPB_SRGN_UNUSED; + + ufshpb_put_map_ctx(hpb, srgn->mctx); + } +} + +static void ufshpb_destroy_region_tbl(struct ufshpb_lu *hpb) +{ + int rgn_idx; + + INFO_MSG("Start"); + + for (rgn_idx = 0; rgn_idx < hpb->rgns_per_lu; rgn_idx++) { + struct ufshpb_region *rgn; + + rgn = hpb->rgn_tbl + rgn_idx; + if (rgn->rgn_state != HPB_RGN_INACTIVE) { + rgn->rgn_state = HPB_RGN_INACTIVE; + + ufshpb_destroy_subregion_tbl(hpb, rgn); + } + + kfree(rgn->srgn_tbl); + } + + ufshpb_table_mempool_remove(hpb); + kfree(hpb->rgn_tbl); + + INFO_MSG("End"); +} + +void ufshpb_remove(struct ufsf_feature *ufsf, int state) +{ + struct ufshpb_lu *hpb; + int lun; + + INFO_MSG("start release"); + ufshpb_set_state(ufsf, HPB_FAILED); + + INFO_MSG("kref count (%d)", + atomic_read(&ufsf->hpb_kref.refcount.refs)); + + seq_scan_lu(lun) { + hpb = ufsf->hpb_lup[lun]; + + INFO_MSG("lun (%d) (%p)", lun, hpb); + + ufsf->hpb_lup[lun] = NULL; + + if (!hpb) + continue; + + ufshpb_cancel_jobs(hpb); + + ufshpb_destroy_region_tbl(hpb); + if (hpb->alloc_mctx != 0) + WARN_MSG("warning: alloc_mctx (%d)", hpb->alloc_mctx); + + ufshpb_map_req_mempool_remove(hpb); + + ufshpb_pre_req_mempool_remove(hpb); + + ufshpb_remove_sysfs(hpb); + + kfree(hpb); + } + + ufshpb_set_state(ufsf, state); + + INFO_MSG("end release"); +} + +void ufshpb_reset(struct ufsf_feature *ufsf) +{ + ufshpb_set_state(ufsf, HPB_PRESENT); +} + +void ufshpb_reset_host(struct ufsf_feature *ufsf) +{ + struct ufshpb_lu *hpb; + int lun; + + ufshpb_set_state(ufsf, HPB_RESET); + seq_scan_lu(lun) { + hpb = ufsf->hpb_lup[lun]; + if (hpb) { + INFO_MSG("UFSHPB lun %d reset", lun); + ufshpb_cancel_jobs(hpb); + ufshpb_drop_retry_list(hpb); + ufshpb_drop_rsp_lists(hpb); + } + } +} + +static inline int ufshpb_probe_lun_done(struct ufsf_feature *ufsf) +{ + return (ufsf->num_lu == ufsf->slave_conf_cnt); +} + +void ufshpb_init_handler(struct work_struct *work) +{ + struct ufsf_feature *ufsf; + int ret = 0; + + ufsf = container_of(work, struct ufsf_feature, hpb_init_work); + + INFO_MSG("probe_done(%d)", ufshpb_probe_lun_done(ufsf)); + + ret = wait_event_timeout(ufsf->hpb_wait, + ufshpb_probe_lun_done(ufsf), + msecs_to_jiffies(10000)); + if (ret == 0) + ERR_MSG("Probing LU is not fully complete."); + + INFO_MSG("probe_done(%d) ret(%d)", ufshpb_probe_lun_done(ufsf), ret); + INFO_MSG("HPB_INIT_START"); + + ret = ufshpb_init(ufsf); + if (ret) + ERR_MSG("UFSHPB driver init failed. (%d)", ret); +} + +void ufshpb_suspend(struct ufsf_feature *ufsf) +{ + struct ufshpb_lu *hpb; + int lun; + + seq_scan_lu(lun) { + hpb = ufsf->hpb_lup[lun]; + if (hpb) { + INFO_MSG("ufshpb_lu %d goto suspend", lun); + INFO_MSG("ufshpb_lu %d changes suspend state", lun); + ufshpb_set_state(ufsf, HPB_SUSPEND); + ufshpb_cancel_jobs(hpb); + } + } +} + +void ufshpb_resume(struct ufsf_feature *ufsf) +{ + struct ufshpb_lu *hpb; + int lun; + + seq_scan_lu(lun) { + hpb = ufsf->hpb_lup[lun]; + if (hpb) { + bool do_task_work = false; + bool do_retry_work = false; + + ufshpb_set_state(ufsf, HPB_PRESENT); + + do_task_work = !ufshpb_is_empty_rsp_lists(hpb); + do_retry_work = + !list_empty_careful(&hpb->lh_map_req_retry); + + INFO_MSG("ufshpb_lu %d resume. do_task_work %d retry %d", + lun, do_task_work, do_retry_work); + + if (do_task_work) + schedule_work(&hpb->task_work); + if (do_retry_work) + schedule_delayed_work(&hpb->retry_work, + msecs_to_jiffies(100)); + } + } +} + +static void ufshpb_stat_reset(struct ufshpb_lu *hpb) +{ + atomic64_set(&hpb->hit, 0); + atomic64_set(&hpb->miss, 0); + atomic64_set(&hpb->rb_noti_cnt, 0); + atomic64_set(&hpb->rb_active_cnt, 0); + atomic64_set(&hpb->rb_inactive_cnt, 0); + atomic64_set(&hpb->map_req_cnt, 0); + atomic64_set(&hpb->map_compl_cnt, 0); + atomic64_set(&hpb->pre_req_cnt, 0); +} + +/* SYSFS functions */ +static ssize_t ufshpb_sysfs_prep_disable_show(struct ufshpb_lu *hpb, char *buf) +{ + int ret; + + ret = snprintf(buf, PAGE_SIZE, "force_hpb_read_disable %d\n", + hpb->force_disable); + + INFO_MSG("force_hpb_read_disable %d", hpb->force_disable); + return ret; +} + +static ssize_t ufshpb_sysfs_prep_disable_store(struct ufshpb_lu *hpb, + const char *buf, size_t cnt) +{ + unsigned long value; + + if (kstrtoul(buf, 0, &value)) + return -EINVAL; + + if (value > 1) + return -EINVAL; + + if (value == 1) + hpb->force_disable = true; + else if (value == 0) + hpb->force_disable = false; + + INFO_MSG("force_hpb_read_disable %d", hpb->force_disable); + + return cnt; +} + +static ssize_t ufshpb_sysfs_map_disable_show(struct ufshpb_lu *hpb, char *buf) +{ + int ret; + + ret = snprintf(buf, PAGE_SIZE, "force_map_req_disable %d\n", + hpb->force_map_req_disable); + + INFO_MSG("force_map_req_disable %d", hpb->force_map_req_disable); + + return ret; +} + +static ssize_t ufshpb_sysfs_map_disable_store(struct ufshpb_lu *hpb, + const char *buf, size_t cnt) +{ + unsigned long value; + + if (kstrtoul(buf, 0, &value)) + return -EINVAL; + + if (value > 1) + return -EINVAL; + + if (value == 1) + hpb->force_map_req_disable = true; + else if (value == 0) + hpb->force_map_req_disable = false; + + INFO_MSG("force_map_req_disable %d", hpb->force_map_req_disable); + + return cnt; +} + +static ssize_t ufshpb_sysfs_throttle_map_req_show(struct ufshpb_lu *hpb, + char *buf) +{ + int ret; + +#if defined(CONFIG_HPB_DEBUG) + ret = snprintf(buf, PAGE_SIZE, "throttle_map_req %d (inflight: %d)\n", + hpb->throttle_map_req, hpb->num_inflight_map_req); +#else + ret = snprintf(buf, PAGE_SIZE, "throttle_map_req %d\n", + hpb->throttle_map_req); +#endif + + INFO_MSG("throttle_map_req %d", hpb->throttle_map_req); + + return ret; +} + +static ssize_t ufshpb_sysfs_throttle_map_req_store(struct ufshpb_lu *hpb, + const char *buf, size_t cnt) +{ + unsigned long throttle_map_req; + + if (kstrtoul(buf, 0, &throttle_map_req)) + return -EINVAL; + + if (throttle_map_req > hpb->qd) + return -EINVAL; + + if ((int)throttle_map_req < 0 || (int)throttle_map_req > hpb->qd) + return -EINVAL; + + hpb->throttle_map_req = (int)throttle_map_req; + + INFO_MSG("throttle_map_req %d", hpb->throttle_map_req); + + return cnt; +} + +static ssize_t ufshpb_sysfs_throttle_pre_req_show(struct ufshpb_lu *hpb, + char *buf) +{ + int ret; + +#if defined(CONFIG_HPB_DEBUG) + ret = snprintf(buf, PAGE_SIZE, "throttle_pre_req %d (inflight: %d)\n", + hpb->throttle_pre_req, hpb->num_inflight_pre_req); +#else + ret = snprintf(buf, PAGE_SIZE, "throttle_pre_req %d\n", + hpb->throttle_pre_req); +#endif + + INFO_MSG("throttle_pre_req %d", hpb->throttle_pre_req); + + return ret; +} + +static ssize_t ufshpb_sysfs_throttle_pre_req_store(struct ufshpb_lu *hpb, + const char *buf, size_t cnt) +{ + unsigned long throttle_pre_req; + + if (kstrtoul(buf, 0, &throttle_pre_req)) + return -EINVAL; + + if (throttle_pre_req > hpb->qd) + return -EINVAL; + + if ((int)throttle_pre_req < 0 || (int)throttle_pre_req > hpb->qd) + return -EINVAL; + + hpb->throttle_pre_req = (int)throttle_pre_req; + + INFO_MSG("throttle_pre_req %d", hpb->throttle_pre_req); + + return cnt; +} + +static ssize_t ufshpb_sysfs_pre_req_min_tr_len_show(struct ufshpb_lu *hpb, + char *buf) +{ + int ret; + + ret = snprintf(buf, PAGE_SIZE, "%d", hpb->pre_req_min_tr_len); + INFO_MSG("pre_req min transfer len %d", hpb->pre_req_min_tr_len); + + return ret; +} + +static ssize_t ufshpb_sysfs_pre_req_min_tr_len_store(struct ufshpb_lu *hpb, + const char *buf, + size_t count) +{ + unsigned long val; + + if (kstrtoul(buf, 0, &val)) + return -EINVAL; + + if (val < 0) + val = 0; + + if (hpb->pre_req_max_tr_len < val || val < HPB_MULTI_CHUNK_LOW) + INFO_MSG("value is wrong. pre_req transfer len %d ~ %d", + HPB_MULTI_CHUNK_LOW, hpb->pre_req_max_tr_len); + else + hpb->pre_req_min_tr_len = val; + + INFO_MSG("pre_req min transfer len %d", hpb->pre_req_min_tr_len); + + return count; +} + +static ssize_t ufshpb_sysfs_pre_req_max_tr_len_show(struct ufshpb_lu *hpb, + char *buf) +{ + int ret; + + ret = snprintf(buf, PAGE_SIZE, "%d", hpb->pre_req_max_tr_len); + INFO_MSG("pre_req max transfer len %d", hpb->pre_req_max_tr_len); + + return ret; +} + +static ssize_t ufshpb_sysfs_pre_req_max_tr_len_store(struct ufshpb_lu *hpb, + const char *buf, + size_t count) +{ + unsigned long val; + + if (kstrtoul(buf, 0, &val)) + return -EINVAL; + + if (hpb->pre_req_min_tr_len > val || val > HPB_MULTI_CHUNK_HIGH) { + INFO_MSG("value is wrong. pre_req transfer len %d ~ %d", + hpb->pre_req_min_tr_len, HPB_MULTI_CHUNK_HIGH); + return -EINVAL; + } + + hpb->pre_req_max_tr_len = val; + + INFO_MSG("pre_req max transfer len %d", hpb->pre_req_max_tr_len); + + return count; +} + +static ssize_t ufshpb_sysfs_pre_req_requeue_cnt_show(struct ufshpb_lu *hpb, + char *buf) +{ + int ret; + + ret = snprintf(buf, PAGE_SIZE, "%d", hpb->pre_req_requeue_cnt); + INFO_MSG("pre_req requeue cnt %d", hpb->pre_req_requeue_cnt); + + return ret; +} + +static ssize_t ufshpb_sysfs_pre_req_requeue_cnt_store(struct ufshpb_lu *hpb, + const char *buf, + size_t count) +{ + unsigned long val; + + if (kstrtoul(buf, 0, &val)) + return -EINVAL; + + if (val < 0) + val = 0; + + hpb->pre_req_requeue_cnt = val; + + INFO_MSG("pre_req requeue cnt %d", hpb->pre_req_requeue_cnt); + + return count; +} + +#if defined(CONFIG_HPB_DEBUG) +static ssize_t ufshpb_sysfs_debug_show(struct ufshpb_lu *hpb, char *buf) +{ + int ret; + + ret = snprintf(buf, PAGE_SIZE, "debug %d\n", hpb->debug); + + INFO_MSG("debug %d", hpb->debug); + + return ret; +} + +static ssize_t ufshpb_sysfs_debug_store(struct ufshpb_lu *hpb, + const char *buf, size_t cnt) +{ + unsigned long debug; + + if (kstrtoul(buf, 0, &debug)) + return -EINVAL; + + if (debug > 1) + return -EINVAL; + + if (debug == 1) + hpb->debug = 1; + else + hpb->debug = 0; + + INFO_MSG("debug %d", hpb->debug); + + return cnt; +} +#endif + +static ssize_t ufshpb_sysfs_rsp_list_cnt_show(struct ufshpb_lu *hpb, char *buf) +{ + long long act_cnt, inact_cnt; + int ret; + + act_cnt = atomic64_read(&hpb->act_rsp_list_cnt); + inact_cnt = atomic64_read(&hpb->inact_rsp_list_cnt); + + ret = snprintf(buf, PAGE_SIZE, + "act_rsp_list_cnt %lld inact_rsp_list_cnt %lld\n", + act_cnt, inact_cnt); + + INFO_MSG("%s", buf); + + return ret; +} + +#if defined(CONFIG_HPB_ERR_INJECTION) +static ssize_t ufshpb_sysfs_err_injection_select_show(struct ufshpb_lu *hpb, + char *buf) +{ + char error_injection[8]; + int ret; + + if (hpb->err_injection_select == HPB_ERR_INJECTION_DISABLE) + strlcpy(error_injection, "disable", 8); + else if (hpb->err_injection_select == HPB_ERR_INJECTION_BITFLIP) + strlcpy(error_injection, "bitflip", 8); + else if (hpb->err_injection_select == HPB_ERR_INJECTION_OFFSET) + strlcpy(error_injection, "offset", 7); + else if (hpb->err_injection_select == HPB_ERR_INJECTION_RANDOM) + strlcpy(error_injection, "random", 7); + + ret = snprintf(buf, PAGE_SIZE, "err_injection_select %s", + error_injection); + + INFO_MSG("err_injection_select %s", error_injection); + + return ret; +} + +static ssize_t ufshpb_sysfs_err_injection_select_store(struct ufshpb_lu *hpb, + const char *buf, + size_t cnt) +{ + char error_injection[8]; + int ret; + + error_injection[7] = '\0'; + + ret = sscanf(buf, "%7s", error_injection); + if (!ret) { + INFO_MSG("input failed..."); + return cnt; + } + + if (!strncmp(error_injection, "disable", 7)) + hpb->err_injection_select = HPB_ERR_INJECTION_DISABLE; + else if (!strncmp(error_injection, "bitflip", 7)) + hpb->err_injection_select = HPB_ERR_INJECTION_BITFLIP; + else if (!strncmp(error_injection, "offset", 6)) + hpb->err_injection_select = HPB_ERR_INJECTION_OFFSET; + else if (!strncmp(error_injection, "random", 6)) + hpb->err_injection_select = HPB_ERR_INJECTION_RANDOM; + else { + hpb->err_injection_select = HPB_ERR_INJECTION_DISABLE; + strlcpy(error_injection, "disable", 8); + } + + INFO_MSG("err_injection_select %s", error_injection); + + return cnt; +} + +static ssize_t ufshpb_sysfs_err_injection_bitflip_show(struct ufshpb_lu *hpb, + char *buf) +{ + int ret; + + ret = snprintf(buf, PAGE_SIZE, "err_injection_bitflip %llx", + hpb->err_injection_bitflip); + + return ret; +} + +static ssize_t ufshpb_sysfs_err_injection_bitflip_store(struct ufshpb_lu *hpb, + const char *buf, + size_t cnt) +{ + unsigned long long err_injection_bitflip; + + if (kstrtoull(buf, 0, &err_injection_bitflip)) + return -EINVAL; + + hpb->err_injection_bitflip = err_injection_bitflip; + + INFO_MSG("err_injection_bitflip %llx", hpb->err_injection_bitflip); + + return cnt; +} + +static ssize_t ufshpb_sysfs_err_injection_offset_show(struct ufshpb_lu *hpb, + char *buf) +{ + int ret; + + ret = snprintf(buf, PAGE_SIZE, "err_injection_offset %lld\n", + hpb->err_injection_offset); + + INFO_MSG("err_injection_offset %lld", hpb->err_injection_offset); + + return ret; +} + +static ssize_t ufshpb_sysfs_err_injection_offset_store(struct ufshpb_lu *hpb, + const char *buf, + size_t cnt) +{ + unsigned long long err_injection_offset; + + if (kstrtoull(buf, 0, &err_injection_offset)) + return -EINVAL; + + hpb->err_injection_offset = err_injection_offset; + + INFO_MSG("err_injection_offset %lld", hpb->err_injection_offset); + + return cnt; +} + +static ssize_t ufshpb_sysfs_err_injection_random_show(struct ufshpb_lu *hpb, + char *buf) +{ + int ret; + + ret = snprintf(buf, PAGE_SIZE, "err_injection_random %d\n", + hpb->err_injection_random); + + INFO_MSG("err_injection_random %d", hpb->err_injection_random); + + return ret; +} + +static ssize_t ufshpb_sysfs_err_injection_random_store(struct ufshpb_lu *hpb, + const char *buf, + size_t cnt) +{ + unsigned long err_injection_random; + + if (kstrtoul(buf, 0, &err_injection_random)) + return -EINVAL; + + if (err_injection_random > 1) + return -EINVAL; + + hpb->err_injection_random = err_injection_random; + + INFO_MSG("err_injection_random %d", hpb->err_injection_random); + + return cnt; +} +#endif + +static ssize_t ufshpb_sysfs_version_show(struct ufshpb_lu *hpb, char *buf) +{ + int ret; + + ret = snprintf(buf, PAGE_SIZE, "HPB version %.4X D/D version %.6X%s\n", + hpb->ufsf->hpb_dev_info.version, + UFSHPB_DD_VER, UFSHPB_DD_VER_POST); + + INFO_MSG("HPB version %.4X D/D version %.6X%s", + hpb->ufsf->hpb_dev_info.version, + UFSHPB_DD_VER, UFSHPB_DD_VER_POST); + + return ret; +} + +static ssize_t ufshpb_sysfs_hit_show(struct ufshpb_lu *hpb, char *buf) +{ + long long hit_cnt; + int ret; + + hit_cnt = atomic64_read(&hpb->hit); + + ret = snprintf(buf, PAGE_SIZE, "hit_count %lld\n", hit_cnt); + + INFO_MSG("hit_count %lld", hit_cnt); + + return ret; +} + +static ssize_t ufshpb_sysfs_miss_show(struct ufshpb_lu *hpb, char *buf) +{ + long long miss_cnt; + int ret; + + miss_cnt = atomic64_read(&hpb->miss); + + ret = snprintf(buf, PAGE_SIZE, "miss_count %lld\n", miss_cnt); + + INFO_MSG("miss_count %lld", miss_cnt); + + return ret; +} + +static ssize_t ufshpb_sysfs_map_req_show(struct ufshpb_lu *hpb, char *buf) +{ + long long rb_noti_cnt, rb_active_cnt, rb_inactive_cnt; + long long map_req_cnt, map_compl_cnt; + int ret; + + rb_noti_cnt = atomic64_read(&hpb->rb_noti_cnt); + rb_active_cnt = atomic64_read(&hpb->rb_active_cnt); + rb_inactive_cnt = atomic64_read(&hpb->rb_inactive_cnt); + map_req_cnt = atomic64_read(&hpb->map_req_cnt); + map_compl_cnt = atomic64_read(&hpb->map_compl_cnt); + + ret = snprintf(buf, PAGE_SIZE, + "rb_noti %lld ACT %lld INACT %lld map_req_count %lld map_compl_count %lld\n", + rb_noti_cnt, rb_active_cnt, rb_inactive_cnt, + map_req_cnt, map_compl_cnt); + + INFO_MSG("rb_noti %lld ACT %lld INACT %lld map_req_count %lld map_compl_count %lld", + rb_noti_cnt, rb_active_cnt, rb_inactive_cnt, + map_req_cnt, map_compl_cnt); + + return ret; +} + +static ssize_t ufshpb_sysfs_pre_req_show(struct ufshpb_lu *hpb, char *buf) +{ + long long pre_req_cnt; + int ret; + + pre_req_cnt = atomic64_read(&hpb->pre_req_cnt); + + ret = snprintf(buf, PAGE_SIZE, "pre_req_count %lld\n", pre_req_cnt); + + INFO_MSG("pre_req_count %lld", pre_req_cnt); + + return ret; +} + +static ssize_t ufshpb_sysfs_region_stat_show(struct ufshpb_lu *hpb, char *buf) +{ + int ret, pin_cnt = 0, act_cnt = 0, inact_cnt = 0, rgn_idx; + enum HPB_RGN_STATE state; + + for (rgn_idx = 0; rgn_idx < hpb->rgns_per_lu; rgn_idx++) { + state = hpb->rgn_tbl[rgn_idx].rgn_state; + if (state == HPB_RGN_PINNED) + pin_cnt++; + else if (state == HPB_RGN_ACTIVE) + act_cnt++; + else if (state == HPB_RGN_INACTIVE) + inact_cnt++; + } + + ret = snprintf(buf, PAGE_SIZE, + "Total %d pinned %d active %d inactive %d\n", + hpb->rgns_per_lu, pin_cnt, act_cnt, inact_cnt); + + INFO_MSG("Total %d pinned %d active %d inactive %d", + hpb->rgns_per_lu, pin_cnt, act_cnt, inact_cnt); + + return ret; +} + +static ssize_t ufshpb_sysfs_count_reset_store(struct ufshpb_lu *hpb, + const char *buf, size_t cnt) +{ + unsigned long debug; + + if (kstrtoul(buf, 0, &debug)) + return -EINVAL; + + if (debug != 1) + return -EINVAL; + + INFO_MSG("Stat Init"); + + ufshpb_stat_reset(hpb); + + return cnt; +} + +static ssize_t ufshpb_sysfs_info_lba_store(struct ufshpb_lu *hpb, + const char *buf, size_t cnt) +{ + struct ufshpb_region *rgn; + struct ufshpb_subregion *srgn; + u64 ppn = 0; + unsigned long value, lpn, flags; + int rgn_idx, srgn_idx, srgn_offset, error = 0; + + if (kstrtoul(buf, 0, &value)) { + ERR_MSG("kstrtoul error"); + return -EINVAL; + } + + if (value >= hpb->lu_num_blocks * SECTORS_PER_BLOCK) { + ERR_MSG("value %lu >= lu_num_blocks %d error", + value, hpb->lu_num_blocks * SECTORS_PER_BLOCK); + return -EINVAL; + } + + lpn = value / SECTORS_PER_BLOCK; + + ufshpb_get_pos_from_lpn(hpb, lpn, &rgn_idx, &srgn_idx, &srgn_offset); + + rgn = hpb->rgn_tbl + rgn_idx; + srgn = rgn->srgn_tbl + srgn_idx; + + spin_lock_irqsave(&hpb->hpb_lock, flags); + INFO_MSG("lba %lu lpn %lu region %d state %d subregion %d state %d", + value, lpn, rgn_idx, rgn->rgn_state, srgn_idx, + srgn->srgn_state); + + if (!ufshpb_valid_srgn(rgn, srgn)) { + INFO_MSG("[region %d subregion %d] has not valid hpb info.", + rgn_idx, srgn_idx); + goto out; + } + + if (!srgn->mctx) { + INFO_MSG("mctx is NULL"); + goto out; + } + + ppn = ufshpb_get_ppn(srgn->mctx, srgn_offset, &error); + if (error) { + INFO_MSG("getting ppn is fail from a page."); + goto out; + } + + INFO_MSG("ppn %llx is_dirty %d", ppn, + ufshpb_ppn_dirty_check(hpb, lpn, 1)); +out: + spin_unlock_irqrestore(&hpb->hpb_lock, flags); + return cnt; +} + +static ssize_t ufshpb_sysfs_info_region_store(struct ufshpb_lu *hpb, + const char *buf, size_t cnt) +{ + unsigned long rgn_idx; + int srgn_idx; + + if (kstrtoul(buf, 0, &rgn_idx)) + return -EINVAL; + + if (rgn_idx >= hpb->rgns_per_lu) { + ERR_MSG("error region %ld max %d", rgn_idx, hpb->rgns_per_lu); + return -EINVAL; + } + + INFO_MSG("(region state : PINNED=%d ACTIVE=%d INACTIVE=%d)", + HPB_RGN_PINNED, HPB_RGN_ACTIVE, HPB_RGN_INACTIVE); + + INFO_MSG("region %ld state %d", rgn_idx, + hpb->rgn_tbl[rgn_idx].rgn_state); + + for (srgn_idx = 0; srgn_idx < hpb->rgn_tbl[rgn_idx].srgn_cnt; + srgn_idx++) { + INFO_MSG("--- subregion %d state %d", srgn_idx, + hpb->rgn_tbl[rgn_idx].srgn_tbl[srgn_idx].srgn_state); + } + + return cnt; +} + +static ssize_t ufshpb_sysfs_ufshpb_release_store(struct ufshpb_lu *hpb, + const char *buf, size_t cnt) +{ + unsigned long value; + + INFO_MSG("start release function"); + + if (kstrtoul(buf, 0, &value)) { + ERR_MSG("kstrtoul error"); + return -EINVAL; + } + + if (value == 0xab) { + INFO_MSG("magic number %lu release start", value); + goto err_out; + } else { + INFO_MSG("wrong magic number %lu", value); + return -EINVAL; + } + + return cnt; +err_out: + INFO_MSG("ref_cnt %d", + atomic_read(&hpb->ufsf->hpb_kref.refcount.refs)); + ufshpb_failed(hpb, __func__); + + return cnt; +} + +static struct ufshpb_sysfs_entry ufshpb_sysfs_entries[] = { + __ATTR(hpb_read_disable, 0644, + ufshpb_sysfs_prep_disable_show, ufshpb_sysfs_prep_disable_store), + __ATTR(map_cmd_disable, 0644, + ufshpb_sysfs_map_disable_show, ufshpb_sysfs_map_disable_store), + __ATTR(throttle_map_req, 0644, + ufshpb_sysfs_throttle_map_req_show, + ufshpb_sysfs_throttle_map_req_store), + __ATTR(throttle_pre_req, 0644, + ufshpb_sysfs_throttle_pre_req_show, + ufshpb_sysfs_throttle_pre_req_store), + __ATTR(pre_req_min_tr_len, 0644, + ufshpb_sysfs_pre_req_min_tr_len_show, + ufshpb_sysfs_pre_req_min_tr_len_store), + __ATTR(pre_req_max_tr_len, 0644, + ufshpb_sysfs_pre_req_max_tr_len_show, + ufshpb_sysfs_pre_req_max_tr_len_store), + __ATTR(pre_req_requeue_cnt, 0644, + ufshpb_sysfs_pre_req_requeue_cnt_show, + ufshpb_sysfs_pre_req_requeue_cnt_store), +#if defined(CONFIG_HPB_DEBUG) + __ATTR(debug, 0644, + ufshpb_sysfs_debug_show, ufshpb_sysfs_debug_store), +#endif + __ATTR(rsp_list_count, 0444, ufshpb_sysfs_rsp_list_cnt_show, NULL), +#if defined(CONFIG_HPB_ERR_INJECTION) + __ATTR(err_injection_select, 0644, + ufshpb_sysfs_err_injection_select_show, + ufshpb_sysfs_err_injection_select_store), + __ATTR(err_injection_bitflip, 0644, + ufshpb_sysfs_err_injection_bitflip_show, + ufshpb_sysfs_err_injection_bitflip_store), + __ATTR(err_injection_offset, 0644, + ufshpb_sysfs_err_injection_offset_show, + ufshpb_sysfs_err_injection_offset_store), + __ATTR(err_injection_random, 0644, + ufshpb_sysfs_err_injection_random_show, + ufshpb_sysfs_err_injection_random_store), +#endif + __ATTR(hpb_version, 0444, ufshpb_sysfs_version_show, NULL), + __ATTR(hit_count, 0444, ufshpb_sysfs_hit_show, NULL), + __ATTR(miss_count, 0444, ufshpb_sysfs_miss_show, NULL), + __ATTR(map_req_count, 0444, ufshpb_sysfs_map_req_show, NULL), + __ATTR(pre_req_count, 0444, ufshpb_sysfs_pre_req_show, NULL), + __ATTR(region_stat_count, 0444, ufshpb_sysfs_region_stat_show, NULL), + __ATTR(count_reset, 0200, NULL, ufshpb_sysfs_count_reset_store), + __ATTR(get_info_from_lba, 0200, NULL, ufshpb_sysfs_info_lba_store), + __ATTR(get_info_from_region, 0200, NULL, + ufshpb_sysfs_info_region_store), + __ATTR(release, 0200, NULL, ufshpb_sysfs_ufshpb_release_store), + __ATTR_NULL +}; + +static ssize_t ufshpb_attr_show(struct kobject *kobj, struct attribute *attr, + char *page) +{ + struct ufshpb_sysfs_entry *entry; + struct ufshpb_lu *hpb; + ssize_t error; + + entry = container_of(attr, struct ufshpb_sysfs_entry, attr); + hpb = container_of(kobj, struct ufshpb_lu, kobj); + + if (!entry->show) + return -EIO; + + mutex_lock(&hpb->sysfs_lock); + error = entry->show(hpb, page); + mutex_unlock(&hpb->sysfs_lock); + return error; +} + +static ssize_t ufshpb_attr_store(struct kobject *kobj, struct attribute *attr, + const char *page, size_t len) +{ + struct ufshpb_sysfs_entry *entry; + struct ufshpb_lu *hpb; + ssize_t error; + + entry = container_of(attr, struct ufshpb_sysfs_entry, attr); + hpb = container_of(kobj, struct ufshpb_lu, kobj); + + if (!entry->store) + return -EIO; + + mutex_lock(&hpb->sysfs_lock); + error = entry->store(hpb, page, len); + mutex_unlock(&hpb->sysfs_lock); + return error; +} + +static const struct sysfs_ops ufshpb_sysfs_ops = { + .show = ufshpb_attr_show, + .store = ufshpb_attr_store, +}; + +static struct kobj_type ufshpb_ktype = { + .sysfs_ops = &ufshpb_sysfs_ops, + .release = NULL, +}; + +static int ufshpb_create_sysfs(struct ufsf_feature *ufsf, struct ufshpb_lu *hpb) +{ + struct device *dev = ufsf->hba->dev; + struct ufshpb_sysfs_entry *entry; + int err; + + hpb->sysfs_entries = ufshpb_sysfs_entries; + + ufshpb_stat_reset(hpb); + atomic64_set(&hpb->act_rsp_list_cnt, 0); + atomic64_set(&hpb->inact_rsp_list_cnt, 0); + + kobject_init(&hpb->kobj, &ufshpb_ktype); + mutex_init(&hpb->sysfs_lock); + + INFO_MSG("ufshpb creates sysfs lu %d %p dev->kobj %p", hpb->lun, + &hpb->kobj, &dev->kobj); + + err = kobject_add(&hpb->kobj, kobject_get(&dev->kobj), + "ufshpb_lu%d", hpb->lun); + if (!err) { + for (entry = hpb->sysfs_entries; entry->attr.name != NULL; + entry++) { + INFO_MSG("ufshpb_lu%d sysfs attr creates: %s", + hpb->lun, entry->attr.name); + if (sysfs_create_file(&hpb->kobj, &entry->attr)) + break; + } + INFO_MSG("ufshpb_lu%d sysfs adds uevent", hpb->lun); + kobject_uevent(&hpb->kobj, KOBJ_ADD); + } + + return err; +} + +static inline void ufshpb_remove_sysfs(struct ufshpb_lu *hpb) +{ + kobject_uevent(&hpb->kobj, KOBJ_REMOVE); + INFO_MSG("ufshpb removes sysfs lu %d %p ", hpb->lun, &hpb->kobj); + kobject_del(&hpb->kobj); +} diff --git a/drivers/scsi/ufs/ufshpb.h b/drivers/scsi/ufs/ufshpb.h new file mode 100755 index 000000000000..47271fa6ba33 --- /dev/null +++ b/drivers/scsi/ufs/ufshpb.h @@ -0,0 +1,346 @@ +/* + * Universal Flash Storage Host Performance Booster + * + * Copyright (C) 2017-2018 Samsung Electronics Co., Ltd. + * + * Authors: + * Yongmyung Lee + * Jinyoung Choi + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * See the COPYING file in the top-level directory or visit + * + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * This program is provided "AS IS" and "WITH ALL FAULTS" and + * without warranty of any kind. You are solely responsible for + * determining the appropriateness of using and distributing + * the program and assume all risks associated with your exercise + * of rights with respect to the program, including but not limited + * to infringement of third party rights, the risks and costs of + * program errors, damage to or loss of data, programs or equipment, + * and unavailability or interruption of operations. Under no + * circumstances will the contributor of this Program be liable for + * any damages of any kind arising from your use or distribution of + * this program. + * + * The Linux Foundation chooses to take subject only to the GPLv2 + * license terms, and distributes only under these terms. + */ + +#ifndef _UFSHPB_H_ +#define _UFSHPB_H_ + +#include +#include +#include +#include +#include +#include + +#if defined(CONFIG_HPB_ERR_INJECTION) +#include +#endif + +#include "../../../block/blk.h" +#include "../../../block/blk-mq.h" +#include "../../../block/blk-mq-sched.h" +#include "../scsi_priv.h" + +/* Version info*/ +#define UFSHPB_VER 0x0201 +#define UFSHPB_DD_VER 0x020800 +#define UFSHPB_DD_VER_POST UFSFEATURE_DD_VER_POST + +/* Constant value*/ +#define MAX_ACTIVE_NUM 2 +#define MAX_INACTIVE_NUM 2 + +#define HPB_ENTRY_SIZE 0x08 +#define HPB_ENTREIS_PER_OS_PAGE (OS_PAGE_SIZE / HPB_ENTRY_SIZE) + +#define RETRY_DELAY_MS 5000 + +/* HPB Support Chunk Size */ +#define HPB_MULTI_CHUNK_LOW 9 +#define HPB_MULTI_CHUNK_HIGH 128 +#define MAX_HPB_CONTEXT_ID 0x7f + +/* Description */ +#define UFS_FEATURE_SUPPORT_HPB_BIT 0x80 + +/* Response UPIU types */ +#define HPB_RSP_NONE 0x00 +#define HPB_RSP_REQ_REGION_UPDATE 0x01 + +/* Vender defined OPCODE */ +#define UFSHPB_READ_BUFFER 0xF9 +#define UFSHPB_WRITE_BUFFER 0xFA + +#define UFSHPB_GROUP_NUMBER 0x11 +#define UFSHPB_RB_ID_READ 0x01 +#define UFSHPB_WB_ID_PREFETCH 0x02 +#define TRANSFER_LEN 0x01 + +#define DEV_DATA_SEG_LEN 0x14 +#define DEV_SENSE_SEG_LEN 0x12 +#define DEV_DES_TYPE 0x80 +#define DEV_ADDITIONAL_LEN 0x10 + +#define PINNED_NOT_SET (-1) +#define DEFAULT_WB_REQUEUE_CNT 3 + +/* + * UFSHPB DEBUG + */ + +#if defined(CONFIG_HPB_DEBUG) +#define HPB_DEBUG(hpb, msg, args...) \ + do { if (hpb && hpb->debug) \ + printk(KERN_ERR "%s:%d " msg "\n", \ + __func__, __LINE__, ##args); \ + } while (0) + +#define HPB_FTRACE(hpb, msg, args...) \ + do { if (hpb && hpb->debug) \ + trace_printk("%04d\t " msg "\n", \ + __LINE__, ##args); \ + } while (0) +#else +#define HPB_DEBUG(hpb, msg, args...) do { } while (0) +#define HPB_FTRACE(hpb, msg, args...) do { } while (0) +#endif + +enum UFSHPB_STATE { + HPB_PRESENT = 1, + HPB_SUSPEND = 2, + HPB_FAILED = -2, + HPB_NEED_INIT = 0, + HPB_RESET = -3, +}; + +enum HPB_RGN_STATE { + HPB_RGN_INACTIVE, + HPB_RGN_ACTIVE, + HPB_RGN_PINNED, +}; + +enum HPB_SRGN_STATE { + HPB_SRGN_UNUSED, + HPB_SRGN_DIRTY, + HPB_SRGN_CLEAN, + HPB_SRGN_ISSUED, +}; + +#if defined(CONFIG_HPB_ERR_INJECTION) +enum HPB_ERR_INJECTION_SELECT { + HPB_ERR_INJECTION_DISABLE, + HPB_ERR_INJECTION_BITFLIP, + HPB_ERR_INJECTION_OFFSET, + HPB_ERR_INJECTION_RANDOM, +}; +#endif + +struct ufshpb_dev_info { + int num_lu; + int version; + int rgn_size; + int srgn_size; +}; + +struct ufshpb_active_field { + __be16 active_rgn; + __be16 active_srgn; +} __packed; + +struct ufshpb_rsp_field { + __be16 sense_data_len; + u8 desc_type; + u8 additional_len; + u8 hpb_type; + u8 reserved; + u8 active_rgn_cnt; + u8 inactive_rgn_cnt; + struct ufshpb_active_field hpb_active_field[2]; + __be16 hpb_inactive_field[2]; +} __packed; + +struct ufshpb_map_ctx { + struct page **m_page; + unsigned int *ppn_dirty; + + struct list_head list_table; +}; + +struct ufshpb_subregion { + struct ufshpb_map_ctx *mctx; + enum HPB_SRGN_STATE srgn_state; + int rgn_idx; + int srgn_idx; + + /* below information is used by rsp_list */ + struct list_head list_act_srgn; +}; + +struct ufshpb_region { + struct ufshpb_subregion *srgn_tbl; + enum HPB_RGN_STATE rgn_state; + int rgn_idx; + int srgn_cnt; + + /* below information is used by rsp_list */ + struct list_head list_inact_rgn; + + /* below information is used by lru */ + struct list_head list_lru_rgn; +}; + +struct ufshpb_req { + struct request *req; + struct bio *bio; + struct ufshpb_lu *hpb; + struct list_head list_req; + void (*end_io)(struct request *rq, int err); + void *end_io_data; + char sense[SCSI_SENSE_BUFFERSIZE]; + + union { + struct { + struct ufshpb_map_ctx *mctx; + unsigned int rgn_idx; + unsigned int srgn_idx; + unsigned int lun; + } rb; + struct { + struct page *m_page; + unsigned int len; + unsigned long lpn; + } wb; + }; +}; + +enum selection_type { + LRU = 1, +}; + +struct victim_select_info { + int selection_type; + struct list_head lh_lru_rgn; + int max_lru_active_cnt; /* supported hpb #region - pinned #region */ + atomic64_t active_cnt; +}; + +struct ufshpb_lu { + struct ufsf_feature *ufsf; + int lun; + int qd; + struct ufshpb_region *rgn_tbl; + + spinlock_t hpb_lock; + + spinlock_t retry_list_lock; + struct ufshpb_req *map_req; + int num_inflight_map_req; + int throttle_map_req; + struct list_head lh_map_req_free; + struct list_head lh_map_req_retry; + struct list_head lh_map_ctx_free; + + spinlock_t rsp_list_lock; + struct list_head lh_pinned_srgn; + struct list_head lh_act_srgn; + struct list_head lh_inact_rgn; + + struct kobject kobj; + struct mutex sysfs_lock; + struct ufshpb_sysfs_entry *sysfs_entries; + + struct ufshpb_req *pre_req; + int num_inflight_pre_req; + int throttle_pre_req; + struct list_head lh_pre_req_free; + int ctx_id_ticket; + int pre_req_min_tr_len; + int pre_req_max_tr_len; + int pre_req_requeue_cnt; + + struct work_struct pinned_work; + struct delayed_work retry_work; + struct work_struct task_work; + + /* for selecting victim */ + struct victim_select_info lru_info; + + int lu_max_active_rgns; + int lu_pinned_rgn_startidx; + int lu_pinned_end_offset; + int srgns_per_lu; + int rgns_per_lu; + int srgns_per_rgn; + int srgn_mem_size; + int entries_per_rgn_shift; + int entries_per_rgn_mask; + int entries_per_srgn; + int entries_per_srgn_shift; + int entries_per_srgn_mask; + int dwords_per_srgn; + unsigned long long srgn_unit_size; + int mpage_bytes; + int mpages_per_srgn; + int lu_num_blocks; + + /* for debug */ + int alloc_mctx; + int debug_free_table; + bool force_disable; + bool force_map_req_disable; + bool debug; + atomic64_t hit; + atomic64_t miss; + atomic64_t rb_noti_cnt; + atomic64_t rb_active_cnt; + atomic64_t rb_inactive_cnt; + atomic64_t map_req_cnt; + atomic64_t map_compl_cnt; + atomic64_t pre_req_cnt; + + atomic64_t act_rsp_list_cnt; + atomic64_t inact_rsp_list_cnt; +#if defined(CONFIG_HPB_ERR_INJECTION) + enum HPB_ERR_INJECTION_SELECT err_injection_select; + u64 err_injection_bitflip; + u64 err_injection_offset; + int err_injection_random; +#endif +}; + +struct ufshpb_sysfs_entry { + struct attribute attr; + ssize_t (*show)(struct ufshpb_lu *hpb, char *buf); + ssize_t (*store)(struct ufshpb_lu *hpb, const char *buf, size_t count); +}; + +struct ufs_hba; +struct ufshcd_lrb; + +int ufshpb_get_state(struct ufsf_feature *ufsf); +void ufshpb_set_state(struct ufsf_feature *ufsf, int state); +void ufshpb_get_dev_info(struct ufsf_feature *ufsf, u8 *desc_buf); +void ufshpb_get_geo_info(struct ufsf_feature *ufsf, u8 *geo_buf); +void ufshpb_get_lu_info(struct ufsf_feature *ufsf, int lun, u8 *unit_buf); +void ufshpb_init_handler(struct work_struct *work); +void ufshpb_rsp_upiu(struct ufsf_feature *ufsf, struct ufshcd_lrb *lrbp); +void ufshpb_resume(struct ufsf_feature *ufsf); +void ufshpb_suspend(struct ufsf_feature *ufsf); +void ufshpb_reset_host(struct ufsf_feature *ufsf); +void ufshpb_reset(struct ufsf_feature *ufsf); +void ufshpb_prep_fn(struct ufsf_feature *ufsf, struct ufshcd_lrb *lrbp); +void ufshpb_remove(struct ufsf_feature *ufsf, int state); +#endif /* End of Header */ diff --git a/drivers/scsi/ufs/ufshpb_skh.c b/drivers/scsi/ufs/ufshpb_skh.c new file mode 100644 index 000000000000..afa7cb0fb866 --- /dev/null +++ b/drivers/scsi/ufs/ufshpb_skh.c @@ -0,0 +1,3275 @@ +/* + * Universal Flash Storage Host Performance Booster + * + * Original work Copyright (C) 2017-2018 Samsung Electronics Co., Ltd. + * Modified work Copyright (C) 2018, Google, Inc. + * Modified work Copyright (C) 2019 SK hynix + * + * Original work Authors: + * Yongmyung Lee + * Jinyoung Choi + * Modifier + * Kihyun Cho + * Sengman Shin + * Jinsu Lee + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * See the COPYING file in the top-level directory or visit + * + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * This program is provided "AS IS" and "WITH ALL FAULTS" and + * without warranty of any kind. You are solely responsible for + * determining the appropriateness of using and distributing + * the program and assume all risks associated with your exercise + * of rights with respect to the program, including but not limited + * to infringement of third party rights, the risks and costs of + * program errors, damage to or loss of data, programs or equipment, + * and unavailability or interruption of operations. Under no + * circumstances will the contributor of this Program be liable for + * any damages of any kind arising from your use or distribution of + * this program. + * + * The Linux Foundation chooses to take subject only to the GPLv2 + * license terms, and distributes only under these terms. + */ + +#include +#include +#include +#include +#include + +#include "../../../block/blk.h" + +#include "ufs.h" +#include "ufshcd.h" +#include "ufshpb_skh.h" +#include + +u32 skhpb_debug_mask = SKHPB_LOG_ERR | SKHPB_LOG_INFO; +//u32 skhpb_debug_mask = SKHPB_LOG_ERR | SKHPB_LOG_INFO | SKHPB_LOG_DEBUG | SKHPB_LOG_HEX; +int debug_map_req = SKHPB_MAP_RSP_DISABLE; + +/* + * debug variables + */ +static int skhpb_alloc_mctx; + +/* + * define global constants + */ +static int skhpb_sects_per_blk_shift; +static int skhpb_bits_per_dword_shift; +static int skhpb_bits_per_dword_mask; + +static int skhpb_create_sysfs(struct ufs_hba *hba, struct skhpb_lu *hpb); +static int skhpb_check_lru_evict(struct skhpb_lu *hpb, struct skhpb_region *cb); +static void skhpb_error_handler(struct work_struct *work); +static void skhpb_evict_region(struct skhpb_lu *hpb, + struct skhpb_region *cb); +static void skhpb_purge_active_block(struct skhpb_lu *hpb); +static int skhpb_set_map_req(struct skhpb_lu *hpb, + int region, int subregion, struct skhpb_map_ctx *mctx, + struct skhpb_rsp_info *rsp_info, + enum SKHPB_BUFFER_MODE flag); +static void skhpb_rsp_map_cmd_req(struct skhpb_lu *hpb, + struct skhpb_rsp_info *rsp_info); +static void skhpb_map_loading_trigger(struct skhpb_lu *hpb, + bool only_pinned, bool do_work_handler); + +static inline void skhpb_purge_active_page(struct skhpb_lu *hpb, + struct skhpb_subregion *cp, int state); + +static void skhpb_hit_lru_info(struct skhpb_victim_select_info *lru_info, + struct skhpb_region *cb); + +static inline void skhpb_get_bit_offset( + struct skhpb_lu *hpb, int subregion_offset, + int *dword, int *offset) +{ + *dword = subregion_offset >> skhpb_bits_per_dword_shift; + *offset = subregion_offset & skhpb_bits_per_dword_mask; +} + +/* called with hpb_lock (irq) */ +static bool skhpb_ppn_dirty_check(struct skhpb_lu *hpb, + struct skhpb_subregion *cp, int subregion_offset) +{ + bool is_dirty = false; + unsigned int bit_dword, bit_offset; + + if (!cp->mctx->ppn_dirty) + return true; + + skhpb_get_bit_offset(hpb, subregion_offset, + &bit_dword, &bit_offset); + is_dirty = cp->mctx->ppn_dirty[bit_dword] & (1 << bit_offset) ? true : false; + + return is_dirty; +} + +static void skhpb_ppn_prep(struct skhpb_lu *hpb, + struct ufshcd_lrb *lrbp, skhpb_t ppn, + unsigned int sector_len) +{ + unsigned char cmd[16] = { 0 }; + + if (hpb->hba->skhpb_quirk & SKHPB_QUIRK_USE_READ_16_FOR_ENCRYPTION) + cmd[0] = READ_16; + else + cmd[0] = SKHPB_READ; + cmd[2] = lrbp->cmd->cmnd[2]; + cmd[3] = lrbp->cmd->cmnd[3]; + cmd[4] = lrbp->cmd->cmnd[4]; + cmd[5] = lrbp->cmd->cmnd[5]; + put_unaligned(ppn, (u64 *)&cmd[6]); + cmd[14] = (u8)(sector_len >> skhpb_sects_per_blk_shift); //Transfer length + if (hpb->hba->skhpb_quirk & SKHPB_QUIRK_USE_READ_16_FOR_ENCRYPTION) + cmd[15] = 0x01; //Control + else + cmd[15] = 0x00; //Control + + memcpy(lrbp->cmd->cmnd, cmd, MAX_CDB_SIZE); + memcpy(lrbp->ucd_req_ptr->sc.cdb, cmd, MAX_CDB_SIZE); + + //To verify the values within READ command + /* SKHPB_DRIVER_HEXDUMP("[HPB] HPB READ ", 16, 1, cmd, sizeof(cmd), 1); */ +} + +static inline void skhpb_set_dirty_bits(struct skhpb_lu *hpb, + struct skhpb_region *cb, struct skhpb_subregion *cp, + int dword, int offset, unsigned int count) +{ + const unsigned long mask = ((1UL << count) - 1) & 0xffffffff; + + if (cb->region_state == SKHPB_REGION_INACTIVE) + return; + + BUG_ON(!cp->mctx); + cp->mctx->ppn_dirty[dword] |= (mask << offset); +} + +/* called with hpb_lock (irq) */ +static void skhpb_set_dirty(struct skhpb_lu *hpb, + struct ufshcd_lrb *lrbp, int region, + int subregion, int subregion_offset) +{ + struct skhpb_region *cb; + struct skhpb_subregion *cp; + int count; + int bit_count, bit_dword, bit_offset; + + count = blk_rq_sectors(lrbp->cmd->request) >> skhpb_sects_per_blk_shift; + skhpb_get_bit_offset(hpb, subregion_offset, + &bit_dword, &bit_offset); + + do { + bit_count = min(count, SKHPB_BITS_PER_DWORD - bit_offset); + + cb = hpb->region_tbl + region; + cp = cb->subregion_tbl + subregion; + + skhpb_set_dirty_bits(hpb, cb, cp, + bit_dword, bit_offset, bit_count); + + bit_offset = 0; + bit_dword++; + + if (bit_dword == hpb->dwords_per_subregion) { + bit_dword = 0; + subregion++; + + if (subregion == hpb->subregions_per_region) { + subregion = 0; + region++; + } + } + + count -= bit_count; + } while (count); +} + +static inline bool skhpb_is_encrypted_lrbp(struct ufshcd_lrb *lrbp) +{ + /* To do + * UTRD_CRYPTO_ENABLE is not define at SM8350 + */ + //return (lrbp->utr_descriptor_ptr->header.dword_0 & UTRD_CRYPTO_ENABLE); + return true; +} + +static inline enum SKHPB_CMD skhpb_get_cmd(struct ufshcd_lrb *lrbp) +{ + unsigned char cmd = lrbp->cmd->cmnd[0]; + + if (cmd == READ_10 || cmd == READ_16) + return SKHPB_CMD_READ; + if (cmd == WRITE_10 || cmd == WRITE_16) + return SKHPB_CMD_WRITE; + if (cmd == UNMAP) + return SKHPB_CMD_DISCARD; + return SKHPB_CMD_OTHERS; +} + +static inline void skhpb_get_pos_from_lpn(struct skhpb_lu *hpb, + unsigned int lpn, int *region, int *subregion, int *offset) +{ + int region_offset; + + *region = lpn >> hpb->entries_per_region_shift; + region_offset = lpn & hpb->entries_per_region_mask; + *subregion = region_offset >> hpb->entries_per_subregion_shift; + *offset = region_offset & hpb->entries_per_subregion_mask; +} + +static inline bool skhpb_check_region_subregion_validity(struct skhpb_lu *hpb, + int region, int subregion) +{ + struct skhpb_region *cb; + + if (region >= hpb->regions_per_lu){ + SKHPB_DRIVER_E("[HCM] Out of REGION range - region[%d]:MAX[%d]\n", + region, hpb->regions_per_lu); + return false; + } + cb = hpb->region_tbl + region; + if (subregion >= cb->subregion_count){ + SKHPB_DRIVER_E("[HCM] Out of SUBREGION range - subregion[i]:MAX[%d]\n", + subregion, cb->subregion_count); + return false; + } + return true; +} + + +static inline skhpb_t skhpb_get_ppn(struct skhpb_map_ctx *mctx, int pos) +{ + skhpb_t *ppn_table; + int index, offset; + + index = pos / SKHPB_ENTREIS_PER_OS_PAGE; + offset = pos % SKHPB_ENTREIS_PER_OS_PAGE; + + ppn_table = page_address(mctx->m_page[index]); + return ppn_table[offset]; +} + + +#if defined(SKHPB_READ_LARGE_CHUNK_SUPPORT) +static bool skhpb_subregion_dirty_check(struct skhpb_lu *hpb, struct skhpb_subregion *cp, + int subregion_offset, int reqBlkCnt) +{ + unsigned int bit_dword, bit_offset; + unsigned int tmp; + int checkCnt; + + if (!cp->mctx) + return true; + + if (!cp->mctx->ppn_dirty) + return true; + + skhpb_get_bit_offset(hpb, subregion_offset, + &bit_dword, &bit_offset); + + while (true) { + checkCnt = SKHPB_BITS_PER_DWORD - bit_offset; + if (cp->mctx->ppn_dirty[bit_dword]) { + tmp = cp->mctx->ppn_dirty[bit_dword] << bit_offset; + if (SKHPB_BITS_PER_DWORD - reqBlkCnt > 0) + tmp = tmp >> (SKHPB_BITS_PER_DWORD - reqBlkCnt); + if (tmp) + return true; + } + reqBlkCnt -= checkCnt; + if (reqBlkCnt <= 0) + break; + bit_dword++; + if (bit_dword >= hpb->ppn_dirties_per_subregion) + break; + bit_offset = 0; + } + return false; +} + +static bool skhpb_lc_dirty_check(struct skhpb_lu *hpb, unsigned int lpn, unsigned int rq_sectors) +{ + int reg; + int subReg; + struct skhpb_region *cb; + struct skhpb_subregion *cp; + unsigned long cur_lpn = lpn; + int subRegOffset; + int reqBlkCnt = rq_sectors >> skhpb_sects_per_blk_shift; + + do { + skhpb_get_pos_from_lpn(hpb, cur_lpn, ®, &subReg, &subRegOffset); + if (!skhpb_check_region_subregion_validity(hpb, reg, subReg)) + return true; + cb = hpb->region_tbl + reg; + cp = cb->subregion_tbl + subReg; + + if (cb->region_state == SKHPB_REGION_INACTIVE || + cp->subregion_state != SKHPB_SUBREGION_CLEAN) { + atomic64_inc(&hpb->lc_reg_subreg_miss); + return true; + } + + if (skhpb_subregion_dirty_check(hpb, cp, subRegOffset, reqBlkCnt)) { + //SKHPB_DRIVER_D("[NORMAL READ] DIRTY: Region(%d), SubRegion(%d) \n", reg, subReg); + atomic64_inc(&hpb->lc_entry_dirty_miss); + return true; + } + + if (hpb->entries_per_subregion < subRegOffset + reqBlkCnt) { + reqBlkCnt -= (hpb->entries_per_subregion - subRegOffset); + } else { + reqBlkCnt = 0; + } + + cur_lpn += reqBlkCnt; + } while(reqBlkCnt); + + return false; +} +#endif + +void skhpb_prep_fn(struct ufs_hba *hba, struct ufshcd_lrb *lrbp) +{ + struct skhpb_lu *hpb; + struct skhpb_region *cb; + struct skhpb_subregion *cp; + unsigned int lpn; + skhpb_t ppn = 0; + int region, subregion, subregion_offset; + const struct request *rq; + unsigned long long rq_pos; + unsigned int rq_sectors; + unsigned char cmd; + unsigned long flags; + + /* WKLU could not be HPB-LU */ + if (lrbp->lun >= UFS_UPIU_MAX_GENERAL_LUN) + return; + + hpb = hba->skhpb_lup[lrbp->lun]; + if (!hpb || !hpb->lu_hpb_enable) + return; + + if (hpb->force_hpb_read_disable) + return; + + cmd = skhpb_get_cmd(lrbp); + if (cmd == SKHPB_CMD_OTHERS) + return; + /* + * TODO: check if ICE is not supported or not. + * + * if (cmd==SKHPB_CMD_READ && skhpb_is_encrypted_lrbp(lrbp)) + * return; + */ + rq = lrbp->cmd->request; + rq_pos = blk_rq_pos(rq); + rq_sectors = blk_rq_sectors(rq); + + lpn = rq_pos / SKHPB_SECTORS_PER_BLOCK; + skhpb_get_pos_from_lpn( + hpb, lpn, ®ion, &subregion, &subregion_offset); + if (!skhpb_check_region_subregion_validity(hpb, region, subregion)) + return; + cb = hpb->region_tbl + region; + cp = cb->subregion_tbl + subregion; + + if (cmd == SKHPB_CMD_WRITE || + cmd == SKHPB_CMD_DISCARD) { + if (cb->region_state == SKHPB_REGION_INACTIVE) { + return; + } + spin_lock_irqsave(&hpb->hpb_lock, flags); + skhpb_set_dirty(hpb, lrbp, region, subregion, + subregion_offset); + spin_unlock_irqrestore(&hpb->hpb_lock, flags); + return; + } + +#if defined(SKHPB_READ_LARGE_CHUNK_SUPPORT) + if (((rq_sectors & (SKHPB_SECTORS_PER_BLOCK - 1)) != 0) || + rq_sectors > + (SKHPB_READ_LARGE_CHUNK_MAX_BLOCK_COUNT << skhpb_sects_per_blk_shift)) { +#else + if (rq_sectors != SKHPB_SECTORS_PER_BLOCK) { +#endif + atomic64_inc(&hpb->size_miss); + return; + } + + spin_lock_irqsave(&hpb->hpb_lock, flags); + if (cb->region_state == SKHPB_REGION_INACTIVE) { + atomic64_inc(&hpb->region_miss); + spin_unlock_irqrestore(&hpb->hpb_lock, flags); + return; + } else if (cp->subregion_state != SKHPB_SUBREGION_CLEAN){ + atomic64_inc(&hpb->subregion_miss); + spin_unlock_irqrestore(&hpb->hpb_lock, flags); + return; + } + if (rq_sectors <= SKHPB_SECTORS_PER_BLOCK) { + if (skhpb_ppn_dirty_check(hpb, cp, subregion_offset)) { + atomic64_inc(&hpb->entry_dirty_miss); + spin_unlock_irqrestore(&hpb->hpb_lock, flags); + return; + } + } +#if defined(SKHPB_READ_LARGE_CHUNK_SUPPORT) + else { + if (skhpb_lc_dirty_check(hpb, lpn, rq_sectors)) { + spin_unlock_irqrestore(&hpb->hpb_lock, flags); + return; + } + atomic64_inc(&hpb->lc_hit); + } +#endif + ppn = skhpb_get_ppn(cp->mctx, subregion_offset); + spin_unlock_irqrestore(&hpb->hpb_lock, flags); + + //SKHPB_DRIVER_D("XXX ufs ppn %016llx, lba %u\n", (unsigned long long)ppn, lpn); + skhpb_ppn_prep(hpb, lrbp, ppn, rq_sectors); + atomic64_inc(&hpb->hit); + return; +} + +static int skhpb_clean_dirty_bitmap( + struct skhpb_lu *hpb, struct skhpb_subregion *cp) +{ + struct skhpb_region *cb; + + cb = hpb->region_tbl + cp->region; + + /* if mctx is null, active block had been evicted out */ + if (cb->region_state == SKHPB_REGION_INACTIVE || !cp->mctx) { + SKHPB_DRIVER_D("%d - %d error already evicted\n", + cp->region, cp->subregion); + return -EINVAL; + } + + memset(cp->mctx->ppn_dirty, 0x00, + hpb->entries_per_subregion / BITS_PER_BYTE); + return 0; +} + +static void skhpb_clean_active_subregion( + struct skhpb_lu *hpb, struct skhpb_subregion *cp) +{ + struct skhpb_region *cb; + + cb = hpb->region_tbl + cp->region; + + /* if mctx is null, active block had been evicted out */ + if (cb->region_state == SKHPB_REGION_INACTIVE || !cp->mctx) { + SKHPB_DRIVER_D("%d - %d clean already evicted\n", + cp->region, cp->subregion); + return; + } + cp->subregion_state = SKHPB_SUBREGION_CLEAN; +} + +static void skhpb_error_active_subregion( + struct skhpb_lu *hpb, struct skhpb_subregion *cp) +{ + struct skhpb_region *cb; + + cb = hpb->region_tbl + cp->region; + + /* if mctx is null, active block had been evicted out */ + if (cb->region_state == SKHPB_REGION_INACTIVE || !cp->mctx) { + SKHPB_DRIVER_E("%d - %d evicted\n", cp->region, cp->subregion); + return; + } + cp->subregion_state = SKHPB_SUBREGION_DIRTY; +} + +static void skhpb_map_compl_process(struct skhpb_lu *hpb, + struct skhpb_map_req *map_req) +{ + unsigned long flags; + + SKHPB_MAP_REQ_TIME(map_req, map_req->RSP_end, 1); + SKHPB_DRIVER_D("SKHPB RB COMPL BUFFER %d - %d\n", map_req->region, map_req->subregion); + + spin_lock_irqsave(&hpb->hpb_lock, flags); + skhpb_clean_active_subregion(hpb, + hpb->region_tbl[map_req->region].subregion_tbl + + map_req->subregion); + spin_unlock_irqrestore(&hpb->hpb_lock, flags); +} + +static void skhpb_map_inactive_compl_process(struct skhpb_lu *hpb, + struct skhpb_map_req *map_req) +{ + SKHPB_DRIVER_D("SKHPB WB COMPL BUFFER %d\n", map_req->region); + SKHPB_MAP_REQ_TIME(map_req, map_req->RSP_end, 1); +} + +/* + * Must held rsp_list_lock before enter this function + */ +static struct skhpb_rsp_info *skhpb_get_req_info(struct skhpb_lu *hpb) +{ + struct skhpb_rsp_info *rsp_info = + list_first_entry_or_null(&hpb->lh_rsp_info_free, + struct skhpb_rsp_info, + list_rsp_info); + if (!rsp_info) { + SKHPB_DRIVER_E("there is no rsp_info"); + return NULL; + } + list_del(&rsp_info->list_rsp_info); + memset(rsp_info, 0x00, sizeof(struct skhpb_rsp_info)); + + INIT_LIST_HEAD(&rsp_info->list_rsp_info); + + return rsp_info; +} + +static void skhpb_map_req_compl_fn(struct request *req, blk_status_t error) +{ + struct skhpb_map_req *map_req = req->end_io_data; + struct ufs_hba *hba; + struct skhpb_lu *hpb; + struct scsi_sense_hdr sshdr; + struct skhpb_region *cb; + struct scsi_request *scsireq = scsi_req(req); + unsigned long flags; + + /* shut up "bio leak" warning */ + memcpy(map_req->sense, scsireq->sense, SCSI_SENSE_BUFFERSIZE); + req->bio = NULL; + blk_put_request(req); + + hpb = map_req->hpb; + hba = hpb->hba; + cb = hpb->region_tbl + map_req->region; + + if (hba->skhpb_state != SKHPB_PRESENT) + goto free_map_req; + + if (!error) { + skhpb_map_compl_process(hpb, map_req); + goto free_map_req; + } + + SKHPB_DRIVER_E("error number %d ( %d - %d )\n", + error, map_req->region, map_req->subregion); + scsi_normalize_sense(map_req->sense, + SCSI_SENSE_BUFFERSIZE, &sshdr); + SKHPB_DRIVER_E("code %x sense_key %x asc %x ascq %x\n", + sshdr.response_code, + sshdr.sense_key, sshdr.asc, sshdr.ascq); + SKHPB_DRIVER_E("byte4 %x byte5 %x byte6 %x additional_len %x\n", + sshdr.byte4, sshdr.byte5, + sshdr.byte6, sshdr.additional_length); + atomic64_inc(&hpb->rb_fail); + + if (sshdr.sense_key == ILLEGAL_REQUEST) { + if (sshdr.asc == 0x00 && sshdr.ascq == 0x16) { + /* OPERATION IN PROGRESS */ + SKHPB_DRIVER_E("retry rb %d - %d", + map_req->region, map_req->subregion); + + spin_lock(&hpb->map_list_lock); + INIT_LIST_HEAD(&map_req->list_map_req); + list_add_tail(&map_req->list_map_req, &hpb->lh_map_req_retry); + spin_unlock(&hpb->map_list_lock); + + schedule_delayed_work(&hpb->skhpb_map_req_retry_work, + msecs_to_jiffies(5000)); + return; + } + } + // Only change subregion status at here. + // Do not put map_ctx, it will re-use when it is activated again. + spin_lock_irqsave(&hpb->hpb_lock, flags); + skhpb_error_active_subregion(hpb, cb->subregion_tbl + map_req->subregion); + spin_unlock_irqrestore(&hpb->hpb_lock, flags); +free_map_req: + spin_lock(&hpb->map_list_lock); + INIT_LIST_HEAD(&map_req->list_map_req); + list_add_tail(&map_req->list_map_req, &hpb->lh_map_req_free); + spin_unlock(&hpb->map_list_lock); + atomic64_dec(&hpb->alloc_map_req_cnt); +} + +static void skhpb_map_inactive_req_compl_fn( + struct request *req, blk_status_t error) +{ + struct skhpb_map_req *map_req = req->end_io_data; + struct ufs_hba *hba; + struct skhpb_lu *hpb; + struct scsi_sense_hdr sshdr; + struct skhpb_region *cb; + struct scsi_request *scsireq = scsi_req(req); + unsigned long flags; + + /* shut up "bio leak" warning */ + memcpy(map_req->sense, scsireq->sense, SCSI_SENSE_BUFFERSIZE); + req->bio = NULL; + blk_put_request(req); + + hpb = map_req->hpb; + hba = hpb->hba; + cb = hpb->region_tbl + map_req->region; + + if (hba->skhpb_state != SKHPB_PRESENT) + goto free_map_req; + + if (!error) { + skhpb_map_inactive_compl_process(hpb, map_req); + goto free_map_req; + } + + SKHPB_DRIVER_E("error number %d ( %d - %d )", + error, map_req->region, map_req->subregion); + scsi_normalize_sense(map_req->sense, SCSI_SENSE_BUFFERSIZE, + &sshdr); + SKHPB_DRIVER_E("code %x sense_key %x asc %x ascq %x", + sshdr.response_code, + sshdr.sense_key, sshdr.asc, sshdr.ascq); + SKHPB_DRIVER_E("byte4 %x byte5 %x byte6 %x additional_len %x", + sshdr.byte4, sshdr.byte5, + sshdr.byte6, sshdr.additional_length); + atomic64_inc(&hpb->rb_fail); + + if (sshdr.sense_key == ILLEGAL_REQUEST) { + if (cb->is_pinned) { + SKHPB_DRIVER_E("WRITE_BUFFER is not allowed on pinned area: region#%d", + cb->region); + } else { + if (sshdr.asc == 0x00 && sshdr.ascq == 0x16) { + /* OPERATION IN PROGRESS */ + SKHPB_DRIVER_E("retry wb %d", map_req->region); + + spin_lock(&hpb->map_list_lock); + INIT_LIST_HEAD(&map_req->list_map_req); + list_add_tail(&map_req->list_map_req, &hpb->lh_map_req_retry); + spin_unlock(&hpb->map_list_lock); + + schedule_delayed_work(&hpb->skhpb_map_req_retry_work, + msecs_to_jiffies(5000)); + return; + } + } + } + // Only change subregion status at here. + // Do not put map_ctx, it will re-use when it is activated again. + spin_lock_irqsave(&hpb->hpb_lock, flags); + skhpb_error_active_subregion(hpb, cb->subregion_tbl + map_req->subregion); + spin_unlock_irqrestore(&hpb->hpb_lock, flags); +free_map_req: + spin_lock(&hpb->map_list_lock); + INIT_LIST_HEAD(&map_req->list_map_req); + list_add_tail(&map_req->list_map_req, &hpb->lh_map_req_free); + spin_unlock(&hpb->map_list_lock); + atomic64_dec(&hpb->alloc_map_req_cnt); +} + +static int skhpb_execute_req_dev_ctx(struct skhpb_lu *hpb, + unsigned char *cmd, void *buf, int length) +{ + unsigned long flags; + struct scsi_sense_hdr sshdr; + struct scsi_device *sdp; + struct ufs_hba *hba = hpb->hba; + int ret = 0; + + spin_lock_irqsave(hba->host->host_lock, flags); + sdp = hba->sdev_ufs_lu[hpb->lun]; + if (sdp) { + ret = scsi_device_get(sdp); + if (!ret && !scsi_device_online(sdp)) { + ret = -ENODEV; + scsi_device_put(sdp); + } else if (!ret) { + hba->issue_ioctl = true; + } + } else { + ret = -ENODEV; + } + spin_unlock_irqrestore(hba->host->host_lock, flags); + if (ret) + return ret; + + ret = scsi_execute_req(sdp, cmd, DMA_FROM_DEVICE, + buf, length, &sshdr, + msecs_to_jiffies(30000), 3, NULL); + spin_lock_irqsave(hba->host->host_lock, flags); + hba->issue_ioctl = false; + spin_unlock_irqrestore(hba->host->host_lock, flags); + scsi_device_put(sdp); + return ret; +} + +static inline void skhpb_set_read_dev_ctx_cmd(unsigned char *cmd, int lba, + int length) +{ + cmd[0] = READ_10; + cmd[1] = 0x02; + cmd[2] = SKHPB_GET_BYTE_3(lba); + cmd[3] = SKHPB_GET_BYTE_2(lba); + cmd[4] = SKHPB_GET_BYTE_1(lba); + cmd[5] = SKHPB_GET_BYTE_0(lba); + cmd[6] = SKHPB_GET_BYTE_2(length); + cmd[7] = SKHPB_GET_BYTE_1(length); + cmd[8] = SKHPB_GET_BYTE_0(length); +} + +int skhpb_issue_req_dev_ctx(struct skhpb_lu *hpb, unsigned char *buf, + int buf_length) +{ + unsigned char cmd[10] = { 0 }; + int cmd_len = buf_length >> PAGE_SHIFT; + int ret = 0; + + skhpb_set_read_dev_ctx_cmd(cmd, 0x48504230, cmd_len); + + ret = skhpb_execute_req_dev_ctx(hpb, cmd, buf, buf_length); + if (ret < 0) + SKHPB_DRIVER_E("failed with err %d\n", ret); + return ret; +} + +static inline void skhpb_set_write_buf_cmd(unsigned char *cmd, int region) +{ + cmd[0] = SKHPB_WRITE_BUFFER; + cmd[1] = 0x01; + cmd[2] = SKHPB_GET_BYTE_1(region); + cmd[3] = SKHPB_GET_BYTE_0(region); + cmd[4] = 0x00; + cmd[5] = 0x00; + cmd[6] = 0x00; + cmd[7] = 0x00; + cmd[8] = 0x00; + cmd[9] = 0x00; + + //To verify the values within WRITE_BUFFER command + SKHPB_DRIVER_HEXDUMP("[HPB] WRITE BUFFER ", 16, 1, cmd, 10, 1); +} + +static inline void skhpb_set_read_buf_cmd(unsigned char *cmd, + int region, int subregion, int subregion_mem_size) +{ + cmd[0] = SKHPB_READ_BUFFER; + cmd[1] = 0x01; + cmd[2] = SKHPB_GET_BYTE_1(region); + cmd[3] = SKHPB_GET_BYTE_0(region); + cmd[4] = SKHPB_GET_BYTE_1(subregion); + cmd[5] = SKHPB_GET_BYTE_0(subregion); + cmd[6] = SKHPB_GET_BYTE_2(subregion_mem_size); + cmd[7] = SKHPB_GET_BYTE_1(subregion_mem_size); + cmd[8] = SKHPB_GET_BYTE_0(subregion_mem_size); + cmd[9] = 0x00; + + //To verify the values within READ_BUFFER command + SKHPB_DRIVER_HEXDUMP("[HPB] READ BUFFER ", 16, 1, cmd, 10, 1); +} + +static int skhpb_add_bio_page(struct skhpb_lu *hpb, + struct request_queue *q, struct bio *bio, struct bio_vec *bvec, + struct skhpb_map_ctx *mctx) +{ + struct page *page = NULL; + int i, ret; + + bio_init(bio, bvec, hpb->mpages_per_subregion); + + for (i = 0; i < hpb->mpages_per_subregion; i++) { + page = mctx->m_page[i]; + if (!page) + return -ENOMEM; + + ret = bio_add_pc_page(q, bio, page, hpb->mpage_bytes, 0); + if (ret != hpb->mpage_bytes) { + SKHPB_DRIVER_E("error ret %d\n", ret); + return -EINVAL; + } + } + return 0; +} + +static int skhpb_map_req_issue( + struct skhpb_lu *hpb, struct skhpb_map_req *map_req) +{ + struct request_queue *q = hpb->hba->sdev_ufs_lu[hpb->lun]->request_queue; + struct request *req; + struct scsi_request *scsireq; + unsigned char cmd[10] = { 0 }; + int ret; + + if (map_req->rwbuffer_flag == W_BUFFER) + skhpb_set_write_buf_cmd(cmd, map_req->region); + else + skhpb_set_read_buf_cmd(cmd, map_req->region, map_req->subregion, + map_req->subregion_mem_size); + if (map_req->rwbuffer_flag == W_BUFFER) + req = blk_get_request(q, REQ_OP_SCSI_OUT, BLK_MQ_REQ_PREEMPT); + else + req = blk_get_request(q, REQ_OP_SCSI_IN, BLK_MQ_REQ_PREEMPT); + if (IS_ERR(req)) { + int rv = PTR_ERR(req); + + if (map_req->rwbuffer_flag == W_BUFFER) + SKHPB_DRIVER_E("blk_get_request errno %d, \ + retry #%d, WRITE BUFFER %d\n", + rv, map_req->retry_cnt, map_req->region); + else + SKHPB_DRIVER_E("blk_get_request errno %d, \ + retry #%d, READ BUFFER %d:%d\n", + rv, map_req->retry_cnt, + map_req->region, map_req->subregion); + + if (map_req->retry_cnt == 10) { + /* give up */ + return rv; + } + + spin_lock_bh(&hpb->map_list_lock); + list_add_tail(&map_req->list_map_req, &hpb->lh_map_req_retry); + spin_unlock_bh(&hpb->map_list_lock); + + schedule_delayed_work(&hpb->skhpb_map_req_retry_work, msecs_to_jiffies(10)); + + return 0; + } + scsireq = scsi_req(req); + + scsireq->cmd_len = COMMAND_SIZE(cmd[0]); + BUG_ON(scsireq->cmd_len > sizeof(scsireq->__cmd)); + scsireq->cmd = scsireq->__cmd; + memcpy(scsireq->cmd, cmd, scsireq->cmd_len); + + req->rq_flags |= RQF_QUIET | RQF_PREEMPT; + req->timeout = msecs_to_jiffies(30000); + req->end_io_data = map_req; + + if (map_req->rwbuffer_flag == R_BUFFER) { + ret = skhpb_add_bio_page( + hpb, q, &map_req->bio, map_req->bvec, map_req->mctx); + if (ret) { + SKHPB_DRIVER_E("skhpb_add_bio_page_error %d\n", ret); + goto out_put_request; + } + map_req->pbio = &map_req->bio; + blk_rq_append_bio(req, &map_req->pbio); + } + SKHPB_DRIVER_D("issue map_request: %d - %d\n", + map_req->region, map_req->subregion); + + SKHPB_MAP_REQ_TIME(map_req, map_req->RSP_issue, 0); + if( hpb->hpb_control_mode == HOST_CTRL_MODE) { + if (map_req->rwbuffer_flag == W_BUFFER) + blk_execute_rq_nowait( + q, NULL, req, 0, skhpb_map_inactive_req_compl_fn); + else + blk_execute_rq_nowait(q, NULL, req, 0, skhpb_map_req_compl_fn); + } else + blk_execute_rq_nowait(q, NULL, req, 1, skhpb_map_req_compl_fn); + + if (map_req->rwbuffer_flag == W_BUFFER) + atomic64_inc(&hpb->w_map_req_cnt); + else + atomic64_inc(&hpb->map_req_cnt); + + return 0; + +out_put_request: + blk_put_request(req); + return ret; +} + +static int skhpb_set_map_req(struct skhpb_lu *hpb, + int region, int subregion, struct skhpb_map_ctx *mctx, + struct skhpb_rsp_info *rsp_info, + enum SKHPB_BUFFER_MODE flag) +{ + bool last = hpb->region_tbl[region].subregion_tbl[subregion].last; + struct skhpb_map_req *map_req; + + spin_lock_bh(&hpb->map_list_lock); + map_req = list_first_entry_or_null(&hpb->lh_map_req_free, + struct skhpb_map_req, + list_map_req); + if (!map_req) { + SKHPB_DRIVER_E("There is no map_req\n"); + spin_unlock_bh(&hpb->map_list_lock); + return -ENOMEM; + } + list_del(&map_req->list_map_req); + spin_unlock_bh(&hpb->map_list_lock); + atomic64_inc(&hpb->alloc_map_req_cnt); + + memset(map_req, 0x00, sizeof(struct skhpb_map_req)); + + map_req->hpb = hpb; + map_req->region = region; + map_req->subregion = subregion; + map_req->subregion_mem_size = + last ? hpb->last_subregion_mem_size : hpb->subregion_mem_size; + map_req->mctx = mctx; + map_req->lun = hpb->lun; + map_req->RSP_start = rsp_info->RSP_start; + if (flag == W_BUFFER) { + map_req->rwbuffer_flag = W_BUFFER; + } else + map_req->rwbuffer_flag = R_BUFFER; + + if(skhpb_map_req_issue(hpb, map_req)) { + SKHPB_DRIVER_E("issue Failed!!!\n"); + return -ENOMEM; + } + + return 0; +} + +static struct skhpb_map_ctx *skhpb_get_map_ctx(struct skhpb_lu *hpb) +{ + struct skhpb_map_ctx *mctx; + + mctx = list_first_entry_or_null(&hpb->lh_map_ctx, + struct skhpb_map_ctx, list_table); + if (mctx) { + list_del_init(&mctx->list_table); + hpb->debug_free_table--; + return mctx; + } + return ERR_PTR(-ENOMEM); +} + +static inline void skhpb_add_lru_info(struct skhpb_victim_select_info *lru_info, + struct skhpb_region *cb) +{ + cb->region_state = SKHPB_REGION_ACTIVE; + list_add_tail(&cb->list_region, &lru_info->lru); + atomic64_inc(&lru_info->active_count); +} + +static inline int skhpb_add_region(struct skhpb_lu *hpb, + struct skhpb_region *cb) +{ + struct skhpb_victim_select_info *lru_info; + int subregion; + int err = 0; + + lru_info = &hpb->lru_info; + + //SKHPB_DRIVER_D("E->active region: %d", cb->region); + + for (subregion = 0; subregion < cb->subregion_count; subregion++) { + struct skhpb_subregion *cp; + + cp = cb->subregion_tbl + subregion; + cp->mctx = skhpb_get_map_ctx(hpb); + if (IS_ERR(cp->mctx)) { + err = PTR_ERR(cp->mctx); + goto out; + } + cp->subregion_state = SKHPB_SUBREGION_DIRTY; + } + if (!cb->is_pinned) + skhpb_add_lru_info(lru_info, cb); + + atomic64_inc(&hpb->region_add); +out: + if (err) + SKHPB_DRIVER_E("get mctx failed. err %d subregion %d free_table %d\n", + err, subregion, hpb->debug_free_table); + return err; +} + +static inline void skhpb_put_map_ctx( + struct skhpb_lu *hpb, struct skhpb_map_ctx *mctx) +{ + list_add(&mctx->list_table, &hpb->lh_map_ctx); + hpb->debug_free_table++; +} + +static inline void skhpb_purge_active_page(struct skhpb_lu *hpb, + struct skhpb_subregion *cp, int state) +{ + if (state == SKHPB_SUBREGION_UNUSED) { + skhpb_put_map_ctx(hpb, cp->mctx); + cp->mctx = NULL; + } + cp->subregion_state = state; +} + +static inline void skhpb_cleanup_lru_info( + struct skhpb_victim_select_info *lru_info, + struct skhpb_region *cb) +{ + list_del_init(&cb->list_region); + cb->region_state = SKHPB_REGION_INACTIVE; + cb->hit_count = 0; + atomic64_dec(&lru_info->active_count); +} + +static inline void skhpb_evict_region(struct skhpb_lu *hpb, + struct skhpb_region *cb) +{ + struct skhpb_victim_select_info *lru_info; + struct skhpb_subregion *cp; + int subregion; + + // If the maximum value is exceeded at the time of region addition, + // it may have already been processed. + if (cb->region_state == SKHPB_REGION_INACTIVE) { + SKHPB_DRIVER_D("Region:%d was already inactivated.\n", + cb->region); + return; + } + lru_info = &hpb->lru_info; + + //SKHPB_DRIVER_D("C->EVICT region: %d\n", cb->region); + + skhpb_cleanup_lru_info(lru_info, cb); + atomic64_inc(&hpb->region_evict); + for (subregion = 0; subregion < cb->subregion_count; subregion++) { + cp = cb->subregion_tbl + subregion; + + skhpb_purge_active_page(hpb, cp, SKHPB_SUBREGION_UNUSED); + } +} + +static void skhpb_hit_lru_info(struct skhpb_victim_select_info *lru_info, + struct skhpb_region *cb) +{ + switch (lru_info->selection_type) { + case TYPE_LRU: + list_move_tail(&cb->list_region, &lru_info->lru); + break; + case TYPE_LFU: + if (cb->hit_count != 0xffffffff) + cb->hit_count++; + + list_move_tail(&cb->list_region, &lru_info->lru); + break; + default: + break; + } +} + +static struct skhpb_region *skhpb_victim_lru_info( + struct skhpb_victim_select_info *lru_info) +{ + struct skhpb_region *cb; + struct skhpb_region *victim_cb = NULL; + u32 hit_count = 0xffffffff; + + switch (lru_info->selection_type) { + case TYPE_LRU: + victim_cb = list_first_entry(&lru_info->lru, + struct skhpb_region, list_region); + break; + case TYPE_LFU: + list_for_each_entry(cb, &lru_info->lru, list_region) { + if (hit_count > cb->hit_count) { + hit_count = cb->hit_count; + victim_cb = cb; + } + } + break; + default: + break; + } + return victim_cb; +} + +static int skhpb_check_lru_evict(struct skhpb_lu *hpb, struct skhpb_region *cb) +{ + struct skhpb_victim_select_info *lru_info = &hpb->lru_info; + struct skhpb_region *victim_cb; + unsigned long flags; + if (cb->is_pinned) + return 0; + + spin_lock_irqsave(&hpb->hpb_lock, flags); + if (!list_empty(&cb->list_region)) { + skhpb_hit_lru_info(lru_info, cb); + goto out; + } + + if (cb->region_state != SKHPB_REGION_INACTIVE) + goto out; + + if (atomic64_read(&lru_info->active_count) == lru_info->max_lru_active_count) { + + victim_cb = skhpb_victim_lru_info(lru_info); + + if (!victim_cb) { + SKHPB_DRIVER_E("SKHPB victim_cb is NULL\n"); + goto unlock_error; + } + SKHPB_DRIVER_D("max lru case. victim : %d\n", victim_cb->region); + skhpb_evict_region(hpb, victim_cb); + } + if (skhpb_add_region(hpb, cb)) { + SKHPB_DRIVER_E("SKHPB memory allocation failed\n"); + goto unlock_error; + } + + +out: + spin_unlock_irqrestore(&hpb->hpb_lock, flags); + return 0; +unlock_error: + spin_unlock_irqrestore(&hpb->hpb_lock, flags); + return -ENOMEM; +} + +static int skhpb_evict_load_region(struct skhpb_lu *hpb, + struct skhpb_rsp_info *rsp_info) +{ + struct skhpb_region *cb; + int region, iter; + unsigned long flags; + + for (iter = 0; iter < rsp_info->inactive_cnt; iter++) { + region = rsp_info->inactive_list.region[iter]; + cb = hpb->region_tbl + region; + + if (cb->is_pinned) { + /* + * Pinned active-block should not drop-out. + * But if so, it would treat error as critical, + * and it will run skhpb_eh_work + */ + SKHPB_DRIVER_E("SKHPB pinned active-block drop-out error\n"); + return -ENOMEM; + } + if (list_empty(&cb->list_region)) + continue; + + spin_lock_irqsave(&hpb->hpb_lock, flags); + skhpb_evict_region(hpb, cb); + spin_unlock_irqrestore(&hpb->hpb_lock, flags); + } + + for (iter = 0; iter < rsp_info->active_cnt; iter++) { + region = rsp_info->active_list.region[iter]; + cb = hpb->region_tbl + region; + if(skhpb_check_lru_evict(hpb, cb)) + return -ENOMEM; + } + return 0; +} + +static inline struct skhpb_rsp_field *skhpb_get_hpb_rsp( + struct ufshcd_lrb *lrbp) +{ + return (struct skhpb_rsp_field *)&lrbp->ucd_rsp_ptr->sr.sense_data_len; +} + +static void skhpb_rsp_map_cmd_req(struct skhpb_lu *hpb, + struct skhpb_rsp_info *rsp_info) +{ + struct skhpb_region *cb; + struct skhpb_subregion *cp; + int region, subregion; + int iter; + int ret; + unsigned long flags; + + /* + * Before Issue read buffer CMD for active active block, + * prepare the memory from memory pool. + */ + ret = skhpb_evict_load_region(hpb, rsp_info); + if (ret) { + SKHPB_DRIVER_E("region evict/load failed. ret %d\n", ret); + goto wakeup_ee_worker; + } + for (iter = 0; iter < rsp_info->active_cnt; iter++) { + region = rsp_info->active_list.region[iter]; + subregion = rsp_info->active_list.subregion[iter]; + cb = hpb->region_tbl + region; + + if (region >= hpb->regions_per_lu || + subregion >= cb->subregion_count) { + SKHPB_DRIVER_E("skhpb issue-map %d - %d range error\n", + region, subregion); + goto wakeup_ee_worker; + } + + cp = cb->subregion_tbl + subregion; + + /* + * if subregion_state set SKHPB_SUBREGION_ISSUED, + * active_page has already been added to list, + * so it just ends function. + */ + spin_lock_irqsave(&hpb->hpb_lock, flags); + if (cp->subregion_state == SKHPB_SUBREGION_ISSUED) { + spin_unlock_irqrestore(&hpb->hpb_lock, flags); + continue; + } + + cp->subregion_state = SKHPB_SUBREGION_ISSUED; + + ret = skhpb_clean_dirty_bitmap(hpb, cp); + + spin_unlock_irqrestore(&hpb->hpb_lock, flags); + + if (ret) + continue; + + if (!hpb->hba->sdev_ufs_lu[hpb->lun] || + !hpb->hba->sdev_ufs_lu[hpb->lun]->request_queue) + return; + ret = skhpb_set_map_req(hpb, region, subregion, + cp->mctx, rsp_info, R_BUFFER); + SKHPB_DRIVER_D("SEND READ_BUFFER - Region:%d, SubRegion:%d\n", + region, subregion); + if (ret) { + SKHPB_DRIVER_E("skhpb_set_map_req error %d\n", ret); + goto wakeup_ee_worker; + } + } + return; + +wakeup_ee_worker: + hpb->hba->skhpb_state = SKHPB_FAILED; + schedule_work(&hpb->hba->skhpb_eh_work); +} + +/* routine : isr (ufs) */ +void skhpb_rsp_upiu(struct ufs_hba *hba, struct ufshcd_lrb *lrbp) +{ + struct skhpb_lu *hpb; + struct skhpb_rsp_field *rsp_field; + struct skhpb_rsp_info *rsp_info; + int data_seg_len, num, blk_idx, update_alert; + + update_alert = be32_to_cpu(lrbp->ucd_rsp_ptr->header.dword_2) + & MASK_RSP_UPIU_HPB_UPDATE_ALERT; + data_seg_len = be32_to_cpu(lrbp->ucd_rsp_ptr->header.dword_2) + & MASK_RSP_UPIU_DATA_SEG_LEN; + if (!update_alert || !data_seg_len) { + bool do_tasklet = false; + + if (lrbp->lun >= UFS_UPIU_MAX_GENERAL_LUN) + return; + + hpb = hba->skhpb_lup[lrbp->lun]; + if (!hpb) + return; + + spin_lock(&hpb->rsp_list_lock); + do_tasklet = !list_empty(&hpb->lh_rsp_info); + spin_unlock(&hpb->rsp_list_lock); + + if (do_tasklet) + schedule_work(&hpb->skhpb_rsp_work); + return; + } + rsp_field = skhpb_get_hpb_rsp(lrbp); + if ((get_unaligned_be16(rsp_field->sense_data_len + 0) + != SKHPB_DEV_SENSE_SEG_LEN) || + rsp_field->desc_type != SKHPB_DEV_DES_TYPE || + rsp_field->additional_len != SKHPB_DEV_ADDITIONAL_LEN || + rsp_field->hpb_type == SKHPB_RSP_NONE || + rsp_field->active_region_cnt > SKHPB_MAX_ACTIVE_NUM || + rsp_field->inactive_region_cnt > SKHPB_MAX_INACTIVE_NUM) + return; + + if (rsp_field->lun >= UFS_UPIU_MAX_GENERAL_LUN) { + SKHPB_DRIVER_E("lun is not general = %d", rsp_field->lun); + return; + } + hpb = hba->skhpb_lup[rsp_field->lun]; + if (!hpb) { + SKHPB_DRIVER_E("UFS-LU%d is not SKHPB LU\n", rsp_field->lun); + return; + } + if (hpb->force_map_req_disable) + return; + SKHPB_DRIVER_D("HPB-Info Noti: %d LUN: %d Seg-Len %d, Req_type = %d\n", + rsp_field->hpb_type, rsp_field->lun, + data_seg_len, rsp_field->hpb_type); + if (!hpb->lu_hpb_enable) { + SKHPB_DRIVER_E("LU(%d) not HPB-LU\n", rsp_field->lun); + return; + } + //To verify the values within RESPONSE UPIU. + SKHPB_DRIVER_HEXDUMP("[HPB] RESP UPIU ", 16, 1, + lrbp->ucd_rsp_ptr, sizeof(struct utp_upiu_rsp), 1); + //If wLUMaxActiveHPBRegions == wNumHPBPinnedRegions + if (hpb->lru_info.max_lru_active_count == 0){ + SKHPB_DRIVER_D("max_lru_active_count is 0"); + return; + } + + switch (rsp_field->hpb_type) { + case SKHPB_RSP_REQ_REGION_UPDATE: + atomic64_inc(&hpb->rb_noti_cnt); + WARN_ON(data_seg_len != SKHPB_DEV_DATA_SEG_LEN); + + spin_lock(&hpb->rsp_list_lock); + rsp_info = skhpb_get_req_info(hpb); + spin_unlock(&hpb->rsp_list_lock); + if (!rsp_info) + return; + rsp_info->type = SKHPB_RSP_REQ_REGION_UPDATE; + + for (num = 0; num < rsp_field->active_region_cnt; num++) { + blk_idx = num * SKHPB_PER_ACTIVE_INFO_BYTES; + rsp_info->active_list.region[num] = + get_unaligned_be16(rsp_field->hpb_active_field + blk_idx); + rsp_info->active_list.subregion[num] = + get_unaligned_be16(rsp_field->hpb_active_field + + blk_idx + 2); + SKHPB_DRIVER_D("active num: %d, #block: %d, page#: %d\n", + num + 1, + rsp_info->active_list.region[num], + rsp_info->active_list.subregion[num]); + } + rsp_info->active_cnt = num; + + for (num = 0; num < rsp_field->inactive_region_cnt; num++) { + blk_idx = num * SKHPB_PER_INACTIVE_INFO_BYTES; + rsp_info->inactive_list.region[num] = + get_unaligned_be16(rsp_field->hpb_inactive_field + blk_idx); + SKHPB_DRIVER_D("inactive num: %d, #block: %d\n", + num + 1, rsp_info->inactive_list.region[num]); + } + rsp_info->inactive_cnt = num; + + SKHPB_DRIVER_D("active cnt: %d, inactive cnt: %d\n", + rsp_info->active_cnt, rsp_info->inactive_cnt); + SKHPB_DRIVER_D("add_list %p -> %p\n", + rsp_info, &hpb->lh_rsp_info); + + spin_lock(&hpb->rsp_list_lock); + list_add_tail(&rsp_info->list_rsp_info, &hpb->lh_rsp_info); + spin_unlock(&hpb->rsp_list_lock); + + schedule_work(&hpb->skhpb_rsp_work); + break; + + case SKHPB_RSP_HPB_RESET: + for (num = 0 ; num < UFS_UPIU_MAX_GENERAL_LUN ; num++) + { + hpb = hba->skhpb_lup[num]; + if (!hpb || !hpb->lu_hpb_enable) + continue; + atomic64_inc(&hpb->reset_noti_cnt); + + spin_lock(&hpb->rsp_list_lock); + rsp_info = skhpb_get_req_info(hpb); + spin_unlock(&hpb->rsp_list_lock); + if (!rsp_info) + return; + + rsp_info->type = SKHPB_RSP_HPB_RESET; + + spin_lock(&hpb->rsp_list_lock); + list_add_tail(&rsp_info->list_rsp_info, &hpb->lh_rsp_info); + spin_unlock(&hpb->rsp_list_lock); + + schedule_work(&hpb->skhpb_rsp_work); + } + break; + + default: + SKHPB_DRIVER_E("hpb_type is not available : %d\n", + rsp_field->hpb_type); + break; + } +} + +static int skhpb_read_desc(struct ufs_hba *hba, + u8 desc_id, u8 desc_index, u8 *desc_buf, u32 size) +{ + int err = 0; + + err = ufshcd_query_descriptor_retry(hba, UPIU_QUERY_OPCODE_READ_DESC, + desc_id, desc_index, 0, desc_buf, &size); + if (err) { + SKHPB_DRIVER_E("reading Device Desc failed. err = %d\n", err); + } + return err; +} + +static int skhpb_read_device_desc( + struct ufs_hba *hba, u8 *desc_buf, u32 size) +{ + return skhpb_read_desc(hba, QUERY_DESC_IDN_DEVICE, 0, desc_buf, size); +} + +static int skhpb_read_geo_desc(struct ufs_hba *hba, u8 *desc_buf, u32 size) +{ + return skhpb_read_desc(hba, QUERY_DESC_IDN_GEOMETRY, 0, + desc_buf, size); +} + +static int skhpb_read_unit_desc(struct ufs_hba *hba, int lun, + u8 *desc_buf, u32 size) +{ + return skhpb_read_desc(hba, QUERY_DESC_IDN_UNIT, + lun, desc_buf, size); +} + +static inline void skhpb_add_subregion_to_req_list(struct skhpb_lu *hpb, + struct skhpb_subregion *cp) +{ + list_add_tail(&cp->list_subregion, &hpb->lh_subregion_req); + cp->subregion_state = SKHPB_SUBREGION_ISSUED; +} + +static int skhpb_execute_req(struct skhpb_lu *hpb, unsigned char *cmd, + struct skhpb_subregion *cp) +{ + struct ufs_hba *hba = hpb->hba; + struct scsi_device *sdp; + struct request_queue *q; + struct request *req; + struct scsi_request *scsireq; + struct bio bio; + struct bio *pbio = &bio; + struct scsi_sense_hdr sshdr; + unsigned long flags; + int ret = 0; + + spin_lock_irqsave(hba->host->host_lock, flags); + sdp = hba->sdev_ufs_lu[hpb->lun]; + if (sdp) { + ret = scsi_device_get(sdp); + if (!ret && !scsi_device_online(sdp)) { + ret = -ENODEV; + scsi_device_put(sdp); + } + } else { + ret = -ENODEV; + } + spin_unlock_irqrestore(hba->host->host_lock, flags); + if (ret) + return ret; + + q = sdp->request_queue; + + req = blk_get_request(q, REQ_OP_SCSI_IN, BLK_MQ_REQ_PREEMPT); + if (IS_ERR(req)) { + ret = PTR_ERR(req); + goto out_put; + } + scsireq = scsi_req(req); + + scsireq->cmd_len = COMMAND_SIZE(cmd[0]); + if (scsireq->cmd_len > sizeof(scsireq->__cmd)) { + scsireq->cmd = kmalloc(scsireq->cmd_len, __GFP_RECLAIM); + if (!scsireq->cmd) { + ret = -ENOMEM; + goto out_put_request; + } + } else { + scsireq->cmd = scsireq->__cmd; + } + memcpy(scsireq->cmd, cmd, scsireq->cmd_len); + + scsireq->retries = 3; + req->timeout = msecs_to_jiffies(10000); + req->rq_flags |= RQF_QUIET | RQF_PREEMPT; + + ret = skhpb_add_bio_page(hpb, q, &bio, hpb->bvec, cp->mctx); + if (ret) + goto out_put_scsi_req; + blk_rq_append_bio(req, &pbio); + + blk_execute_rq(q, NULL, req, 1); + + if (scsireq->result) { + scsi_normalize_sense(scsireq->sense, SCSI_SENSE_BUFFERSIZE, &sshdr); + SKHPB_DRIVER_E("code %x sense_key %x asc %x ascq %x", + sshdr.response_code, sshdr.sense_key, sshdr.asc, + sshdr.ascq); + SKHPB_DRIVER_E("byte4 %x byte5 %x byte6 %x additional_len %x", + sshdr.byte4, sshdr.byte5, sshdr.byte6, + sshdr.additional_length); + spin_lock_irqsave(&hpb->hpb_lock, flags); + skhpb_error_active_subregion(hpb, cp); + spin_unlock_irqrestore(&hpb->hpb_lock, flags); + ret = -EIO; + } else { + spin_lock_irqsave(&hpb->hpb_lock, flags); + ret = skhpb_clean_dirty_bitmap(hpb, cp); + spin_unlock_irqrestore(&hpb->hpb_lock, flags); + if (ret) { + SKHPB_DRIVER_E("skhpb_clean_dirty_bitmap error %d", ret); + } + ret = 0; + } + +out_put_scsi_req: + scsi_req_free_cmd(scsireq); +out_put_request: + blk_put_request(req); +out_put: + scsi_device_put(sdp); + return ret; +} + +static int skhpb_issue_map_req_from_list(struct skhpb_lu *hpb) +{ + struct skhpb_subregion *cp, *next_cp; + int ret; + unsigned long flags; + + LIST_HEAD(req_list); + + spin_lock_irqsave(&hpb->hpb_lock, flags); + list_splice_init(&hpb->lh_subregion_req, &req_list); + spin_unlock_irqrestore(&hpb->hpb_lock, flags); + + list_for_each_entry_safe(cp, next_cp, &req_list, list_subregion) { + int subregion_mem_size = + cp->last ? hpb->last_subregion_mem_size : hpb->subregion_mem_size; + unsigned char cmd[10] = { 0 }; + + skhpb_set_read_buf_cmd(cmd, cp->region, cp->subregion, + subregion_mem_size); + + SKHPB_DRIVER_D("issue map_request: %d - %d/%d\n", + cp->region, cp->subregion, subregion_mem_size); + + ret = skhpb_execute_req(hpb, cmd, cp); + if (ret < 0) { + SKHPB_DRIVER_E("region %d sub %d failed with err %d", + cp->region, cp->subregion, ret); + continue; + } + + spin_lock_irqsave(&hpb->hpb_lock, flags); + skhpb_clean_active_subregion(hpb, cp); + list_del_init(&cp->list_subregion); + spin_unlock_irqrestore(&hpb->hpb_lock, flags); + } + + return 0; +} + +static void skhpb_pinned_work_handler(struct work_struct *work) +{ + struct delayed_work *dwork = to_delayed_work(work); + struct skhpb_lu *hpb = + container_of(dwork, struct skhpb_lu, skhpb_pinned_work); + int ret; + + SKHPB_DRIVER_D("worker start\n"); + + if (!list_empty(&hpb->lh_subregion_req)) { + pm_runtime_get_sync(SKHPB_DEV(hpb)); + ret = skhpb_issue_map_req_from_list(hpb); + /* + * if its function failed at init time, + * skhpb-device will request map-req, + * so it is not critical-error, and just finish work-handler + */ + if (ret) + SKHPB_DRIVER_E("failed map-issue. ret %d\n", ret); + pm_runtime_mark_last_busy(SKHPB_DEV(hpb)); + pm_runtime_put_noidle(SKHPB_DEV(hpb)); + } + + SKHPB_DRIVER_D("worker end\n"); +} + +static void skhpb_map_req_retry_work_handler(struct work_struct *work) +{ + struct skhpb_lu *hpb; + struct delayed_work *dwork = to_delayed_work(work); + struct skhpb_map_req *map_req; + int ret = 0; + + LIST_HEAD(retry_list); + + hpb = container_of(dwork, struct skhpb_lu, skhpb_map_req_retry_work); + SKHPB_DRIVER_D("retry worker start"); + + spin_lock_bh(&hpb->map_list_lock); + list_splice_init(&hpb->lh_map_req_retry, &retry_list); + spin_unlock_bh(&hpb->map_list_lock); + + while (1) { + map_req = list_first_entry_or_null(&retry_list, + struct skhpb_map_req, list_map_req); + if (!map_req) { + SKHPB_DRIVER_D("There is no map_req"); + break; + } + list_del(&map_req->list_map_req); + + map_req->retry_cnt++; + + ret = skhpb_map_req_issue(hpb, map_req); + if (ret) { + SKHPB_DRIVER_E("skhpb_map_req_issue error %d", ret); + goto wakeup_ee_worker; + } + } + SKHPB_DRIVER_D("worker end"); + return; + +wakeup_ee_worker: + hpb->hba->skhpb_state = SKHPB_FAILED; + schedule_work(&hpb->hba->skhpb_eh_work); +} + +static void skhpb_delayed_rsp_work_handler(struct work_struct *work) +{ + struct skhpb_lu *hpb = container_of(work, struct skhpb_lu, skhpb_rsp_work); + struct skhpb_rsp_info *rsp_info; + unsigned long flags; + + SKHPB_DRIVER_D("rsp_work enter"); + + while (1) { + if(hpb->hba->clk_gating.is_suspended) { + SKHPB_DRIVER_D("rsp_work break"); + break; + } + spin_lock_irqsave(&hpb->rsp_list_lock, flags); + rsp_info = list_first_entry_or_null(&hpb->lh_rsp_info, + struct skhpb_rsp_info, list_rsp_info); + if (!rsp_info) { + spin_unlock_irqrestore(&hpb->rsp_list_lock, flags); + break; + } + + SKHPB_RSP_TIME(rsp_info->RSP_start); + + list_del_init(&rsp_info->list_rsp_info); + spin_unlock_irqrestore(&hpb->rsp_list_lock, flags); + + switch (rsp_info->type) { + case SKHPB_RSP_REQ_REGION_UPDATE: + skhpb_rsp_map_cmd_req(hpb, rsp_info); + break; + + case SKHPB_RSP_HPB_RESET: + skhpb_purge_active_block(hpb); + skhpb_map_loading_trigger(hpb, true, false); + break; + + default: + break; + } + + spin_lock_irqsave(&hpb->rsp_list_lock, flags); + list_add_tail(&rsp_info->list_rsp_info, &hpb->lh_rsp_info_free); + spin_unlock_irqrestore(&hpb->rsp_list_lock, flags); + } + SKHPB_DRIVER_D("rsp_work end"); +} + +static void skhpb_init_constant(void) +{ + skhpb_sects_per_blk_shift = ffs(SKHPB_BLOCK) - ffs(SKHPB_SECTOR); + SKHPB_DRIVER_D("skhpb_sects_per_blk_shift: %u %u\n", + skhpb_sects_per_blk_shift, ffs(SKHPB_SECTORS_PER_BLOCK) - 1); + + skhpb_bits_per_dword_shift = ffs(SKHPB_BITS_PER_DWORD) - 1; + skhpb_bits_per_dword_mask = SKHPB_BITS_PER_DWORD - 1; + SKHPB_DRIVER_D("bits_per_dword %u shift %u mask 0x%X\n", + SKHPB_BITS_PER_DWORD, skhpb_bits_per_dword_shift, skhpb_bits_per_dword_mask); +} + +static void skhpb_table_mempool_remove(struct skhpb_lu *hpb) +{ + struct skhpb_map_ctx *mctx, *next; + int i; + + /* + * the mctx in the lh_map_ctx has been allocated completely. + */ + list_for_each_entry_safe(mctx, next, &hpb->lh_map_ctx, list_table) { + list_del(&mctx->list_table); + + for (i = 0; i < hpb->mpages_per_subregion; i++) + __free_page(mctx->m_page[i]); + + kvfree(mctx->ppn_dirty); + kfree(mctx->m_page); + kfree(mctx); + skhpb_alloc_mctx--; + } +} + +static int skhpb_init_pinned_active_block(struct skhpb_lu *hpb, + struct skhpb_region *cb) +{ + struct skhpb_subregion *cp; + int subregion, j; + int err = 0; + unsigned long flags; + + for (subregion = 0 ; subregion < cb->subregion_count ; subregion++) { + cp = cb->subregion_tbl + subregion; + + cp->mctx = skhpb_get_map_ctx(hpb); + if (IS_ERR(cp->mctx)) { + err = PTR_ERR(cp->mctx); + goto release; + } + spin_lock_irqsave(&hpb->hpb_lock, flags); + skhpb_add_subregion_to_req_list(hpb, cp); + spin_unlock_irqrestore(&hpb->hpb_lock, flags); + } + return 0; + +release: + for (j = 0 ; j < subregion ; j++) { + cp = cb->subregion_tbl + j; + skhpb_put_map_ctx(hpb, cp->mctx); + } + return err; +} + +static inline bool skhpb_is_pinned( + struct skhpb_lu_desc *lu_desc, int region) +{ + if (lu_desc->lu_hpb_pinned_end_offset != -1 && + region >= lu_desc->hpb_pinned_region_startidx && + region <= lu_desc->lu_hpb_pinned_end_offset) + return true; + + return false; +} + +static inline void skhpb_init_jobs(struct skhpb_lu *hpb) +{ + INIT_DELAYED_WORK(&hpb->skhpb_pinned_work, skhpb_pinned_work_handler); + INIT_DELAYED_WORK(&hpb->skhpb_map_req_retry_work, skhpb_map_req_retry_work_handler); + INIT_WORK(&hpb->skhpb_rsp_work, skhpb_delayed_rsp_work_handler); +} + +static inline void skhpb_cancel_jobs(struct skhpb_lu *hpb) +{ + cancel_delayed_work_sync(&hpb->skhpb_pinned_work); + cancel_work_sync(&hpb->skhpb_rsp_work); + cancel_delayed_work_sync(&hpb->skhpb_map_req_retry_work); +} + +static void skhpb_init_subregion_tbl(struct skhpb_lu *hpb, + struct skhpb_region *cb, bool last_region) +{ + int subregion; + + for (subregion = 0 ; subregion < cb->subregion_count ; subregion++) { + struct skhpb_subregion *cp = cb->subregion_tbl + subregion; + + cp->region = cb->region; + cp->subregion = subregion; + cp->subregion_state = SKHPB_SUBREGION_UNUSED; + cp->last = (last_region && subregion == cb->subregion_count - 1); + } +} + +static inline int skhpb_alloc_subregion_tbl(struct skhpb_lu *hpb, + struct skhpb_region *cb, int subregion_count) +{ + cb->subregion_tbl = kzalloc(sizeof(struct skhpb_subregion) * subregion_count, + GFP_KERNEL); + if (!cb->subregion_tbl) + return -ENOMEM; + + cb->subregion_count = subregion_count; + + return 0; +} + +static int skhpb_table_mempool_init(struct skhpb_lu *hpb, + int num_regions, int subregions_per_region, + int entry_count, int entry_byte) +{ + int i, j; + struct skhpb_map_ctx *mctx = NULL; + + for (i = 0 ; i < num_regions * subregions_per_region ; i++) { + mctx = kzalloc(sizeof(struct skhpb_map_ctx), GFP_KERNEL); + if (!mctx) + goto release_mem; + + mctx->m_page = kzalloc(sizeof(struct page *) * + hpb->mpages_per_subregion, GFP_KERNEL); + if (!mctx->m_page) + goto release_mem; + + mctx->ppn_dirty = kvzalloc(entry_count / BITS_PER_BYTE, GFP_KERNEL); + if (!mctx->ppn_dirty) + goto release_mem; + + for (j = 0; j < hpb->mpages_per_subregion; j++) { + mctx->m_page[j] = alloc_page(GFP_KERNEL | __GFP_ZERO); + if (!mctx->m_page[j]) + goto release_mem; + } + /* SKSKHPB_DRIVER_D("[%d] mctx->m_page %p get_order %d\n", i, + mctx->m_page, get_order(hpb->mpages_per_subregion)); */ + + INIT_LIST_HEAD(&mctx->list_table); + list_add(&mctx->list_table, &hpb->lh_map_ctx); + + hpb->debug_free_table++; + } + + skhpb_alloc_mctx = num_regions * subregions_per_region; + /* SKSKHPB_DRIVER_D("number of mctx %d %d %d. debug_free_table %d\n", + num_regions * subregions_per_region, num_regions, + subregions_per_region, hpb->debug_free_table); */ + return 0; + +release_mem: + /* + * mctxs already added in lh_map_ctx will be removed + * in the caller function. + */ + if (!mctx) + goto out; + + if (mctx->m_page) { + for (j = 0; j < hpb->mpages_per_subregion; j++) + if (mctx->m_page[j]) + __free_page(mctx->m_page[j]); + kfree(mctx->m_page); + } + kvfree(mctx->ppn_dirty); + kfree(mctx); +out: + return -ENOMEM; +} + +static int skhpb_req_mempool_init(struct ufs_hba *hba, + struct skhpb_lu *hpb, int queue_depth) +{ + struct skhpb_rsp_info *rsp_info = NULL; + struct skhpb_map_req *map_req = NULL; + int i; + + if (!queue_depth) { + queue_depth = hba->nutrs; + SKHPB_DRIVER_E("lu_queue_depth is 0. we use device's queue info.\n"); + SKHPB_DRIVER_E("hba->nutrs = %d\n", hba->nutrs); + } + INIT_LIST_HEAD(&hpb->lh_rsp_info_free); + INIT_LIST_HEAD(&hpb->lh_map_req_free); + INIT_LIST_HEAD(&hpb->lh_map_req_retry); + + hpb->rsp_info = vzalloc(queue_depth * sizeof(struct skhpb_rsp_info)); + if (!hpb->rsp_info) + goto release_mem; + + hpb->map_req = vzalloc(queue_depth * sizeof(struct skhpb_map_req)); + if (!hpb->map_req) + goto release_mem; + + for (i = 0; i < queue_depth; i++) { + rsp_info = hpb->rsp_info + i; + INIT_LIST_HEAD(&rsp_info->list_rsp_info); + list_add_tail(&rsp_info->list_rsp_info, &hpb->lh_rsp_info_free); + } + + for (i = 0; i < queue_depth; i++) { + map_req = hpb->map_req + i; + INIT_LIST_HEAD(&map_req->list_map_req); + list_add_tail(&map_req->list_map_req, &hpb->lh_map_req_free); + } + + return 0; + +release_mem: + return -ENOMEM; +} + +static void skhpb_init_lu_constant(struct skhpb_lu *hpb, + struct skhpb_lu_desc *lu_desc, + struct skhpb_func_desc *func_desc) +{ + unsigned long long region_unit_size, region_mem_size; + unsigned long long last_subregion_unit_size; + int entries_per_region; + + /* From descriptors */ + region_unit_size = SKHPB_SECTOR * (1ULL << func_desc->hpb_region_size); + region_mem_size = region_unit_size / SKHPB_BLOCK * SKHPB_ENTRY_SIZE; + + hpb->subregion_unit_size = SKHPB_SECTOR * (1ULL << func_desc->hpb_subregion_size); + hpb->subregion_mem_size = hpb->subregion_unit_size / + SKHPB_BLOCK * SKHPB_ENTRY_SIZE; + if (func_desc->hpb_region_size == func_desc->hpb_subregion_size) + hpb->identical_size = true; + else + hpb->identical_size = false; + + last_subregion_unit_size = + lu_desc->lu_logblk_cnt * + (1ULL << lu_desc->lu_logblk_size) % hpb->subregion_unit_size; + hpb->last_subregion_mem_size = + last_subregion_unit_size / SKHPB_BLOCK * SKHPB_ENTRY_SIZE; + if (hpb->last_subregion_mem_size == 0) + hpb->last_subregion_mem_size = hpb->subregion_mem_size; + hpb->hpb_ver = func_desc->hpb_ver; + hpb->lu_max_active_regions = lu_desc->lu_max_active_hpb_regions; + hpb->lru_info.max_lru_active_count = + lu_desc->lu_max_active_hpb_regions + - lu_desc->lu_num_hpb_pinned_regions; + + /* relation : lu <-> region <-> sub region <-> entry */ + hpb->lu_num_blocks = lu_desc->lu_logblk_cnt; + entries_per_region = region_mem_size / SKHPB_ENTRY_SIZE; + hpb->entries_per_subregion = hpb->subregion_mem_size / SKHPB_ENTRY_SIZE; + hpb->subregions_per_region = region_mem_size / hpb->subregion_mem_size; + + hpb->hpb_control_mode = func_desc->hpb_control_mode; +#if defined(SKHPB_READ_LARGE_CHUNK_SUPPORT) + hpb->ppn_dirties_per_subregion = + hpb->entries_per_subregion/BITS_PER_PPN_DIRTY; + /* SKSKHPB_DRIVER_D("ppn_dirties_per_subregion:%d, %d \n", + hpb->ppn_dirties_per_subregion, BITS_PER_PPN_DIRTY); */ +#endif + /* + * 1. regions_per_lu + * = (lu_num_blocks * 4096) / region_unit_size + * = (lu_num_blocks * SKHPB_ENTRY_SIZE) / region_mem_size + * = lu_num_blocks / (region_mem_size / SKHPB_ENTRY_SIZE) + * + * 2. regions_per_lu = lu_num_blocks / subregion_mem_size (is trik...) + * if SKHPB_ENTRY_SIZE != subregions_per_region, it is error. + */ + hpb->regions_per_lu = (hpb->lu_num_blocks + + (region_mem_size / SKHPB_ENTRY_SIZE) - 1) + / (region_mem_size / SKHPB_ENTRY_SIZE); + hpb->subregions_per_lu = (hpb->lu_num_blocks + + (hpb->subregion_mem_size / SKHPB_ENTRY_SIZE) - 1) + / (hpb->subregion_mem_size / SKHPB_ENTRY_SIZE); + + /* mempool info */ + hpb->mpage_bytes = PAGE_SIZE; + hpb->mpages_per_subregion = hpb->subregion_mem_size / hpb->mpage_bytes; + + /* Bitmask Info. */ + hpb->dwords_per_subregion = hpb->entries_per_subregion / SKHPB_BITS_PER_DWORD; + hpb->entries_per_region_shift = ffs(entries_per_region) - 1; + hpb->entries_per_region_mask = entries_per_region - 1; + hpb->entries_per_subregion_shift = ffs(hpb->entries_per_subregion) - 1; + hpb->entries_per_subregion_mask = hpb->entries_per_subregion - 1; + + SKHPB_DRIVER_I("===== Device Descriptor =====\n"); + SKHPB_DRIVER_I("hpb_region_size = %d, hpb_subregion_size = %d\n", + func_desc->hpb_region_size, + func_desc->hpb_subregion_size); + SKHPB_DRIVER_I("===== Constant Values =====\n"); + SKHPB_DRIVER_I("region_unit_size = %llu, region_mem_size %llu\n", + region_unit_size, region_mem_size); + SKHPB_DRIVER_I("subregion_unit_size = %llu, subregion_mem_size %d\n", + hpb->subregion_unit_size, hpb->subregion_mem_size); + SKHPB_DRIVER_I("last_subregion_mem_size = %d\n", hpb->last_subregion_mem_size); + SKHPB_DRIVER_I("lu_num_blks = %llu, reg_per_lu = %d, subreg_per_lu = %d\n", + hpb->lu_num_blocks, hpb->regions_per_lu, + hpb->subregions_per_lu); + SKHPB_DRIVER_I("subregions_per_region = %d\n", + hpb->subregions_per_region); + SKHPB_DRIVER_I("entries_per_region %u shift %u mask 0x%X\n", + entries_per_region, hpb->entries_per_region_shift, + hpb->entries_per_region_mask); + SKHPB_DRIVER_I("entries_per_subregion %u shift %u mask 0x%X\n", + hpb->entries_per_subregion, + hpb->entries_per_subregion_shift, + hpb->entries_per_subregion_mask); + SKHPB_DRIVER_I("mpages_per_subregion : %d\n", + hpb->mpages_per_subregion); + SKHPB_DRIVER_I("===================================\n"); +} + +static int skhpb_lu_hpb_init(struct ufs_hba *hba, struct skhpb_lu *hpb, + struct skhpb_func_desc *func_desc, + struct skhpb_lu_desc *lu_desc, int lun, + bool *do_work_lun) +{ + struct skhpb_region *cb; + struct skhpb_subregion *cp; + int region, subregion; + int total_subregion_count, subregion_count; + int ret, j; + + *do_work_lun = false; + + spin_lock_init(&hpb->hpb_lock); + spin_lock_init(&hpb->rsp_list_lock); + spin_lock_init(&hpb->map_list_lock); + + /* init lru information */ + INIT_LIST_HEAD(&hpb->lru_info.lru); + hpb->lru_info.selection_type = TYPE_LRU; + + INIT_LIST_HEAD(&hpb->lh_subregion_req); + INIT_LIST_HEAD(&hpb->lh_rsp_info); + INIT_LIST_HEAD(&hpb->lh_map_ctx); + + hpb->lu_hpb_enable = true; + + skhpb_init_lu_constant(hpb, lu_desc, func_desc); + + hpb->region_tbl = vzalloc(sizeof(struct skhpb_region) * hpb->regions_per_lu); + if (!hpb->region_tbl) + return -ENOMEM; + + SKHPB_DRIVER_D("active_block_table bytes: %lu\n", + (sizeof(struct skhpb_region) * hpb->regions_per_lu)); + + ret = skhpb_table_mempool_init(hpb, + lu_desc->lu_max_active_hpb_regions, + hpb->subregions_per_region, + hpb->entries_per_subregion, SKHPB_ENTRY_SIZE); + if (ret) { + SKHPB_DRIVER_E("ppn table mempool init fail!\n"); + goto release_mempool; + } + + ret = skhpb_req_mempool_init(hba, hpb, lu_desc->lu_queue_depth); + if (ret) { + SKHPB_DRIVER_E("rsp_info_mempool init fail!\n"); + goto release_mempool; + } + + total_subregion_count = hpb->subregions_per_lu; + + skhpb_init_jobs(hpb); + + SKHPB_DRIVER_D("total_subregion_count: %d\n", total_subregion_count); + for (region = 0, subregion_count = 0, + total_subregion_count = hpb->subregions_per_lu; + region < hpb->regions_per_lu; + region++, total_subregion_count -= subregion_count) { + cb = hpb->region_tbl + region; + cb->region = region; + + /* init lru region information*/ + INIT_LIST_HEAD(&cb->list_region); + cb->hit_count = 0; + + subregion_count = min(total_subregion_count, + hpb->subregions_per_region); + /* SKSKHPB_DRIVER_D("total: %d subregion_count: %d\n", + total_subregion_count, subregion_count); */ + + ret = skhpb_alloc_subregion_tbl(hpb, cb, subregion_count); + if (ret) + goto release_region_cp; + skhpb_init_subregion_tbl(hpb, cb, region == hpb->regions_per_lu - 1); + + if (skhpb_is_pinned(lu_desc, region)) { + SKHPB_DRIVER_D("region: %d PINNED %d ~ %d\n", + region, lu_desc->hpb_pinned_region_startidx, + lu_desc->lu_hpb_pinned_end_offset); + ret = skhpb_init_pinned_active_block(hpb, cb); + if (ret) + goto release_region_cp; + *do_work_lun = true; + cb->is_pinned = true; + cb->region_state = SKHPB_REGION_ACTIVE; + } else { + /* SKSKHPB_DRIVER_D("region: %d inactive\n", cb->region); */ + cb->is_pinned = false; + cb->region_state = SKHPB_REGION_INACTIVE; + } + } + if (total_subregion_count != 0) { + SKHPB_DRIVER_E("error total_subregion_count: %d\n", + total_subregion_count); + goto release_region_cp; + } + hpb->hba = hba; + hpb->lun = lun; + /* + * even if creating sysfs failed, skhpb could run normally. + * so we don't deal with error handling + */ + skhpb_create_sysfs(hba, hpb); + return 0; + +release_region_cp: + for (j = 0 ; j < region ; j++) { + cb = hpb->region_tbl + j; + if (cb->subregion_tbl) { + for (subregion = 0; subregion < cb->subregion_count; + subregion++) { + cp = cb->subregion_tbl + subregion; + + if (cp->mctx) + skhpb_put_map_ctx(hpb, cp->mctx); + } + kfree(cb->subregion_tbl); + } + } + +release_mempool: + skhpb_table_mempool_remove(hpb); + *do_work_lun = false; + return ret; +} + +static int skhpb_get_hpb_lu_desc(struct ufs_hba *hba, + struct skhpb_lu_desc *lu_desc, int lun) +{ + int ret; + u8 logical_buf[SKHPB_QUERY_DESC_UNIT_MAX_SIZE] = { 0 }; + + ret = skhpb_read_unit_desc(hba, lun, logical_buf, + SKHPB_QUERY_DESC_UNIT_MAX_SIZE); + if (ret) { + SKHPB_DRIVER_E("read unit desc failed. ret %d\n", ret); + return ret; + } + + lu_desc->lu_queue_depth = logical_buf[UNIT_DESC_PARAM_LU_Q_DEPTH]; + + // 2^log, ex) 0x0C = 4KB + lu_desc->lu_logblk_size = logical_buf[UNIT_DESC_PARAM_LOGICAL_BLK_SIZE]; + lu_desc->lu_logblk_cnt = + get_unaligned_be64(&logical_buf[UNIT_DESC_PARAM_LOGICAL_BLK_COUNT]); + + if (logical_buf[UNIT_DESC_PARAM_LU_ENABLE] == LU_HPB_ENABLE) + lu_desc->lu_hpb_enable = true; + else + lu_desc->lu_hpb_enable = false; + + lu_desc->lu_max_active_hpb_regions = + get_unaligned_be16(logical_buf + + UNIT_DESC_HPB_LU_MAX_ACTIVE_REGIONS); + lu_desc->hpb_pinned_region_startidx = + get_unaligned_be16(logical_buf + + UNIT_DESC_HPB_LU_PIN_REGION_START_OFFSET); + lu_desc->lu_num_hpb_pinned_regions = + get_unaligned_be16(logical_buf + + UNIT_DESC_HPB_LU_NUM_PIN_REGIONS); + + if (lu_desc->lu_hpb_enable) { + SKHPB_DRIVER_D("LUN(%d) [0A] bLogicalBlockSize %d\n", + lun, lu_desc->lu_logblk_size); + SKHPB_DRIVER_D("LUN(%d) [0B] qLogicalBlockCount %llu\n", + lun, lu_desc->lu_logblk_cnt); + SKHPB_DRIVER_D("LUN(%d) [03] bLuEnable %d\n", + lun, logical_buf[UNIT_DESC_PARAM_LU_ENABLE]); + SKHPB_DRIVER_D("LUN(%d) [06] bLuQueueDepth %d\n", + lun, lu_desc->lu_queue_depth); + SKHPB_DRIVER_D("LUN(%d) [23:24] wLUMaxActiveHPBRegions %d\n", + lun, lu_desc->lu_max_active_hpb_regions); + SKHPB_DRIVER_D("LUN(%d) [25:26] wHPBPinnedRegionStartIdx %d\n", + lun, lu_desc->hpb_pinned_region_startidx); + SKHPB_DRIVER_D("LUN(%d) [27:28] wNumHPBPinnedRegions %d\n", + lun, lu_desc->lu_num_hpb_pinned_regions); + } + + if (lu_desc->lu_num_hpb_pinned_regions > 0) { + lu_desc->lu_hpb_pinned_end_offset = + lu_desc->hpb_pinned_region_startidx + + lu_desc->lu_num_hpb_pinned_regions - 1; + } else + lu_desc->lu_hpb_pinned_end_offset = -1; + + if (lu_desc->lu_hpb_enable) + SKHPB_DRIVER_I("Enable, LU: %d, MAX_REGION: %d, PIN: %d - %d\n", + lun, + lu_desc->lu_max_active_hpb_regions, + lu_desc->hpb_pinned_region_startidx, + lu_desc->lu_num_hpb_pinned_regions); + return 0; +} + +static void skhpb_quirk_setup(struct ufs_hba *hba, + struct skhpb_func_desc *desc) +{ + if (hba->dev_quirks & SKHPB_QUIRK_PURGE_HINT_INFO_WHEN_SLEEP) { + hba->skhpb_quirk |= SKHPB_QUIRK_PURGE_HINT_INFO_WHEN_SLEEP; + SKHPB_DRIVER_I("QUIRK set PURGE_HINT_INFO_WHEN_SLEEP\n"); + } + if (desc->hpb_ver >= 0x0101) { + hba->skhpb_quirk |= SKHPB_QUIRK_USE_READ_16_FOR_ENCRYPTION; + SKHPB_DRIVER_I("QUIRK set USE_READ_16_FOR_ENCRYPTION\n"); + } + if (desc->hpb_ver == 0x0102) { + hba->skhpb_quirk |= SKHPB_QUIRK_ALWAYS_DEVICE_CONTROL_MODE; + SKHPB_DRIVER_I("QUIRK set ALWAYS_DEVICE_CONTROL_MODE\n"); + } +} + +static int skhpb_read_dev_desc_support(struct ufs_hba *hba, + struct skhpb_func_desc *desc) +{ + u8 desc_buf[SKHPB_QUERY_DESC_DEVICE_MAX_SIZE]; + int err; + + err = skhpb_read_device_desc(hba, desc_buf, + SKHPB_QUERY_DESC_DEVICE_MAX_SIZE); + if (err) + return err; + + if (desc_buf[DEVICE_DESC_PARAM_UFS_FEAT] & + SKHPB_UFS_FEATURE_SUPPORT_HPB_BIT) { + hba->skhpb_feat |= SKHPB_UFS_FEATURE_SUPPORT_HPB_BIT; + SKHPB_DRIVER_I("FeaturesSupport= support\n"); + } else { + SKHPB_DRIVER_I("FeaturesSupport= not support\n"); + return -ENODEV; + } + + desc->lu_cnt = desc_buf[DEVICE_DESC_PARAM_NUM_LU]; + SKHPB_DRIVER_D("Dev LU count= %d\n", desc->lu_cnt); + + desc->spec_ver = + (u16)SKHPB_SHIFT_BYTE_1(desc_buf[DEVICE_DESC_PARAM_SPEC_VER]) | + (u16)SKHPB_SHIFT_BYTE_0(desc_buf[DEVICE_DESC_PARAM_SPEC_VER + 1]); + SKHPB_DRIVER_I("Dev Spec Ver= %x.%x\n", + SKHPB_GET_BYTE_1(desc->spec_ver), + SKHPB_GET_BYTE_0(desc->spec_ver) ); + + desc->hpb_ver = + (u16)SKHPB_SHIFT_BYTE_1(desc_buf[DEVICE_DESC_PARAM_HPB_VER]) | + (u16)SKHPB_SHIFT_BYTE_0(desc_buf[DEVICE_DESC_PARAM_HPB_VER + 1]); + SKHPB_DRIVER_I("Dev Ver= %x.%x.%x, DD Ver= %x.%x.%x\n", + (desc->hpb_ver >> 8) & 0xf, + (desc->hpb_ver >> 4) & 0xf, + (desc->hpb_ver >> 0) & 0xf, + SKHPB_GET_BYTE_2(SKHPB_DD_VER), + SKHPB_GET_BYTE_1(SKHPB_DD_VER), + SKHPB_GET_BYTE_0(SKHPB_DD_VER)); + + skhpb_quirk_setup(hba, desc); + + if (hba->skhpb_quirk & SKHPB_QUIRK_ALWAYS_DEVICE_CONTROL_MODE) + desc->hpb_control_mode = DEV_CTRL_MODE; + else + desc->hpb_control_mode = (u8)desc_buf[DEVICE_DESC_PARAM_HPB_CONTROL]; + + + SKHPB_DRIVER_I("HPB Control Mode = %s", + (desc->hpb_control_mode)?"DEV MODE":"HOST MODE"); + if (desc->hpb_control_mode == HOST_CTRL_MODE) { + SKHPB_DRIVER_E("Driver does not support Host Control Mode"); + return -ENODEV; + } + hba->hpb_control_mode = desc->hpb_control_mode; + return 0; +} + +static int skhpb_read_geo_desc_support(struct ufs_hba *hba, + struct skhpb_func_desc *desc) +{ + int err; + u8 geometry_buf[SKHPB_QUERY_DESC_GEOMETRY_MAX_SIZE]; + + err = skhpb_read_geo_desc(hba, geometry_buf, + SKHPB_QUERY_DESC_GEOMETRY_MAX_SIZE); + if (err) + return err; + + desc->hpb_region_size = geometry_buf[GEOMETRY_DESC_HPB_REGION_SIZE]; + desc->hpb_number_lu = geometry_buf[GEOMETRY_DESC_HPB_NUMBER_LU]; + desc->hpb_subregion_size = + geometry_buf[GEOMETRY_DESC_HPB_SUBREGION_SIZE]; + desc->hpb_device_max_active_regions = + get_unaligned_be16(geometry_buf + + GEOMETRY_DESC_HPB_DEVICE_MAX_ACTIVE_REGIONS); + + SKHPB_DRIVER_D("[48] bHPBRegionSize %u\n", desc->hpb_region_size); + SKHPB_DRIVER_D("[49] bHPBNumberLU %u\n", desc->hpb_number_lu); + SKHPB_DRIVER_D("[4A] bHPBSubRegionSize %u\n", desc->hpb_subregion_size); + SKHPB_DRIVER_D("[4B:4C] wDeviceMaxActiveHPBRegions %u\n", + desc->hpb_device_max_active_regions); + + if (desc->hpb_number_lu == 0) { + SKHPB_DRIVER_E("HPB is not supported\n"); + return -ENODEV; + } + /* for activation */ + hba->skhpb_max_regions = desc->hpb_device_max_active_regions; + return 0; +} + +int skhpb_control_validation(struct ufs_hba *hba, + struct skhpb_config_desc *config) +{ + unsigned int num_regions = 0; + int lun; + + if (!(hba->skhpb_feat & SKHPB_UFS_FEATURE_SUPPORT_HPB_BIT)) + return -ENOTSUPP; + + for (lun = 0 ; lun < UFS_UPIU_MAX_GENERAL_LUN ; lun++) { + unsigned char *unit = config->unit[lun]; + + if (unit[SKHPB_CONF_LU_ENABLE] >= LU_SET_MAX) + return -EINVAL; + + /* total should not exceed max_active_regions */ + num_regions += unit[SKHPB_CONF_ACTIVE_REGIONS] << 8; + num_regions += unit[SKHPB_CONF_ACTIVE_REGIONS + 1]; + if (num_regions > hba->skhpb_max_regions) + return -EINVAL; + } + return 0; +} + +static int skhpb_init(struct ufs_hba *hba) +{ + struct skhpb_func_desc func_desc; + int lun, ret, retries; + int hpb_dev = 0; + bool do_work; + + pm_runtime_get_sync(hba->dev); + ret = skhpb_read_dev_desc_support(hba, &func_desc); + if (ret) + goto out_state; + + ret = skhpb_read_geo_desc_support(hba, &func_desc); + if (ret) + goto out_state; + + for (retries = 0; retries < 20; retries++) { + if (!hba->lrb_in_use) { + ret = ufshcd_query_flag(hba, + UPIU_QUERY_OPCODE_SET_FLAG, + QUERY_FLAG_IDN_HPB_RESET, 0, + NULL); + if (!ret) { + SKHPB_DRIVER_I("Query fHPBReset is successfully sent retries = %d\n", retries); + break; + } + } + else + msleep(200); + } + if (ret == 0) { + bool fHPBReset = true; + + ufshcd_query_flag_retry(hba, + UPIU_QUERY_OPCODE_READ_FLAG, + QUERY_FLAG_IDN_HPB_RESET, 0, + &fHPBReset); + if (fHPBReset) + SKHPB_DRIVER_I("fHPBReset still set\n"); + } + + skhpb_init_constant(); + + do_work = false; + for (lun = 0 ; lun < UFS_UPIU_MAX_GENERAL_LUN ; lun++) { + struct skhpb_lu_desc lu_desc; + bool do_work_lun; + + ret = skhpb_get_hpb_lu_desc(hba, &lu_desc, lun); + if (ret) + goto out_state; + + if (lu_desc.lu_hpb_enable == false) + continue; + + hba->skhpb_lup[lun] = kzalloc(sizeof(struct skhpb_lu), + GFP_KERNEL); + if (!hba->skhpb_lup[lun]) { + ret = -ENOMEM; + goto out_free_mem; + } + + ret = skhpb_lu_hpb_init(hba, hba->skhpb_lup[lun], + &func_desc, &lu_desc, lun, &do_work_lun); + if (ret) { + if (ret == -ENODEV) + continue; + else + goto out_free_mem; + } + do_work |= do_work_lun; + hba->skhpb_quicklist_lu_enable[hpb_dev] = lun; + SKHPB_DRIVER_D("skhpb_quicklist_lu_enable[%d] = %d\n", hpb_dev, lun); + hpb_dev++; + } + + if (hpb_dev) + goto done; + + goto out_free_mem; + +done: + INIT_WORK(&hba->skhpb_eh_work, skhpb_error_handler); + hba->skhpb_state = SKHPB_PRESENT; + hba->issue_ioctl = false; + pm_runtime_mark_last_busy(hba->dev); + pm_runtime_put_noidle(hba->dev); + + if (do_work) { + for (lun = 0; lun < UFS_UPIU_MAX_GENERAL_LUN; lun++) { + struct skhpb_lu *hpb = hba->skhpb_lup[lun]; + + if (hpb) + schedule_delayed_work(&hpb->skhpb_pinned_work, 0); + } + } + + return 0; + +out_free_mem: + skhpb_release(hba, SKHPB_NOT_SUPPORTED); +out_state: + hba->skhpb_state = SKHPB_NOT_SUPPORTED; + pm_runtime_mark_last_busy(hba->dev); + pm_runtime_put_noidle(hba->dev); + return ret; +} + +static void skhpb_map_loading_trigger(struct skhpb_lu *hpb, + bool only_pinned, bool do_work_handler) +{ + int region, subregion; + unsigned long flags; + + if (do_work_handler) + goto work_out; + flush_delayed_work(&hpb->skhpb_pinned_work); + for (region = 0 ; region < hpb->regions_per_lu ; region++) { + struct skhpb_region *cb; + + cb = hpb->region_tbl + region; + + if (cb->region_state != SKHPB_REGION_ACTIVE && + !cb->is_pinned) + continue; + + if ((only_pinned && cb->is_pinned) || + !only_pinned) { + spin_lock_irqsave(&hpb->hpb_lock, flags); + for (subregion = 0; subregion < cb->subregion_count; + subregion++) + skhpb_add_subregion_to_req_list(hpb, + cb->subregion_tbl + subregion); + spin_unlock_irqrestore(&hpb->hpb_lock, flags); + do_work_handler = true; + } + } +work_out: + if (do_work_handler) + schedule_delayed_work(&hpb->skhpb_pinned_work, 0); +} + +static void skhpb_purge_active_block(struct skhpb_lu *hpb) +{ + int region, subregion; + int state; + struct skhpb_region *cb; + struct skhpb_subregion *cp; + unsigned long flags; + + spin_lock_irqsave(&hpb->hpb_lock, flags); + for (region = 0 ; region < hpb->regions_per_lu ; region++) { + cb = hpb->region_tbl + region; + + if (cb->region_state == SKHPB_REGION_INACTIVE) { + continue; + } + + if (cb->is_pinned){ + state = SKHPB_SUBREGION_DIRTY; + } else if (cb->region_state == SKHPB_REGION_ACTIVE) { + state = SKHPB_SUBREGION_UNUSED; + skhpb_cleanup_lru_info(&hpb->lru_info, cb); + } else { + SKHPB_DRIVER_E("Unsupported state of region\n"); + continue; + } + + SKHPB_DRIVER_D("region %d state %d dft %d\n", + region, state, + hpb->debug_free_table); + for (subregion = 0 ; subregion < cb->subregion_count; + subregion++) { + cp = cb->subregion_tbl + subregion; + + skhpb_purge_active_page(hpb, cp, state); + } + SKHPB_DRIVER_D("region %d state %d dft %d\n", + region, state, hpb->debug_free_table); + } + spin_unlock_irqrestore(&hpb->hpb_lock, flags); +} + +static void skhpb_retrieve_rsp_info(struct skhpb_lu *hpb) +{ + struct skhpb_rsp_info *rsp_info; + unsigned long flags; + + while (1) { + spin_lock_irqsave(&hpb->rsp_list_lock, flags); + rsp_info = list_first_entry_or_null(&hpb->lh_rsp_info, + struct skhpb_rsp_info, list_rsp_info); + if (!rsp_info) { + spin_unlock_irqrestore(&hpb->rsp_list_lock, flags); + break; + } + list_move_tail(&rsp_info->list_rsp_info, + &hpb->lh_rsp_info_free); + spin_unlock_irqrestore(&hpb->rsp_list_lock, flags); + SKHPB_DRIVER_D("add_list %p -> %p", + &hpb->lh_rsp_info_free, rsp_info); + } +} + +static void skhpb_probe(struct ufs_hba *hba) +{ + struct skhpb_lu *hpb; + int lu; + + for (lu = 0 ; lu < UFS_UPIU_MAX_GENERAL_LUN ; lu++) { + hpb = hba->skhpb_lup[lu]; + + if (hpb && hpb->lu_hpb_enable) { + skhpb_cancel_jobs(hpb); + skhpb_retrieve_rsp_info(hpb); + skhpb_purge_active_block(hpb); + SKHPB_DRIVER_I("SKHPB lun %d reset\n", lu); +// tasklet_init(&hpb->hpb_work_handler, +// skhpb_work_handler_fn, (unsigned long)hpb); + } + } + hba->skhpb_state = SKHPB_PRESENT; +} + +static void skhpb_destroy_subregion_tbl(struct skhpb_lu *hpb, + struct skhpb_region *cb) +{ + int subregion; + + for (subregion = 0 ; subregion < cb->subregion_count ; subregion++) { + struct skhpb_subregion *cp; + + cp = cb->subregion_tbl + subregion; + cp->subregion_state = SKHPB_SUBREGION_UNUSED; + skhpb_put_map_ctx(hpb, cp->mctx); + } + + kfree(cb->subregion_tbl); +} + +static void skhpb_destroy_region_tbl(struct skhpb_lu *hpb) +{ + int region; + + if (!hpb->region_tbl) + return; + + for (region = 0 ; region < hpb->regions_per_lu ; region++) { + struct skhpb_region *cb; + + cb = hpb->region_tbl + region; + if (cb->region_state == SKHPB_REGION_ACTIVE) { + cb->region_state = SKHPB_REGION_INACTIVE; + + skhpb_destroy_subregion_tbl(hpb, cb); + } + } + vfree(hpb->region_tbl); +} + +void skhpb_suspend(struct ufs_hba *hba) +{ + int lun; + unsigned long flags; + struct skhpb_lu *hpb; + struct skhpb_rsp_info *rsp_info; + struct skhpb_map_req *map_req; + + if (hba->skhpb_quirk & SKHPB_QUIRK_PURGE_HINT_INFO_WHEN_SLEEP) { + for (lun = 0 ; lun < UFS_UPIU_MAX_GENERAL_LUN ; lun++) { + hpb = hba->skhpb_lup[lun]; + if (!hpb) + continue; + + while (1) { + spin_lock_irqsave(&hpb->rsp_list_lock, flags); + rsp_info = list_first_entry_or_null(&hpb->lh_rsp_info, + struct skhpb_rsp_info, list_rsp_info); + if (!rsp_info) { + spin_unlock_irqrestore(&hpb->rsp_list_lock, flags); + break; + } + list_del_init(&rsp_info->list_rsp_info); + list_add_tail(&rsp_info->list_rsp_info, &hpb->lh_rsp_info_free); + atomic64_inc(&hpb->canceled_resp); + spin_unlock_irqrestore(&hpb->rsp_list_lock, flags); + } + while (1) { + spin_lock_bh(&hpb->map_list_lock); + map_req = list_first_entry_or_null(&hpb->lh_map_req_retry, + struct skhpb_map_req, list_map_req); + if (!map_req) { + spin_unlock_bh(&hpb->map_list_lock); + break; + } + list_del_init(&map_req->list_map_req); + list_add_tail(&map_req->list_map_req, &hpb->lh_map_req_free); + atomic64_inc(&hpb->canceled_map_req); + spin_unlock_bh(&hpb->map_list_lock); + } + } + } + for (lun = 0 ; lun < UFS_UPIU_MAX_GENERAL_LUN ; lun++) { + hpb = hba->skhpb_lup[lun]; + if (!hpb) + continue; + skhpb_cancel_jobs(hpb); + } +} + +void skhpb_resume(struct ufs_hba *hba) { + int lun; + struct skhpb_lu *hpb; + + for (lun = 0 ; lun < UFS_UPIU_MAX_GENERAL_LUN ; lun++) { + hpb = hba->skhpb_lup[lun]; + if (!hpb) + continue; + spin_lock_bh(&hpb->map_list_lock); + if (!list_empty(&hpb->lh_map_req_retry)) + schedule_delayed_work(&hpb->skhpb_map_req_retry_work, msecs_to_jiffies(1000)); + spin_unlock_bh(&hpb->map_list_lock); + } +} + +void skhpb_release(struct ufs_hba *hba, int state) +{ + int lun; + + hba->skhpb_state = SKHPB_FAILED; + + for (lun = 0 ; lun < UFS_UPIU_MAX_GENERAL_LUN ; lun++) { + struct skhpb_lu *hpb = hba->skhpb_lup[lun]; + + if (!hpb) + continue; + + hba->skhpb_lup[lun] = NULL; + + if (!hpb->lu_hpb_enable) + continue; + + hpb->lu_hpb_enable = false; + + skhpb_cancel_jobs(hpb); + + skhpb_destroy_region_tbl(hpb); + skhpb_table_mempool_remove(hpb); + + vfree(hpb->rsp_info); + vfree(hpb->map_req); + + kobject_uevent(&hpb->kobj, KOBJ_REMOVE); + kobject_del(&hpb->kobj); // TODO --count & del? + + kfree(hpb); + } + + if (skhpb_alloc_mctx != 0) + SKHPB_DRIVER_E("warning: skhpb_alloc_mctx %d", skhpb_alloc_mctx); + + hba->skhpb_state = state; +} + +void skhpb_init_handler(struct work_struct *work) +{ + struct delayed_work *dwork = to_delayed_work(work); + struct ufs_hba *hba = + container_of(dwork, struct ufs_hba, skhpb_init_work); + struct Scsi_Host *shost = hba->host; + bool async_scan; + + if (hba->skhpb_state == SKHPB_NOT_SUPPORTED) + return; + + spin_lock_irq(shost->host_lock); + async_scan = shost->async_scan; + spin_unlock_irq(shost->host_lock); + if (async_scan) { + schedule_delayed_work(dwork, msecs_to_jiffies(100)); + return; + } + + if (hba->skhpb_state == SKHPB_NEED_INIT) { + int err = skhpb_init(hba); + + if (hba->skhpb_state == SKHPB_NOT_SUPPORTED) + SKHPB_DRIVER_E("Run without HPB - err=%d\n", err); + } else if (hba->skhpb_state == SKHPB_RESET) { + skhpb_probe(hba); + } +} + +void ufshcd_init_hpb(struct ufs_hba *hba) +{ + int lun; + + hba->skhpb_feat = 0; + hba->skhpb_quirk = 0; + hba->skhpb_state = SKHPB_NEED_INIT; + for (lun = 0 ; lun < UFS_UPIU_MAX_GENERAL_LUN ; lun++) { + hba->skhpb_lup[lun] = NULL; + hba->sdev_ufs_lu[lun] = NULL; + hba->skhpb_quicklist_lu_enable[lun] = SKHPB_U8_MAX; + } + + INIT_DELAYED_WORK(&hba->skhpb_init_work, skhpb_init_handler); +} + +static void skhpb_error_handler(struct work_struct *work) +{ + struct ufs_hba *hba; + + hba = container_of(work, struct ufs_hba, skhpb_eh_work); + + SKHPB_DRIVER_E("SKHPB driver runs without SKHPB\n"); + SKHPB_DRIVER_E("SKHPB will be removed from the kernel\n"); + + skhpb_release(hba, SKHPB_FAILED); +} + +static void skhpb_stat_init(struct skhpb_lu *hpb) +{ + atomic64_set(&hpb->hit, 0); + atomic64_set(&hpb->size_miss, 0); + atomic64_set(&hpb->region_miss, 0); + atomic64_set(&hpb->subregion_miss, 0); + atomic64_set(&hpb->entry_dirty_miss, 0); + atomic64_set(&hpb->w_map_req_cnt, 0); +#if defined(SKHPB_READ_LARGE_CHUNK_SUPPORT) + atomic64_set(&hpb->lc_entry_dirty_miss, 0); + atomic64_set(&hpb->lc_reg_subreg_miss, 0); + atomic64_set(&hpb->lc_hit, 0); +#endif + atomic64_set(&hpb->rb_noti_cnt, 0); + atomic64_set(&hpb->reset_noti_cnt, 0); + atomic64_set(&hpb->map_req_cnt, 0); + atomic64_set(&hpb->region_evict, 0); + atomic64_set(&hpb->region_add, 0); + atomic64_set(&hpb->rb_fail, 0); + atomic64_set(&hpb->canceled_resp, 0); + atomic64_set(&hpb->canceled_map_req, 0); + atomic64_set(&hpb->alloc_map_req_cnt, 0); +} + + +static ssize_t skhpb_sysfs_info_from_region_store(struct skhpb_lu *hpb, + const char *buf, size_t count) +{ + unsigned long long value; + int region, subregion; + struct skhpb_region *cb; + struct skhpb_subregion *cp; + + if (kstrtoull(buf, 0, &value)) { + SKHPB_DRIVER_E( "kstrtoul error\n"); + return -EINVAL; + } + + if (value >= hpb->regions_per_lu) { + SKHPB_DRIVER_E("value %llu >= regions_per_lu %d error\n", + value, hpb->regions_per_lu); + return -EINVAL; + } + + region = (int)value; + cb = hpb->region_tbl + region; + + SKHPB_DRIVER_I("get_info_from_region[%d]=", region); + SKHPB_DRIVER_I("region %u state %s", region, + ((cb->region_state == SKHPB_REGION_INACTIVE) ? "INACTIVE" : + ((cb->region_state == SKHPB_REGION_ACTIVE) ? "ACTIVE" : "INVALID")) + ); + for (subregion = 0; subregion < cb->subregion_count; subregion++) { + cp = cb->subregion_tbl + subregion; + SKHPB_DRIVER_I("subregion %u state %s", subregion, + ((cp->subregion_state == SKHPB_SUBREGION_UNUSED) ? "UNUSED" : + ((cp->subregion_state == SKHPB_SUBREGION_DIRTY) ? "DIRTY" : + ((cp->subregion_state == SKHPB_SUBREGION_CLEAN) ? "CLEAN" : + (cp->subregion_state == SKHPB_SUBREGION_ISSUED) ? + "ISSUED" : "INVALID"))) + ); + } + return count; +} + +static ssize_t skhpb_sysfs_info_from_lba_store(struct skhpb_lu *hpb, + const char *buf, size_t count) +{ + skhpb_t ppn; + unsigned long long value; + unsigned int lpn; + int region, subregion, subregion_offset; + struct skhpb_region *cb; + struct skhpb_subregion *cp; + unsigned long flags; + int dirty; + + if (kstrtoull(buf, 0, &value)) { + SKHPB_DRIVER_E("kstrtoul error\n"); + return -EINVAL; + } + + if (value > hpb->lu_num_blocks * SKHPB_SECTORS_PER_BLOCK) { + SKHPB_DRIVER_E("value %llu > lu_num_blocks %llu error\n", + value, hpb->lu_num_blocks); + return -EINVAL; + } + lpn = value / SKHPB_SECTORS_PER_BLOCK; + + skhpb_get_pos_from_lpn(hpb, lpn, ®ion, &subregion, + &subregion_offset); + if(!skhpb_check_region_subregion_validity(hpb, region, subregion)) + return -EINVAL; + cb = hpb->region_tbl + region; + cp = cb->subregion_tbl + subregion; + + if (cb->region_state != SKHPB_REGION_INACTIVE) { + ppn = skhpb_get_ppn(cp->mctx, subregion_offset); + spin_lock_irqsave(&hpb->hpb_lock, flags); + dirty = skhpb_ppn_dirty_check(hpb, cp, subregion_offset); + spin_unlock_irqrestore(&hpb->hpb_lock, flags); + } else { + ppn = 0; + dirty = -1; + } + SKHPB_DRIVER_I("get_info_from_lba[%llu]=", value); + SKHPB_DRIVER_I("sector %llu region %d state %s subregion %d state %s", + value, region, + ((cb->region_state == SKHPB_REGION_INACTIVE) ? "INACTIVE" : + ((cb->region_state == SKHPB_REGION_ACTIVE) ? "ACTIVE" : "INVALID")), + subregion, + ((cp->subregion_state == SKHPB_SUBREGION_UNUSED) ? "UNUSED" : + ((cp->subregion_state == SKHPB_SUBREGION_DIRTY) ? "DIRTY" : + ((cp->subregion_state == SKHPB_SUBREGION_CLEAN) ? "CLEAN" : + (cp->subregion_state == SKHPB_SUBREGION_ISSUED) ? + "ISSUED":"INVALID"))) + ); + SKHPB_DRIVER_I("sector %llu lpn %u ppn %llx dirty %d", + value, lpn, ppn, dirty); + return count; +} + +static ssize_t skhpb_sysfs_map_req_show(struct skhpb_lu *hpb, char *buf) +{ + return snprintf(buf, PAGE_SIZE, + "map_req_count[RB_NOTI RESET_NOTI MAP_REQ]= %lld %lld %lld\n", + (long long)atomic64_read(&hpb->rb_noti_cnt), + (long long)atomic64_read(&hpb->reset_noti_cnt), + (long long)atomic64_read(&hpb->map_req_cnt)); +} + +static ssize_t skhpb_sysfs_count_reset_store(struct skhpb_lu *hpb, + const char *buf, size_t count) +{ + unsigned long debug; + + if (kstrtoul(buf, 0, &debug)) + return -EINVAL; + + skhpb_stat_init(hpb); + + return count; +} + +static ssize_t skhpb_sysfs_add_evict_show(struct skhpb_lu *hpb, char *buf) +{ + return snprintf(buf, PAGE_SIZE, "add_evict_count[ADD EVICT]= %lld %lld\n", + (long long)atomic64_read(&hpb->region_add), + (long long)atomic64_read(&hpb->region_evict)); +} + +static ssize_t skhpb_sysfs_statistics_show(struct skhpb_lu *hpb, char *buf) +{ + long long size_miss, region_miss, subregion_miss, entry_dirty_miss, + hit, miss_all, rb_fail, canceled_resp, canceled_map_req; +#if defined(SKHPB_READ_LARGE_CHUNK_SUPPORT) + long long lc_dirty_miss = 0, lc_state_miss = 0, lc_hit = 0; +#endif + int count = 0; + + hit = atomic64_read(&hpb->hit); + size_miss = atomic64_read(&hpb->size_miss); + region_miss = atomic64_read(&hpb->region_miss); + subregion_miss = atomic64_read(&hpb->subregion_miss); + entry_dirty_miss = atomic64_read(&hpb->entry_dirty_miss); + rb_fail = atomic64_read(&hpb->rb_fail); + canceled_resp = atomic64_read(&hpb->canceled_resp); + canceled_map_req = atomic64_read(&hpb->canceled_map_req); + +#if defined(SKHPB_READ_LARGE_CHUNK_SUPPORT) + lc_dirty_miss = atomic64_read(&hpb->lc_entry_dirty_miss); + lc_state_miss = atomic64_read(&hpb->lc_reg_subreg_miss); + lc_hit = atomic64_read(&hpb->lc_hit); + entry_dirty_miss += lc_dirty_miss; + subregion_miss += lc_state_miss; +#endif + miss_all = size_miss + region_miss + subregion_miss + entry_dirty_miss; + count += snprintf(buf + count, PAGE_SIZE, + "Total: %lld\nHit_Counts: %lld\n", + hit + miss_all, hit); + count += snprintf(buf + count, PAGE_SIZE, + "Miss_Counts[ALL SIZE REG SUBREG DIRTY]= %lld %lld %lld %lld %lld\n", + miss_all, size_miss, region_miss, subregion_miss, entry_dirty_miss); + +#if defined(SKHPB_READ_LARGE_CHUNK_SUPPORT) + count += snprintf(buf + count, PAGE_SIZE, + "LARG_CHNK Hit_Count: %lld, miss_count[ALL DIRTY REG_SUBREG]= %lld %lld %lld\n", + lc_hit, lc_dirty_miss + lc_state_miss, lc_dirty_miss, lc_state_miss); +#endif + if (hpb->hba->skhpb_quirk & SKHPB_QUIRK_PURGE_HINT_INFO_WHEN_SLEEP) + count += snprintf(buf + count, PAGE_SIZE, + "READ_BUFFER miss_count[RB_FAIL CANCEL_RESP CANCEL_MAP]= %lld %lld %lld\n", + rb_fail, canceled_resp, canceled_map_req); + else + count += snprintf(buf + count, PAGE_SIZE, + "READ_BUFFER miss_count[RB_FAIL]= %lld\n", rb_fail); + return count; +} + +static ssize_t skhpb_sysfs_version_show(struct skhpb_lu *hpb, char *buf) +{ + return snprintf(buf, PAGE_SIZE, + "HPBversion[HPB DD]= %x.%x.%x %x.%x.%x\n", + (hpb->hpb_ver >> 8) & 0xf, + (hpb->hpb_ver >> 4) & 0xf, + (hpb->hpb_ver >> 0) & 0xf, + SKHPB_GET_BYTE_2(SKHPB_DD_VER), + SKHPB_GET_BYTE_1(SKHPB_DD_VER), + SKHPB_GET_BYTE_0(SKHPB_DD_VER)); +} + +static ssize_t skhpb_sysfs_active_count_show( + struct skhpb_lu *hpb, char *buf) +{ + struct skhpb_region *cb; + int region; + int pinned_cnt = 0; + + for (region = 0 ; region < hpb->regions_per_lu ; region++) { + cb = hpb->region_tbl + region; + + if (cb->is_pinned && cb->region_state == SKHPB_REGION_ACTIVE) + pinned_cnt++; + } + return snprintf(buf, PAGE_SIZE, + "active_count[ACTIVE_CNT PINNED_CNT]= %ld %d\n", + atomic64_read(&hpb->lru_info.active_count), pinned_cnt); +} + +static ssize_t skhpb_sysfs_hpb_disable_show( + struct skhpb_lu *hpb, char *buf) +{ + return snprintf(buf, PAGE_SIZE, + "[0] BOTH_enabled= %d\ + \n[1] ONLY_HPB_BUFFER_disabled= %d\ + \n[2] ONLY_HPB_READ_disabled= %d\ + \n[3] BOTH_disabled= %d\n", + (hpb->force_map_req_disable == 0 && + hpb->force_hpb_read_disable == 0)?1:0, + hpb->force_map_req_disable, + hpb->force_hpb_read_disable, + (hpb->force_map_req_disable == 1 && + hpb->force_hpb_read_disable == 1)?1:0); +} + +static ssize_t skhpb_sysfs_hpb_disable_store(struct skhpb_lu *hpb, + const char *buf, size_t count) +{ + unsigned long value; + + if (kstrtoul(buf, 0, &value)) + return -EINVAL; + + if (value > 3) { + SKHPB_DRIVER_E("Error, Only [0-3] is valid:\ + \n[0] BOTH_enable\ + \n[1] ONLY_HPB_BUFFER_disable\ + \n[2] ONLY_HPB_READ_disable\ + \n[3] BOTH_disabie\n"); + return -EINVAL; + } + + hpb->force_map_req_disable = value & 0x1; + hpb->force_hpb_read_disable = value & 0x2; + return count; +} + +static ssize_t skhpb_sysfs_hpb_reset_store(struct skhpb_lu *hpb, + const char *buf, size_t count) +{ + unsigned long value; + unsigned int doorbel; + struct skhpb_rsp_info *rsp_info; + unsigned long flags; + int ret, retries, lun; + struct skhpb_lu *hpb_lu; + + if (kstrtoul(buf, 0, &value)) + return -EINVAL; + if (!value) + return count; + + for (retries = 1; retries <= 20; retries++) { + pm_runtime_get_sync(SKHPB_DEV(hpb)); + ufshcd_hold(hpb->hba, false); + doorbel = ufshcd_readl(hpb->hba, REG_UTP_TRANSFER_REQ_DOOR_BELL); + ufshcd_release(hpb->hba); + pm_runtime_mark_last_busy(SKHPB_DEV(hpb)); + pm_runtime_put_noidle(SKHPB_DEV(hpb)); + + if (!doorbel) { + pm_runtime_get_sync(SKHPB_DEV(hpb)); + ret = ufshcd_query_flag_retry(hpb->hba, + UPIU_QUERY_OPCODE_SET_FLAG, + QUERY_FLAG_IDN_HPB_RESET, 0, + NULL); + pm_runtime_mark_last_busy(SKHPB_DEV(hpb)); + pm_runtime_put_noidle(SKHPB_DEV(hpb)); + if(!ret) { + SKHPB_DRIVER_I("Query fHPBReset is successfully sent\n"); + break; + } + } + SKHPB_DRIVER_I("fHPBReset failed and will retry[%d] after some time, DOORBELL: 0x%x\n", + retries, doorbel); + msleep(200); + } + if (ret == 0) { + bool fHPBReset = true; + pm_runtime_get_sync(SKHPB_DEV(hpb)); + ufshcd_query_flag_retry(hpb->hba, + UPIU_QUERY_OPCODE_READ_FLAG, + QUERY_FLAG_IDN_HPB_RESET, 0, + &fHPBReset); + pm_runtime_mark_last_busy(SKHPB_DEV(hpb)); + pm_runtime_put_noidle(SKHPB_DEV(hpb)); + if (fHPBReset) + SKHPB_DRIVER_E("fHPBReset is still in progress at device, but keep going\n"); + } else { + SKHPB_DRIVER_E("Fail to set fHPBReset flag\n"); + goto out; + } + flush_delayed_work(&hpb->skhpb_pinned_work); + for (lun = 0 ; lun < UFS_UPIU_MAX_GENERAL_LUN ; lun++) + { + hpb_lu = hpb->hba->skhpb_lup[lun]; + if (!hpb_lu || !hpb_lu->lu_hpb_enable) + continue; + + skhpb_stat_init(hpb_lu); + + spin_lock_irqsave(&hpb_lu->rsp_list_lock, flags); + rsp_info = skhpb_get_req_info(hpb_lu); + spin_unlock_irqrestore(&hpb_lu->rsp_list_lock, flags); + if (!rsp_info) + goto out; + rsp_info->type = SKHPB_RSP_HPB_RESET; + SKHPB_RSP_TIME(rsp_info->RSP_start); + spin_lock_irqsave(&hpb_lu->rsp_list_lock, flags); + list_add_tail(&rsp_info->list_rsp_info, &hpb_lu->lh_rsp_info); + spin_unlock_irqrestore(&hpb_lu->rsp_list_lock, flags); + SKHPB_DRIVER_I("Host HPB reset start LU%d - fHPBReset\n", lun); + schedule_work(&hpb_lu->skhpb_rsp_work); + } +out: + return count; +} + +static ssize_t skhpb_sysfs_debug_log_show( + struct skhpb_lu *hpb, char *buf) +{ + int value = skhpb_debug_mask; + + if (value == (SKHPB_LOG_OFF)) { + value = SKHPB_LOG_LEVEL_OFF; + } else if (value == (SKHPB_LOG_ERR)) { + value = SKHPB_LOG_LEVEL_ERR; + } else if (value == (SKHPB_LOG_ERR | SKHPB_LOG_INFO)) { + value = SKHPB_LOG_LEVEL_INFO; + } else if (value == (SKHPB_LOG_ERR | SKHPB_LOG_INFO | SKHPB_LOG_DEBUG)) { + value = SKHPB_LOG_LEVEL_DEBUG; + } else if (value == (SKHPB_LOG_ERR | SKHPB_LOG_INFO | SKHPB_LOG_DEBUG | SKHPB_LOG_HEX)) { + value = SKHPB_LOG_LEVEL_HEX; + } + + return snprintf(buf, PAGE_SIZE, "[0] : LOG_LEVEL_OFF\ + \n[1] : LOG_LEVEL_ERR\ + \n[2] : LOG_LEVEL_INFO\ + \n[3] : LOG_LEVEL_DEBUG\ + \n[4] : LOG_LEVEL_HEX\ + \n-----------------------\ + \nLog-Level = %d\n", value); +} + +static ssize_t skhpb_sysfs_debug_log_store(struct skhpb_lu *hpb, + const char *buf, size_t count) +{ + unsigned long value; + + if (kstrtoul(buf, 0, &value)) + return -EINVAL; + + if (value > SKHPB_LOG_LEVEL_HEX) { + SKHPB_DRIVER_E("Error, Only [0-4] is valid:\ + \n[0] : LOG_LEVEL_OFF\ + \n[1] : LOG_LEVEL_ERR\ + \n[2] : LOG_LEVEL_INFO\ + \n[3] : LOG_LEVEL_DEBUG\ + \n[4] : LOG_LEVEL_HEX\n"); + return -EINVAL; + } + + if (value == SKHPB_LOG_LEVEL_OFF) { + skhpb_debug_mask = SKHPB_LOG_OFF; + } else if (value == SKHPB_LOG_LEVEL_ERR) { + skhpb_debug_mask = SKHPB_LOG_ERR; + } else if (value == SKHPB_LOG_LEVEL_INFO) { + skhpb_debug_mask = SKHPB_LOG_ERR | SKHPB_LOG_INFO; + } else if (value == SKHPB_LOG_LEVEL_DEBUG) { + skhpb_debug_mask = SKHPB_LOG_ERR | SKHPB_LOG_INFO | SKHPB_LOG_DEBUG; + } else if (value == SKHPB_LOG_LEVEL_HEX) { + skhpb_debug_mask = SKHPB_LOG_ERR | SKHPB_LOG_INFO | SKHPB_LOG_DEBUG | SKHPB_LOG_HEX; + } + + return count; +} + +static ssize_t skhpb_sysfs_rsp_time_show(struct skhpb_lu *hpb, char *buf) +{ + return snprintf(buf, PAGE_SIZE, + "map_req_time: %s\n", (debug_map_req ? "Enable":"Disable")); +} + +static ssize_t skhpb_sysfs_rsp_time_store(struct skhpb_lu *hpb, + const char *buf, size_t count) +{ + unsigned long value; + + if (kstrtoul(buf, 0, &value)) + return -EINVAL; + + if (value > 1) + return count; + + debug_map_req = value; + return count; +} + +static struct skhpb_sysfs_entry skhpb_sysfs_entries[] = { + __ATTR(hpb_disable, 0644, + skhpb_sysfs_hpb_disable_show, + skhpb_sysfs_hpb_disable_store), + __ATTR(HPBVersion, 0444, skhpb_sysfs_version_show, NULL), + __ATTR(statistics, 0444, skhpb_sysfs_statistics_show, NULL), + __ATTR(active_count, 0444, skhpb_sysfs_active_count_show, NULL), + __ATTR(add_evict_count, 0444, skhpb_sysfs_add_evict_show, NULL), + __ATTR(count_reset, 0200, NULL, skhpb_sysfs_count_reset_store), + __ATTR(map_req_count, 0444, skhpb_sysfs_map_req_show, NULL), + __ATTR(get_info_from_region, 0200, NULL, + skhpb_sysfs_info_from_region_store), + __ATTR(get_info_from_lba, 0200, NULL, skhpb_sysfs_info_from_lba_store), + __ATTR(hpb_reset, 0200, NULL, skhpb_sysfs_hpb_reset_store), + __ATTR(debug_log, 0644, + skhpb_sysfs_debug_log_show, + skhpb_sysfs_debug_log_store), + __ATTR(response_time, 0644, + skhpb_sysfs_rsp_time_show, + skhpb_sysfs_rsp_time_store), + __ATTR_NULL +}; + +static ssize_t skhpb_attr_show(struct kobject *kobj, + struct attribute *attr, char *page) +{ + struct skhpb_sysfs_entry *entry; + struct skhpb_lu *hpb; + ssize_t error; + + entry = container_of(attr, + struct skhpb_sysfs_entry, attr); + hpb = container_of(kobj, struct skhpb_lu, kobj); + + if (!entry->show) + return -EIO; + + mutex_lock(&hpb->sysfs_lock); + error = entry->show(hpb, page); + mutex_unlock(&hpb->sysfs_lock); + return error; +} + +static ssize_t skhpb_attr_store(struct kobject *kobj, + struct attribute *attr, + const char *page, size_t length) +{ + struct skhpb_sysfs_entry *entry; + struct skhpb_lu *hpb; + ssize_t error; + + entry = container_of(attr, struct skhpb_sysfs_entry, attr); + hpb = container_of(kobj, struct skhpb_lu, kobj); + + if (!entry->store) + return -EIO; + + mutex_lock(&hpb->sysfs_lock); + error = entry->store(hpb, page, length); + mutex_unlock(&hpb->sysfs_lock); + return error; +} + +static const struct sysfs_ops skhpb_sysfs_ops = { + .show = skhpb_attr_show, + .store = skhpb_attr_store, +}; + +static struct kobj_type skhpb_ktype = { + .sysfs_ops = &skhpb_sysfs_ops, + .release = NULL, +}; + +static int skhpb_create_sysfs(struct ufs_hba *hba, + struct skhpb_lu *hpb) +{ + struct device *dev = hba->dev; + struct skhpb_sysfs_entry *entry; + int err; + + hpb->sysfs_entries = skhpb_sysfs_entries; + + skhpb_stat_init(hpb); + + kobject_init(&hpb->kobj, &skhpb_ktype); + mutex_init(&hpb->sysfs_lock); + + err = kobject_add(&hpb->kobj, kobject_get(&dev->kobj), + "ufshpb_lu%d", hpb->lun); + if (!err) { + for (entry = hpb->sysfs_entries; + entry->attr.name != NULL ; entry++) { + if (sysfs_create_file(&hpb->kobj, &entry->attr)) + break; + } + kobject_uevent(&hpb->kobj, KOBJ_ADD); + } + return err; +} diff --git a/drivers/scsi/ufs/ufshpb_skh.h b/drivers/scsi/ufs/ufshpb_skh.h new file mode 100644 index 000000000000..5359dda13a52 --- /dev/null +++ b/drivers/scsi/ufs/ufshpb_skh.h @@ -0,0 +1,518 @@ +/* + * Universal Flash Storage Host Performance Booster + * + * Original work Copyright (C) 2017-2018 Samsung Electronics Co., Ltd. + * Modified work Copyright (C) 2018, Google, Inc. + * Modified work Copyright (C) 2019 SK hynix + * + * Origianl work Authors: + * Yongmyung Lee + * Jinyoung Choi + * Modifier + * Kihyun Cho + * Sengman Shin + * Jinsu Lee + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * See the COPYING file in the top-level directory or visit + * + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * This program is provided "AS IS" and "WITH ALL FAULTS" and + * without warranty of any kind. You are solely responsible for + * determining the appropriateness of using and distributing + * the program and assume all risks associated with your exercise + * of rights with respect to the program, including but not limited + * to infringement of third party rights, the risks and costs of + * program errors, damage to or loss of data, programs or equipment, + * and unavailability or interruption of operations. Under no + * circumstances will the contributor of this Program be liable for + * any damages of any kind arising from your use or distribution of + * this program. + * + * The Linux Foundation chooses to take subject only to the GPLv2 + * license terms, and distributes only under these terms. + */ + +#ifndef _SKHPB_H_ +#define _SKHPB_H_ + +#include +#include +#include + +/* Version info*/ +#define SKHPB_DD_VER 0x010506 + +/* QUIRKs */ +/* Use READ16 instead of HPB_READ command, + * This is workaround solution to countmeasure QCT ICE issue. */ +#define SKHPB_QUIRK_USE_READ_16_FOR_ENCRYPTION (1 << 0) + +/* This quirk makes HPB driver always works as Devie Control Mode. + * To cover old Configuration descriptor format which interpret + * the bHPBControl field as RESERVED. */ +#define SKHPB_QUIRK_ALWAYS_DEVICE_CONTROL_MODE (1 << 1) + +/* Discard SubRegion activation hint information that has been processed, + * when the host enters RPM/SPM sleep. + * Must not be set the bit in ufs_quirks.h.*/ +#define SKHPB_QUIRK_PURGE_HINT_INFO_WHEN_SLEEP (1 << 20) + +/* Constant value*/ +#define SKHPB_SECTOR 512 +#define SKHPB_BLOCK 4096 +#define SKHPB_SECTORS_PER_BLOCK (SKHPB_BLOCK / SKHPB_SECTOR) +#define SKHPB_BITS_PER_DWORD 32 +#define SKHPB_MAX_ACTIVE_NUM 2 +#define SKHPB_MAX_INACTIVE_NUM 2 + +#define SKHPB_ENTRY_SIZE 0x08 +#define SKHPB_ENTREIS_PER_OS_PAGE (PAGE_SIZE / SKHPB_ENTRY_SIZE) + +/* Description */ +#define SKHPB_UFS_FEATURE_SUPPORT_HPB_BIT 0x80 + +#define SKHPB_QUERY_DESC_DEVICE_MAX_SIZE 0x43 +#define SKHPB_QUERY_DESC_CONFIGURAION_MAX_SIZE 0xE6 +#define SKHPB_QUERY_DESC_UNIT_MAX_SIZE 0x29 +#define SKHPB_QUERY_DESC_GEOMETRY_MAX_SIZE 0x4D + +/* Configuration for HPB */ +#define SKHPB_CONF_LU_ENABLE 0x00 +#define SKHPB_CONF_ACTIVE_REGIONS 0x10 +#define SKHPB_CONF_PINNED_START 0x12 +#define SKHPB_CONF_PINNED_NUM 0x14 + +/* Parameter Macros */ +#define SKHPB_DEV(h) ((h)->hba->dev) +#define SKHPB_MAX_BVEC_SIZE 128 + +/* Use for HPB activate */ +#define SKHPB_CONFIG_LEN 0xd0 + +#define SKHPB_READ_LARGE_CHUNK_SUPPORT +#define SKHPB_READ_LARGE_CHUNK_MAX_BLOCK_COUNT (128) //TRANSFER LENGTH: 8bit + +typedef u64 skhpb_t; + +enum skhpb_lu_set { + LU_DISABLE = 0x00, + LU_ENABLE = 0x01, + LU_HPB_ENABLE = 0x02, + LU_SET_MAX, +}; + +struct skhpb_config_desc { + unsigned char conf_dev_desc[16]; + unsigned char unit[UFS_UPIU_MAX_GENERAL_LUN][24]; +}; + +/* Response UPIU types */ +#define SKHPB_RSP_NONE 0x00 +#define SKHPB_RSP_REQ_REGION_UPDATE 0x01 +#define SKHPB_RSP_HPB_RESET 0x02 +#define SKHPB_PER_ACTIVE_INFO_BYTES 4 +#define SKHPB_PER_INACTIVE_INFO_BYTES 2 + +/* Vender defined OPCODE */ +#define SKHPB_READ 0xF8 +#define SKHPB_READ_BUFFER 0xF9 +#define SKHPB_WRITE_BUFFER 0xFA + +#define SKHPB_DEV_DATA_SEG_LEN 0x14 +#define SKHPB_DEV_SENSE_SEG_LEN 0x12 +#define SKHPB_DEV_DES_TYPE 0x80 +#define SKHPB_DEV_ADDITIONAL_LEN 0x10 + +/* BYTE SHIFT */ +#define SKHPB_ZERO_BYTE_SHIFT 0 +#define SKHPB_ONE_BYTE_SHIFT 8 +#define SKHPB_TWO_BYTE_SHIFT 16 +#define SKHPB_THREE_BYTE_SHIFT 24 + +#define SKHPB_SHIFT_BYTE_0(num) ((num) << SKHPB_ZERO_BYTE_SHIFT) +#define SKHPB_SHIFT_BYTE_1(num) ((num) << SKHPB_ONE_BYTE_SHIFT) + +#define SKHPB_GET_BYTE_0(num) (((num) >> SKHPB_ZERO_BYTE_SHIFT) & 0xff) +#define SKHPB_GET_BYTE_1(num) (((num) >> SKHPB_ONE_BYTE_SHIFT) & 0xff) +#define SKHPB_GET_BYTE_2(num) (((num) >> SKHPB_TWO_BYTE_SHIFT) & 0xff) +#define SKHPB_GET_BYTE_3(num) (((num) >> SKHPB_THREE_BYTE_SHIFT) & 0xff) + +#define REGION_UNIT_SIZE(bit_offset) (0x01 << (bit_offset)) + +enum SKHPB_STATE { + SKHPB_PRESENT = 1, + SKHPB_NOT_SUPPORTED = -1, + SKHPB_FAILED = -2, + SKHPB_NEED_INIT = 0, + SKHPB_RESET = -3, +}; + +enum SKHPB_BUFFER_MODE { + R_BUFFER = 0, + W_BUFFER = 1, +}; + +enum SKHPB_CMD { + SKHPB_CMD_READ = 0, + SKHPB_CMD_WRITE = 1, + SKHPB_CMD_DISCARD = 2, + SKHPB_CMD_OTHERS = 3, +}; + +enum SKHPB_REGION_STATE { + SKHPB_REGION_INACTIVE, + SKHPB_REGION_ACTIVE, +}; + +enum SKHPB_SUBREGION_STATE { + SKHPB_SUBREGION_UNUSED, + SKHPB_SUBREGION_DIRTY, + SKHPB_SUBREGION_CLEAN, + SKHPB_SUBREGION_ISSUED, +}; + +enum SKHPB_CONTROL_MODE { + HOST_CTRL_MODE = 0, + DEV_CTRL_MODE = 1, +}; + +enum SKHPB_RST_TIME { + SKHPB_MAP_RSP_DISABLE = 0, + SKHPB_MAP_RSP_ENABLE = 1, +}; + +struct skhpb_func_desc { + /*** Device Descriptor ***/ + /* 06h bNumberLU */ + int lu_cnt; + /* 10h wSpecVersion */ + u16 spec_ver; + /* 40h HPB Version */ + u16 hpb_ver; + /* 42h HPB control mode */ + u8 hpb_control_mode; + + /*** Geometry Descriptor ***/ + /* 48h bHPBRegionSize (UNIT: 512KB) */ + u8 hpb_region_size; + /* 49h bHPBNumberLU */ + u8 hpb_number_lu; + /* 4Ah bHPBSubRegionSize */ + u8 hpb_subregion_size; + /* 4B:4Ch wDeviceMaxActiveHPBRegions */ + u16 hpb_device_max_active_regions; +}; + +struct skhpb_lu_desc { + /*** Unit Descriptor ****/ + /* 03h bLUEnable */ + int lu_enable; + /* 06h lu queue depth info*/ + int lu_queue_depth; + /* 0Ah bLogicalBlockSize. default 0x0C = 4KB */ + int lu_logblk_size; + /* 0Bh qLogicalBlockCount. same as the read_capacity ret val. */ + u64 lu_logblk_cnt; + + /* 23h:24h wLUMaxActiveHPBRegions */ + u16 lu_max_active_hpb_regions; + /* 25h:26h wHPBPinnedRegionStartIdx */ + u16 hpb_pinned_region_startidx; + /* 27h:28h wNumHPBPinnedRegions */ + u16 lu_num_hpb_pinned_regions; + + + /* if 03h value is 02h, hpb_enable is set. */ + bool lu_hpb_enable; + + int lu_hpb_pinned_end_offset; +}; + +struct skhpb_rsp_active_list { + u16 region[SKHPB_MAX_ACTIVE_NUM]; + u16 subregion[SKHPB_MAX_ACTIVE_NUM]; +}; + +struct skhpb_rsp_inactive_list { + u16 region[SKHPB_MAX_INACTIVE_NUM]; +}; + +struct skhpb_rsp_update_entry { + unsigned int lpn; + skhpb_t ppn; +}; + +struct skhpb_rsp_info { + int type; + int active_cnt; + int inactive_cnt; + struct skhpb_rsp_active_list active_list; + struct skhpb_rsp_inactive_list inactive_list; + + __u64 RSP_start; + __u64 RSP_tasklet_enter; + + struct list_head list_rsp_info; +}; + +struct skhpb_rsp_field { + u8 sense_data_len[2]; + u8 desc_type; + u8 additional_len; + u8 hpb_type; + u8 lun; + u8 active_region_cnt; + u8 inactive_region_cnt; + u8 hpb_active_field[8]; + u8 hpb_inactive_field[4]; +}; + +struct skhpb_map_ctx { + struct page **m_page; + unsigned int *ppn_dirty; + + struct list_head list_table; +}; + +struct skhpb_subregion { + struct skhpb_map_ctx *mctx; + enum SKHPB_SUBREGION_STATE subregion_state; + int region; + int subregion; + bool last; + struct list_head list_subregion; +}; + +struct skhpb_region { + struct skhpb_subregion *subregion_tbl; + enum SKHPB_REGION_STATE region_state; + bool is_pinned; + int region; + int subregion_count; + /*below information is used by lru*/ + struct list_head list_region; + int hit_count; +}; + +struct skhpb_map_req { + struct skhpb_lu *hpb; + struct skhpb_map_ctx *mctx; + struct bio bio; + struct bio *pbio; + struct bio_vec bvec[SKHPB_MAX_BVEC_SIZE]; + void (*end_io)(struct request *rq, int err); + void *end_io_data; + int region; + int subregion; + int subregion_mem_size; + int lun; + int retry_cnt; + + /* for debug : RSP Profiling */ + __u64 RSP_start; // get the request from device + __u64 RSP_issue; // issue scsi cmd + __u64 RSP_end; // complete the request + + char sense[SCSI_SENSE_BUFFERSIZE]; + + struct list_head list_map_req; + int rwbuffer_flag; +}; + +enum SKHPB_SELECTION_TYPE { + TYPE_LRU = 1, + TYPE_LFU = 2, +}; + +struct skhpb_victim_select_info { + int selection_type; + struct list_head lru; + int max_lru_active_count; // supported hpb #region - pinned #region + atomic64_t active_count; +}; + +struct skhpb_lu { + struct skhpb_region *region_tbl; + struct skhpb_rsp_info *rsp_info; + struct skhpb_map_req *map_req; + + struct list_head lh_map_ctx; + struct list_head lh_subregion_req; + struct list_head lh_rsp_info; + + struct list_head lh_rsp_info_free; + struct list_head lh_map_req_free; + struct list_head lh_map_req_retry; + int debug_free_table; + + bool lu_hpb_enable; + + struct delayed_work skhpb_pinned_work; + struct delayed_work skhpb_map_req_retry_work; + struct work_struct skhpb_rsp_work; + struct bio_vec bvec[SKHPB_MAX_BVEC_SIZE]; + + int subregions_per_lu; + int regions_per_lu; + int subregion_mem_size; + int last_subregion_mem_size; + + /* for selecting victim */ + struct skhpb_victim_select_info lru_info; + + int hpb_ver; + int lu_max_active_regions; + + int entries_per_subregion; + int entries_per_subregion_shift; + int entries_per_subregion_mask; + + int entries_per_region_shift; + int entries_per_region_mask; + int subregions_per_region; + + int dwords_per_subregion; + unsigned long long subregion_unit_size; + bool identical_size; + +#define BITS_PER_PPN_DIRTY (BITS_PER_BYTE * sizeof(unsigned int)) + int ppn_dirties_per_subregion; + + int mpage_bytes; + int mpages_per_subregion; + + /* for debug constant variables */ + unsigned long long lu_num_blocks; + + int lun; + + struct ufs_hba *hba; + + spinlock_t hpb_lock; + spinlock_t rsp_list_lock; + spinlock_t map_list_lock; + + struct kobject kobj; + struct mutex sysfs_lock; + struct skhpb_sysfs_entry *sysfs_entries; + + bool hpb_control_mode; + + /* for debug */ + bool force_hpb_read_disable; + bool force_map_req_disable; + bool read_buf_debug; + atomic64_t hit; + atomic64_t size_miss; + atomic64_t region_miss; + atomic64_t subregion_miss; + atomic64_t entry_dirty_miss; + atomic64_t rb_noti_cnt; + atomic64_t rb_fail; + atomic64_t reset_noti_cnt; + atomic64_t w_map_req_cnt; +#if defined(SKHPB_READ_LARGE_CHUNK_SUPPORT) + atomic64_t lc_entry_dirty_miss; + atomic64_t lc_reg_subreg_miss; + atomic64_t lc_hit; +#endif + atomic64_t map_req_cnt; + atomic64_t region_add; + atomic64_t region_evict; + atomic64_t canceled_resp; + atomic64_t canceled_map_req; + atomic64_t alloc_map_req_cnt; +}; + +struct skhpb_sysfs_entry { + struct attribute attr; + ssize_t (*show)(struct skhpb_lu *hpb, char *buf); + ssize_t (*store)(struct skhpb_lu *hpb, const char *, size_t); +}; + +struct ufshcd_lrb; + +void ufshcd_init_hpb(struct ufs_hba *hba); +void skhpb_init_handler(struct work_struct *work); +void skhpb_prep_fn(struct ufs_hba *hba, struct ufshcd_lrb *lrbp); +void skhpb_rsp_upiu(struct ufs_hba *hba, struct ufshcd_lrb *lrbp); +void skhpb_suspend(struct ufs_hba *hba); +void skhpb_resume(struct ufs_hba *hba); +void skhpb_release(struct ufs_hba *hba, int state); +int skhpb_issue_req_dev_ctx(struct skhpb_lu *hpb, unsigned char *buf, + int buf_length); +int skhpb_control_validation(struct ufs_hba *hba, + struct skhpb_config_desc *config); + +extern u32 skhpb_debug_mask; +extern int debug_map_req; +enum SKHPB_LOG_LEVEL { + SKHPB_LOG_LEVEL_OFF = 0, + SKHPB_LOG_LEVEL_ERR = 1, + SKHPB_LOG_LEVEL_INFO = 2, + SKHPB_LOG_LEVEL_DEBUG = 3, + SKHPB_LOG_LEVEL_HEX = 4, +}; +enum SKHPB_LOG_MASK { + SKHPB_LOG_OFF = SKHPB_LOG_LEVEL_OFF, /* 0 */ + SKHPB_LOG_ERR = (1U << SKHPB_LOG_LEVEL_ERR), /* 2 */ + SKHPB_LOG_INFO = (1U << SKHPB_LOG_LEVEL_INFO), /* 4 */ + SKHPB_LOG_DEBUG = (1U << SKHPB_LOG_LEVEL_DEBUG), /* 8 */ + SKHPB_LOG_HEX = (1U << SKHPB_LOG_LEVEL_HEX), /* 16 */ +}; +#define SKHPB_DRIVER_E(fmt, args...) \ + do { \ + if(likely(skhpb_debug_mask & SKHPB_LOG_ERR)) \ + pr_err("[HPB E][%s:%d] " fmt, __func__, __LINE__, ##args); \ + } while (0) + +#define SKHPB_DRIVER_I(fmt, args...) \ + do { \ + if(unlikely(skhpb_debug_mask & SKHPB_LOG_INFO)) \ + pr_err("[HPB][%s:%d] " fmt, __func__, __LINE__, ##args); \ + } while (0) + +#define SKHPB_DRIVER_D(fmt, args...) \ + do { \ + if(unlikely(skhpb_debug_mask & SKHPB_LOG_DEBUG)) \ + printk(KERN_DEBUG "[HPB][%s:%d] " fmt, __func__, __LINE__, ##args); \ + } while (0) + +#define SKHPB_DRIVER_HEXDUMP(fmt, args...) \ + do { \ + if(unlikely(skhpb_debug_mask & SKHPB_LOG_HEX)) { \ + print_hex_dump(KERN_DEBUG, fmt, DUMP_PREFIX_ADDRESS, ##args); \ + } \ + } while (0) + +#define SKHPB_MAP_REQ_TIME(map_req, val, print) \ + do { \ + if(unlikely(debug_map_req)) { \ + val = ktime_to_us(ktime_get()); \ + if(print) { \ + SKHPB_DRIVER_I("SKHPB COMPL BUFFER %d - %d\n", \ + map_req->region, map_req->subregion); \ + SKHPB_DRIVER_I("start~issue = %lluus, issue~end = %lluus\n", \ + map_req->RSP_issue - map_req->RSP_start, \ + map_req->RSP_end - map_req->RSP_issue); \ + } \ + } \ + } while (0) + +#define SKHPB_RSP_TIME(val) \ + do { \ + if(unlikely(debug_map_req)) { \ + val = ktime_to_us(ktime_get()); \ + } \ + } while(0) + +#endif /* End of Header */ diff --git a/drivers/scsi/ufs/ufstw.c b/drivers/scsi/ufs/ufstw.c new file mode 100755 index 000000000000..b9853b118ae1 --- /dev/null +++ b/drivers/scsi/ufs/ufstw.c @@ -0,0 +1,900 @@ +/* + * Universal Flash Storage Turbo Write + * + * Copyright (C) 2017-2018 Samsung Electronics Co., Ltd. + * + * Authors: + * Yongmyung Lee + * Jinyoung Choi + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * See the COPYING file in the top-level directory or visit + * + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * This program is provided "AS IS" and "WITH ALL FAULTS" and + * without warranty of any kind. You are solely responsible for + * determining the appropriateness of using and distributing + * the program and assume all risks associated with your exercise + * of rights with respect to the program, including but not limited + * to infringement of third party rights, the risks and costs of + * program errors, damage to or loss of data, programs or equipment, + * and unavailability or interruption of operations. Under no + * circumstances will the contributor of this Program be liable for + * any damages of any kind arising from your use or distribution of + * this program. + * + * The Linux Foundation chooses to take subject only to the GPLv2 + * license terms, and distributes only under these terms. + */ + +#include "ufshcd.h" +#include "ufstw.h" + +static int ufstw_create_sysfs(struct ufsf_feature *ufsf, struct ufstw_lu *tw); + +inline int ufstw_get_state(struct ufsf_feature *ufsf) +{ + return atomic_read(&ufsf->tw_state); +} + +inline void ufstw_set_state(struct ufsf_feature *ufsf, int state) +{ + atomic_set(&ufsf->tw_state, state); +} + +static int ufstw_is_not_present(struct ufsf_feature *ufsf) +{ + enum UFSTW_STATE cur_state = ufstw_get_state(ufsf); + + if (cur_state != TW_PRESENT) { + INFO_MSG("tw_state != TW_PRESENT (%d)", cur_state); + return -ENODEV; + } + return 0; +} + +#define FLAG_IDN_NAME(idn) \ + (idn == QUERY_FLAG_IDN_TW_EN ? "tw_enable" : \ + idn == QUERY_FLAG_IDN_TW_BUF_FLUSH_EN ? "flush_enable" : \ + idn == QUERY_FLAG_IDN_TW_FLUSH_DURING_HIBERN ? \ + "flush_hibern" : "unknown") + +#define ATTR_IDN_NAME(idn) \ + (idn == QUERY_ATTR_IDN_TW_FLUSH_STATUS ? "flush_status" : \ + idn == QUERY_ATTR_IDN_TW_AVAIL_BUF_SIZE ? "avail_buffer_size" :\ + idn == QUERY_ATTR_IDN_TW_BUF_LIFETIME_EST ? "lifetime_est" : \ + idn == QUERY_ATTR_IDN_TW_CURR_BUF_SIZE ? "current_buf_size" : \ + "unknown") + +static int ufstw_read_lu_attr(struct ufstw_lu *tw, u8 idn, u32 *attr_val) +{ + struct ufs_hba *hba = tw->ufsf->hba; + int err = 0, lun; + u32 val; + + lun = (tw->lun == TW_LU_SHARED) ? 0 : tw->lun; + err = ufsf_query_attr_retry(hba, UPIU_QUERY_OPCODE_READ_ATTR, idn, + (u8)lun, &val); + if (err) { + ERR_MSG("read attr [0x%.2X](%s) failed. (%d)", idn, + ATTR_IDN_NAME(idn), err); + goto out; + } + + *attr_val = val; + + INFO_MSG("read attr LUN(%d) [0x%.2X](%s) success (%u)", + lun, idn, ATTR_IDN_NAME(idn), *attr_val); +out: + return err; +} + +static int ufstw_set_lu_flag(struct ufstw_lu *tw, u8 idn, bool *flag_res) +{ + struct ufs_hba *hba = tw->ufsf->hba; + int err = 0, lun; + + lun = (tw->lun == TW_LU_SHARED) ? 0 : tw->lun; + err = ufsf_query_flag_retry(hba, UPIU_QUERY_OPCODE_SET_FLAG, idn, + (u8)lun, NULL); + if (err) { + ERR_MSG("set flag [0x%.2X](%s) failed. (%d)", idn, + FLAG_IDN_NAME(idn), err); + goto out; + } + + *flag_res = true; + + INFO_MSG("set flag LUN(%d) [0x%.2X](%s) success. (%u)", + lun, idn, FLAG_IDN_NAME(idn), *flag_res); +out: + return err; +} + +static int ufstw_clear_lu_flag(struct ufstw_lu *tw, u8 idn, bool *flag_res) +{ + struct ufs_hba *hba = tw->ufsf->hba; + int err = 0, lun; + + lun = (tw->lun == TW_LU_SHARED) ? 0 : tw->lun; + err = ufsf_query_flag_retry(hba, UPIU_QUERY_OPCODE_CLEAR_FLAG, idn, + (u8)lun, NULL); + if (err) { + ERR_MSG("clear flag [0x%.2X](%s) failed. (%d)", idn, + FLAG_IDN_NAME(idn), err); + goto out; + } + + *flag_res = false; + + INFO_MSG("clear flag LUN(%d) [0x%.2X](%s) success. (%u)", + lun, idn, FLAG_IDN_NAME(idn), *flag_res); +out: + return err; +} + +static int ufstw_read_lu_flag(struct ufstw_lu *tw, u8 idn, bool *flag_res) +{ + struct ufs_hba *hba = tw->ufsf->hba; + int err = 0, lun; + bool val; + + lun = (tw->lun == TW_LU_SHARED) ? 0 : tw->lun; + err = ufsf_query_flag_retry(hba, UPIU_QUERY_OPCODE_READ_FLAG, idn, + (u8)lun, &val); + if (err) { + ERR_MSG("read flag [0x%.2X](%s) failed. (%d)", idn, + FLAG_IDN_NAME(idn), err); + goto out; + } + + *flag_res = val; + + INFO_MSG("read flag LUN(%d) [0x%.2X](%s) success. (%u)", + lun, idn, FLAG_IDN_NAME(idn), *flag_res); +out: + return err; +} + +static inline bool ufstw_is_write_lrbp(struct ufshcd_lrb *lrbp) +{ + if (lrbp->cmd->cmnd[0] == WRITE_10 || lrbp->cmd->cmnd[0] == WRITE_16) + return true; + + return false; +} + +static void ufstw_switch_disable_state(struct ufstw_lu *tw) +{ + int err = 0; + + WARN_MSG("dTurboWriteBUfferLifeTImeEst (0x%.2X)", tw->lifetime_est); + WARN_MSG("tw-mode will change to disable-mode"); + + mutex_lock(&tw->sysfs_lock); + ufstw_set_state(tw->ufsf, TW_FAILED); + mutex_unlock(&tw->sysfs_lock); + + if (tw->tw_enable) { + pm_runtime_get_sync(tw->ufsf->hba->dev); + err = ufstw_clear_lu_flag(tw, QUERY_FLAG_IDN_TW_EN, + &tw->tw_enable); + pm_runtime_put_sync(tw->ufsf->hba->dev); + if (err) + WARN_MSG("tw_enable flag clear failed"); + } +} + +static int ufstw_check_lifetime_not_guarantee(struct ufstw_lu *tw) +{ + bool disable_flag = false; + + if (tw->lifetime_est & MASK_UFSTW_LIFETIME_NOT_GUARANTEE) { + if (tw->lun == TW_LU_SHARED) + WARN_MSG("lun-shared lifetime_est[31] (1)"); + else + WARN_MSG("lun %d lifetime_est[31] (1)", + tw->lun); + + WARN_MSG("Device not guarantee the lifetime of TW Buffer"); +#if defined(CONFIG_UFSTW_IGNORE_GUARANTEE_BIT) + WARN_MSG("but we will ignore them for PoC"); +#else + disable_flag = true; +#endif + } + + if (disable_flag || + (tw->lifetime_est & ~MASK_UFSTW_LIFETIME_NOT_GUARANTEE) >= + UFSTW_MAX_LIFETIME_VALUE) { + ufstw_switch_disable_state(tw); + return -ENODEV; + } + + return 0; +} + +static void ufstw_lifetime_work_fn(struct work_struct *work) +{ + struct ufstw_lu *tw; + int ret; + + tw = container_of(work, struct ufstw_lu, tw_lifetime_work); + + if (ufstw_is_not_present(tw->ufsf)) + return; + + pm_runtime_get_sync(tw->ufsf->hba->dev); + ret = ufstw_read_lu_attr(tw, QUERY_ATTR_IDN_TW_BUF_LIFETIME_EST, + &tw->lifetime_est); + pm_runtime_put_sync(tw->ufsf->hba->dev); + if (ret) + return; + + ufstw_check_lifetime_not_guarantee(tw); +} + +void ufstw_prep_fn(struct ufsf_feature *ufsf, struct ufshcd_lrb *lrbp) +{ + struct ufstw_lu *tw; + + if (!lrbp || !ufsf_is_valid_lun(lrbp->lun)) + return; + + if (!ufstw_is_write_lrbp(lrbp)) + return; + + tw = ufsf->tw_lup[lrbp->lun]; + if (!tw) + return; + + if (!tw->tw_enable) + return; + + spin_lock_bh(&tw->lifetime_lock); + tw->stat_write_sec += blk_rq_sectors(lrbp->cmd->request); + + if (tw->stat_write_sec > UFSTW_LIFETIME_SECT) { + tw->stat_write_sec = 0; + spin_unlock_bh(&tw->lifetime_lock); + schedule_work(&tw->tw_lifetime_work); + return; + } + spin_unlock_bh(&tw->lifetime_lock); + + TMSG(tw->ufsf, lrbp->lun, "%s:%d tw_lifetime_work %u", + __func__, __LINE__, tw->stat_write_sec); +} + +static inline void ufstw_init_lu_jobs(struct ufstw_lu *tw) +{ + INIT_WORK(&tw->tw_lifetime_work, ufstw_lifetime_work_fn); +} + +static inline void ufstw_cancel_lu_jobs(struct ufstw_lu *tw) +{ + int ret; + + ret = cancel_work_sync(&tw->tw_lifetime_work); + INFO_MSG("cancel_work_sync(tw_lifetime_work) ufstw_lu[%d] (%d)", + tw->lun, ret); +} + +static inline int ufstw_version_check(struct ufstw_dev_info *tw_dev_info) +{ + INFO_MSG("Support TW Spec : Driver = %.4X, Device = %.4X", + UFSTW_VER, tw_dev_info->tw_ver); + + INFO_MSG("TW Driver Version : %.6X%s", UFSTW_DD_VER, + UFSTW_DD_VER_POST); + + if (tw_dev_info->tw_ver != UFSTW_VER) + return -ENODEV; + + return 0; +} + +void ufstw_get_dev_info(struct ufsf_feature *ufsf, u8 *desc_buf) +{ + struct ufstw_dev_info *tw_dev_info = &ufsf->tw_dev_info; + + if (LI_EN_32(&desc_buf[DEVICE_DESC_PARAM_EX_FEAT_SUP]) & + UFS_FEATURE_SUPPORT_TW_BIT) { + INFO_MSG("bUFSExFeaturesSupport: TW is set"); + } else { + ERR_MSG("bUFSExFeaturesSupport: TW not support"); + ufstw_set_state(ufsf, TW_FAILED); + return; + } + tw_dev_info->tw_buf_no_reduct = + desc_buf[DEVICE_DESC_PARAM_TW_RETURN_TO_USER]; + tw_dev_info->tw_buf_type = desc_buf[DEVICE_DESC_PARAM_TW_BUF_TYPE]; + tw_dev_info->tw_shared_buf_alloc_units = + LI_EN_32(&desc_buf[DEVICE_DESC_PARAM_TW_SHARED_BUF_ALLOC_UNITS]); + tw_dev_info->tw_ver = LI_EN_16(&desc_buf[DEVICE_DESC_PARAM_TW_VER]); + + if (ufstw_version_check(tw_dev_info)) { + ERR_MSG("TW Spec Version mismatch. TW disabled"); + ufstw_set_state(ufsf, TW_FAILED); + return; + } + + INFO_MSG("tw_dev [53] bTurboWriteBufferNoUserSpaceReductionEn (%u)", + tw_dev_info->tw_buf_no_reduct); + INFO_MSG("tw_dev [54] bTurboWriteBufferType (%u)", + tw_dev_info->tw_buf_type); + INFO_MSG("tw_dev [55] dNumSharedTUrboWriteBufferAllocUnits (%u)", + tw_dev_info->tw_shared_buf_alloc_units); + + if (tw_dev_info->tw_buf_type == TW_BUF_TYPE_SHARED && + tw_dev_info->tw_shared_buf_alloc_units == 0) { + ERR_MSG("TW use shared buffer. But alloc unit is (0)"); + ufstw_set_state(ufsf, TW_FAILED); + return; + } +} + +void ufstw_get_geo_info(struct ufsf_feature *ufsf, u8 *geo_buf) +{ + struct ufstw_dev_info *tw_dev_info = &ufsf->tw_dev_info; + + tw_dev_info->tw_number_lu = geo_buf[GEOMETRY_DESC_TW_NUMBER_LU]; + if (tw_dev_info->tw_number_lu == 0) { + ERR_MSG("Turbo Write is not supported"); + ufstw_set_state(ufsf, TW_FAILED); + return; + } + + INFO_MSG("tw_geo [4F:52] dTurboWriteBufferMaxNAllocUnits (%u)", + LI_EN_32(&geo_buf[GEOMETRY_DESC_TW_MAX_SIZE])); + INFO_MSG("tw_geo [53] bDeviceMaxTurboWriteLUs (%u)", + tw_dev_info->tw_number_lu); + INFO_MSG("tw_geo [54] bTurboWriteBufferCapAdjFac (%u)", + geo_buf[GEOMETRY_DESC_TW_CAP_ADJ_FAC]); + INFO_MSG("tw_geo [55] bSupportedTWBufferUserSpaceReductionTypes (%u)", + geo_buf[GEOMETRY_DESC_TW_SUPPORT_USER_REDUCTION_TYPES]); + INFO_MSG("tw_geo [56] bSupportedTurboWriteBufferTypes (%u)", + geo_buf[GEOMETRY_DESC_TW_SUPPORT_BUF_TYPE]); +} + +static void ufstw_alloc_shared_lu(struct ufsf_feature *ufsf) +{ + struct ufstw_lu *tw; + + tw = kzalloc(sizeof(struct ufstw_lu), GFP_KERNEL); + if (!tw) { + ERR_MSG("ufstw_lu[shared] memory alloc failed"); + return; + } + + tw->lun = TW_LU_SHARED; + tw->ufsf = ufsf; + ufsf->tw_lup[0] = tw; + + INFO_MSG("ufstw_lu[shared] is TurboWrite-Enabled"); +} + +static void ufstw_get_lu_info(struct ufsf_feature *ufsf, int lun, u8 *lu_buf) +{ + struct ufsf_lu_desc lu_desc; + struct ufstw_lu *tw; + + lu_desc.tw_lu_buf_size = + LI_EN_32(&lu_buf[UNIT_DESC_TW_LU_WRITE_BUFFER_ALLOC_UNIT]); + + ufsf->tw_lup[lun] = NULL; + + if (lu_desc.tw_lu_buf_size) { + ufsf->tw_lup[lun] = + kzalloc(sizeof(struct ufstw_lu), GFP_KERNEL); + if (!ufsf->tw_lup[lun]) { + ERR_MSG("ufstw_lu[%d] memory alloc faild", lun); + return; + } + + tw = ufsf->tw_lup[lun]; + tw->ufsf = ufsf; + tw->lun = lun; + INFO_MSG("ufstw_lu[%d] [29:2C] dLUNumTWBufferAllocUnits (%u)", + lun, lu_desc.tw_lu_buf_size); + INFO_MSG("ufstw_lu[%d] is TurboWrite-Enabled.", lun); + } else { + INFO_MSG("ufstw_lu[%d] [29:2C] dLUNumTWBufferAllocUnits (%u)", + lun, lu_desc.tw_lu_buf_size); + INFO_MSG("ufstw_lu[%d] is TurboWrite-disabled", lun); + } +} + +inline void ufstw_alloc_lu(struct ufsf_feature *ufsf, + int lun, u8 *lu_buf) +{ + if (ufsf->tw_dev_info.tw_buf_type == TW_BUF_TYPE_SHARED && + !ufsf->tw_lup[0]) + ufstw_alloc_shared_lu(ufsf); + else if (ufsf->tw_dev_info.tw_buf_type == TW_BUF_TYPE_LU) + ufstw_get_lu_info(ufsf, lun, lu_buf); +} + +static inline void ufstw_print_lu_flag_attr(struct ufstw_lu *tw) +{ + char lun_str[20] = { 0 }; + + if (tw->lun == TW_LU_SHARED) + snprintf(lun_str, 7, "shared"); + else + snprintf(lun_str, 2, "%d", tw->lun); + + INFO_MSG("tw_flag ufstw_lu[%s] IDN (0x%.2X) tw_enable (%d)", + lun_str, QUERY_FLAG_IDN_TW_EN, tw->tw_enable); + INFO_MSG("tw_flag ufstw_lu[%s] IDN (0x%.2X) flush_enable (%d)", + lun_str, QUERY_FLAG_IDN_TW_BUF_FLUSH_EN, + tw->flush_enable); + INFO_MSG("tw_flag ufstw_lu[%s] IDN (0x%.2X) flush_hibern (%d)", + lun_str, QUERY_FLAG_IDN_TW_FLUSH_DURING_HIBERN, + tw->flush_during_hibern_enter); + + INFO_MSG("tw_attr ufstw_lu[%s] IDN (0x%.2X) flush_status (%u)", + lun_str, QUERY_ATTR_IDN_TW_FLUSH_STATUS, tw->flush_status); + INFO_MSG("tw_attr ufstw_lu[%s] IDN (0x%.2X) buffer_size (%u)", + lun_str, QUERY_ATTR_IDN_TW_AVAIL_BUF_SIZE, + tw->available_buffer_size); + INFO_MSG("tw_attr ufstw_lu[%s] IDN (0x%.2X) buffer_lifetime (0x%.2X)", + lun_str, QUERY_ATTR_IDN_TW_BUF_LIFETIME_EST, + tw->lifetime_est); +} + +static inline void ufstw_lu_update(struct ufstw_lu *tw) +{ + /* Flag */ + pm_runtime_get_sync(tw->ufsf->hba->dev); + if (ufstw_read_lu_flag(tw, QUERY_FLAG_IDN_TW_EN, &tw->tw_enable)) + goto error_put; + + if (ufstw_read_lu_flag(tw, QUERY_FLAG_IDN_TW_BUF_FLUSH_EN, + &tw->flush_enable)) + goto error_put; + + if (ufstw_read_lu_flag(tw, QUERY_FLAG_IDN_TW_FLUSH_DURING_HIBERN, + &tw->flush_during_hibern_enter)) + goto error_put; + + /* Attribute */ + if (ufstw_read_lu_attr(tw, QUERY_ATTR_IDN_TW_FLUSH_STATUS, + &tw->flush_status)) + goto error_put; + + if (ufstw_read_lu_attr(tw, QUERY_ATTR_IDN_TW_AVAIL_BUF_SIZE, + &tw->available_buffer_size)) + goto error_put; + + ufstw_read_lu_attr(tw, QUERY_ATTR_IDN_TW_BUF_LIFETIME_EST, + &tw->lifetime_est); +error_put: + pm_runtime_put_sync(tw->ufsf->hba->dev); +} + +static int ufstw_lu_init(struct ufsf_feature *ufsf, int lun) +{ + struct ufstw_lu *tw; + int ret = 0; + + if (lun == TW_LU_SHARED) + tw = ufsf->tw_lup[0]; + else + tw = ufsf->tw_lup[lun]; + + tw->ufsf = ufsf; + spin_lock_init(&tw->lifetime_lock); + + ufstw_lu_update(tw); + + ret = ufstw_check_lifetime_not_guarantee(tw); + if (ret) + goto err_out; + + ufstw_print_lu_flag_attr(tw); + + tw->stat_write_sec = 0; + + ufstw_init_lu_jobs(tw); + +#if defined(CONFIG_UFSTW_BOOT_ENABLED) + pm_runtime_get_sync(ufsf->hba->dev); + ufstw_set_lu_flag(tw, QUERY_FLAG_IDN_TW_EN, &tw->tw_enable); + ufstw_set_lu_flag(tw, QUERY_FLAG_IDN_TW_FLUSH_DURING_HIBERN, + &tw->flush_during_hibern_enter); + pm_runtime_put_sync(ufsf->hba->dev); +#endif + ret = ufstw_create_sysfs(ufsf, tw); + if (ret) + ERR_MSG("create sysfs failed"); +err_out: + return ret; +} + +void ufstw_init(struct ufsf_feature *ufsf) +{ + int lun, ret = 0; + unsigned int tw_enabled_lun = 0; + + INFO_MSG("init start.. tw_state (%d)", ufstw_get_state(ufsf)); + + if (ufsf->tw_dev_info.tw_buf_type == TW_BUF_TYPE_SHARED) { + if (!ufsf->tw_lup[0]) { + ERR_MSG("tw_lup memory allocation failed"); + goto out; + } + BUG_ON(ufsf->tw_lup[0]->lun != TW_LU_SHARED); + + ret = ufstw_lu_init(ufsf, TW_LU_SHARED); + if (ret) + goto out_free_mem; + + INFO_MSG("ufstw_lu[shared] working"); + tw_enabled_lun++; + } else { + seq_scan_lu(lun) { + if (!ufsf->tw_lup[lun]) + continue; + + ret = ufstw_lu_init(ufsf, lun); + if (ret) + goto out_free_mem; + + INFO_MSG("ufstw_lu[%d] working", lun); + tw_enabled_lun++; + } + + if (tw_enabled_lun > ufsf->tw_dev_info.tw_number_lu) { + ERR_MSG("lu count mismatched"); + goto out_free_mem; + } + } + + if (tw_enabled_lun == 0) { + ERR_MSG("tw_enabled_lun count zero"); + goto out_free_mem; + } + + ufstw_set_state(ufsf, TW_PRESENT); + return; +out_free_mem: + seq_scan_lu(lun) { + kfree(ufsf->tw_lup[lun]); + ufsf->tw_lup[lun] = NULL; + } +out: + ERR_MSG("Turbo write initialization failed"); + + ufstw_set_state(ufsf, TW_FAILED); +} + +static inline void ufstw_remove_sysfs(struct ufstw_lu *tw) +{ + int ret; + + ret = kobject_uevent(&tw->kobj, KOBJ_REMOVE); + INFO_MSG("kobject removed (%d)", ret); + kobject_del(&tw->kobj); +} + +void ufstw_remove(struct ufsf_feature *ufsf) +{ + struct ufstw_lu *tw; + int lun; + + dump_stack(); + INFO_MSG("start release"); + + ufstw_set_state(ufsf, TW_FAILED); + + if (ufsf->tw_dev_info.tw_buf_type == TW_BUF_TYPE_SHARED) { + tw = ufsf->tw_lup[0]; + INFO_MSG("ufstw_lu[shared] %p", tw); + ufsf->tw_lup[0] = NULL; + ufstw_cancel_lu_jobs(tw); + ufstw_remove_sysfs(tw); + kfree(tw); + } else { + seq_scan_lu(lun) { + tw = ufsf->tw_lup[lun]; + INFO_MSG("ufstw_lu[%d] %p", lun, tw); + + if (!tw) + continue; + ufsf->tw_lup[lun] = NULL; + ufstw_cancel_lu_jobs(tw); + ufstw_remove_sysfs(tw); + kfree(tw); + } + } + + INFO_MSG("end release"); +} + +static void ufstw_reset_query_handling(struct ufstw_lu *tw) +{ + int ret; + + pm_runtime_get_sync(tw->ufsf->hba->dev); + if (tw->tw_enable) { + ret = ufstw_set_lu_flag(tw, QUERY_FLAG_IDN_TW_EN, + &tw->tw_enable); + if (ret) + tw->tw_enable = false; + } + + if (tw->flush_enable) { + ret = ufstw_set_lu_flag(tw, QUERY_FLAG_IDN_TW_BUF_FLUSH_EN, + &tw->flush_enable); + if (ret) + tw->flush_enable = false; + } + + if (tw->flush_during_hibern_enter) { + ret = ufstw_set_lu_flag(tw, + QUERY_FLAG_IDN_TW_FLUSH_DURING_HIBERN, + &tw->flush_during_hibern_enter); + if (ret) + tw->flush_during_hibern_enter = false; + } + + pm_runtime_put_sync(tw->ufsf->hba->dev); +} + +void ufstw_reset_host(struct ufsf_feature *ufsf) +{ + struct ufstw_lu *tw; + int lun; + + if (ufstw_is_not_present(ufsf)) + return; + + ufstw_set_state(ufsf, TW_RESET); + if (ufsf->tw_dev_info.tw_buf_type == TW_BUF_TYPE_SHARED) { + tw = ufsf->tw_lup[0]; + + INFO_MSG("ufstw_lu[shared] cancel jobs"); + ufstw_cancel_lu_jobs(tw); + } else { + seq_scan_lu(lun) { + tw = ufsf->tw_lup[lun]; + if (!tw) + continue; + + INFO_MSG("ufstw_lu[%d] cancel jobs", lun); + ufstw_cancel_lu_jobs(tw); + } + } +} + +void ufstw_reset(struct ufsf_feature *ufsf, bool resume) +{ + struct ufstw_lu *tw; + int lun; + + INFO_MSG("ufstw reset start. reason: %s", + resume ? "resume" : "reset"); + if (ufstw_get_state(ufsf) != TW_RESET) { + ERR_MSG("tw_state error (%d)", ufstw_get_state(ufsf)); + return; + } + + ufstw_set_state(ufsf, TW_PRESENT); + + if (ufsf->tw_dev_info.tw_buf_type == TW_BUF_TYPE_SHARED) { + tw = ufsf->tw_lup[0]; + + INFO_MSG("ufstw_lu[shared] reset"); + ufstw_reset_query_handling(tw); + } else { + seq_scan_lu(lun) { + tw = ufsf->tw_lup[lun]; + if (!tw) + continue; + + INFO_MSG("ufstw_lu[%d] reset", lun); + ufstw_reset_query_handling(tw); + } + } + + INFO_MSG("ufstw reset finish"); +} + +#define ufstw_sysfs_attr_show_func(_query, _name, _IDN, hex) \ +static ssize_t ufstw_sysfs_show_##_name(struct ufstw_lu *tw, char *buf) \ +{ \ + int ret; \ + \ + if (ufstw_is_not_present(tw->ufsf)) \ + return -ENODEV; \ + \ + pm_runtime_get_sync(tw->ufsf->hba->dev); \ + ret = ufstw_read_lu_##_query(tw, _IDN, &tw->_name); \ + pm_runtime_put_sync(tw->ufsf->hba->dev); \ + if (ret) \ + return -ENODEV; \ + \ + INFO_MSG("read "#_query" "#_name" %u (0x%X)", \ + tw->_name, tw->_name); \ + if (hex) \ + return snprintf(buf, PAGE_SIZE, "0x%.2X\n", tw->_name); \ + return snprintf(buf, PAGE_SIZE, "%u\n", tw->_name); \ +} + +#define ufstw_sysfs_attr_store_func(_name, _IDN) \ +static ssize_t ufstw_sysfs_store_##_name(struct ufstw_lu *tw, \ + const char *buf, \ + size_t count) \ +{ \ + unsigned long val; \ + ssize_t ret = count; \ + \ + if (kstrtoul(buf, 0, &val)) \ + return -EINVAL; \ + \ + if (!(val == 0 || val == 1)) \ + return -EINVAL; \ + \ + INFO_MSG("val %lu", val); \ + if (ufstw_is_not_present(tw->ufsf)) \ + return -ENODEV; \ + \ + pm_runtime_get_sync(tw->ufsf->hba->dev); \ + if (val) { \ + if (ufstw_set_lu_flag(tw, _IDN, &tw->_name)) \ + ret = -ENODEV; \ + } else { \ + if (ufstw_clear_lu_flag(tw, _IDN, &tw->_name)) \ + ret = -ENODEV; \ + } \ + pm_runtime_put_sync(tw->ufsf->hba->dev); \ + \ + INFO_MSG(#_name " query success"); \ + return ret; \ +} + +ufstw_sysfs_attr_show_func(flag, tw_enable, QUERY_FLAG_IDN_TW_EN, 0); +ufstw_sysfs_attr_store_func(tw_enable, QUERY_FLAG_IDN_TW_EN); +ufstw_sysfs_attr_show_func(flag, flush_enable, + QUERY_FLAG_IDN_TW_BUF_FLUSH_EN, 0); +ufstw_sysfs_attr_store_func(flush_enable, QUERY_FLAG_IDN_TW_BUF_FLUSH_EN); +ufstw_sysfs_attr_show_func(flag, flush_during_hibern_enter, + QUERY_FLAG_IDN_TW_FLUSH_DURING_HIBERN, 0); +ufstw_sysfs_attr_store_func(flush_during_hibern_enter, + QUERY_FLAG_IDN_TW_FLUSH_DURING_HIBERN); + +ufstw_sysfs_attr_show_func(attr, flush_status, + QUERY_ATTR_IDN_TW_FLUSH_STATUS, 0); +ufstw_sysfs_attr_show_func(attr, available_buffer_size, + QUERY_ATTR_IDN_TW_AVAIL_BUF_SIZE, 0); +ufstw_sysfs_attr_show_func(attr, lifetime_est, + QUERY_ATTR_IDN_TW_BUF_LIFETIME_EST, 1); +ufstw_sysfs_attr_show_func(attr, curr_buffer_size, + QUERY_ATTR_IDN_TW_CURR_BUF_SIZE, 0); + +#define ufstw_sysfs_attr_ro(_name) __ATTR(_name, 0444,\ + ufstw_sysfs_show_##_name, NULL) +#define ufstw_sysfs_attr_rw(_name) __ATTR(_name, 0644,\ + ufstw_sysfs_show_##_name, \ + ufstw_sysfs_store_##_name) + +static struct ufstw_sysfs_entry ufstw_sysfs_entries[] = { + /* Flag */ + ufstw_sysfs_attr_rw(tw_enable), + ufstw_sysfs_attr_rw(flush_enable), + ufstw_sysfs_attr_rw(flush_during_hibern_enter), + /* Attribute */ + ufstw_sysfs_attr_ro(flush_status), + ufstw_sysfs_attr_ro(available_buffer_size), + ufstw_sysfs_attr_ro(lifetime_est), + ufstw_sysfs_attr_ro(curr_buffer_size), + __ATTR_NULL +}; + +static ssize_t ufstw_attr_show(struct kobject *kobj, struct attribute *attr, + char *page) +{ + struct ufstw_sysfs_entry *entry; + struct ufstw_lu *tw; + ssize_t error; + + entry = container_of(attr, struct ufstw_sysfs_entry, attr); + if (!entry->show) + return -EIO; + + tw = container_of(kobj, struct ufstw_lu, kobj); + + mutex_lock(&tw->sysfs_lock); + error = entry->show(tw, page); + mutex_unlock(&tw->sysfs_lock); + return error; +} + +static ssize_t ufstw_attr_store(struct kobject *kobj, struct attribute *attr, + const char *page, size_t length) +{ + struct ufstw_sysfs_entry *entry; + struct ufstw_lu *tw; + ssize_t error; + + entry = container_of(attr, struct ufstw_sysfs_entry, attr); + if (!entry->store) + return -EIO; + + tw = container_of(kobj, struct ufstw_lu, kobj); + + mutex_lock(&tw->sysfs_lock); + error = entry->store(tw, page, length); + mutex_unlock(&tw->sysfs_lock); + return error; +} + +static const struct sysfs_ops ufstw_sysfs_ops = { + .show = ufstw_attr_show, + .store = ufstw_attr_store, +}; + +static struct kobj_type ufstw_ktype = { + .sysfs_ops = &ufstw_sysfs_ops, + .release = NULL, +}; + +static int ufstw_create_sysfs(struct ufsf_feature *ufsf, struct ufstw_lu *tw) +{ + struct device *dev = ufsf->hba->dev; + struct ufstw_sysfs_entry *entry; + int err; + char lun_str[20] = { 0 }; + + tw->sysfs_entries = ufstw_sysfs_entries; + + kobject_init(&tw->kobj, &ufstw_ktype); + mutex_init(&tw->sysfs_lock); + + if (tw->lun == TW_LU_SHARED) { + snprintf(lun_str, 6, "ufstw"); + INFO_MSG("ufstw creates sysfs ufstw-shared"); + } else { + snprintf(lun_str, 10, "ufstw_lu%d", tw->lun); + INFO_MSG("ufstw creates sysfs ufstw_lu%d", tw->lun); + } + + err = kobject_add(&tw->kobj, kobject_get(&dev->kobj), lun_str); + if (!err) { + for (entry = tw->sysfs_entries; entry->attr.name != NULL; + entry++) { + if (tw->lun == TW_LU_SHARED) + INFO_MSG("ufstw-shared sysfs attr creates: %s", + entry->attr.name); + else + INFO_MSG("ufstw_lu(%d) sysfs attr creates: %s", + tw->lun, entry->attr.name); + + err = sysfs_create_file(&tw->kobj, &entry->attr); + if (err) { + ERR_MSG("create entry(%s) failed", + entry->attr.name); + goto kobj_del; + } + } + kobject_uevent(&tw->kobj, KOBJ_ADD); + } else { + ERR_MSG("kobject_add failed"); + } + + return err; +kobj_del: + err = kobject_uevent(&tw->kobj, KOBJ_REMOVE); + INFO_MSG("kobject removed (%d)", err); + kobject_del(&tw->kobj); + return -EINVAL; +} diff --git a/drivers/scsi/ufs/ufstw.h b/drivers/scsi/ufs/ufstw.h new file mode 100755 index 000000000000..2d3bfc8c7d4a --- /dev/null +++ b/drivers/scsi/ufs/ufstw.h @@ -0,0 +1,126 @@ +/* + * Universal Flash Storage tw Write + * + * Copyright (C) 2017-2018 Samsung Electronics Co., Ltd. + * + * Authors: + * Yongmyung Lee + * Jinyoung Choi + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * See the COPYING file in the top-level directory or visit + * + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * This program is provided "AS IS" and "WITH ALL FAULTS" and + * without warranty of any kind. You are solely responsible for + * determining the appropriateness of using and distributing + * the program and assume all risks associated with your exercise + * of rights with respect to the program, including but not limited + * to infringement of third party rights, the risks and costs of + * program errors, damage to or loss of data, programs or equipment, + * and unavailability or interruption of operations. Under no + * circumstances will the contributor of this Program be liable for + * any damages of any kind arising from your use or distribution of + * this program. + * + * The Linux Foundation chooses to take subject only to the GPLv2 + * license terms, and distributes only under these terms. + */ + +#ifndef _UFSTW_H_ +#define _UFSTW_H_ + +#include +#include +#include +#include +#include + +#include "../../../block/blk.h" + +#define UFSTW_VER 0x0110 +#define UFSTW_DD_VER 0x010300 +#define UFSTW_DD_VER_POST UFSFEATURE_DD_VER_POST + +#define UFSTW_LIFETIME_SECT 2097152 /* 1GB */ +#define UFSTW_MAX_LIFETIME_VALUE 0x0B +#define MASK_UFSTW_LIFETIME_NOT_GUARANTEE 0x80 +#define UFS_FEATURE_SUPPORT_TW_BIT 0x100 + +#define TW_LU_SHARED -1 + +enum UFSTW_STATE { + TW_NEED_INIT = 0, + TW_PRESENT = 1, + TW_RESET = -3, + TW_FAILED = -4 +}; + +enum { + TW_BUF_TYPE_LU = 0, + TW_BUF_TYPE_SHARED, +}; + +struct ufstw_dev_info { + /* from Device Descriptor */ + u16 tw_ver; + u32 tw_shared_buf_alloc_units; + u8 tw_buf_no_reduct; + u8 tw_buf_type; + + /* from Geometry Descriptor */ + u8 tw_number_lu; +}; + +struct ufstw_lu { + struct ufsf_feature *ufsf; + + int lun; + + bool tw_enable; + bool flush_enable; + bool flush_during_hibern_enter; + + unsigned int flush_status; + unsigned int available_buffer_size; + unsigned int curr_buffer_size; + + unsigned int lifetime_est; + spinlock_t lifetime_lock; + u32 stat_write_sec; + struct work_struct tw_lifetime_work; + + /* for sysfs */ + struct kobject kobj; + struct mutex sysfs_lock; + struct ufstw_sysfs_entry *sysfs_entries; +}; + +struct ufstw_sysfs_entry { + struct attribute attr; + ssize_t (*show)(struct ufstw_lu *tw, char *buf); + ssize_t (*store)(struct ufstw_lu *tw, const char *buf, size_t count); +}; + +struct ufshcd_lrb; + +int ufstw_get_state(struct ufsf_feature *ufsf); +void ufstw_set_state(struct ufsf_feature *ufsf, int state); +void ufstw_get_dev_info(struct ufsf_feature *ufsf, u8 *desc_buf); +void ufstw_get_geo_info(struct ufsf_feature *ufsf, u8 *geo_buf); +void ufstw_alloc_lu(struct ufsf_feature *ufsf, int lun, u8 *lu_buf); + +void ufstw_prep_fn(struct ufsf_feature *ufsf, struct ufshcd_lrb *lrbp); +void ufstw_init(struct ufsf_feature *ufsf); +void ufstw_remove(struct ufsf_feature *ufsf); +void ufstw_reset_host(struct ufsf_feature *ufsf); +void ufstw_reset(struct ufsf_feature *ufsf, bool resume); +#endif /* End of Header */ diff --git a/include/scsi/scsi_cmnd.h b/include/scsi/scsi_cmnd.h index dcb0763fb219..b53ecf55ceb9 100644 --- a/include/scsi/scsi_cmnd.h +++ b/include/scsi/scsi_cmnd.h @@ -147,6 +147,9 @@ struct scsi_cmnd { ANDROID_KABI_RESERVE(2); ANDROID_KABI_RESERVE(3); ANDROID_KABI_RESERVE(4); +#if defined(CONFIG_UFSFEATURE) && defined(CONFIG_UFSHPB) + u8 requeue_cnt; +#endif }; /*