From 7cbd556e90be48b88fdb06dd6053180e36b3f9a7 Mon Sep 17 00:00:00 2001 From: Sarthak Garg Date: Tue, 18 Aug 2020 13:21:32 +0530 Subject: [PATCH] mtd: msm_qpic_nand: Add snapshot of QPIC nand driver Add snapshot of QPIC nand driver and it's supported files. This is a snapshot of the QPIC nand driver as of kernel msm-4.14 'commit 5cc17e2cfb19 ("mtd: msm_qpic_nand: Disable Pagescope on sdxprairie")'. Change-Id: I994f4432ddd417743a1d7bf27b57755adc90b702 Signed-off-by: Sarthak Garg --- drivers/mtd/devices/Kconfig | 13 + drivers/mtd/devices/Makefile | 1 + drivers/mtd/devices/msm_qpic_nand.c | 4432 +++++++++++++++++++++++++++ drivers/mtd/devices/msm_qpic_nand.h | 430 +++ drivers/mtd/nand/raw/nand_ids.c | 21 + 5 files changed, 4897 insertions(+) create mode 100644 drivers/mtd/devices/msm_qpic_nand.c create mode 100644 drivers/mtd/devices/msm_qpic_nand.h diff --git a/drivers/mtd/devices/Kconfig b/drivers/mtd/devices/Kconfig index f96287c4b789..1694127a532c 100644 --- a/drivers/mtd/devices/Kconfig +++ b/drivers/mtd/devices/Kconfig @@ -51,6 +51,19 @@ config MTD_MS02NV say M here and read . The module will be called ms02-nv. +config MTD_MSM_QPIC_NAND + tristate "MSM QPIC NAND Device Support" + depends on MTD && (ARCH_QCOM || ARCH_MSM) && !MTD_MSM_NAND + select CRC16 + select BITREVERSE + select MTD_NAND_IDS + default n + help + Support for NAND controller in Qualcomm Technologies, Inc. + Parallel Interface controller (QPIC). This new controller + supports BAM mode and BCH error correction mechanism. Based on the + device capabilities either 4 bit or 8 bit BCH ECC will be used. + config MTD_DATAFLASH tristate "Support for AT45xxx DataFlash" depends on SPI_MASTER diff --git a/drivers/mtd/devices/Makefile b/drivers/mtd/devices/Makefile index 991c8d12c016..886944cd929c 100644 --- a/drivers/mtd/devices/Makefile +++ b/drivers/mtd/devices/Makefile @@ -11,6 +11,7 @@ obj-$(CONFIG_MTD_MS02NV) += ms02-nv.o obj-$(CONFIG_MTD_MTDRAM) += mtdram.o obj-$(CONFIG_MTD_LART) += lart.o obj-$(CONFIG_MTD_BLOCK2MTD) += block2mtd.o +obj-$(CONFIG_MTD_MSM_QPIC_NAND) += msm_qpic_nand.o obj-$(CONFIG_MTD_DATAFLASH) += mtd_dataflash.o obj-$(CONFIG_MTD_MCHP23K256) += mchp23k256.o obj-$(CONFIG_MTD_SPEAR_SMI) += spear_smi.o diff --git a/drivers/mtd/devices/msm_qpic_nand.c b/drivers/mtd/devices/msm_qpic_nand.c new file mode 100644 index 000000000000..9dcb277170f7 --- /dev/null +++ b/drivers/mtd/devices/msm_qpic_nand.c @@ -0,0 +1,4432 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2007 Google, Inc. + * Copyright (c) 2012-2020 The Linux Foundation. All rights reserved. + */ + +#include "msm_qpic_nand.h" + +#define QPIC_BAM_DEFAULT_IPC_LOGLVL 2 + +/* The driver supports devices upto 4K page */ +#define MAX_CW_PER_PAGE 8 +/* + * Max descriptors needed for erase, read, write operations. + * Usually, this is (2 * MAX_CW_PER_PAGE). + */ +#define MAX_DESC 16 +#define SMEM_AARM_PARTITION_TABLE 9 +#define SMEM_APPS 0 + +/* + * Get the DMA memory for requested amount of size. It returns the pointer + * to free memory available from the allocated pool. Returns NULL if there + * is no free memory. + */ +static void *msm_nand_get_dma_buffer(struct msm_nand_chip *chip, size_t size) +{ + uint32_t bitmask, free_bitmask, old_bitmask; + uint32_t need_mask, current_need_mask; + int free_index; + + need_mask = (1UL << DIV_ROUND_UP(size, MSM_NAND_DMA_BUFFER_SLOT_SZ)) + - 1; + bitmask = atomic_read(&chip->dma_buffer_busy); + free_bitmask = ~bitmask; + if (free_bitmask == 0) + return NULL; + + do { + free_index = __ffs(free_bitmask); + current_need_mask = need_mask << free_index; + + if (size + free_index * MSM_NAND_DMA_BUFFER_SLOT_SZ >= + MSM_NAND_DMA_BUFFER_SIZE) + return NULL; + + if ((bitmask & current_need_mask) == 0) { + old_bitmask = + atomic_cmpxchg(&chip->dma_buffer_busy, + bitmask, + bitmask | current_need_mask); + if (old_bitmask == bitmask) + return chip->dma_virt_addr + + free_index * MSM_NAND_DMA_BUFFER_SLOT_SZ; + free_bitmask = 0;/* force return */ + } + /* current free range was too small, clear all free bits */ + /* below the top busy bit within current_need_mask */ + free_bitmask &= + ~(~0U >> (32 - fls(bitmask & current_need_mask))); + } while (free_bitmask); + + return NULL; +} + +/* + * Releases the DMA memory used to the free pool and also wakes up any user + * thread waiting on wait queue for free memory to be available. + */ +static void msm_nand_release_dma_buffer(struct msm_nand_chip *chip, + void *buffer, size_t size) +{ + int index; + uint32_t used_mask; + + used_mask = (1UL << DIV_ROUND_UP(size, MSM_NAND_DMA_BUFFER_SLOT_SZ)) + - 1; + index = ((uint8_t *)buffer - chip->dma_virt_addr) / + MSM_NAND_DMA_BUFFER_SLOT_SZ; + atomic_sub(used_mask << index, &chip->dma_buffer_busy); + + wake_up(&chip->dma_wait_queue); +} + +/* + * Calculates page address of the buffer passed, offset of buffer within + * that page and then maps it for DMA by calling dma_map_page(). + */ +static dma_addr_t msm_nand_dma_map(struct device *dev, void *addr, size_t size, + enum dma_data_direction dir) +{ + struct page *page; + unsigned long offset = (unsigned long)addr & ~PAGE_MASK; + + if (virt_addr_valid(addr)) + page = virt_to_page(addr); + else { + if (WARN_ON(size + offset > PAGE_SIZE)) + return ~0; + page = vmalloc_to_page(addr); + } + return dma_map_page(dev, page, offset, size, dir); +} + +static int msm_nand_setup_clocks_and_bus_bw(struct msm_nand_info *info, + bool vote) +{ + return 0; +} + +#ifdef CONFIG_PM +static int msm_nand_runtime_suspend(struct device *dev) +{ + int ret = 0; + struct msm_nand_info *info = dev_get_drvdata(dev); + + ret = msm_nand_setup_clocks_and_bus_bw(info, false); + + return ret; +} + +static int msm_nand_runtime_resume(struct device *dev) +{ + int ret = 0; + struct msm_nand_info *info = dev_get_drvdata(dev); + + ret = msm_nand_setup_clocks_and_bus_bw(info, true); + + return ret; +} + +static void msm_nand_print_rpm_info(struct device *dev) +{ + pr_err("RPM: runtime_status=%d, usage_count=%d, is_suspended=%d, disable_depth=%d, runtime_error=%d, request_pending=%d, request=%d\n", + dev->power.runtime_status, atomic_read(&dev->power.usage_count), + dev->power.is_suspended, dev->power.disable_depth, + dev->power.runtime_error, dev->power.request_pending, + dev->power.request); +} +#else +static int msm_nand_runtime_suspend(struct device *dev) +{ + return 0; +} + +static int msm_nand_runtime_resume(struct device *dev) +{ + return 0; +} + +static void msm_nand_print_rpm_info(struct device *dev) +{ +} +#endif + +#ifdef CONFIG_PM +static int msm_nand_suspend(struct device *dev) +{ + int ret = 0; + + if (!pm_runtime_suspended(dev)) + ret = msm_nand_runtime_suspend(dev); + + return ret; +} + +static int msm_nand_resume(struct device *dev) +{ + int ret = 0; + + if (!pm_runtime_suspended(dev)) + ret = msm_nand_runtime_resume(dev); + + return ret; +} +#else +static int msm_nand_suspend(struct device *dev) +{ + return 0; +} + +static int msm_nand_resume(struct device *dev) +{ + return 0; +} +#endif + +#ifdef CONFIG_PM +static int msm_nand_get_device(struct device *dev) +{ + int ret = 0; + + ret = pm_runtime_get_sync(dev); + if (ret < 0) { + pr_err("Failed to resume with %d\n", ret); + msm_nand_print_rpm_info(dev); + } else { /* Reset to success */ + ret = 0; + } + return ret; +} + +static int msm_nand_put_device(struct device *dev) +{ + int ret = 0; + + pm_runtime_mark_last_busy(dev); + ret = pm_runtime_put_autosuspend(dev); + if (ret < 0) { + pr_err("Failed to suspend with %d\n", ret); + msm_nand_print_rpm_info(dev); + } else { /* Reset to success */ + ret = 0; + } + return ret; +} +#else +static int msm_nand_get_device(struct device *dev) +{ + return 0; +} + +static int msm_nand_put_device(struct device *dev) +{ + return 0; +} +#endif + +static int msm_nand_bus_register(struct platform_device *pdev, + struct msm_nand_info *info) +{ + return 0; +} + +static void msm_nand_bus_unregister(struct msm_nand_info *info) +{ +} + +/* + * Wrapper function to prepare a single SPS command element with the data + * that is passed to this function. + */ +static inline void msm_nand_prep_ce(struct sps_command_element *ce, + uint32_t addr, uint32_t command, uint32_t data) +{ + ce->addr = addr; + ce->command = (command & WRITE) ? (uint32_t) SPS_WRITE_COMMAND : + (uint32_t) SPS_READ_COMMAND; + ce->data = data; + ce->mask = 0xFFFFFFFF; +} + +static int msm_nand_sps_get_iovec(struct sps_pipe *pipe, uint32_t indx, + unsigned int cnt, struct sps_iovec *iovec) +{ + int ret = 0; + + do { + do { + ret = sps_get_iovec((pipe), (iovec)); + } while (((iovec)->addr == 0x0) && ((iovec)->size == 0x0)); + if (ret) + return ret; + } while (--(cnt)); + return ret; +} + +/* + * Wrapper function to prepare a single command descriptor with a single + * SPS command element with the data that is passed to this function. + * + * Since for any command element it is a must to have this flag + * SPS_IOVEC_FLAG_CMD, this function by default updates this flag for a + * command element that is passed and thus, the caller need not explicilty + * pass this flag. The other flags must be passed based on the need. If a + * command element doesn't have any other flag, then 0 can be passed to flags. + */ +static inline void msm_nand_prep_single_desc(struct msm_nand_sps_cmd *sps_cmd, + uint32_t addr, uint32_t command, + uint32_t data, uint32_t flags) +{ + msm_nand_prep_ce(&sps_cmd->ce, addr, command, data); + sps_cmd->flags = SPS_IOVEC_FLAG_CMD | flags; +} +/* + * Read a single NANDc register as mentioned by its parameter addr. The return + * value indicates whether read is successful or not. The register value read + * is stored in val. + */ +static int msm_nand_flash_rd_reg(struct msm_nand_info *info, uint32_t addr, + uint32_t *val) +{ + int ret = 0, submitted_num_desc = 1; + struct msm_nand_sps_cmd *cmd; + struct msm_nand_chip *chip = &info->nand_chip; + struct { + struct msm_nand_sps_cmd cmd; + uint32_t data; + } *dma_buffer; + struct sps_iovec iovec_temp; + + wait_event(chip->dma_wait_queue, (dma_buffer = msm_nand_get_dma_buffer( + chip, sizeof(*dma_buffer)))); + cmd = &dma_buffer->cmd; + msm_nand_prep_single_desc(cmd, addr, READ, msm_virt_to_dma(chip, + &dma_buffer->data), SPS_IOVEC_FLAG_INT); + + mutex_lock(&info->lock); + ret = msm_nand_get_device(chip->dev); + if (ret) + goto out; + ret = sps_transfer_one(info->sps.cmd_pipe.handle, + msm_virt_to_dma(chip, &cmd->ce), + sizeof(struct sps_command_element), NULL, cmd->flags); + if (ret) { + pr_err("failed to submit command %x ret %d\n", addr, ret); + msm_nand_put_device(chip->dev); + goto out; + } + ret = msm_nand_sps_get_iovec(info->sps.cmd_pipe.handle, + info->sps.cmd_pipe.index, submitted_num_desc, + &iovec_temp); + if (ret) { + pr_err("Failed to get iovec for pipe %d: (ret%d)\n", + (info->sps.cmd_pipe.index), ret); + goto out; + } + ret = msm_nand_put_device(chip->dev); + if (ret) + goto out; + *val = dma_buffer->data; +out: + mutex_unlock(&info->lock); + msm_nand_release_dma_buffer(chip, dma_buffer, sizeof(*dma_buffer)); + return ret; +} + +/* + * Read the Flash ID from the Nand Flash Device. The return value < 0 + * indicates failure. When successful, the Flash ID is stored in parameter + * read_id. + */ +#define READID_CMDS 5 +static int msm_nand_flash_read_id(struct msm_nand_info *info, + bool read_onfi_signature, uint32_t *read_id, + uint32_t *read_id2) +{ + int err = 0, i = 0; + struct msm_nand_sps_cmd *cmd; + struct sps_iovec *iovec; + struct sps_iovec iovec_temp; + struct msm_nand_chip *chip = &info->nand_chip; + /* + * The following 5 commands are required to read id - + * write commands - addr0, flash, exec + * read_commands - read_id, read_id2 + */ + struct { + struct sps_transfer xfer; + struct sps_iovec cmd_iovec[READID_CMDS]; + struct msm_nand_sps_cmd cmd[READID_CMDS]; + uint32_t data[READID_CMDS]; + } *dma_buffer; + + wait_event(chip->dma_wait_queue, (dma_buffer = msm_nand_get_dma_buffer + (chip, sizeof(*dma_buffer)))); + if (read_onfi_signature) + dma_buffer->data[0] = FLASH_READ_ONFI_SIGNATURE_ADDRESS; + else + dma_buffer->data[0] = FLASH_READ_DEVICE_ID_ADDRESS; + + dma_buffer->data[1] = EXTENDED_FETCH_ID | MSM_NAND_CMD_FETCH_ID; + dma_buffer->data[2] = 1; + dma_buffer->data[3] = 0xeeeeeeee; + dma_buffer->data[4] = 0xeeeeeeee; + + cmd = dma_buffer->cmd; + msm_nand_prep_single_desc(cmd, MSM_NAND_ADDR0(info), WRITE, + dma_buffer->data[0], SPS_IOVEC_FLAG_LOCK); + cmd++; + + msm_nand_prep_single_desc(cmd, MSM_NAND_FLASH_CMD(info), WRITE, + dma_buffer->data[1], 0); + cmd++; + + msm_nand_prep_single_desc(cmd, MSM_NAND_EXEC_CMD(info), WRITE, + dma_buffer->data[2], SPS_IOVEC_FLAG_NWD); + cmd++; + + msm_nand_prep_single_desc(cmd, MSM_NAND_READ_ID(info), READ, + msm_virt_to_dma(chip, &dma_buffer->data[3]), 0); + cmd++; + + msm_nand_prep_single_desc(cmd, MSM_NAND_READ_ID2(info), READ, + msm_virt_to_dma(chip, &dma_buffer->data[4]), + SPS_IOVEC_FLAG_UNLOCK | SPS_IOVEC_FLAG_INT); + cmd++; + + WARN_ON(cmd - dma_buffer->cmd > READID_CMDS); + dma_buffer->xfer.iovec_count = (cmd - dma_buffer->cmd); + dma_buffer->xfer.iovec = dma_buffer->cmd_iovec; + dma_buffer->xfer.iovec_phys = msm_virt_to_dma(chip, + &dma_buffer->cmd_iovec); + iovec = dma_buffer->xfer.iovec; + + for (i = 0; i < dma_buffer->xfer.iovec_count; i++) { + iovec->addr = msm_virt_to_dma(chip, &dma_buffer->cmd[i].ce); + iovec->size = sizeof(struct sps_command_element); + iovec->flags = dma_buffer->cmd[i].flags; + iovec++; + } + + mutex_lock(&info->lock); + err = msm_nand_get_device(chip->dev); + if (err) + goto out; + err = sps_transfer(info->sps.cmd_pipe.handle, &dma_buffer->xfer); + if (err) { + pr_err("Failed to submit commands %d\n", err); + msm_nand_put_device(chip->dev); + goto out; + } + err = msm_nand_sps_get_iovec(info->sps.cmd_pipe.handle, + info->sps.cmd_pipe.index, dma_buffer->xfer.iovec_count, + &iovec_temp); + + if (err) { + pr_err("Failed to get iovec for pipe %d: (err:%d)\n", + (info->sps.cmd_pipe.index), err); + goto out; + } + pr_debug("Read ID register value 0x%x\n", dma_buffer->data[3]); + if (!read_onfi_signature) + pr_debug("nandid: %x maker %02x device %02x\n", + dma_buffer->data[3], dma_buffer->data[3] & 0xff, + (dma_buffer->data[3] >> 8) & 0xff); + *read_id = dma_buffer->data[3]; + if (read_id2) { + pr_debug("Extended Read ID register value 0x%x\n", + dma_buffer->data[4]); + *read_id2 = dma_buffer->data[4]; + } + err = msm_nand_put_device(chip->dev); +out: + mutex_unlock(&info->lock); + msm_nand_release_dma_buffer(chip, dma_buffer, sizeof(*dma_buffer)); + return err; +} + +/* + * Contains data for common configuration registers that must be programmed + * for every NANDc operation. + */ +struct msm_nand_common_cfgs { + uint32_t cmd; + uint32_t addr0; + uint32_t addr1; + uint32_t cfg0; + uint32_t cfg1; +}; + +/* + * Function to prepare SPS command elements to write into NANDc configuration + * registers as per the data defined in struct msm_nand_common_cfgs. This is + * required for the following NANDc operations - Erase, Bad Block checking + * and for reading ONFI parameter page. + */ +static void msm_nand_prep_cfg_cmd_desc(struct msm_nand_info *info, + struct msm_nand_common_cfgs data, + struct msm_nand_sps_cmd **curr_cmd) +{ + struct msm_nand_sps_cmd *cmd; + + cmd = *curr_cmd; + msm_nand_prep_single_desc(cmd, MSM_NAND_FLASH_CMD(info), WRITE, + data.cmd, SPS_IOVEC_FLAG_LOCK); + cmd++; + + msm_nand_prep_single_desc(cmd, MSM_NAND_ADDR0(info), WRITE, + data.addr0, 0); + cmd++; + + msm_nand_prep_single_desc(cmd, MSM_NAND_ADDR1(info), WRITE, + data.addr1, 0); + cmd++; + + msm_nand_prep_single_desc(cmd, MSM_NAND_DEV0_CFG0(info), WRITE, + data.cfg0, 0); + cmd++; + + msm_nand_prep_single_desc(cmd, MSM_NAND_DEV0_CFG1(info), WRITE, + data.cfg1, 0); + cmd++; + *curr_cmd = cmd; +} + +/* + * Function to check the CRC integrity check on ONFI parameter page read. + * For ONFI parameter page read, the controller ECC will be disabled. Hence, + * it is mandatory to manually compute CRC and check it against the value + * stored within ONFI page. + */ +static uint16_t msm_nand_flash_onfi_crc_check(uint8_t *buffer, uint16_t count) +{ + int i; + uint16_t result; + + for (i = 0; i < count; i++) + buffer[i] = bitrev8(buffer[i]); + + result = bitrev16(crc16(bitrev16(0x4f4e), buffer, count)); + + for (i = 0; i < count; i++) + buffer[i] = bitrev8(buffer[i]); + + return result; +} + +/* + * Structure that contains NANDc register data for commands required + * for reading ONFI parameter page. + */ +struct msm_nand_flash_onfi_data { + struct msm_nand_common_cfgs cfg; + uint32_t exec; + uint32_t ecc_bch_cfg; +}; + +struct version { + uint16_t nand_major; + uint16_t nand_minor; + uint16_t qpic_major; + uint16_t qpic_minor; +}; + +static int msm_nand_version_check(struct msm_nand_info *info, + struct version *nandc_version) +{ + uint32_t qpic_ver = 0, nand_ver = 0; + int err = 0; + + /* Lookup the version to identify supported features */ + err = msm_nand_flash_rd_reg(info, MSM_NAND_VERSION(info), + &nand_ver); + if (err) { + pr_err("Failed to read NAND_VERSION, err=%d\n", err); + goto out; + } + nandc_version->nand_major = (nand_ver & MSM_NAND_VERSION_MAJOR_MASK) >> + MSM_NAND_VERSION_MAJOR_SHIFT; + nandc_version->nand_minor = (nand_ver & MSM_NAND_VERSION_MINOR_MASK) >> + MSM_NAND_VERSION_MINOR_SHIFT; + + err = msm_nand_flash_rd_reg(info, MSM_NAND_QPIC_VERSION(info), + &qpic_ver); + if (err) { + pr_err("Failed to read QPIC_VERSION, err=%d\n", err); + goto out; + } + nandc_version->qpic_major = (qpic_ver & MSM_NAND_VERSION_MAJOR_MASK) >> + MSM_NAND_VERSION_MAJOR_SHIFT; + nandc_version->qpic_minor = (qpic_ver & MSM_NAND_VERSION_MINOR_MASK) >> + MSM_NAND_VERSION_MINOR_SHIFT; + pr_info("nand_major:%d, nand_minor:%d, qpic_major:%d, qpic_minor:%d\n", + nandc_version->nand_major, nandc_version->nand_minor, + nandc_version->qpic_major, nandc_version->qpic_minor); +out: + return err; +} + +/* + * Function to identify whether the attached NAND flash device is + * complaint to ONFI spec or not. If yes, then it reads the ONFI parameter + * page to get the device parameters. + */ +#define ONFI_CMDS 10 +static int msm_nand_flash_onfi_probe(struct msm_nand_info *info) +{ + struct msm_nand_chip *chip = &info->nand_chip; + struct flash_identification *flash = &info->flash_dev; + uint32_t crc_chk_count = 0, page_address = 0; + int ret = 0, i = 0, submitted_num_desc = 1; + + /* SPS parameters */ + struct msm_nand_sps_cmd *cmd, *curr_cmd; + struct sps_iovec *iovec; + struct sps_iovec iovec_temp; + uint32_t rdata; + + /* ONFI Identifier/Parameter Page parameters */ + uint8_t *onfi_param_info_buf = NULL; + dma_addr_t dma_addr_param_info = 0; + struct onfi_param_page *onfi_param_page_ptr; + struct msm_nand_flash_onfi_data data; + uint32_t onfi_signature = 0; + + /* + * The following 9 commands are required to get onfi parameters - + * flash, addr0, addr1, cfg0, cfg1, dev0_ecc_cfg, + * read_loc_0, exec, flash_status (read cmd). + */ + struct { + struct sps_transfer xfer; + struct sps_iovec cmd_iovec[ONFI_CMDS]; + struct msm_nand_sps_cmd cmd[ONFI_CMDS]; + uint32_t flash_status; + } *dma_buffer; + + + /* Lookup the version to identify supported features */ + struct version nandc_version = {0}; + + ret = msm_nand_version_check(info, &nandc_version); + if (!ret && !(nandc_version.nand_major == 1 && + nandc_version.nand_minor >= 5 && + nandc_version.qpic_major == 1 && + nandc_version.qpic_minor >= 5)) { + ret = -EPERM; + goto out; + } + wait_event(chip->dma_wait_queue, (onfi_param_info_buf = + msm_nand_get_dma_buffer(chip, ONFI_PARAM_INFO_LENGTH))); + dma_addr_param_info = msm_virt_to_dma(chip, onfi_param_info_buf); + + wait_event(chip->dma_wait_queue, (dma_buffer = msm_nand_get_dma_buffer + (chip, sizeof(*dma_buffer)))); + + ret = msm_nand_flash_read_id(info, 1, &onfi_signature, NULL); + if (ret < 0) { + pr_err("Failed to read ONFI signature\n"); + goto free_dma; + } + if (onfi_signature != ONFI_PARAMETER_PAGE_SIGNATURE) { + ret = -EIO; + goto free_dma; + } + + memset(&data, 0, sizeof(struct msm_nand_flash_onfi_data)); + + /* Lookup the partition to which apps has access to */ + for (i = 0; i < FLASH_PTABLE_MAX_PARTS_V4; i++) { + if (mtd_part[i].name && !strcmp("boot", mtd_part[i].name)) { + page_address = mtd_part[i].offset << 6; + break; + } + } + if (!page_address) { + pr_err("%s: no apps partition found in smem\n", __func__); + ret = -EPERM; + goto free_dma; + } + data.cfg.cmd = MSM_NAND_CMD_PAGE_READ_ONFI; + data.exec = 1; + data.cfg.addr0 = (page_address << 16) | + FLASH_READ_ONFI_PARAMETERS_ADDRESS; + data.cfg.addr1 = (page_address >> 16) & 0xFF; + data.cfg.cfg0 = MSM_NAND_CFG0_RAW_ONFI_PARAM_INFO; + data.cfg.cfg1 = MSM_NAND_CFG1_RAW_ONFI_PARAM_INFO; + data.ecc_bch_cfg = 1 << ECC_CFG_ECC_DISABLE; + dma_buffer->flash_status = 0xeeeeeeee; + + curr_cmd = cmd = dma_buffer->cmd; + msm_nand_prep_cfg_cmd_desc(info, data.cfg, &curr_cmd); + + cmd = curr_cmd; + msm_nand_prep_single_desc(cmd, MSM_NAND_DEV0_ECC_CFG(info), WRITE, + data.ecc_bch_cfg, 0); + cmd++; + + rdata = (0 << 0) | (ONFI_PARAM_INFO_LENGTH << 16) | (1 << 31); + msm_nand_prep_single_desc(cmd, MSM_NAND_READ_LOCATION_0(info), WRITE, + rdata, 0); + cmd++; + + if (chip->qpic_version >= 2) { + msm_nand_prep_single_desc(cmd, + MSM_NAND_READ_LOCATION_LAST_CW_0(info), WRITE, + rdata, 0); + cmd++; + } + + msm_nand_prep_single_desc(cmd, MSM_NAND_EXEC_CMD(info), WRITE, + data.exec, SPS_IOVEC_FLAG_NWD); + cmd++; + + msm_nand_prep_single_desc(cmd, MSM_NAND_FLASH_STATUS(info), READ, + msm_virt_to_dma(chip, &dma_buffer->flash_status), + SPS_IOVEC_FLAG_UNLOCK | SPS_IOVEC_FLAG_INT); + cmd++; + + WARN_ON(cmd - dma_buffer->cmd > ONFI_CMDS); + dma_buffer->xfer.iovec_count = (cmd - dma_buffer->cmd); + dma_buffer->xfer.iovec = dma_buffer->cmd_iovec; + dma_buffer->xfer.iovec_phys = msm_virt_to_dma(chip, + &dma_buffer->cmd_iovec); + iovec = dma_buffer->xfer.iovec; + + for (i = 0; i < dma_buffer->xfer.iovec_count; i++) { + iovec->addr = msm_virt_to_dma(chip, + &dma_buffer->cmd[i].ce); + iovec->size = sizeof(struct sps_command_element); + iovec->flags = dma_buffer->cmd[i].flags; + iovec++; + } + mutex_lock(&info->lock); + ret = msm_nand_get_device(chip->dev); + if (ret) + goto unlock_mutex; + /* Submit data descriptor */ + ret = sps_transfer_one(info->sps.data_prod.handle, dma_addr_param_info, + ONFI_PARAM_INFO_LENGTH, NULL, SPS_IOVEC_FLAG_INT); + if (ret) { + pr_err("Failed to submit data descriptors %d\n", ret); + goto put_dev; + } + /* Submit command descriptors */ + ret = sps_transfer(info->sps.cmd_pipe.handle, + &dma_buffer->xfer); + if (ret) { + pr_err("Failed to submit commands %d\n", ret); + goto put_dev; + } + + ret = msm_nand_sps_get_iovec(info->sps.cmd_pipe.handle, + info->sps.cmd_pipe.index, dma_buffer->xfer.iovec_count, + &iovec_temp); + + if (ret) { + pr_err("Failed to get iovec for pipe %d: (ret:%d)\n", + (info->sps.cmd_pipe.index), ret); + goto put_dev; + } + ret = msm_nand_sps_get_iovec(info->sps.data_prod.handle, + info->sps.data_prod.index, submitted_num_desc, + &iovec_temp); + if (ret) { + pr_err("Failed to get iovec for pipe %d: (ret:%d)\n", + (info->sps.data_prod.index), ret); + goto put_dev; + } + + ret = msm_nand_put_device(chip->dev); + mutex_unlock(&info->lock); + if (ret) + goto free_dma; + + /* Check for flash status errors */ + if (dma_buffer->flash_status & (FS_OP_ERR | FS_MPU_ERR)) { + pr_err("MPU/OP err (0x%x) is set\n", dma_buffer->flash_status); + ret = -EIO; + goto free_dma; + } + + for (crc_chk_count = 0; crc_chk_count < ONFI_PARAM_INFO_LENGTH + / ONFI_PARAM_PAGE_LENGTH; crc_chk_count++) { + onfi_param_page_ptr = + (struct onfi_param_page *) + (&(onfi_param_info_buf + [ONFI_PARAM_PAGE_LENGTH * + crc_chk_count])); + if (msm_nand_flash_onfi_crc_check( + (uint8_t *)onfi_param_page_ptr, + ONFI_PARAM_PAGE_LENGTH - 2) == + onfi_param_page_ptr->integrity_crc) { + break; + } + } + if (crc_chk_count >= ONFI_PARAM_INFO_LENGTH + / ONFI_PARAM_PAGE_LENGTH) { + pr_err("CRC Check failed on param page\n"); + ret = -EIO; + goto free_dma; + } + ret = msm_nand_flash_read_id(info, 0, &flash->flash_id, NULL); + if (ret < 0) { + pr_err("Failed to read flash ID\n"); + goto free_dma; + } + flash->widebus = onfi_param_page_ptr->features_supported & 0x01; + flash->pagesize = onfi_param_page_ptr->number_of_data_bytes_per_page; + flash->blksize = onfi_param_page_ptr->number_of_pages_per_block * + flash->pagesize; + flash->oobsize = onfi_param_page_ptr->number_of_spare_bytes_per_page; + flash->density = onfi_param_page_ptr->number_of_blocks_per_logical_unit + * flash->blksize; + flash->ecc_correctability = + onfi_param_page_ptr->number_of_bits_ecc_correctability; + + pr_info("Found an ONFI compliant device %s\n", + onfi_param_page_ptr->device_model); + /* + * Temporary hack for MT29F4G08ABC device. + * Since the device is not properly adhering + * to ONFi specification it is reporting + * as 16 bit device though it is 8 bit device!!! + */ + if (!strcmp(onfi_param_page_ptr->device_model, "MT29F4G08ABC")) + flash->widebus = 0; + goto free_dma; +put_dev: + msm_nand_put_device(chip->dev); +unlock_mutex: + mutex_unlock(&info->lock); +free_dma: + msm_nand_release_dma_buffer(chip, dma_buffer, sizeof(*dma_buffer)); + msm_nand_release_dma_buffer(chip, onfi_param_info_buf, + ONFI_PARAM_INFO_LENGTH); +out: + return ret; +} + +/* + * Structure that contains read/write parameters required for reading/writing + * from/to a page. + */ +struct msm_nand_rw_params { + uint32_t page; + uint32_t page_count; + uint32_t sectordatasize; + uint32_t sectoroobsize; + uint32_t cwperpage; + uint32_t oob_len_cmd; + uint32_t oob_len_data; + uint32_t start_sector; + uint32_t oob_col; + dma_addr_t data_dma_addr; + dma_addr_t oob_dma_addr; + dma_addr_t ecc_dma_addr; + dma_addr_t data_dma_addr_curr; + dma_addr_t oob_dma_addr_curr; + dma_addr_t ecc_dma_addr_curr; + bool read; +}; + +/* + * Structure that contains NANDc register data required for reading/writing + * from/to a page. + */ +struct msm_nand_rw_reg_data { + uint32_t cmd; + uint32_t addr0; + uint32_t addr1; + uint32_t cfg0; + uint32_t cfg1; + uint32_t ecc_bch_cfg; + uint32_t exec; + uint32_t ecc_cfg; + uint32_t clrfstatus; + uint32_t clrrstatus; +}; + +/* + * Function that validates page read/write MTD parameters received from upper + * layers such as MTD/YAFFS2 and returns error for any unsupported operations + * by the driver. In case of success, it also maps the data and oob buffer + * received for DMA. + */ +static int msm_nand_validate_mtd_params(struct mtd_info *mtd, bool read, + loff_t offset, + struct mtd_oob_ops *ops, + struct msm_nand_rw_params *args) +{ + struct msm_nand_info *info = mtd->priv; + struct msm_nand_chip *chip = &info->nand_chip; + int err = 0; + + pr_debug("========================================================\n"); + pr_debug("offset 0x%llx mode %d\ndatbuf 0x%pK datlen 0x%x\n", + offset, ops->mode, ops->datbuf, ops->len); + pr_debug("oobbuf 0x%pK ooblen 0x%x\n", ops->oobbuf, ops->ooblen); + + if (ops->mode == MTD_OPS_PLACE_OOB) { + pr_err("MTD_OPS_PLACE_OOB is not supported\n"); + err = -EINVAL; + goto out; + } + + if (mtd->writesize == PAGE_SIZE_2K) + args->page = offset >> 11; + + if (mtd->writesize == PAGE_SIZE_4K) + args->page = offset >> 12; + + args->oob_len_cmd = ops->ooblen; + args->oob_len_data = ops->ooblen; + args->cwperpage = (mtd->writesize >> 9); + args->read = (read ? true : false); + + if (offset & (mtd->writesize - 1)) { + pr_err("unsupported offset 0x%llx\n", offset); + err = -EINVAL; + goto out; + } + + if (!read && !ops->datbuf) { + pr_err("No data buffer provided for write!!\n"); + err = -EINVAL; + goto out; + } + + if (ops->mode == MTD_OPS_RAW) { + if (!ops->datbuf) { + pr_err("No data buffer provided for RAW mode\n"); + err = -EINVAL; + goto out; + } else if ((ops->len % (mtd->writesize + + mtd->oobsize)) != 0) { + pr_err("unsupported data len %d for RAW mode\n", + ops->len); + err = -EINVAL; + goto out; + } + args->page_count = ops->len / (mtd->writesize + mtd->oobsize); + + } else if (ops->mode == MTD_OPS_AUTO_OOB) { + if (ops->datbuf && (ops->len % mtd->writesize) != 0) { + /* when ops->datbuf is NULL, ops->len can be ooblen */ + pr_err("unsupported data len %d for AUTO mode\n", + ops->len); + err = -EINVAL; + goto out; + } + if (read && ops->oobbuf && !ops->datbuf) { + args->start_sector = args->cwperpage - 1; + args->page_count = ops->ooblen / mtd->oobavail; + if ((args->page_count == 0) && (ops->ooblen)) + args->page_count = 1; + } else if (ops->datbuf) { + args->page_count = ops->len / mtd->writesize; + } + } + + if (ops->datbuf) { + if (read) + memset(ops->datbuf, 0xFF, ops->len); + args->data_dma_addr_curr = args->data_dma_addr = + msm_nand_dma_map(chip->dev, ops->datbuf, ops->len, + (read ? DMA_FROM_DEVICE : DMA_TO_DEVICE)); + if (dma_mapping_error(chip->dev, args->data_dma_addr)) { + pr_err("dma mapping failed for 0x%pK\n", ops->datbuf); + err = -EIO; + goto out; + } + } + if (ops->oobbuf) { + if (read) + memset(ops->oobbuf, 0xFF, ops->ooblen); + args->oob_dma_addr_curr = args->oob_dma_addr = + msm_nand_dma_map(chip->dev, ops->oobbuf, ops->ooblen, + (read ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE)); + if (dma_mapping_error(chip->dev, args->oob_dma_addr)) { + pr_err("dma mapping failed for 0x%pK\n", ops->oobbuf); + err = -EIO; + goto dma_map_oobbuf_failed; + } + } + goto out; +dma_map_oobbuf_failed: + if (ops->datbuf) + dma_unmap_page(chip->dev, args->data_dma_addr, ops->len, + (read ? DMA_FROM_DEVICE : DMA_TO_DEVICE)); +out: + return err; +} + +/* + * Function that updates NANDc register data (struct msm_nand_rw_reg_data) + * required for page read/write. + */ +static void msm_nand_update_rw_reg_data(struct msm_nand_chip *chip, + struct mtd_oob_ops *ops, + struct msm_nand_rw_params *args, + struct msm_nand_rw_reg_data *data) +{ + if (args->read) { + if (ops->mode != MTD_OPS_RAW) { + data->cmd = MSM_NAND_CMD_PAGE_READ_ECC; + data->cfg0 = + (chip->cfg0 & ~(7U << CW_PER_PAGE)) | + (((args->cwperpage-1) - args->start_sector) + << CW_PER_PAGE); + data->cfg1 = chip->cfg1; + data->ecc_bch_cfg = chip->ecc_bch_cfg; + } else { + data->cmd = MSM_NAND_CMD_PAGE_READ_ALL; + data->cfg0 = + (chip->cfg0_raw & ~(7U << CW_PER_PAGE)) | + (((args->cwperpage-1) - args->start_sector) + << CW_PER_PAGE); + data->cfg1 = chip->cfg1_raw; + data->ecc_bch_cfg = chip->ecc_cfg_raw; + } + + } else { + if (ops->mode != MTD_OPS_RAW) { + data->cmd = MSM_NAND_CMD_PRG_PAGE; + data->cfg0 = chip->cfg0; + data->cfg1 = chip->cfg1; + data->ecc_bch_cfg = chip->ecc_bch_cfg; + } else { + data->cmd = MSM_NAND_CMD_PRG_PAGE_ALL; + data->cfg0 = chip->cfg0_raw; + data->cfg1 = chip->cfg1_raw; + data->ecc_bch_cfg = chip->ecc_cfg_raw; + } + data->clrfstatus = MSM_NAND_RESET_FLASH_STS; + data->clrrstatus = MSM_NAND_RESET_READ_STS; + } + data->exec = 1; + data->ecc_cfg = chip->ecc_buf_cfg; +} + +/* + * Function to prepare series of SPS command descriptors required for a page + * read/write operation. + */ +static void msm_nand_prep_rw_cmd_desc(struct mtd_oob_ops *ops, + struct msm_nand_rw_params *args, + struct msm_nand_rw_reg_data *data, + struct msm_nand_info *info, + uint32_t curr_cw, + struct msm_nand_rw_cmd_desc *cmd_list, + uint32_t *cw_desc_cnt, + uint32_t ecc_parity_bytes) +{ + struct msm_nand_chip *chip = &info->nand_chip; + uint32_t rdata; + /* read_location register parameters */ + uint32_t offset, size, last_read; + struct sps_command_element *curr_ce, *start_ce; + uint32_t *flags_ptr, *num_ce_ptr; + + if (curr_cw == args->start_sector) { + curr_ce = start_ce = &cmd_list->setup_desc.ce[0]; + num_ce_ptr = &cmd_list->setup_desc.num_ce; + flags_ptr = &cmd_list->setup_desc.flags; + *flags_ptr = CMD_LCK; + cmd_list->count = 1; + msm_nand_prep_ce(curr_ce, MSM_NAND_FLASH_CMD(info), WRITE, + data->cmd); + curr_ce++; + + msm_nand_prep_ce(curr_ce, MSM_NAND_ADDR0(info), WRITE, + data->addr0); + curr_ce++; + + msm_nand_prep_ce(curr_ce, MSM_NAND_ADDR1(info), WRITE, + data->addr1); + curr_ce++; + + msm_nand_prep_ce(curr_ce, MSM_NAND_DEV0_CFG0(info), WRITE, + data->cfg0); + curr_ce++; + + msm_nand_prep_ce(curr_ce, MSM_NAND_DEV0_CFG1(info), WRITE, + data->cfg1); + curr_ce++; + + msm_nand_prep_ce(curr_ce, MSM_NAND_DEV0_ECC_CFG(info), WRITE, + data->ecc_bch_cfg); + curr_ce++; + + msm_nand_prep_ce(curr_ce, MSM_NAND_EBI2_ECC_BUF_CFG(info), + WRITE, data->ecc_cfg); + curr_ce++; + + if (!args->read) { + msm_nand_prep_ce(curr_ce, MSM_NAND_FLASH_STATUS(info), + WRITE, data->clrfstatus); + curr_ce++; + goto sub_exec_cmd; + } else { + msm_nand_prep_ce(curr_ce, + MSM_NAND_ERASED_CW_DETECT_CFG(info), + WRITE, CLR_ERASED_PAGE_DET); + curr_ce++; + msm_nand_prep_ce(curr_ce, + MSM_NAND_ERASED_CW_DETECT_CFG(info), + WRITE, SET_ERASED_PAGE_DET); + curr_ce++; + } + } else { + curr_ce = start_ce = &cmd_list->cw_desc[*cw_desc_cnt].ce[0]; + num_ce_ptr = &cmd_list->cw_desc[*cw_desc_cnt].num_ce; + flags_ptr = &cmd_list->cw_desc[*cw_desc_cnt].flags; + *cw_desc_cnt += 1; + *flags_ptr = CMD; + cmd_list->count++; + } + if (!args->read) + goto sub_exec_cmd; + + if (ops->mode == MTD_OPS_RAW) { + if (ecc_parity_bytes) { + rdata = (BYTES_517 << 0) | (ecc_parity_bytes << 16) + | (1 << 31); + msm_nand_prep_ce(curr_ce, + MSM_NAND_READ_LOCATION_0(info), + WRITE, rdata); + curr_ce++; + if (chip->qpic_version >= 2) { + msm_nand_prep_ce(curr_ce, + MSM_NAND_READ_LOCATION_LAST_CW_0(info), + WRITE, rdata); + curr_ce++; + } + } else { + rdata = (0 << 0) | (chip->cw_size << 16) | (1 << 31); + msm_nand_prep_ce(curr_ce, + MSM_NAND_READ_LOCATION_0(info), + WRITE, rdata); + curr_ce++; + if (chip->qpic_version >= 2) { + msm_nand_prep_ce(curr_ce, + MSM_NAND_READ_LOCATION_LAST_CW_0(info), + WRITE, rdata); + curr_ce++; + } + } + } + if (ops->mode == MTD_OPS_AUTO_OOB) { + if (ops->datbuf) { + offset = 0; + size = (curr_cw < (args->cwperpage - 1)) ? 516 : + (512 - ((args->cwperpage - 1) << 2)); + last_read = (curr_cw < (args->cwperpage - 1)) ? 1 : + (ops->oobbuf ? 0 : 1); + rdata = (offset << 0) | (size << 16) | + (last_read << 31); + + msm_nand_prep_ce(curr_ce, + MSM_NAND_READ_LOCATION_0(info), + WRITE, + rdata); + curr_ce++; + if (chip->qpic_version >= 2) { + msm_nand_prep_ce(curr_ce, + MSM_NAND_READ_LOCATION_LAST_CW_0(info), + WRITE, rdata); + curr_ce++; + } + } + if (curr_cw == (args->cwperpage - 1) && ops->oobbuf) { + offset = 512 - ((args->cwperpage - 1) << 2); + size = (args->cwperpage) << 2; + if (size > args->oob_len_cmd) + size = args->oob_len_cmd; + args->oob_len_cmd -= size; + last_read = 1; + rdata = (offset << 0) | (size << 16) | + (last_read << 31); + + if (!ops->datbuf) { + msm_nand_prep_ce(curr_ce, + MSM_NAND_READ_LOCATION_0(info), + WRITE, rdata); + curr_ce++; + if (chip->qpic_version >= 2) { + msm_nand_prep_ce(curr_ce, + MSM_NAND_READ_LOCATION_LAST_CW_0(info), + WRITE, rdata); + curr_ce++; + } + } else { + msm_nand_prep_ce(curr_ce, + MSM_NAND_READ_LOCATION_1(info), + WRITE, rdata); + curr_ce++; + if (chip->qpic_version >= 2) { + msm_nand_prep_ce(curr_ce, + MSM_NAND_READ_LOCATION_LAST_CW_1(info), + WRITE, rdata); + curr_ce++; + } + } + } + } +sub_exec_cmd: + *flags_ptr |= NWD; + msm_nand_prep_ce(curr_ce, MSM_NAND_EXEC_CMD(info), WRITE, data->exec); + curr_ce++; + + *num_ce_ptr = curr_ce - start_ce; +} + +/* + * Function to prepare and submit SPS data descriptors required for a page + * read/write operation. + */ +static int msm_nand_submit_rw_data_desc(struct mtd_oob_ops *ops, + struct msm_nand_rw_params *args, + struct msm_nand_info *info, + uint32_t curr_cw, + uint32_t ecc_parity_bytes) +{ + struct msm_nand_chip *chip = &info->nand_chip; + struct sps_pipe *data_pipe_handle; + uint32_t sectordatasize, sectoroobsize; + uint32_t sps_flags = 0; + int err = 0; + + if (args->read) + data_pipe_handle = info->sps.data_prod.handle; + else + data_pipe_handle = info->sps.data_cons.handle; + + if (ops->mode == MTD_OPS_RAW) { + if (ecc_parity_bytes && args->read) { + if (curr_cw == (args->cwperpage - 1)) + sps_flags |= SPS_IOVEC_FLAG_INT; + + /* read only ecc bytes */ + err = sps_transfer_one(data_pipe_handle, + args->ecc_dma_addr_curr, + ecc_parity_bytes, NULL, + sps_flags); + if (err) + goto out; + args->ecc_dma_addr_curr += ecc_parity_bytes; + } else { + sectordatasize = chip->cw_size; + if (!args->read) + sps_flags = SPS_IOVEC_FLAG_EOT; + if (curr_cw == (args->cwperpage - 1)) + sps_flags |= SPS_IOVEC_FLAG_INT; + + err = sps_transfer_one(data_pipe_handle, + args->data_dma_addr_curr, + sectordatasize, NULL, + sps_flags); + if (err) + goto out; + args->data_dma_addr_curr += sectordatasize; + } + } else if (ops->mode == MTD_OPS_AUTO_OOB) { + if (ops->datbuf) { + sectordatasize = (curr_cw < (args->cwperpage - 1)) + ? 516 : (512 - ((args->cwperpage - 1) << 2)); + + if (!args->read) { + sps_flags = SPS_IOVEC_FLAG_EOT; + if (curr_cw == (args->cwperpage - 1) && + ops->oobbuf) + sps_flags = 0; + } + if ((curr_cw == (args->cwperpage - 1)) && !ops->oobbuf) + sps_flags |= SPS_IOVEC_FLAG_INT; + + err = sps_transfer_one(data_pipe_handle, + args->data_dma_addr_curr, + sectordatasize, NULL, + sps_flags); + if (err) + goto out; + args->data_dma_addr_curr += sectordatasize; + } + + if (ops->oobbuf && (curr_cw == (args->cwperpage - 1))) { + sectoroobsize = args->cwperpage << 2; + if (sectoroobsize > args->oob_len_data) + sectoroobsize = args->oob_len_data; + + if (!args->read) + sps_flags |= SPS_IOVEC_FLAG_EOT; + sps_flags |= SPS_IOVEC_FLAG_INT; + err = sps_transfer_one(data_pipe_handle, + args->oob_dma_addr_curr, + sectoroobsize, NULL, + sps_flags); + if (err) + goto out; + args->oob_dma_addr_curr += sectoroobsize; + args->oob_len_data -= sectoroobsize; + } + } +out: + return err; +} + +/* + * + * Function to prepare series of SPS command descriptors required for a page + * read operation with enhanced read pagescope feature. + */ +static void msm_nand_prep_read_cmd_desc_pagescope(struct mtd_oob_ops *ops, + struct msm_nand_rw_params *args, + struct msm_nand_rw_reg_data *data, + struct msm_nand_info *info, + struct msm_nand_rw_cmd_desc *cmd_list, + uint32_t ecc_parity_bytes) +{ + struct msm_nand_chip *chip = &info->nand_chip; + uint32_t rdata; + /* read_location register parameters */ + uint32_t offset, size, last_read; + struct sps_command_element *curr_ce, *start_ce; + uint32_t *flags_ptr, *num_ce_ptr; + uint32_t auto_status_value = 0x0; + + curr_ce = start_ce = &cmd_list->setup_desc.ce[0]; + num_ce_ptr = &cmd_list->setup_desc.num_ce; + flags_ptr = &cmd_list->setup_desc.flags; + *flags_ptr = CMD_LCK; + cmd_list->count = 1; + + auto_status_value = (NAND_FLASH_STATUS_EN | + NANDC_BUFFER_STATUS_EN | + NAND_ERASED_CW_DETECT_STATUS_EN | + NAND_FLASH_STATUS_LAST_CW_EN | + NANDC_BUFFER_STATUS_LAST_CW_EN | + NAND_ERASED_CW_DETECT_STATUS_LAST_CW_EN); + + msm_nand_prep_ce(curr_ce, MSM_NAND_DEV0_CFG0(info), WRITE, + data->cfg0); + curr_ce++; + + msm_nand_prep_ce(curr_ce, MSM_NAND_DEV0_CFG1(info), WRITE, + data->cfg1); + curr_ce++; + + msm_nand_prep_ce(curr_ce, MSM_NAND_DEV0_ECC_CFG(info), WRITE, + data->ecc_bch_cfg); + curr_ce++; + + msm_nand_prep_ce(curr_ce, MSM_NAND_AUTO_STATUS_EN(info), WRITE, + auto_status_value); + curr_ce++; + + msm_nand_prep_ce(curr_ce, MSM_NAND_ADDR0(info), WRITE, + data->addr0); + curr_ce++; + + msm_nand_prep_ce(curr_ce, MSM_NAND_ADDR1(info), WRITE, + data->addr1); + curr_ce++; + *num_ce_ptr = curr_ce - start_ce; + + /* Prepare next set of command descriptors */ + + curr_ce = start_ce = &cmd_list->cw_desc[0].ce[0]; + num_ce_ptr = &cmd_list->cw_desc[0].num_ce; + flags_ptr = &cmd_list->cw_desc[0].flags; + *flags_ptr = CMD; + cmd_list->count++; + + if (ops->mode == MTD_OPS_RAW) { + if (ecc_parity_bytes) { + rdata = (BYTES_517 << 0) | (ecc_parity_bytes << 16) + | (1 << 31); + msm_nand_prep_ce(curr_ce, + MSM_NAND_READ_LOCATION_0(info), + WRITE, rdata); + curr_ce++; + + msm_nand_prep_ce(curr_ce, + MSM_NAND_READ_LOCATION_LAST_CW_0(info), + WRITE, rdata); + curr_ce++; + + } else { + rdata = (0 << 0) | (chip->cw_size << 16) | (1 << 31); + msm_nand_prep_ce(curr_ce, + MSM_NAND_READ_LOCATION_0(info), + WRITE, rdata); + curr_ce++; + msm_nand_prep_ce(curr_ce, + MSM_NAND_READ_LOCATION_LAST_CW_0(info), + WRITE, rdata); + curr_ce++; + + } + } + if (ops->mode == MTD_OPS_AUTO_OOB) { + if (ops->datbuf) { + offset = 0; + size = BYTES_516; + last_read = 1; + rdata = (offset << 0) | (size << 16) | + (last_read << 31); + msm_nand_prep_ce(curr_ce, + MSM_NAND_READ_LOCATION_0(info), + WRITE, rdata); + curr_ce++; + size = (BYTES_512 - ((args->cwperpage - 1) << 2)); + last_read = (ops->oobbuf ? 0 : 1); + rdata = (offset << 0) | (size << 16) | + (last_read << 31); + msm_nand_prep_ce(curr_ce, + MSM_NAND_READ_LOCATION_LAST_CW_0(info), + WRITE, rdata); + curr_ce++; + } + + if (ops->oobbuf) { + last_read = 1; + offset = BYTES_512 - ((args->cwperpage - 1) << 2); + size = (args->cwperpage) << 2; + if (size > args->oob_len_cmd) + size = args->oob_len_cmd; + args->oob_len_cmd -= size; + rdata = (offset << 0) | (size << 16) | + (last_read << 31); + + if (!ops->datbuf) { + msm_nand_prep_ce(curr_ce, + MSM_NAND_READ_LOCATION_0(info), + WRITE, rdata); + curr_ce++; + msm_nand_prep_ce(curr_ce, + MSM_NAND_READ_LOCATION_LAST_CW_0(info), + WRITE, rdata); + curr_ce++; + } else { + msm_nand_prep_ce(curr_ce, + MSM_NAND_READ_LOCATION_LAST_CW_1(info), + WRITE, rdata); + curr_ce++; + } + } + } + + msm_nand_prep_ce(curr_ce, MSM_NAND_FLASH_CMD(info), WRITE, + data->cmd); + curr_ce++; + + *flags_ptr |= NWD; + msm_nand_prep_ce(curr_ce, MSM_NAND_EXEC_CMD(info), WRITE, data->exec); + curr_ce++; + *num_ce_ptr = curr_ce - start_ce; +} + +/* + * Function to submit read status descriptors to + * Data Producer Status Pipe during enhanced read Pagescope feature. + */ +static int msm_nand_submit_read_status_desc(struct mtd_oob_ops *ops, + struct msm_nand_rw_params *args, + struct msm_nand_info *info, + uint32_t curr_cw, + struct msm_nand_read_status_desc *status_desc) +{ + + struct msm_nand_chip *chip = &info->nand_chip; + struct sps_pipe *data_pipe_handle = NULL; + uint32_t sps_flags = 0; + int err = 0; + + /* + * As per QPIC2.0 HPG, Data Producer Status Pipe is used + * only to submit status descriptors for read page operations. + */ + if (args->read) + data_pipe_handle = info->sps.data_prod_stat.handle; + + if (ops->mode == MTD_OPS_RAW) { + if (args->read) { + if (curr_cw == (args->cwperpage - 1)) + sps_flags |= SPS_IOVEC_FLAG_INT; + + err = sps_transfer_one(data_pipe_handle, + msm_virt_to_dma(chip, status_desc), + sizeof(*status_desc), NULL, sps_flags); + if (err) + goto out; + } + } else if (ops->mode == MTD_OPS_AUTO_OOB) { + if (ops->datbuf) { + if ((curr_cw == (args->cwperpage - 1)) && !ops->oobbuf) + sps_flags |= SPS_IOVEC_FLAG_INT; + + err = sps_transfer_one(data_pipe_handle, + msm_virt_to_dma(chip, status_desc), + sizeof(*status_desc), NULL, sps_flags); + if (err) + goto out; + } + if (ops->oobbuf && (curr_cw == (args->cwperpage - 1))) { + status_desc++; + sps_flags |= SPS_IOVEC_FLAG_INT; + err = sps_transfer_one(data_pipe_handle, + msm_virt_to_dma(chip, status_desc), + sizeof(*status_desc), NULL, sps_flags); + if (err) + goto out; + } + } +out: + if (err) + pr_err("Failed to submit status descriptor for codeword=%d\n", + curr_cw); + return err; +} + +/* + * Refer msm_nand_is_erased_page() for comments. + * It only differs from it in using pagescope read commands. + */ +static int msm_nand_is_erased_page_ps(struct mtd_info *mtd, loff_t from, + struct mtd_oob_ops *ops, + struct msm_nand_rw_params *rw_params, + bool *erased_page) +{ + struct msm_nand_info *info = mtd->priv; + struct msm_nand_chip *chip = &info->nand_chip; + uint32_t cwperpage = (mtd->writesize >> 9); + int err, submitted_num_desc = 0; + uint32_t n = 0, num_zero_bits = 0, total_ecc_byte_cnt; + struct msm_nand_rw_reg_data data; + struct sps_iovec *iovec; + struct msm_nand_sps_cmd *sps_cmd; + struct sps_iovec iovec_temp; + struct mtd_oob_ops raw_ops; + + /* + * The following commands will be sent only once, for every single + * page read operation using pagescope feature - addr0, addr1, + * dev0_cfg0, dev0_cfg1, dev0_ecc_cfg, auto_status, flash, + * read_location_0, read_location_1, read_location_last_cw_0, + * read_location_last_cw_1, exec. + */ + struct msm_nand_rw_cmd_desc *cmd_list = NULL; + struct msm_nand_read_status_desc *status_desc = NULL; + uint32_t flash_cmd = 0x0; + struct { + struct msm_nand_sps_cmd cmd; + struct sps_transfer xfer; + struct sps_iovec cmd_iovec[MAX_DESC]; + struct { + uint32_t count; + struct msm_nand_cmd_setup_desc setup_desc; + struct msm_nand_cmd_cw_desc cw_desc[MAX_DESC]; + } cmd_list; + struct { + uint32_t flash_status; + uint32_t buffer_status; + uint32_t erased_cw_status; + /* This extra +1 is for oobbuf case */ + } result[MAX_CW_PER_PAGE + 1]; + } *dma_buffer; + uint8_t *ecc, *ecc_temp; + + total_ecc_byte_cnt = (chip->ecc_parity_bytes * cwperpage); + memcpy(&raw_ops, ops, sizeof(struct mtd_oob_ops)); + raw_ops.mode = MTD_OPS_RAW; + ecc = kzalloc(total_ecc_byte_cnt, GFP_KERNEL); + + wait_event(chip->dma_wait_queue, (dma_buffer = msm_nand_get_dma_buffer( + chip, sizeof(*dma_buffer)))); + + sps_cmd = &dma_buffer->cmd; + memset(&data, 0, sizeof(struct msm_nand_rw_reg_data)); + msm_nand_update_rw_reg_data(chip, &raw_ops, rw_params, &data); + + if (rw_params->read) { + if (raw_ops.mode != MTD_OPS_RAW) + data.cmd = MSM_NAND_CMD_PAGE_READ_ECC_PS; + else + data.cmd = MSM_NAND_CMD_PAGE_READ_ALL_PS; + } + + cmd_list = (struct msm_nand_rw_cmd_desc *)&dma_buffer->cmd_list; + status_desc = + (struct msm_nand_read_status_desc *)&dma_buffer->result[0]; + + /* map the ecc for dma operations */ + rw_params->ecc_dma_addr_curr = rw_params->ecc_dma_addr = + dma_map_single(chip->dev, ecc, total_ecc_byte_cnt, + DMA_FROM_DEVICE); + + data.addr0 = (rw_params->page << 16) | rw_params->oob_col; + data.addr1 = (rw_params->page >> 16) & 0xff; + for (n = rw_params->start_sector; n < cwperpage; n++) { + dma_buffer->result[n].flash_status = 0xeeeeeeee; + dma_buffer->result[n].buffer_status = 0xeeeeeeee; + dma_buffer->result[n].erased_cw_status = 0xeeeeee00; + } + msm_nand_prep_read_cmd_desc_pagescope(&raw_ops, rw_params, &data, + info, cmd_list, chip->ecc_parity_bytes); + + dma_buffer->xfer.iovec_count = cmd_list->count; + dma_buffer->xfer.iovec = dma_buffer->cmd_iovec; + dma_buffer->xfer.iovec_phys = msm_virt_to_dma(chip, + &dma_buffer->cmd_iovec); + iovec = dma_buffer->xfer.iovec; + + iovec->addr = msm_virt_to_dma(chip, + &cmd_list->setup_desc.ce[0]); + iovec->size = sizeof(struct sps_command_element) * + cmd_list->setup_desc.num_ce; + iovec->flags = cmd_list->setup_desc.flags; + iovec++; + for (n = 0; n < (cmd_list->count - 1); n++) { + iovec->addr = msm_virt_to_dma(chip, + &cmd_list->cw_desc[n].ce[0]); + iovec->size = sizeof(struct sps_command_element) * + cmd_list->cw_desc[n].num_ce; + iovec->flags = cmd_list->cw_desc[n].flags; + iovec++; + } + mutex_lock(&info->lock); + err = msm_nand_get_device(chip->dev); + if (err) + goto unlock_mutex; + /* Submit data descriptors */ + for (n = rw_params->start_sector; n < cwperpage; n++) { + err = msm_nand_submit_rw_data_desc(&raw_ops, + rw_params, info, n, + chip->ecc_parity_bytes); + if (err) { + pr_err("Failed to submit data descs %d\n", err); + panic("error in nand driver\n"); + goto put_dev; + } + } + submitted_num_desc = cwperpage - rw_params->start_sector; + /* Submit Data Status Descriptors */ + for (n = rw_params->start_sector; n < cwperpage; n++) { + err = msm_nand_submit_read_status_desc(&raw_ops, + rw_params, info, + n, status_desc); + if (err) { + pr_err("Failed to submit data status descs %d\n", err); + panic("error in nand driver\n"); + goto put_dev; + } + status_desc++; + } + + /* Submit command descriptors */ + err = sps_transfer(info->sps.cmd_pipe.handle, + &dma_buffer->xfer); + if (err) { + pr_err("Failed to submit commands %d\n", err); + goto put_dev; + } + /* Poll for command descriptors completion */ + err = msm_nand_sps_get_iovec(info->sps.cmd_pipe.handle, + info->sps.cmd_pipe.index, + dma_buffer->xfer.iovec_count, + &iovec_temp); + if (err) { + pr_err("Failed to get iovec for pipe %d: (err:%d)\n", + (info->sps.cmd_pipe.index), err); + goto put_dev; + } + /* Poll for data descriptors completion */ + err = msm_nand_sps_get_iovec(info->sps.data_prod.handle, + info->sps.data_prod.index, submitted_num_desc, + &iovec_temp); + if (err) { + pr_err("Failed to get iovec for pipe %d: (err:%d)\n", + (info->sps.data_prod.index), err); + goto put_dev; + } + /* + * Poll for data status descriptors completion + * the number of desc. is same as data desc. + */ + err = msm_nand_sps_get_iovec(info->sps.data_prod_stat.handle, + info->sps.data_prod_stat.index, submitted_num_desc, + &iovec_temp); + if (err) { + pr_err("Failed to get iovec for pipe %d: (err:%d)\n", + (info->sps.data_prod_stat.index), err); + goto put_dev; + } + /* + * There is a H/W BUG in qpic 2.0. You should unlock the command + * pipe only after all the status descriptors are collected on + * status descriptor pipe (pipe#3). + */ + + /* Unlock the command pipe now */ + msm_nand_prep_single_desc(sps_cmd, MSM_NAND_AUTO_STATUS_EN(info), + WRITE, flash_cmd, INT_UNLCK); + err = sps_transfer_one(info->sps.cmd_pipe.handle, + msm_virt_to_dma(chip, &sps_cmd->ce), + sizeof(struct sps_command_element), NULL, + sps_cmd->flags); + if (err) { + pr_err("Failed to unlock cmd desc. pipe: %d\n", err); + goto put_dev; + } + err = msm_nand_sps_get_iovec(info->sps.cmd_pipe.handle, + info->sps.cmd_pipe.index, 1, &iovec_temp); + if (err) { + pr_err("Failed to get iovec for cmd desc. err:%d\n", err); + goto put_dev; + } + err = msm_nand_put_device(chip->dev); + mutex_unlock(&info->lock); + if (err) + goto free_dma; + + pr_debug("addr0: 0x%08x, addr1: 0x%08x\n", data.addr0, data.addr1); + for (n = rw_params->start_sector; n < cwperpage; n++) + pr_debug("cw %d: flash_sts %x buffr_sts %x, erased_cw_status: %x\n", + n, dma_buffer->result[n].flash_status, + dma_buffer->result[n].buffer_status, + dma_buffer->result[n].erased_cw_status); + + goto free_dma; +put_dev: + msm_nand_put_device(chip->dev); +unlock_mutex: + mutex_unlock(&info->lock); +free_dma: + msm_nand_release_dma_buffer(chip, dma_buffer, sizeof(*dma_buffer)); + /* unmap ecc dma memory */ + dma_unmap_single(chip->dev, rw_params->ecc_dma_addr, + total_ecc_byte_cnt, DMA_FROM_DEVICE); + /* check for bit flips in ecc data */ + ecc_temp = ecc; + for (n = rw_params->start_sector; n < cwperpage; n++) { + int last_pos = 0, next_pos = 0; + int ecc_bytes_percw_in_bits = (chip->ecc_parity_bytes * 8); + + do { + last_pos = find_next_zero_bit((void *)ecc_temp, + ecc_bytes_percw_in_bits, next_pos); + + if (last_pos < ecc_bytes_percw_in_bits) + num_zero_bits++; + + if (num_zero_bits > 4) { + *erased_page = false; + goto free_mem; + } + + next_pos = last_pos + 1; + } while (last_pos < ecc_bytes_percw_in_bits); + + num_zero_bits = last_pos = next_pos = 0; + ecc_temp += chip->ecc_parity_bytes; + } + + if ((n == cwperpage) && (num_zero_bits <= 4)) + *erased_page = true; +free_mem: + kfree(ecc); + return err; +} + +/* + * Function that gets called from upper layers such as MTD/YAFFS2 to read a + * page with main or/and spare data. + * Function to be called for enhanced read pagescope feature. + */ +static int msm_nand_read_pagescope(struct mtd_info *mtd, loff_t from, + struct mtd_oob_ops *ops) +{ + struct msm_nand_info *info = mtd->priv; + struct msm_nand_chip *chip = &info->nand_chip; + struct flash_identification *flash_dev = &info->flash_dev; + uint32_t cwperpage = (mtd->writesize >> 9); + int err = 0, pageerr = 0, rawerr = 0, submitted_num_desc = 0; + uint32_t n = 0, pages_read = 0, flash_cmd = 0x0; + uint32_t ecc_errors = 0, total_ecc_errors = 0, ecc_capability; + struct msm_nand_rw_params rw_params; + struct msm_nand_rw_reg_data data; + struct sps_iovec *iovec; + struct msm_nand_sps_cmd *sps_cmd; + struct sps_iovec iovec_temp; + bool erased_page; + uint64_t fix_data_in_pages = 0; + + /* + * The following commands will be sent only once, for every single + * page read operation using pagescope feature - addr0, addr1, + * dev0_cfg0, dev0_cfg1, dev0_ecc_cfg, auto_status, flash, + * read_location_0, read_location_1, read_location_last_cw_0, + * read_location_last_cw_1, exec. + */ + struct { + struct msm_nand_sps_cmd cmd; + struct sps_transfer xfer; + struct sps_iovec cmd_iovec[MAX_DESC]; + struct { + uint32_t count; + struct msm_nand_cmd_setup_desc setup_desc; + struct msm_nand_cmd_cw_desc cw_desc[MAX_DESC]; + } cmd_list; + struct { + uint32_t flash_status; + uint32_t buffer_status; + uint32_t erased_cw_status; + /* This extra +1 is for oobbuf case */ + } result[MAX_CW_PER_PAGE + 1]; + } *dma_buffer; + struct msm_nand_rw_cmd_desc *cmd_list = NULL; + struct msm_nand_read_status_desc *status_desc = NULL; + + memset(&rw_params, 0, sizeof(struct msm_nand_rw_params)); + err = msm_nand_validate_mtd_params(mtd, true, from, ops, &rw_params); + if (err) + goto validate_mtd_params_failed; + + wait_event(chip->dma_wait_queue, (dma_buffer = msm_nand_get_dma_buffer( + chip, sizeof(*dma_buffer)))); + + rw_params.oob_col = rw_params.start_sector * chip->cw_size; + if (chip->cfg1 & (1 << WIDE_FLASH)) + rw_params.oob_col >>= 1; + sps_cmd = &dma_buffer->cmd; + + memset(&data, 0, sizeof(struct msm_nand_rw_reg_data)); + msm_nand_update_rw_reg_data(chip, ops, &rw_params, &data); + + if (rw_params.read) { + if (ops->mode != MTD_OPS_RAW) + data.cmd = MSM_NAND_CMD_PAGE_READ_ECC_PS; + else + data.cmd = MSM_NAND_CMD_PAGE_READ_ALL_PS; + } + + cmd_list = (struct msm_nand_rw_cmd_desc *)&dma_buffer->cmd_list; + status_desc = + (struct msm_nand_read_status_desc *)&dma_buffer->result[0]; + ecc_capability = flash_dev->ecc_capability; + + while (rw_params.page_count-- > 0) { + + erased_page = false; + data.addr0 = (rw_params.page << 16) | rw_params.oob_col; + data.addr1 = (rw_params.page >> 16) & 0xff; + + for (n = rw_params.start_sector; n < cwperpage; n++) { + dma_buffer->result[n].flash_status = 0xeeeeeeee; + dma_buffer->result[n].buffer_status = 0xeeeeeeee; + dma_buffer->result[n].erased_cw_status = 0xeeeeee00; + } + msm_nand_prep_read_cmd_desc_pagescope(ops, &rw_params, + &data, info, + cmd_list, 0); + dma_buffer->xfer.iovec_count = cmd_list->count; + dma_buffer->xfer.iovec = dma_buffer->cmd_iovec; + dma_buffer->xfer.iovec_phys = msm_virt_to_dma(chip, + &dma_buffer->cmd_iovec); + iovec = dma_buffer->xfer.iovec; + + iovec->addr = msm_virt_to_dma(chip, + &cmd_list->setup_desc.ce[0]); + iovec->size = sizeof(struct sps_command_element) * + cmd_list->setup_desc.num_ce; + iovec->flags = cmd_list->setup_desc.flags; + iovec++; + for (n = 0; n < (cmd_list->count - 1); n++) { + iovec->addr = msm_virt_to_dma(chip, + &cmd_list->cw_desc[n].ce[0]); + iovec->size = sizeof(struct sps_command_element) * + cmd_list->cw_desc[n].num_ce; + iovec->flags = cmd_list->cw_desc[n].flags; + iovec++; + } + mutex_lock(&info->lock); + err = msm_nand_get_device(chip->dev); + if (err) + goto unlock_mutex; + /* Submit data descriptors */ + for (n = rw_params.start_sector; n < cwperpage; n++) { + err = msm_nand_submit_rw_data_desc(ops, + &rw_params, info, n, 0); + if (err) { + pr_err("Failed to submit data descs %d\n", err); + panic("error in nand driver\n"); + goto put_dev; + } + } + if (ops->mode == MTD_OPS_RAW) { + submitted_num_desc = cwperpage - rw_params.start_sector; + } else if (ops->mode == MTD_OPS_AUTO_OOB) { + if (ops->datbuf) + submitted_num_desc = cwperpage - + rw_params.start_sector; + if (ops->oobbuf) + submitted_num_desc++; + } + /* Submit Data Status Descriptors */ + for (n = rw_params.start_sector; n < cwperpage; n++) { + err = msm_nand_submit_read_status_desc(ops, + &rw_params, info, + n, status_desc); + if (err) { + pr_err("Failed to submit data status descs %d\n", + err); + panic("error in nand driver\n"); + goto put_dev; + } + status_desc++; + } + /* Submit command descriptors */ + err = sps_transfer(info->sps.cmd_pipe.handle, + &dma_buffer->xfer); + if (err) { + pr_err("Failed to submit commands %d\n", err); + goto put_dev; + } + /* Poll for command descriptors completion */ + err = msm_nand_sps_get_iovec(info->sps.cmd_pipe.handle, + info->sps.cmd_pipe.index, + dma_buffer->xfer.iovec_count, + &iovec_temp); + if (err) { + pr_err("Failed to get iovec for pipe %d: (err: %d)\n", + (info->sps.cmd_pipe.index), err); + goto put_dev; + } + /* Poll for data descriptors completion */ + err = msm_nand_sps_get_iovec(info->sps.data_prod.handle, + info->sps.data_prod.index, submitted_num_desc, + &iovec_temp); + if (err) { + pr_err("Failed to get iovec for pipe %d: (err: %d)\n", + (info->sps.data_prod.index), err); + goto put_dev; + } + /* + * Poll for data status descriptors completion + * the number of desc. is same as data desc. + */ + err = msm_nand_sps_get_iovec(info->sps.data_prod_stat.handle, + info->sps.data_prod_stat.index, submitted_num_desc, + &iovec_temp); + if (err) { + pr_err("Failed to get iovec for pipe %d: (err: %d)\n", + (info->sps.data_prod_stat.index), err); + goto put_dev; + } + /* + * There is a H/W BUG in qpic 2.0. You should unlock the command + * pipe only after all the status descriptors are collected on + * status descriptor pipe (pipe#3). + */ + + /* Unlock the command pipe now */ + msm_nand_prep_single_desc(sps_cmd, + MSM_NAND_AUTO_STATUS_EN(info), + WRITE, flash_cmd, INT_UNLCK); + err = sps_transfer_one(info->sps.cmd_pipe.handle, + msm_virt_to_dma(chip, &sps_cmd->ce), + sizeof(struct sps_command_element), + NULL, sps_cmd->flags); + if (err) { + pr_err("Failed to unlock cmd desc. pipe: %d\n", err); + goto put_dev; + } + err = msm_nand_sps_get_iovec(info->sps.cmd_pipe.handle, + info->sps.cmd_pipe.index, 1, &iovec_temp); + if (err) { + pr_err("Failed to get iovec for cmd desc. err:%d\n", + err); + goto put_dev; + } + err = msm_nand_put_device(chip->dev); + mutex_unlock(&info->lock); + if (err) + goto free_dma; + /* Check for flash status errors */ + pageerr = rawerr = 0; + for (n = rw_params.start_sector; n < cwperpage; n++) { + if (dma_buffer->result[n].flash_status & (FS_OP_ERR | + FS_MPU_ERR)) { + rawerr = -EIO; + /* + * Check if ECC error was due to an erased + * codeword. If so, ignore the error. + * + * NOTE: There is a bug in erased page + * detection hardware block when reading + * only spare data. In order to work around + * this issue, instead of using PAGE_ALL_ERASED + * bit to check for whether a whole page is + * erased or not, we use CODEWORD_ALL_ERASED + * and CODEWORD_ERASED bits together and check + * each codeword that has FP_OP_ERR bit set is + * an erased codeword or not. + */ + if ((dma_buffer->result[n].erased_cw_status & + ERASED_CW) == ERASED_CW) { + /* + * At least one code word is detected + * as an erased code word. + */ + pr_debug("erased codeword detected - ignore ecc error\n"); + continue; + } + pageerr = rawerr; + break; + } + } + /* check for uncorrectable errors */ + if (pageerr) { + for (n = rw_params.start_sector; n < cwperpage; n++) { + if (dma_buffer->result[n].buffer_status & + BS_UNCORRECTABLE_BIT) { + /* + * Check if page is actually + * erased or not. + */ + err = msm_nand_is_erased_page_ps(mtd, + from, ops, + &rw_params, + &erased_page); + if (err) + goto free_dma; + if (!erased_page) { + mtd->ecc_stats.failed++; + pageerr = -EBADMSG; + break; + } + pageerr = 0; + pr_debug("Uncorrectable ECC errors detected on an erased page and has been fixed.\n"); + break; + } + } + } + + if (rawerr && !pageerr && erased_page) { + /* + * This means an erased page had bit flips and now + * those bit-flips need to be cleared in the data + * being sent to upper layers. This will keep track + * of those pages and at the end, the data will be + * fixed before this function returns. + * Note that a whole page worth of data will be fixed + * and this will only handle about 64 pages being read + * at a time i.e. one erase block worth of pages. + */ + fix_data_in_pages |= BIT(rw_params.page_count); + } + /* check for correctable errors */ + if (!rawerr) { + for (n = rw_params.start_sector; n < cwperpage; n++) { + ecc_errors = + dma_buffer->result[n].buffer_status + & BS_CORRECTABLE_ERR_MSK; + if (ecc_errors) { + total_ecc_errors += ecc_errors; + mtd->ecc_stats.corrected += ecc_errors; + } + } + } + if (pageerr && (pageerr != -EUCLEAN || err == 0)) + err = pageerr; + + if (rawerr && !pageerr) { + pr_debug("%llx %x %x empty page\n", + (loff_t)rw_params.page * mtd->writesize, + ops->len, ops->ooblen); + } else { + for (n = rw_params.start_sector; n < cwperpage; n++) + pr_debug("cw %d: flash_sts %x buffr_sts %x, erased_cw_status: %x, pageerr: %d, rawerr: %d\n", + n, dma_buffer->result[n].flash_status, + dma_buffer->result[n].buffer_status, + dma_buffer->result[n].erased_cw_status, + pageerr, rawerr); + } + if (err && err != -EUCLEAN && err != -EBADMSG) + goto free_dma; + pages_read++; + rw_params.page++; + } + goto free_dma; +put_dev: + msm_nand_put_device(chip->dev); +unlock_mutex: + mutex_unlock(&info->lock); +free_dma: + msm_nand_release_dma_buffer(chip, dma_buffer, sizeof(*dma_buffer)); + if (ops->oobbuf) + dma_unmap_page(chip->dev, rw_params.oob_dma_addr, + ops->ooblen, DMA_FROM_DEVICE); + if (ops->datbuf) + dma_unmap_page(chip->dev, rw_params.data_dma_addr, + ops->len, DMA_BIDIRECTIONAL); + /* + * If there were any erased pages detected with ECC errors, then + * it is most likely that the data is not all 0xff. So memset that + * page to all 0xff. + */ + while (fix_data_in_pages) { + int temp_page = 0, oobsize = rw_params.cwperpage << 2; + int count = 0, offset = 0; + + temp_page = fix_data_in_pages & BIT_MASK(0); + fix_data_in_pages = fix_data_in_pages >> 1; + count++; + if (!temp_page) + continue; + offset = (count - 1) * mtd->writesize; + if (ops->datbuf) + memset((ops->datbuf + offset), 0xff, mtd->writesize); + offset = (count - 1) * oobsize; + if (ops->oobbuf) + memset(ops->oobbuf + offset, 0xff, oobsize); + } +validate_mtd_params_failed: + if (ops->mode != MTD_OPS_RAW) + ops->retlen = mtd->writesize * pages_read; + else + ops->retlen = (mtd->writesize + mtd->oobsize) * pages_read; + ops->oobretlen = ops->ooblen - rw_params.oob_len_data; + if (err) + pr_err("0x%llx datalen 0x%x ooblen %x err %d corrected %d\n", + from, ops->datbuf ? ops->len : 0, ops->ooblen, err, + total_ecc_errors); + pr_debug("ret %d, retlen %d oobretlen %d\n", + err, ops->retlen, ops->oobretlen); + return err; +} + +/* + * Read ECC bytes and check whether page is erased or not. + * + * The NAND devices manufactured with newer process node technology are + * susceptible to bit-flips. These bit-flips are easily fixable with the + * ECC engine and ECC information stored on the NAND device. This device + * specific information is found in the data sheet for the NAND device + * and is usually specified as a "number of bit-flips expected per code- + * word". For example, "a single bit-flip per codeword". Also this means + * that the number of ECC errors don't increase over period of time as in + * the past and can't be used to predict a "bad-block about to happen" + * situation anymore. + * + * So what this means to erased pages: + * Since ECC data for an erased page is all 0xFF's, the ECC engine would + * not be able to correct any bit-flips that occur in these newer parts. + * If the NAND controller is unable to identify the erased page due to + * the bit-flips, then there would be "uncorrectable ECC errors" detected + * and would get reported to file system layer (YAFFS2/UBIFS etc) and would + * result in a good block being marked as a bad block and also lead to + * error scenarios. + + * So to handle this, the following will be done by software until newer + * NAND controller hardware is avialable that can detected erased pages + * with bit-flips successfully. + * + * 1. msm_nand_read_oob() calls this function when "uncorrectable ECC + * errors" occur. + * 2. This function then performs a raw read of the page. + * 3. This read is done to extract ECC bytes and not data from that page. + * 4. For each codeword’s ECC data, the following is done + * a. Count number of zero bits + * b. If that count is greater than , then it is + * not an erased page. + * c. Else repeat for next codeword’s ECC data + * d. If all codewords have less than bits of + * zeros, then it’s considered an erased page. + * + * Since "uncorrectable ECC errors" do not occur except for either an + * erased page or in the case of an actual errror, this solution would + * work. + * + */ +static int msm_nand_is_erased_page(struct mtd_info *mtd, loff_t from, + struct mtd_oob_ops *ops, + struct msm_nand_rw_params *rw_params, + bool *erased_page) +{ + struct msm_nand_info *info = mtd->priv; + struct msm_nand_chip *chip = &info->nand_chip; + uint32_t cwperpage = (mtd->writesize >> 9); + int err, submitted_num_desc = 0; + uint32_t n = 0, num_zero_bits = 0, total_ecc_byte_cnt; + struct msm_nand_rw_reg_data data; + struct sps_iovec *iovec; + struct sps_iovec iovec_temp; + struct mtd_oob_ops raw_ops; + + /* + * The following 6 commands will be sent only once for the first + * codeword (CW) - addr0, addr1, dev0_cfg0, dev0_cfg1, + * dev0_ecc_cfg, ebi2_ecc_buf_cfg. The following 6 commands will + * be sent for every CW - flash, read_location_0, read_location_1, + * exec, flash_status and buffer_status. + */ + struct msm_nand_rw_cmd_desc *cmd_list = NULL; + uint32_t cw_desc_cnt = 0; + struct { + struct sps_transfer xfer; + struct sps_iovec cmd_iovec[MAX_DESC]; + struct { + uint32_t count; + struct msm_nand_cmd_setup_desc setup_desc; + struct msm_nand_cmd_cw_desc cw_desc[MAX_DESC - 1]; + } cmd_list; + struct { + uint32_t flash_status; + uint32_t buffer_status; + uint32_t erased_cw_status; + } result[MAX_CW_PER_PAGE]; + } *dma_buffer; + uint8_t *ecc, *ecc_temp; + + pr_debug("========================================================\n"); + total_ecc_byte_cnt = (chip->ecc_parity_bytes * cwperpage); + memcpy(&raw_ops, ops, sizeof(struct mtd_oob_ops)); + raw_ops.mode = MTD_OPS_RAW; + ecc = kzalloc(total_ecc_byte_cnt, GFP_KERNEL); + + wait_event(chip->dma_wait_queue, (dma_buffer = msm_nand_get_dma_buffer( + chip, sizeof(*dma_buffer)))); + + memset(&data, 0, sizeof(struct msm_nand_rw_reg_data)); + msm_nand_update_rw_reg_data(chip, &raw_ops, rw_params, &data); + cmd_list = (struct msm_nand_rw_cmd_desc *)&dma_buffer->cmd_list; + + /* map the ecc for dma operations */ + rw_params->ecc_dma_addr_curr = rw_params->ecc_dma_addr = + dma_map_single(chip->dev, ecc, total_ecc_byte_cnt, + DMA_FROM_DEVICE); + + data.addr0 = (rw_params->page << 16) | rw_params->oob_col; + data.addr1 = (rw_params->page >> 16) & 0xff; + for (n = rw_params->start_sector; n < cwperpage; n++) { + struct sps_command_element *curr_ce, *start_ce; + + dma_buffer->result[n].flash_status = 0xeeeeeeee; + dma_buffer->result[n].buffer_status = 0xeeeeeeee; + dma_buffer->result[n].erased_cw_status = 0xeeeeee00; + + msm_nand_prep_rw_cmd_desc(&raw_ops, rw_params, &data, info, + n, cmd_list, &cw_desc_cnt, + chip->ecc_parity_bytes); + + start_ce = &cmd_list->cw_desc[cw_desc_cnt].ce[0]; + curr_ce = start_ce; + cmd_list->cw_desc[cw_desc_cnt].flags = CMD; + if (n == (cwperpage - 1)) + cmd_list->cw_desc[cw_desc_cnt].flags |= + INT_UNLCK; + cmd_list->count++; + + msm_nand_prep_ce(curr_ce, MSM_NAND_FLASH_STATUS(info), + READ, msm_virt_to_dma(chip, + &dma_buffer->result[n].flash_status)); + curr_ce++; + + msm_nand_prep_ce(curr_ce, MSM_NAND_BUFFER_STATUS(info), + READ, msm_virt_to_dma(chip, + &dma_buffer->result[n].buffer_status)); + curr_ce++; + + msm_nand_prep_ce(curr_ce, + MSM_NAND_ERASED_CW_DETECT_STATUS(info), + READ, msm_virt_to_dma(chip, + &dma_buffer->result[n].erased_cw_status)); + curr_ce++; + cmd_list->cw_desc[cw_desc_cnt++].num_ce = curr_ce - + start_ce; + } + + dma_buffer->xfer.iovec_count = cmd_list->count; + dma_buffer->xfer.iovec = dma_buffer->cmd_iovec; + dma_buffer->xfer.iovec_phys = msm_virt_to_dma(chip, + &dma_buffer->cmd_iovec); + iovec = dma_buffer->xfer.iovec; + + iovec->addr = msm_virt_to_dma(chip, + &cmd_list->setup_desc.ce[0]); + iovec->size = sizeof(struct sps_command_element) * + cmd_list->setup_desc.num_ce; + iovec->flags = cmd_list->setup_desc.flags; + iovec++; + for (n = 0; n < (cmd_list->count - 1); n++) { + iovec->addr = msm_virt_to_dma(chip, + &cmd_list->cw_desc[n].ce[0]); + iovec->size = sizeof(struct sps_command_element) * + cmd_list->cw_desc[n].num_ce; + iovec->flags = cmd_list->cw_desc[n].flags; + iovec++; + } + mutex_lock(&info->lock); + err = msm_nand_get_device(chip->dev); + if (err) + goto unlock_mutex; + /* Submit data descriptors */ + for (n = rw_params->start_sector; n < cwperpage; n++) { + err = msm_nand_submit_rw_data_desc(&raw_ops, + rw_params, info, n, + chip->ecc_parity_bytes); + if (err) { + pr_err("Failed to submit data descs %d\n", err); + panic("error in nand driver\n"); + goto put_dev; + } + } + submitted_num_desc = cwperpage - rw_params->start_sector; + + /* Submit command descriptors */ + err = sps_transfer(info->sps.cmd_pipe.handle, + &dma_buffer->xfer); + if (err) { + pr_err("Failed to submit commands %d\n", err); + goto put_dev; + } + + err = msm_nand_sps_get_iovec(info->sps.cmd_pipe.handle, + info->sps.cmd_pipe.index, + dma_buffer->xfer.iovec_count, + &iovec_temp); + if (err) { + pr_err("Failed to get iovec for pipe %d: (err:%d)\n", + (info->sps.cmd_pipe.index), err); + goto put_dev; + } + err = msm_nand_sps_get_iovec(info->sps.data_prod.handle, + info->sps.data_prod.index, submitted_num_desc, + &iovec_temp); + if (err) { + pr_err("Failed to get iovec for pipe %d: (err:%d)\n", + (info->sps.data_prod.index), err); + goto put_dev; + } + + err = msm_nand_put_device(chip->dev); + mutex_unlock(&info->lock); + if (err) + goto free_dma; + + pr_debug("addr0: 0x%08x, addr1: 0x%08x\n", data.addr0, data.addr1); + for (n = rw_params->start_sector; n < cwperpage; n++) + pr_debug("cw %d: flash_sts %x buffr_sts %x, erased_cw_status: %x\n", + n, dma_buffer->result[n].flash_status, + dma_buffer->result[n].buffer_status, + dma_buffer->result[n].erased_cw_status); + + goto free_dma; +put_dev: + msm_nand_put_device(chip->dev); +unlock_mutex: + mutex_unlock(&info->lock); +free_dma: + msm_nand_release_dma_buffer(chip, dma_buffer, sizeof(*dma_buffer)); + /* umap ecc dma memory */ + dma_unmap_single(chip->dev, rw_params->ecc_dma_addr, + total_ecc_byte_cnt, DMA_FROM_DEVICE); + /* check for bit flips in ecc data */ + ecc_temp = ecc; + for (n = rw_params->start_sector; n < cwperpage; n++) { + int last_pos = 0, next_pos = 0; + int ecc_bytes_percw_in_bits = (chip->ecc_parity_bytes * 8); + + do { + last_pos = find_next_zero_bit((void *)ecc_temp, + ecc_bytes_percw_in_bits, next_pos); + + if (last_pos < ecc_bytes_percw_in_bits) + num_zero_bits++; + + if (num_zero_bits > 4) { + *erased_page = false; + goto free_mem; + } + + next_pos = last_pos + 1; + } while (last_pos < ecc_bytes_percw_in_bits); + + num_zero_bits = last_pos = next_pos = 0; + ecc_temp += chip->ecc_parity_bytes; + } + + if ((n == cwperpage) && (num_zero_bits <= 4)) + *erased_page = true; +free_mem: + kfree(ecc); + pr_debug("========================================================\n"); + return err; +} + +/* + * Function that gets called from upper layers such as MTD/YAFFS2 to read a + * page with main or/and spare data. + */ +static int msm_nand_read_oob(struct mtd_info *mtd, loff_t from, + struct mtd_oob_ops *ops) +{ + struct msm_nand_info *info = mtd->priv; + struct msm_nand_chip *chip = &info->nand_chip; + struct flash_identification *flash_dev = &info->flash_dev; + uint32_t cwperpage = (mtd->writesize >> 9); + int err, pageerr = 0, rawerr = 0, submitted_num_desc = 0; + uint32_t n = 0, pages_read = 0; + uint32_t ecc_errors = 0, total_ecc_errors = 0, ecc_capability; + struct msm_nand_rw_params rw_params; + struct msm_nand_rw_reg_data data; + struct sps_iovec *iovec; + struct sps_iovec iovec_temp; + bool erased_page; + uint64_t fix_data_in_pages = 0; + + /* + * The following 6 commands will be sent only once for the first + * codeword (CW) - addr0, addr1, dev0_cfg0, dev0_cfg1, + * dev0_ecc_cfg, ebi2_ecc_buf_cfg. The following 6 commands will + * be sent for every CW - flash, read_location_0, read_location_1, + * exec, flash_status and buffer_status. + */ + struct { + struct sps_transfer xfer; + struct sps_iovec cmd_iovec[MAX_DESC]; + struct { + uint32_t count; + struct msm_nand_cmd_setup_desc setup_desc; + struct msm_nand_cmd_cw_desc cw_desc[MAX_DESC - 1]; + } cmd_list; + struct { + uint32_t flash_status; + uint32_t buffer_status; + uint32_t erased_cw_status; + } result[MAX_CW_PER_PAGE]; + } *dma_buffer; + struct msm_nand_rw_cmd_desc *cmd_list = NULL; + + memset(&rw_params, 0, sizeof(struct msm_nand_rw_params)); + err = msm_nand_validate_mtd_params(mtd, true, from, ops, &rw_params); + if (err) + goto validate_mtd_params_failed; + + wait_event(chip->dma_wait_queue, (dma_buffer = msm_nand_get_dma_buffer( + chip, sizeof(*dma_buffer)))); + + rw_params.oob_col = rw_params.start_sector * chip->cw_size; + if (chip->cfg1 & (1 << WIDE_FLASH)) + rw_params.oob_col >>= 1; + + memset(&data, 0, sizeof(struct msm_nand_rw_reg_data)); + msm_nand_update_rw_reg_data(chip, ops, &rw_params, &data); + cmd_list = (struct msm_nand_rw_cmd_desc *)&dma_buffer->cmd_list; + + ecc_capability = flash_dev->ecc_capability; + + while (rw_params.page_count-- > 0) { + uint32_t cw_desc_cnt = 0; + + erased_page = false; + data.addr0 = (rw_params.page << 16) | rw_params.oob_col; + data.addr1 = (rw_params.page >> 16) & 0xff; + + for (n = rw_params.start_sector; n < cwperpage; n++) { + struct sps_command_element *curr_ce, *start_ce; + + dma_buffer->result[n].flash_status = 0xeeeeeeee; + dma_buffer->result[n].buffer_status = 0xeeeeeeee; + dma_buffer->result[n].erased_cw_status = 0xeeeeee00; + + msm_nand_prep_rw_cmd_desc(ops, &rw_params, &data, info, + n, cmd_list, &cw_desc_cnt, 0); + + start_ce = &cmd_list->cw_desc[cw_desc_cnt].ce[0]; + curr_ce = start_ce; + cmd_list->cw_desc[cw_desc_cnt].flags = CMD; + if (n == (cwperpage - 1)) + cmd_list->cw_desc[cw_desc_cnt].flags |= + INT_UNLCK; + cmd_list->count++; + + msm_nand_prep_ce(curr_ce, MSM_NAND_FLASH_STATUS(info), + READ, msm_virt_to_dma(chip, + &dma_buffer->result[n].flash_status)); + curr_ce++; + + msm_nand_prep_ce(curr_ce, MSM_NAND_BUFFER_STATUS(info), + READ, msm_virt_to_dma(chip, + &dma_buffer->result[n].buffer_status)); + curr_ce++; + + msm_nand_prep_ce(curr_ce, + MSM_NAND_ERASED_CW_DETECT_STATUS(info), + READ, msm_virt_to_dma(chip, + &dma_buffer->result[n].erased_cw_status)); + curr_ce++; + cmd_list->cw_desc[cw_desc_cnt++].num_ce = curr_ce - + start_ce; + } + + dma_buffer->xfer.iovec_count = cmd_list->count; + dma_buffer->xfer.iovec = dma_buffer->cmd_iovec; + dma_buffer->xfer.iovec_phys = msm_virt_to_dma(chip, + &dma_buffer->cmd_iovec); + iovec = dma_buffer->xfer.iovec; + + iovec->addr = msm_virt_to_dma(chip, + &cmd_list->setup_desc.ce[0]); + iovec->size = sizeof(struct sps_command_element) * + cmd_list->setup_desc.num_ce; + iovec->flags = cmd_list->setup_desc.flags; + iovec++; + for (n = 0; n < (cmd_list->count - 1); n++) { + iovec->addr = msm_virt_to_dma(chip, + &cmd_list->cw_desc[n].ce[0]); + iovec->size = sizeof(struct sps_command_element) * + cmd_list->cw_desc[n].num_ce; + iovec->flags = cmd_list->cw_desc[n].flags; + iovec++; + } + mutex_lock(&info->lock); + err = msm_nand_get_device(chip->dev); + if (err) + goto unlock_mutex; + /* Submit data descriptors */ + for (n = rw_params.start_sector; n < cwperpage; n++) { + err = msm_nand_submit_rw_data_desc(ops, + &rw_params, info, n, 0); + if (err) { + pr_err("Failed to submit data descs %d\n", err); + panic("error in nand driver\n"); + goto put_dev; + } + } + + if (ops->mode == MTD_OPS_RAW) { + submitted_num_desc = cwperpage - rw_params.start_sector; + } else if (ops->mode == MTD_OPS_AUTO_OOB) { + if (ops->datbuf) + submitted_num_desc = cwperpage - + rw_params.start_sector; + if (ops->oobbuf) + submitted_num_desc++; + } + + /* Submit command descriptors */ + err = sps_transfer(info->sps.cmd_pipe.handle, + &dma_buffer->xfer); + if (err) { + pr_err("Failed to submit commands %d\n", err); + goto put_dev; + } + + err = msm_nand_sps_get_iovec(info->sps.cmd_pipe.handle, + info->sps.cmd_pipe.index, + dma_buffer->xfer.iovec_count, + &iovec_temp); + if (err) { + pr_err("Failed to get iovec for pipe %d: (err: %d)\n", + (info->sps.cmd_pipe.index), err); + goto put_dev; + } + err = msm_nand_sps_get_iovec(info->sps.data_prod.handle, + info->sps.data_prod.index, submitted_num_desc, + &iovec_temp); + if (err) { + pr_err("Failed to get iovec for pipe %d: (err: %d)\n", + (info->sps.data_prod.index), err); + goto put_dev; + } + + err = msm_nand_put_device(chip->dev); + mutex_unlock(&info->lock); + if (err) + goto free_dma; + /* Check for flash status errors */ + pageerr = rawerr = 0; + for (n = rw_params.start_sector; n < cwperpage; n++) { + if (dma_buffer->result[n].flash_status & (FS_OP_ERR | + FS_MPU_ERR)) { + rawerr = -EIO; + /* + * Check if ECC error was due to an erased + * codeword. If so, ignore the error. + * + * NOTE: There is a bug in erased page + * detection hardware block when reading + * only spare data. In order to work around + * this issue, instead of using PAGE_ALL_ERASED + * bit to check for whether a whole page is + * erased or not, we use CODEWORD_ALL_ERASED + * and CODEWORD_ERASED bits together and check + * each codeword that has FP_OP_ERR bit set is + * an erased codeword or not. + */ + if ((dma_buffer->result[n].erased_cw_status & + ERASED_CW) == ERASED_CW) { + /* + * At least one code word is detected + * as an erased code word. + */ + pr_debug("erased codeword detected - ignore ecc error\n"); + continue; + } + pageerr = rawerr; + break; + } + } + /* check for uncorrectable errors */ + if (pageerr) { + for (n = rw_params.start_sector; n < cwperpage; n++) { + if (dma_buffer->result[n].buffer_status & + BS_UNCORRECTABLE_BIT) { + /* + * Check if page is actually + * erased or not. + */ + err = msm_nand_is_erased_page(mtd, + from, ops, + &rw_params, + &erased_page); + if (err) + goto free_dma; + if (!erased_page) { + mtd->ecc_stats.failed++; + pageerr = -EBADMSG; + break; + } + pageerr = 0; + pr_debug("Uncorrectable ECC errors detected on an erased page and has been fixed.\n"); + break; + } + } + } + + if (rawerr && !pageerr && erased_page) { + /* + * This means an erased page had bit flips and now + * those bit-flips need to be cleared in the data + * being sent to upper layers. This will keep track + * of those pages and at the end, the data will be + * fixed before this function returns. + * Note that a whole page worth of data will be fixed + * and this will only handle about 64 pages being read + * at a time i.e. one erase block worth of pages. + */ + fix_data_in_pages |= BIT(rw_params.page_count); + } + /* check for correctable errors */ + if (!rawerr) { + for (n = rw_params.start_sector; n < cwperpage; n++) { + ecc_errors = + dma_buffer->result[n].buffer_status + & BS_CORRECTABLE_ERR_MSK; + if (ecc_errors) { + total_ecc_errors += ecc_errors; + mtd->ecc_stats.corrected += ecc_errors; + } + } + } + if (pageerr && (pageerr != -EUCLEAN || err == 0)) + err = pageerr; + + if (rawerr && !pageerr) { + pr_debug("%llx %x %x empty page\n", + (loff_t)rw_params.page * mtd->writesize, + ops->len, ops->ooblen); + } else { + for (n = rw_params.start_sector; n < cwperpage; n++) + pr_debug("cw %d: flash_sts %x buffr_sts %x, erased_cw_status: %x, pageerr: %d, rawerr: %d\n", + n, dma_buffer->result[n].flash_status, + dma_buffer->result[n].buffer_status, + dma_buffer->result[n].erased_cw_status, + pageerr, rawerr); + } + if (err && err != -EUCLEAN && err != -EBADMSG) + goto free_dma; + pages_read++; + rw_params.page++; + } + goto free_dma; +put_dev: + msm_nand_put_device(chip->dev); +unlock_mutex: + mutex_unlock(&info->lock); +free_dma: + msm_nand_release_dma_buffer(chip, dma_buffer, sizeof(*dma_buffer)); + if (ops->oobbuf) + dma_unmap_page(chip->dev, rw_params.oob_dma_addr, + ops->ooblen, DMA_FROM_DEVICE); + if (ops->datbuf) + dma_unmap_page(chip->dev, rw_params.data_dma_addr, + ops->len, DMA_BIDIRECTIONAL); + /* + * If there were any erased pages detected with ECC errors, then + * it is most likely that the data is not all 0xff. So memset that + * page to all 0xff. + */ + while (fix_data_in_pages) { + int temp_page = 0, oobsize = rw_params.cwperpage << 2; + int count = 0, offset = 0; + + temp_page = fix_data_in_pages & BIT_MASK(0); + fix_data_in_pages = fix_data_in_pages >> 1; + count++; + + if (!temp_page) + continue; + + offset = (count - 1) * mtd->writesize; + if (ops->datbuf) + memset((ops->datbuf + offset), 0xff, mtd->writesize); + + offset = (count - 1) * oobsize; + if (ops->oobbuf) + memset(ops->oobbuf + offset, 0xff, oobsize); + } +validate_mtd_params_failed: + if (ops->mode != MTD_OPS_RAW) + ops->retlen = mtd->writesize * pages_read; + else + ops->retlen = (mtd->writesize + mtd->oobsize) * pages_read; + ops->oobretlen = ops->ooblen - rw_params.oob_len_data; + if (err) + pr_err("0x%llx datalen 0x%x ooblen %x err %d corrected %d\n", + from, ops->datbuf ? ops->len : 0, ops->ooblen, err, + total_ecc_errors); + pr_debug("ret %d, retlen %d oobretlen %d\n", + err, ops->retlen, ops->oobretlen); + + pr_debug("========================================================\n"); + return err; +} + +/** + * msm_nand_read_partial_page() - read partial page + * @mtd: pointer to mtd info + * @from: start address of the page + * @ops: pointer to mtd_oob_ops + * + * Reads a page into a bounce buffer and copies the required + * number of bytes to actual buffer. The pages that are aligned + * do not use bounce buffer. + */ +static int msm_nand_read_partial_page(struct mtd_info *mtd, + loff_t from, struct mtd_oob_ops *ops) +{ + int err = 0; + unsigned char *actual_buf; + unsigned char *bounce_buf; + loff_t aligned_from; + loff_t offset; + size_t len; + size_t actual_len, ret_len; + int is_euclean = 0; + int is_ebadmsg = 0; + struct msm_nand_info *info = mtd->priv; + + actual_len = ops->len; + ret_len = 0; + actual_buf = ops->datbuf; + + bounce_buf = kmalloc(mtd->writesize, GFP_KERNEL); + if (!bounce_buf) { + err = -ENOMEM; + goto out; + } + + /* Get start address of page to read from */ + ops->len = mtd->writesize; + offset = from & (mtd->writesize - 1); + aligned_from = from - offset; + + for (;;) { + bool no_copy = false; + + len = mtd->writesize - offset; + if (len > actual_len) + len = actual_len; + + if (offset == 0 && len == mtd->writesize) + no_copy = true; + + if (!virt_addr_valid(actual_buf) && + !is_buffer_in_page(actual_buf, ops->len)) + no_copy = false; + + ops->datbuf = no_copy ? actual_buf : bounce_buf; + if (info->nand_chip.caps & MSM_NAND_CAP_PAGE_SCOPE_READ) + err = msm_nand_read_pagescope(mtd, aligned_from, ops); + else + err = msm_nand_read_oob(mtd, aligned_from, ops); + if (err == -EUCLEAN) { + is_euclean = 1; + err = 0; + } + + if (err == -EBADMSG) { + is_ebadmsg = 1; + err = 0; + } + + if (err < 0) { + /* Clear previously set EUCLEAN / EBADMSG */ + is_euclean = 0; + is_ebadmsg = 0; + ret_len = ops->retlen; + break; + } + + if (!no_copy) + memcpy(actual_buf, bounce_buf + offset, len); + + actual_len -= len; + ret_len += len; + + if (actual_len == 0) + break; + + actual_buf += len; + offset = 0; + aligned_from += mtd->writesize; + } + + ops->retlen = ret_len; + kfree(bounce_buf); +out: + if (is_euclean == 1) + err = -EUCLEAN; + + /* Snub EUCLEAN if we also have EBADMSG */ + if (is_ebadmsg == 1) + err = -EBADMSG; + return err; +} + +/* + * Function that gets called from upper layers such as MTD/YAFFS2 to read a + * page with only main data. + */ +static int msm_nand_read(struct mtd_info *mtd, loff_t from, size_t len, + size_t *retlen, u_char *buf) +{ + int ret; + int is_euclean = 0; + int is_ebadmsg = 0; + struct mtd_oob_ops ops; + unsigned char *bounce_buf = NULL; + struct msm_nand_info *info = mtd->priv; + + ops.mode = MTD_OPS_AUTO_OOB; + ops.retlen = 0; + ops.ooblen = 0; + ops.oobbuf = NULL; + *retlen = 0; + + if (!(from & (mtd->writesize - 1)) && !(len % mtd->writesize)) { + /* + * Handle reading of large size read buffer in vmalloc + * address space that does not fit in an MMU page. + */ + if (!virt_addr_valid(buf) && !is_buffer_in_page(buf, len)) { + ops.len = mtd->writesize; + + bounce_buf = kmalloc(ops.len, GFP_KERNEL); + if (!bounce_buf) { + ret = -ENOMEM; + goto out; + } + + for (;;) { + bool no_copy = false; + + if (!is_buffer_in_page(buf, ops.len)) { + memcpy(bounce_buf, buf, ops.len); + ops.datbuf = (uint8_t *) bounce_buf; + } else { + ops.datbuf = (uint8_t *) buf; + no_copy = true; + } + if (info->nand_chip.caps & + MSM_NAND_CAP_PAGE_SCOPE_READ) { + ret = msm_nand_read_pagescope(mtd, + from, &ops); + } else { + ret = msm_nand_read_oob(mtd, + from, &ops); + } + if (ret == -EUCLEAN) { + is_euclean = 1; + ret = 0; + } + if (ret == -EBADMSG) { + is_ebadmsg = 1; + ret = 0; + } + if (ret < 0) { + /* Clear previously set errors */ + is_euclean = 0; + is_ebadmsg = 0; + break; + } + + + if (!no_copy) + memcpy(buf, bounce_buf, ops.retlen); + + len -= ops.retlen; + *retlen += ops.retlen; + if (len == 0) + break; + buf += ops.retlen; + from += ops.retlen; + + if (len < mtd->writesize) { + ops.len = len; + ops.datbuf = buf; + ret = msm_nand_read_partial_page( + mtd, from, &ops); + *retlen += ops.retlen; + break; + } + } + kfree(bounce_buf); + } else { + ops.len = len; + ops.datbuf = (uint8_t *)buf; + if (info->nand_chip.caps & + MSM_NAND_CAP_PAGE_SCOPE_READ) + ret = msm_nand_read_pagescope(mtd, from, &ops); + else + ret = msm_nand_read_oob(mtd, from, &ops); + *retlen = ops.retlen; + } + } else { + ops.len = len; + ops.datbuf = (uint8_t *)buf; + ret = msm_nand_read_partial_page(mtd, from, &ops); + *retlen = ops.retlen; + } +out: + if (is_euclean == 1) + ret = -EUCLEAN; + + /* Snub EUCLEAN if we also have EBADMSG */ + if (is_ebadmsg == 1) + ret = -EBADMSG; + + return ret; +} + +/* + * Function that gets called from upper layers such as MTD/YAFFS2 to write a + * page with both main and spare data. + */ +static int msm_nand_write_oob(struct mtd_info *mtd, loff_t to, + struct mtd_oob_ops *ops) +{ + struct msm_nand_info *info = mtd->priv; + struct msm_nand_chip *chip = &info->nand_chip; + uint32_t cwperpage = (mtd->writesize >> 9); + uint32_t n, flash_sts, pages_written = 0; + int err = 0, submitted_num_desc = 0; + struct msm_nand_rw_params rw_params; + struct msm_nand_rw_reg_data data; + struct sps_iovec *iovec; + struct sps_iovec iovec_temp; + + /* + * The following 7 commands will be sent only once : + * For first codeword (CW) - addr0, addr1, dev0_cfg0, dev0_cfg1, + * dev0_ecc_cfg, ebi2_ecc_buf_cfg. + * For last codeword (CW) - read_status(write) + * + * The following 4 commands will be sent for every CW : + * flash, exec, flash_status (read), flash_status (write). + */ + struct { + struct sps_transfer xfer; + struct sps_iovec cmd_iovec[MAX_DESC + 1]; + struct { + uint32_t count; + struct msm_nand_cmd_setup_desc setup_desc; + struct msm_nand_cmd_cw_desc cw_desc[MAX_DESC]; + } cmd_list; + struct { + uint32_t flash_status; + } data[MAX_CW_PER_PAGE]; + } *dma_buffer; + struct msm_nand_rw_cmd_desc *cmd_list = NULL; + + memset(&rw_params, 0, sizeof(struct msm_nand_rw_params)); + err = msm_nand_validate_mtd_params(mtd, false, to, ops, &rw_params); + if (err) + goto validate_mtd_params_failed; + + wait_event(chip->dma_wait_queue, (dma_buffer = + msm_nand_get_dma_buffer(chip, sizeof(*dma_buffer)))); + + memset(&data, 0, sizeof(struct msm_nand_rw_reg_data)); + msm_nand_update_rw_reg_data(chip, ops, &rw_params, &data); + cmd_list = (struct msm_nand_rw_cmd_desc *)&dma_buffer->cmd_list; + + while (rw_params.page_count-- > 0) { + uint32_t cw_desc_cnt = 0; + struct sps_command_element *curr_ce, *start_ce; + + data.addr0 = (rw_params.page << 16); + data.addr1 = (rw_params.page >> 16) & 0xff; + + for (n = 0; n < cwperpage ; n++) { + dma_buffer->data[n].flash_status = 0xeeeeeeee; + + msm_nand_prep_rw_cmd_desc(ops, &rw_params, &data, info, + n, cmd_list, &cw_desc_cnt, 0); + + curr_ce = &cmd_list->cw_desc[cw_desc_cnt].ce[0]; + cmd_list->cw_desc[cw_desc_cnt].flags = CMD; + cmd_list->count++; + + msm_nand_prep_ce(curr_ce, MSM_NAND_FLASH_STATUS(info), + READ, msm_virt_to_dma(chip, + &dma_buffer->data[n].flash_status)); + cmd_list->cw_desc[cw_desc_cnt++].num_ce = 1; + } + + start_ce = &cmd_list->cw_desc[cw_desc_cnt].ce[0]; + curr_ce = start_ce; + cmd_list->cw_desc[cw_desc_cnt].flags = CMD_INT_UNLCK; + cmd_list->count++; + msm_nand_prep_ce(curr_ce, MSM_NAND_FLASH_STATUS(info), + WRITE, data.clrfstatus); + curr_ce++; + + msm_nand_prep_ce(curr_ce, MSM_NAND_READ_STATUS(info), + WRITE, data.clrrstatus); + curr_ce++; + cmd_list->cw_desc[cw_desc_cnt++].num_ce = curr_ce - start_ce; + + dma_buffer->xfer.iovec_count = cmd_list->count; + dma_buffer->xfer.iovec = dma_buffer->cmd_iovec; + dma_buffer->xfer.iovec_phys = msm_virt_to_dma(chip, + &dma_buffer->cmd_iovec); + iovec = dma_buffer->xfer.iovec; + + iovec->addr = msm_virt_to_dma(chip, + &cmd_list->setup_desc.ce[0]); + iovec->size = sizeof(struct sps_command_element) * + cmd_list->setup_desc.num_ce; + iovec->flags = cmd_list->setup_desc.flags; + iovec++; + for (n = 0; n < (cmd_list->count - 1); n++) { + iovec->addr = msm_virt_to_dma(chip, + &cmd_list->cw_desc[n].ce[0]); + iovec->size = sizeof(struct sps_command_element) * + cmd_list->cw_desc[n].num_ce; + iovec->flags = cmd_list->cw_desc[n].flags; + iovec++; + } + mutex_lock(&info->lock); + err = msm_nand_get_device(chip->dev); + if (err) + goto unlock_mutex; + /* Submit data descriptors */ + for (n = 0; n < cwperpage; n++) { + err = msm_nand_submit_rw_data_desc(ops, + &rw_params, info, n, 0); + if (err) { + pr_err("Failed to submit data descs %d\n", err); + panic("Error in nand driver\n"); + goto put_dev; + } + } + + if (ops->mode == MTD_OPS_RAW) { + submitted_num_desc = n; + } else if (ops->mode == MTD_OPS_AUTO_OOB) { + if (ops->datbuf) + submitted_num_desc = n; + if (ops->oobbuf) + submitted_num_desc++; + } + + /* Submit command descriptors */ + err = sps_transfer(info->sps.cmd_pipe.handle, + &dma_buffer->xfer); + if (err) { + pr_err("Failed to submit commands %d\n", err); + goto put_dev; + } + + err = msm_nand_sps_get_iovec(info->sps.cmd_pipe.handle, + info->sps.cmd_pipe.index, + dma_buffer->xfer.iovec_count, + &iovec_temp); + if (err) { + pr_err("Failed to get iovec for pipe %d (err:%d)\n", + (info->sps.cmd_pipe.index), err); + goto put_dev; + } + err = msm_nand_sps_get_iovec(info->sps.data_cons.handle, + info->sps.data_cons.index, submitted_num_desc, + &iovec_temp); + if (err) { + pr_err("Failed to get iovec for pipe %d (err:%d)\n", + (info->sps.data_cons.index), err); + goto put_dev; + } + + err = msm_nand_put_device(chip->dev); + mutex_unlock(&info->lock); + if (err) + goto free_dma; + + for (n = 0; n < cwperpage; n++) + pr_debug("write pg %d: flash_status[%d] = %x\n", + rw_params.page, n, + dma_buffer->data[n].flash_status); + + /* Check for flash status errors */ + for (n = 0; n < cwperpage; n++) { + flash_sts = dma_buffer->data[n].flash_status; + if (flash_sts & (FS_OP_ERR | FS_MPU_ERR)) { + pr_err("MPU/OP err (0x%x) set\n", flash_sts); + err = -EIO; + goto free_dma; + } + if (n == (cwperpage - 1)) { + if (!(flash_sts & FS_DEVICE_WP) || + (flash_sts & FS_DEVICE_STS_ERR)) { + pr_err("Dev sts err 0x%x\n", flash_sts); + err = -EIO; + goto free_dma; + } + } + } + pages_written++; + rw_params.page++; + } + goto free_dma; +put_dev: + msm_nand_put_device(chip->dev); +unlock_mutex: + mutex_unlock(&info->lock); +free_dma: + msm_nand_release_dma_buffer(chip, dma_buffer, sizeof(*dma_buffer)); + if (ops->oobbuf) + dma_unmap_page(chip->dev, rw_params.oob_dma_addr, + ops->ooblen, DMA_TO_DEVICE); + if (ops->datbuf) + dma_unmap_page(chip->dev, rw_params.data_dma_addr, + ops->len, DMA_TO_DEVICE); +validate_mtd_params_failed: + if (ops->mode != MTD_OPS_RAW) + ops->retlen = mtd->writesize * pages_written; + else + ops->retlen = (mtd->writesize + mtd->oobsize) * pages_written; + + ops->oobretlen = ops->ooblen - rw_params.oob_len_data; + if (err) + pr_err("to %llx datalen %x ooblen %x failed with err %d\n", + to, ops->len, ops->ooblen, err); + pr_debug("ret %d, retlen %d oobretlen %d\n", + err, ops->retlen, ops->oobretlen); + + pr_debug("================================================\n"); + return err; +} + +/* + * Function that gets called from upper layers such as MTD/YAFFS2 to write a + * page with only main data. + */ +static int msm_nand_write(struct mtd_info *mtd, loff_t to, size_t len, + size_t *retlen, const u_char *buf) +{ + int ret; + struct mtd_oob_ops ops; + unsigned char *bounce_buf = NULL; + + ops.mode = MTD_OPS_AUTO_OOB; + ops.retlen = 0; + ops.ooblen = 0; + ops.oobbuf = NULL; + + /* partial page writes are not supported */ + if ((to & (mtd->writesize - 1)) || (len % mtd->writesize)) { + ret = -EINVAL; + *retlen = ops.retlen; + pr_err("%s: partial page writes are not supported\n", __func__); + goto out; + } + + /* + * Handle writing of large size write buffer in vmalloc + * address space that does not fit in an MMU page. + */ + if (!virt_addr_valid(buf) && !is_buffer_in_page(buf, len)) { + ops.len = mtd->writesize; + + bounce_buf = kmalloc(ops.len, GFP_KERNEL); + if (!bounce_buf) { + ret = -ENOMEM; + goto out; + } + + for (;;) { + if (!is_buffer_in_page(buf, ops.len)) { + memcpy(bounce_buf, buf, ops.len); + ops.datbuf = (uint8_t *) bounce_buf; + } else { + ops.datbuf = (uint8_t *) buf; + } + ret = msm_nand_write_oob(mtd, to, &ops); + if (ret < 0) + break; + + len -= mtd->writesize; + *retlen += mtd->writesize; + if (len == 0) + break; + + buf += mtd->writesize; + to += mtd->writesize; + } + kfree(bounce_buf); + } else { + ops.len = len; + ops.datbuf = (uint8_t *)buf; + ret = msm_nand_write_oob(mtd, to, &ops); + *retlen = ops.retlen; + } +out: + return ret; +} + +/* + * Structure that contains NANDc register data for commands required + * for Erase operation. + */ +struct msm_nand_erase_reg_data { + struct msm_nand_common_cfgs cfg; + uint32_t exec; + uint32_t flash_status; + uint32_t clrfstatus; + uint32_t clrrstatus; +}; + +/* + * Function that gets called from upper layers such as MTD/YAFFS2 to erase a + * block within NAND device. + */ +#define ERASE_CMDS 9 +static int msm_nand_erase(struct mtd_info *mtd, struct erase_info *instr) +{ + int i = 0, err = 0; + struct msm_nand_info *info = mtd->priv; + struct msm_nand_chip *chip = &info->nand_chip; + uint32_t page = 0; + struct msm_nand_sps_cmd *cmd, *curr_cmd; + struct msm_nand_erase_reg_data data; + struct sps_iovec *iovec; + struct sps_iovec iovec_temp; + + /* + * The following 9 commands are required to erase a page - + * flash, addr0, addr1, cfg0, cfg1, exec, flash_status(read), + * flash_status(write), read_status. + */ + struct { + struct sps_transfer xfer; + struct sps_iovec cmd_iovec[ERASE_CMDS]; + struct msm_nand_sps_cmd cmd[ERASE_CMDS]; + uint32_t flash_status; + } *dma_buffer; + + if (mtd->writesize == PAGE_SIZE_2K) + page = instr->addr >> 11; + + if (mtd->writesize == PAGE_SIZE_4K) + page = instr->addr >> 12; + + if (instr->addr & (mtd->erasesize - 1)) { + pr_err("unsupported erase address, 0x%llx\n", instr->addr); + err = -EINVAL; + goto out; + } + if (instr->len != mtd->erasesize) { + pr_err("unsupported erase len, %lld\n", instr->len); + err = -EINVAL; + goto out; + } + + wait_event(chip->dma_wait_queue, (dma_buffer = msm_nand_get_dma_buffer( + chip, sizeof(*dma_buffer)))); + cmd = dma_buffer->cmd; + + memset(&data, 0, sizeof(struct msm_nand_erase_reg_data)); + data.cfg.cmd = MSM_NAND_CMD_BLOCK_ERASE; + data.cfg.addr0 = page; + data.cfg.addr1 = 0; + data.cfg.cfg0 = chip->cfg0 & (~(7 << CW_PER_PAGE)); + data.cfg.cfg1 = chip->cfg1; + data.exec = 1; + dma_buffer->flash_status = 0xeeeeeeee; + data.clrfstatus = MSM_NAND_RESET_FLASH_STS; + data.clrrstatus = MSM_NAND_RESET_READ_STS; + + curr_cmd = cmd; + msm_nand_prep_cfg_cmd_desc(info, data.cfg, &curr_cmd); + + cmd = curr_cmd; + msm_nand_prep_single_desc(cmd, MSM_NAND_EXEC_CMD(info), WRITE, + data.exec, SPS_IOVEC_FLAG_NWD); + cmd++; + + msm_nand_prep_single_desc(cmd, MSM_NAND_FLASH_STATUS(info), READ, + msm_virt_to_dma(chip, &dma_buffer->flash_status), 0); + cmd++; + + msm_nand_prep_single_desc(cmd, MSM_NAND_FLASH_STATUS(info), WRITE, + data.clrfstatus, 0); + cmd++; + + msm_nand_prep_single_desc(cmd, MSM_NAND_READ_STATUS(info), WRITE, + data.clrrstatus, + SPS_IOVEC_FLAG_UNLOCK | SPS_IOVEC_FLAG_INT); + cmd++; + + WARN_ON((cmd - dma_buffer->cmd) > ERASE_CMDS); + dma_buffer->xfer.iovec_count = (cmd - dma_buffer->cmd); + dma_buffer->xfer.iovec = dma_buffer->cmd_iovec; + dma_buffer->xfer.iovec_phys = msm_virt_to_dma(chip, + &dma_buffer->cmd_iovec); + iovec = dma_buffer->xfer.iovec; + + for (i = 0; i < dma_buffer->xfer.iovec_count; i++) { + iovec->addr = msm_virt_to_dma(chip, &dma_buffer->cmd[i].ce); + iovec->size = sizeof(struct sps_command_element); + iovec->flags = dma_buffer->cmd[i].flags; + iovec++; + } + mutex_lock(&info->lock); + err = msm_nand_get_device(chip->dev); + if (err) + goto unlock_mutex; + + err = sps_transfer(info->sps.cmd_pipe.handle, &dma_buffer->xfer); + if (err) { + pr_err("Failed to submit commands %d\n", err); + goto put_dev; + } + err = msm_nand_sps_get_iovec(info->sps.cmd_pipe.handle, + info->sps.cmd_pipe.index, dma_buffer->xfer.iovec_count, + &iovec_temp); + if (err) { + pr_err("Failed to get iovec for pipe %d (err: %d)\n", + (info->sps.cmd_pipe.index), err); + goto put_dev; + } + err = msm_nand_put_device(chip->dev); + if (err) + goto unlock_mutex; + + /* Check for flash status errors */ + if (dma_buffer->flash_status & (FS_OP_ERR | + FS_MPU_ERR | FS_DEVICE_STS_ERR)) { + pr_err("MPU/OP/DEV err (0x%x) set\n", dma_buffer->flash_status); + err = -EIO; + } + if (!(dma_buffer->flash_status & FS_DEVICE_WP)) { + pr_err("Device is write protected\n"); + err = -EIO; + } + if (err) { + pr_err("Erase failed, 0x%llx\n", instr->addr); + instr->fail_addr = instr->addr; + } else { + instr->fail_addr = 0xffffffff; + } + goto unlock_mutex; +put_dev: + msm_nand_put_device(chip->dev); +unlock_mutex: + mutex_unlock(&info->lock); + msm_nand_release_dma_buffer(chip, dma_buffer, sizeof(*dma_buffer)); +out: + return err; +} + +/* + * Structure that contains NANDc register data for commands required + * for checking if a block is bad. + */ +struct msm_nand_blk_isbad_data { + struct msm_nand_common_cfgs cfg; + uint32_t ecc_bch_cfg; + uint32_t exec; + uint32_t read_offset; +}; + +/* + * Function that gets called from upper layers such as MTD/YAFFS2 to check if + * a block is bad. This is done by reading the first page within a block and + * checking whether the bad block byte location contains 0xFF or not. If it + * doesn't contain 0xFF, then it is considered as bad block. + */ +#define ISBAD_CMDS 10 +static int msm_nand_block_isbad(struct mtd_info *mtd, loff_t ofs) +{ + struct msm_nand_info *info = mtd->priv; + struct msm_nand_chip *chip = &info->nand_chip; + int i = 0, ret = 0, bad_block = 0, submitted_num_desc = 1; + uint8_t *buf; + uint32_t page = 0, rdata, cwperpage; + struct msm_nand_sps_cmd *cmd, *curr_cmd; + struct msm_nand_blk_isbad_data data; + struct sps_iovec *iovec; + struct sps_iovec iovec_temp; + /* + * The following 9 commands are required to check bad block - + * flash, addr0, addr1, cfg0, cfg1, ecc_cfg, read_loc_0, + * exec, flash_status(read). + */ + struct { + struct sps_transfer xfer; + struct sps_iovec cmd_iovec[ISBAD_CMDS]; + struct msm_nand_sps_cmd cmd[ISBAD_CMDS]; + uint32_t flash_status; + } *dma_buffer; + + if (mtd->writesize == PAGE_SIZE_2K) + page = ofs >> 11; + + if (mtd->writesize == PAGE_SIZE_4K) + page = ofs >> 12; + + cwperpage = (mtd->writesize >> 9); + + if (ofs > mtd->size) { + pr_err("Invalid offset 0x%llx\n", ofs); + bad_block = -EINVAL; + goto out; + } + if (ofs & (mtd->erasesize - 1)) { + pr_err("unsupported block address, 0x%x\n", (uint32_t)ofs); + bad_block = -EINVAL; + goto out; + } + + wait_event(chip->dma_wait_queue, (dma_buffer = msm_nand_get_dma_buffer( + chip, sizeof(*dma_buffer) + 4))); + buf = (uint8_t *)dma_buffer + sizeof(*dma_buffer); + + cmd = dma_buffer->cmd; + memset(&data, 0, sizeof(struct msm_nand_blk_isbad_data)); + data.cfg.cmd = MSM_NAND_CMD_PAGE_READ_ALL; + data.cfg.cfg0 = chip->cfg0_raw & ~(7U << CW_PER_PAGE); + data.cfg.cfg1 = chip->cfg1_raw; + + if (chip->cfg1 & (1 << WIDE_FLASH)) + data.cfg.addr0 = (page << 16) | + ((chip->cw_size * (cwperpage-1)) >> 1); + else + data.cfg.addr0 = (page << 16) | + (chip->cw_size * (cwperpage-1)); + + data.cfg.addr1 = (page >> 16) & 0xff; + data.ecc_bch_cfg = 1 << ECC_CFG_ECC_DISABLE; + data.exec = 1; + data.read_offset = (mtd->writesize - (chip->cw_size * (cwperpage-1))); + dma_buffer->flash_status = 0xeeeeeeee; + + curr_cmd = cmd; + msm_nand_prep_cfg_cmd_desc(info, data.cfg, &curr_cmd); + + cmd = curr_cmd; + msm_nand_prep_single_desc(cmd, MSM_NAND_DEV0_ECC_CFG(info), WRITE, + data.ecc_bch_cfg, 0); + cmd++; + + rdata = (data.read_offset << 0) | (4 << 16) | (1 << 31); + msm_nand_prep_single_desc(cmd, MSM_NAND_READ_LOCATION_0(info), WRITE, + rdata, 0); + cmd++; + + if (chip->qpic_version >= 2) { + msm_nand_prep_single_desc(cmd, + MSM_NAND_READ_LOCATION_LAST_CW_0(info), WRITE, + rdata, 0); + cmd++; + } + + msm_nand_prep_single_desc(cmd, MSM_NAND_EXEC_CMD(info), WRITE, + data.exec, SPS_IOVEC_FLAG_NWD); + cmd++; + + msm_nand_prep_single_desc(cmd, MSM_NAND_FLASH_STATUS(info), READ, + msm_virt_to_dma(chip, &dma_buffer->flash_status), + SPS_IOVEC_FLAG_INT | SPS_IOVEC_FLAG_UNLOCK); + cmd++; + + WARN_ON(cmd - dma_buffer->cmd > ISBAD_CMDS); + dma_buffer->xfer.iovec_count = (cmd - dma_buffer->cmd); + dma_buffer->xfer.iovec = dma_buffer->cmd_iovec; + dma_buffer->xfer.iovec_phys = msm_virt_to_dma(chip, + &dma_buffer->cmd_iovec); + iovec = dma_buffer->xfer.iovec; + + for (i = 0; i < dma_buffer->xfer.iovec_count; i++) { + iovec->addr = msm_virt_to_dma(chip, &dma_buffer->cmd[i].ce); + iovec->size = sizeof(struct sps_command_element); + iovec->flags = dma_buffer->cmd[i].flags; + iovec++; + } + mutex_lock(&info->lock); + ret = msm_nand_get_device(chip->dev); + if (ret) { + mutex_unlock(&info->lock); + goto free_dma; + } + /* Submit data descriptor */ + ret = sps_transfer_one(info->sps.data_prod.handle, + msm_virt_to_dma(chip, buf), + 4, NULL, SPS_IOVEC_FLAG_INT); + + if (ret) { + pr_err("Failed to submit data desc %d\n", ret); + goto put_dev; + } + /* Submit command descriptor */ + ret = sps_transfer(info->sps.cmd_pipe.handle, &dma_buffer->xfer); + if (ret) { + pr_err("Failed to submit commands %d\n", ret); + goto put_dev; + } + + ret = msm_nand_sps_get_iovec(info->sps.cmd_pipe.handle, + info->sps.cmd_pipe.index, dma_buffer->xfer.iovec_count, + &iovec_temp); + if (ret) { + pr_err("Failed to get iovec for pipe %d (ret: %d)\n", + (info->sps.cmd_pipe.index), ret); + goto put_dev; + } + ret = msm_nand_sps_get_iovec(info->sps.data_prod.handle, + info->sps.data_prod.index, submitted_num_desc, + &iovec_temp); + if (ret) { + pr_err("Failed to get iovec for pipe %d (ret: %d)\n", + (info->sps.data_prod.index), ret); + goto put_dev; + } + + ret = msm_nand_put_device(chip->dev); + mutex_unlock(&info->lock); + if (ret) + goto free_dma; + + /* Check for flash status errors */ + if (dma_buffer->flash_status & (FS_OP_ERR | FS_MPU_ERR)) { + pr_err("MPU/OP err set: %x\n", dma_buffer->flash_status); + bad_block = -EIO; + goto free_dma; + } + + /* Check for bad block marker byte */ + if (chip->cfg1 & (1 << WIDE_FLASH)) { + if (buf[0] != 0xFF || buf[1] != 0xFF) + bad_block = 1; + } else { + if (buf[0] != 0xFF) + bad_block = 1; + } + goto free_dma; +put_dev: + msm_nand_put_device(chip->dev); + mutex_unlock(&info->lock); +free_dma: + msm_nand_release_dma_buffer(chip, dma_buffer, sizeof(*dma_buffer) + 4); +out: + return ret ? ret : bad_block; +} + +/* + * Function that gets called from upper layers such as MTD/YAFFS2 to mark a + * block as bad. This is done by writing the first page within a block with 0, + * thus setting the bad block byte location as well to 0. + */ +static int msm_nand_block_markbad(struct mtd_info *mtd, loff_t ofs) +{ + struct mtd_oob_ops ops; + int ret; + uint8_t *buf; + size_t len; + + if (ofs > mtd->size) { + pr_err("Invalid offset 0x%llx\n", ofs); + ret = -EINVAL; + goto out; + } + if (ofs & (mtd->erasesize - 1)) { + pr_err("unsupported block address, 0x%x\n", (uint32_t)ofs); + ret = -EINVAL; + goto out; + } + len = mtd->writesize + mtd->oobsize; + buf = kzalloc(len, GFP_KERNEL); + if (!buf) { + ret = -ENOMEM; + goto out; + } + ops.mode = MTD_OPS_RAW; + ops.len = len; + ops.retlen = 0; + ops.ooblen = 0; + ops.datbuf = buf; + ops.oobbuf = NULL; + ret = msm_nand_write_oob(mtd, ofs, &ops); + kfree(buf); +out: + return ret; +} + +/* + * Function that scans for the attached NAND device. This fills out all + * the uninitialized function pointers with the defaults. The flash ID is + * read and the mtd/chip structures are filled with the appropriate values. + */ +static int msm_nand_scan(struct mtd_info *mtd) +{ + struct msm_nand_info *info = mtd->priv; + struct msm_nand_chip *chip = &info->nand_chip; + struct flash_identification *supported_flash = &info->flash_dev; + int err = 0; + uint32_t i, j, mtd_writesize = 0; + uint8_t dev_found = 0, wide_bus = 0; + uint32_t manid, devid, devcfg; + uint32_t flash_id = 0, flash_id2 = 0; + uint8_t id_byte[NAND_MAX_ID_LEN]; + uint32_t bad_block_byte, spare_bytes; + struct nand_flash_dev *flashdev = NULL; + const struct nand_manufacturer *flashman = NULL; + + /* Probe the Flash device for ONFI compliance */ + if (!msm_nand_flash_onfi_probe(info)) { + dev_found = 1; + } else { + err = msm_nand_flash_read_id(info, 0, &flash_id, &flash_id2); + if (err < 0) { + pr_err("Failed to read Flash ID\n"); + err = -EINVAL; + goto out; + } + manid = id_byte[0] = flash_id & 0xFF; + devid = id_byte[1] = (flash_id >> 8) & 0xFF; + devcfg = id_byte[3] = (flash_id >> 24) & 0xFF; + id_byte[2] = (flash_id >> 16) & 0xFF; + id_byte[4] = flash_id2 & 0xFF; + id_byte[5] = (flash_id2 >> 8) & 0xFF; + id_byte[6] = (flash_id2 >> 16) & 0xFF; + id_byte[7] = (flash_id2 >> 24) & 0xFF; + + flashman = nand_get_manufacturer(manid); + + for (i = 0; !flashdev; ++i) { + /* + * If id_len is specified for an entry in the nand ids + * array, then at least 4 bytes of the nand id is + * present in the nand ids array - use that to identify + * the nand device first. If that is not present, only + * then fall back to searching the legacy or extended + * ids in the nand ids array. + * The id_len number of bytes in the nand id read from + * the device are checked against those in the nand id + * table for exact match. + */ + if (nand_flash_ids[i].id_len) { + for (j = 0; j < nand_flash_ids[i].id_len; j++) { + if (nand_flash_ids[i].id[j] == + id_byte[j]) + continue; + else + break; + } + if (j == nand_flash_ids[i].id_len) + flashdev = &nand_flash_ids[i]; + } else if (!nand_flash_ids[i].id_len && + nand_flash_ids[i].dev_id == devid) + flashdev = &nand_flash_ids[i]; + } + if (!flashdev || !flashman) { + pr_err("unknown nand flashid=%x manuf=%x devid=%x\n", + flash_id, manid, devid); + err = -ENOENT; + goto out; + } + dev_found = 1; + if (!flashdev->pagesize) { + pr_err("missing page size info - extract from NAND ID\n"); + supported_flash->widebus = devcfg & (1 << 6) ? 1 : 0; + supported_flash->pagesize = 1024 << (devcfg & 0x3); + supported_flash->blksize = (64 * 1024) << + ((devcfg >> 4) & 0x3); + supported_flash->oobsize = (8 << ((devcfg >> 2) & 1)) * + (supported_flash->pagesize >> 9); + } else { + supported_flash->widebus = flashdev->options & + NAND_BUSWIDTH_16 ? 1 : 0; + supported_flash->pagesize = flashdev->pagesize; + supported_flash->blksize = flashdev->erasesize; + supported_flash->oobsize = flashdev->oobsize; + supported_flash->ecc_correctability = + flashdev->ecc.strength_ds; + if (!flashdev->ecc.strength_ds) + pr_err("num ecc correctable bit not specified and defaults to 4 bit BCH\n"); + } + supported_flash->flash_id = flash_id; + supported_flash->density = ((uint64_t)flashdev->chipsize) << 20; + } + + if (dev_found) { + wide_bus = supported_flash->widebus; + mtd->size = supported_flash->density; + mtd->writesize = supported_flash->pagesize; + mtd->oobsize = supported_flash->oobsize; + mtd->erasesize = supported_flash->blksize; + mtd->writebufsize = mtd->writesize; + mtd_writesize = mtd->writesize; + + /* Check whether NAND device support 8bit ECC*/ + if (supported_flash->ecc_correctability >= 8) { + chip->bch_caps = MSM_NAND_CAP_8_BIT_BCH; + supported_flash->ecc_capability = 8; + } else { + chip->bch_caps = MSM_NAND_CAP_4_BIT_BCH; + supported_flash->ecc_capability = 4; + } + + pr_info("NAND Id: 0x%x Buswidth: %dBits Density: %lld MByte\n", + supported_flash->flash_id, (wide_bus) ? 16 : 8, + (mtd->size >> 20)); + pr_info("pagesize: %d Erasesize: %d oobsize: %d (in Bytes)\n", + mtd->writesize, mtd->erasesize, mtd->oobsize); + pr_info("BCH ECC: %d Bit\n", supported_flash->ecc_capability); + } + + chip->cw_size = (chip->bch_caps & MSM_NAND_CAP_8_BIT_BCH) ? 532 : 528; + chip->cfg0 = (((mtd_writesize >> 9) - 1) << CW_PER_PAGE) + | (516 << UD_SIZE_BYTES) + | (0 << DISABLE_STATUS_AFTER_WRITE) + | (5 << NUM_ADDR_CYCLES); + + bad_block_byte = (mtd_writesize - (chip->cw_size * ( + (mtd_writesize >> 9) - 1)) + 1); + chip->cfg1 = (7 << NAND_RECOVERY_CYCLES) + | (0 << CS_ACTIVE_BSY) + | (bad_block_byte << BAD_BLOCK_BYTE_NUM) + | (0 << BAD_BLOCK_IN_SPARE_AREA) + | (2 << WR_RD_BSY_GAP) + | ((wide_bus ? 1 : 0) << WIDE_FLASH) + | (1 << ENABLE_BCH_ECC); + + /* + * For 4bit BCH ECC (default ECC), parity bytes = 7(x8) or 8(x16 I/O) + * For 8bit BCH ECC, parity bytes = 13 (x8) or 14 (x16 I/O). + */ + chip->ecc_parity_bytes = (chip->bch_caps & MSM_NAND_CAP_8_BIT_BCH) ? + (wide_bus ? 14 : 13) : (wide_bus ? 8 : 7); + + spare_bytes = chip->cw_size - (BYTES_512 + chip->ecc_parity_bytes); + chip->cfg0_raw = (((mtd_writesize >> 9) - 1) << CW_PER_PAGE) + | (5 << NUM_ADDR_CYCLES) + | (spare_bytes << SPARE_SIZE_BYTES) + | (BYTES_512 << UD_SIZE_BYTES); + + chip->cfg1_raw = (2 << WR_RD_BSY_GAP) + | (1 << BAD_BLOCK_IN_SPARE_AREA) + | (21 << BAD_BLOCK_BYTE_NUM) + | (0 << CS_ACTIVE_BSY) + | (7 << NAND_RECOVERY_CYCLES) + | ((wide_bus ? 1 : 0) << WIDE_FLASH) + | (1 << DEV0_CFG1_ECC_DISABLE); + + chip->ecc_bch_cfg = (0 << ECC_CFG_ECC_DISABLE) + | (0 << ECC_SW_RESET) + | (516 << ECC_NUM_DATA_BYTES) + | (chip->ecc_parity_bytes << ECC_PARITY_SIZE_BYTES) + | (1 << ECC_FORCE_CLK_OPEN); + + chip->ecc_cfg_raw = (1 << ECC_FORCE_CLK_OPEN) + | (BYTES_512 << ECC_NUM_DATA_BYTES) + | (chip->ecc_parity_bytes << ECC_PARITY_SIZE_BYTES) + | (0 << ECC_SW_RESET) + | (1 << ECC_CFG_ECC_DISABLE); + + if (chip->bch_caps & MSM_NAND_CAP_8_BIT_BCH) { + chip->cfg0 |= (wide_bus ? 0 << SPARE_SIZE_BYTES : + 2 << SPARE_SIZE_BYTES); + chip->ecc_bch_cfg |= (1 << ECC_MODE); + chip->ecc_cfg_raw |= (1 << ECC_MODE); + } else { + chip->cfg0 |= (wide_bus ? 2 << SPARE_SIZE_BYTES : + 4 << SPARE_SIZE_BYTES); + chip->ecc_bch_cfg |= (0 << ECC_MODE); + chip->ecc_cfg_raw |= (0 << ECC_MODE); + } + + chip->ecc_buf_cfg = 0x203; /* No of bytes covered by ECC - 516 bytes */ + + pr_info("CFG0: 0x%08x, CFG1: 0x%08x\n" + " RAWCFG0: 0x%08x, RAWCFG1: 0x%08x\n" + " ECCBUFCFG: 0x%08x, ECCBCHCFG: 0x%08x\n" + " RAWECCCFG: 0x%08x, BAD BLOCK BYTE: 0x%08x\n", + chip->cfg0, chip->cfg1, chip->cfg0_raw, chip->cfg1_raw, + chip->ecc_buf_cfg, chip->ecc_bch_cfg, + chip->ecc_cfg_raw, bad_block_byte); + + if (mtd->writesize == 2048) + mtd->oobavail = 16; + else if (mtd->writesize == 4096) + mtd->oobavail = 32; + else { + pr_err("Unsupported NAND pagesize: 0x%x\n", mtd->writesize); + err = -ENODEV; + goto out; + } + + /* Fill in remaining MTD driver data */ + mtd->type = MTD_NANDFLASH; + mtd->flags = MTD_CAP_NANDFLASH; + mtd->_erase = msm_nand_erase; + mtd->_block_isbad = msm_nand_block_isbad; + mtd->_block_markbad = msm_nand_block_markbad; + mtd->_read = msm_nand_read; + mtd->_write = msm_nand_write; + mtd->owner = THIS_MODULE; +out: + return err; +} + +#define BAM_APPS_PIPE_LOCK_GRP0 0 +#define BAM_APPS_PIPE_LOCK_GRP1 1 +/* + * This function allocates, configures, connects an end point and + * also registers event notification for an end point. It also allocates + * DMA memory for descriptor FIFO of a pipe. + */ +static int msm_nand_init_endpoint(struct msm_nand_info *info, + struct msm_nand_sps_endpt *end_point, + uint32_t pipe_index) +{ + int rc = 0; + struct sps_pipe *pipe_handle; + struct sps_connect *sps_config = &end_point->config; + struct sps_register_event *sps_event = &end_point->event; + + pipe_handle = sps_alloc_endpoint(); + if (!pipe_handle) { + pr_err("sps_alloc_endpoint() failed\n"); + rc = -ENOMEM; + goto out; + } + + rc = sps_get_config(pipe_handle, sps_config); + if (rc) { + pr_err("sps_get_config() failed %d\n", rc); + goto free_endpoint; + } + + if (pipe_index == SPS_DATA_PROD_PIPE_INDEX || + pipe_index == SPS_DATA_PROD_STAT_PIPE_INDEX) { + /* READ CASE: source - BAM; destination - system memory */ + sps_config->source = info->sps.bam_handle; + sps_config->destination = SPS_DEV_HANDLE_MEM; + sps_config->mode = SPS_MODE_SRC; + sps_config->src_pipe_index = pipe_index; + } else if (pipe_index == SPS_DATA_CONS_PIPE_INDEX || + pipe_index == SPS_CMD_CONS_PIPE_INDEX) { + /* WRITE CASE: source - system memory; destination - BAM */ + sps_config->source = SPS_DEV_HANDLE_MEM; + sps_config->destination = info->sps.bam_handle; + sps_config->mode = SPS_MODE_DEST; + sps_config->dest_pipe_index = pipe_index; + } + + sps_config->options = SPS_O_AUTO_ENABLE | SPS_O_POLL | + SPS_O_ACK_TRANSFERS; + + if (pipe_index == SPS_DATA_PROD_PIPE_INDEX || + pipe_index == SPS_DATA_CONS_PIPE_INDEX || + pipe_index == SPS_DATA_PROD_STAT_PIPE_INDEX) + sps_config->lock_group = BAM_APPS_PIPE_LOCK_GRP0; + else if (pipe_index == SPS_CMD_CONS_PIPE_INDEX) + sps_config->lock_group = BAM_APPS_PIPE_LOCK_GRP1; + + /* + * Descriptor FIFO is a cyclic FIFO. If SPS_MAX_DESC_NUM descriptors + * are allowed to be submitted before we get any ack for any of them, + * the descriptor FIFO size should be: (SPS_MAX_DESC_NUM + 1) * + * sizeof(struct sps_iovec). + */ + sps_config->desc.size = (SPS_MAX_DESC_NUM + 1) * + sizeof(struct sps_iovec); + sps_config->desc.base = dmam_alloc_coherent(info->nand_chip.dev, + sps_config->desc.size, + &sps_config->desc.phys_base, + GFP_KERNEL); + if (!sps_config->desc.base) { + pr_err("dmam_alloc_coherent() failed for size %x\n", + sps_config->desc.size); + rc = -ENOMEM; + goto free_endpoint; + } + memset(sps_config->desc.base, 0x00, sps_config->desc.size); + + rc = sps_connect(pipe_handle, sps_config); + if (rc) { + pr_err("sps_connect() failed %d\n", rc); + goto free_endpoint; + } + + sps_event->options = SPS_O_EOT; + sps_event->mode = SPS_TRIGGER_WAIT; + sps_event->user = (void *)info; + + rc = sps_register_event(pipe_handle, sps_event); + if (rc) { + pr_err("sps_register_event() failed %d\n", rc); + goto sps_disconnect; + } + end_point->index = pipe_index; + end_point->handle = pipe_handle; + pr_debug("pipe handle 0x%x for pipe %d\n", (uint32_t)pipe_handle, + pipe_index); + goto out; +sps_disconnect: + sps_disconnect(pipe_handle); +free_endpoint: + sps_free_endpoint(pipe_handle); +out: + return rc; +} + +/* This function disconnects and frees an end point */ +static void msm_nand_deinit_endpoint(struct msm_nand_info *info, + struct msm_nand_sps_endpt *end_point) +{ + sps_disconnect(end_point->handle); + sps_free_endpoint(end_point->handle); +} + +/* + * This function registers BAM device and initializes its end points for + * the following pipes - + * system consumer pipe for data (pipe#0), + * system producer pipe for data (pipe#1), + * system consumer pipe for commands (pipe#2). + */ +static int msm_nand_bam_init(struct msm_nand_info *nand_info) +{ + struct sps_bam_props bam = {0}; + int rc = 0; + struct msm_nand_chip *chip = &nand_info->nand_chip; + + bam.phys_addr = nand_info->bam_phys; + bam.virt_addr = nand_info->bam_base; + bam.irq = nand_info->bam_irq; + /* + * NAND device is accessible from both Apps and Modem processor and + * thus, NANDc and BAM are shared between both the processors. But BAM + * must be enabled and instantiated only once during boot up by + * Trustzone before Modem/Apps is brought out from reset. + * + * This is indicated to SPS driver on Apps by marking flag + * SPS_BAM_MGR_DEVICE_REMOTE. The following are the global + * initializations that will be done by Trustzone - Execution + * Environment, Pipes assignment to Apps/Modem, Pipe Super groups and + * Descriptor summing threshold. + * + * NANDc BAM device supports 2 execution environments - Modem and Apps + * and thus the flag SPS_BAM_MGR_MULTI_EE is set. + */ + bam.manage = SPS_BAM_MGR_DEVICE_REMOTE | SPS_BAM_MGR_MULTI_EE; + bam.ipc_loglevel = QPIC_BAM_DEFAULT_IPC_LOGLVL; + mutex_lock(&nand_info->lock); + rc = msm_nand_get_device(chip->dev); + if (rc) { + pr_err("failed to get the device err:%d\n", rc); + goto out; + } + rc = sps_phy2h(bam.phys_addr, &nand_info->sps.bam_handle); + if (!rc) + goto init_sps_ep; + rc = sps_register_bam_device(&bam, &nand_info->sps.bam_handle); + if (rc) { + pr_err("%s: sps_register_bam_device() failed with %d\n", + __func__, rc); + goto put_dev; + } + pr_info("%s: BAM device registered: bam_handle 0x%lx\n", + __func__, nand_info->sps.bam_handle); +init_sps_ep: + rc = msm_nand_init_endpoint(nand_info, &nand_info->sps.data_prod, + SPS_DATA_PROD_PIPE_INDEX); + if (rc) + goto put_dev; + rc = msm_nand_init_endpoint(nand_info, &nand_info->sps.data_cons, + SPS_DATA_CONS_PIPE_INDEX); + if (rc) + goto deinit_data_prod; + + rc = msm_nand_init_endpoint(nand_info, &nand_info->sps.cmd_pipe, + SPS_CMD_CONS_PIPE_INDEX); + if (rc) + goto deinit_data_cons; + goto put_dev; +deinit_data_cons: + msm_nand_deinit_endpoint(nand_info, &nand_info->sps.data_cons); +deinit_data_prod: + msm_nand_deinit_endpoint(nand_info, &nand_info->sps.data_prod); +put_dev: + rc = msm_nand_put_device(chip->dev); +out: + mutex_unlock(&nand_info->lock); + return rc; +} + +/* + * This function disconnects and frees its end points for all the pipes. + * Since the BAM is shared resource, it is not deregistered as its handle + * might be in use with LCDC. + */ +static void msm_nand_bam_free(struct msm_nand_info *nand_info) +{ + msm_nand_deinit_endpoint(nand_info, &nand_info->sps.data_prod); + msm_nand_deinit_endpoint(nand_info, &nand_info->sps.data_cons); + msm_nand_deinit_endpoint(nand_info, &nand_info->sps.cmd_pipe); + if (nand_info->nand_chip.qpic_version >= 2) + msm_nand_deinit_endpoint(nand_info, + &nand_info->sps.data_prod_stat); +} + +/* This function enables DMA support for the NANDc in BAM mode. */ +static int msm_nand_enable_dma(struct msm_nand_info *info) +{ + struct msm_nand_sps_cmd *sps_cmd; + struct msm_nand_chip *chip = &info->nand_chip; + int ret, submitted_num_desc = 1; + struct sps_iovec iovec_temp; + + wait_event(chip->dma_wait_queue, + (sps_cmd = msm_nand_get_dma_buffer(chip, sizeof(*sps_cmd)))); + + msm_nand_prep_single_desc(sps_cmd, MSM_NAND_CTRL(info), WRITE, + (1 << BAM_MODE_EN), SPS_IOVEC_FLAG_INT); + + mutex_lock(&info->lock); + ret = msm_nand_get_device(chip->dev); + if (ret) + goto out; + + ret = sps_transfer_one(info->sps.cmd_pipe.handle, + msm_virt_to_dma(chip, &sps_cmd->ce), + sizeof(struct sps_command_element), NULL, + sps_cmd->flags); + if (ret) { + pr_err("Failed to submit command: %d\n", ret); + goto put_dev; + } + ret = msm_nand_sps_get_iovec(info->sps.cmd_pipe.handle, + info->sps.cmd_pipe.index, submitted_num_desc, + &iovec_temp); + if (ret) { + pr_err("Failed to get iovec for pipe %d (ret: %d)\n", + (info->sps.cmd_pipe.index), ret); + goto put_dev; + } +put_dev: + ret = msm_nand_put_device(chip->dev); +out: + mutex_unlock(&info->lock); + msm_nand_release_dma_buffer(chip, sps_cmd, sizeof(*sps_cmd)); + return ret; + +} + +static int msm_nand_parse_smem_ptable(int *nr_parts) +{ + + uint32_t i, j; + size_t len = FLASH_PTABLE_HDR_LEN; + struct flash_partition_entry *pentry; + char *delimiter = ":"; + void *temp_ptable = NULL; + char *name = NULL; + + pr_info("Parsing partition table info from SMEM\n"); + temp_ptable = qcom_smem_get(SMEM_APPS, SMEM_AARM_PARTITION_TABLE, &len); + + + if (IS_ERR_OR_NULL(temp_ptable)) { + pr_err("Error reading partition table header\n"); + goto out; + } + + /* Read only the header portion of ptable */ + ptable = *(struct flash_partition_table *)temp_ptable; + + /* Verify ptable magic */ + if (ptable.magic1 != FLASH_PART_MAGIC1 || + ptable.magic2 != FLASH_PART_MAGIC2) { + pr_err("Partition table magic verification failed\n"); + goto out; + } + /* Ensure that # of partitions is less than the max we have allocated */ + if (ptable.numparts > FLASH_PTABLE_MAX_PARTS_V4) { + pr_err("Partition numbers exceed the max limit\n"); + goto out; + } + /* Find out length of partition data based on table version. */ + if (ptable.version <= FLASH_PTABLE_V3) { + len = FLASH_PTABLE_HDR_LEN + FLASH_PTABLE_MAX_PARTS_V3 * + sizeof(struct flash_partition_entry); + } else if (ptable.version == FLASH_PTABLE_V4) { + len = FLASH_PTABLE_HDR_LEN + FLASH_PTABLE_MAX_PARTS_V4 * + sizeof(struct flash_partition_entry); + } else { + pr_err("Unknown ptable version (%d)\n", ptable.version); + goto out; + } + + *nr_parts = ptable.numparts; + + /* + * Now that the partition table header has been parsed, verified + * and the length of the partition table calculated, read the + * complete partition table. + */ + temp_ptable = qcom_smem_get(SMEM_APPS, SMEM_AARM_PARTITION_TABLE, &len); + if (IS_ERR_OR_NULL(temp_ptable)) { + pr_err("Error reading partition table\n"); + goto out; + } + + /* Read only the header portion of ptable */ + ptable = *(struct flash_partition_table *)temp_ptable; + + for (i = 0; i < ptable.numparts; i++) { + pentry = &ptable.part_entry[i]; + if (pentry->name[0] == '\0') + continue; + /* Convert name to lower case and discard the initial chars */ + name = pentry->name; + strsep(&name, delimiter); + mtd_part[i].name = name; + if (!mtd_part[i].name) + mtd_part[i].name = pentry->name; + for (j = 0; j < strlen(mtd_part[i].name); j++) + *((char *)mtd_part[i].name + j) = + tolower(*((char *)mtd_part[i].name + j)); + mtd_part[i].offset = pentry->offset; + mtd_part[i].mask_flags = pentry->attr; + mtd_part[i].size = pentry->length; + pr_debug("%d: %s offs=0x%08x size=0x%08x attr:0x%08x\n", + i, pentry->name, pentry->offset, pentry->length, + pentry->attr); + } + pr_info("SMEM partition table found: ver: %d len: %d\n", + ptable.version, ptable.numparts); + return 0; +out: + return -EINVAL; +} + +#define BOOT_DEV_MASK 0x1E +#define BOOT_DEV_NAND 0x4 + +/* + * This function gets called when its device named msm-nand is added to + * device tree .dts file with all its resources such as physical addresses + * for NANDc and BAM, BAM IRQ. + * + * It also expects the NAND flash partition information to be passed in .dts + * file so that it can parse the partitions by calling MTD function + * mtd_device_parse_register(). + * + */ +static int msm_nand_probe(struct platform_device *pdev) +{ + struct msm_nand_info *info; + struct resource *res; + int i, err, nr_parts; + struct device *dev; + u32 adjustment_offset; + void __iomem *boot_cfg_base; + u32 boot_dev; + struct version qpic_version = {0}; + + res = platform_get_resource_byname(pdev, IORESOURCE_MEM, + "boot_cfg"); + if (res && res->start) { + boot_cfg_base = devm_ioremap(&pdev->dev, res->start, + resource_size(res)); + if (!boot_cfg_base) { + pr_err("ioremap() failed for addr 0x%x size 0x%x\n", + res->start, resource_size(res)); + return -ENOMEM; + } + boot_dev = (readl_relaxed(boot_cfg_base) & BOOT_DEV_MASK) >> 1; + if (boot_dev != BOOT_DEV_NAND) { + pr_err("disabling nand as boot device (%x) is not NAND\n", + boot_dev); + return -ENODEV; + } + } + /* + * The partition information can also be passed from kernel command + * line. Also, the MTD core layer supports adding the whole device as + * one MTD device when no partition information is available at all. + */ + info = devm_kzalloc(&pdev->dev, sizeof(struct msm_nand_info), + GFP_KERNEL); + if (!info) { + err = -ENOMEM; + goto out; + } + res = platform_get_resource_byname(pdev, IORESOURCE_MEM, + "nand_phys"); + if (!res || !res->start) { + pr_err("NAND phys address range is not provided\n"); + err = -ENODEV; + goto out; + } + info->nand_phys = res->start; + + err = of_property_read_u32(pdev->dev.of_node, + "qcom,reg-adjustment-offset", + &adjustment_offset); + if (err) { + pr_err("adjustment_offset not found, err = %d\n", err); + WARN_ON(1); + return err; + } + + info->nand_phys_adjusted = info->nand_phys + adjustment_offset; + + res = platform_get_resource_byname(pdev, IORESOURCE_MEM, + "bam_phys"); + if (!res || !res->start) { + pr_err("BAM phys address range is not provided\n"); + err = -ENODEV; + goto out; + } + info->bam_phys = res->start; + info->bam_base = devm_ioremap(&pdev->dev, res->start, + resource_size(res)); + if (!info->bam_base) { + pr_err("BAM ioremap() failed for addr 0x%x size 0x%x\n", + res->start, resource_size(res)); + err = -ENOMEM; + goto out; + } + + info->bam_irq = platform_get_irq_byname(pdev, "bam_irq"); + if (info->bam_irq < 0) { + pr_err("BAM IRQ is not provided\n"); + err = -ENODEV; + goto out; + } + + info->mtd.name = dev_name(&pdev->dev); + info->mtd.priv = info; + info->mtd.owner = THIS_MODULE; + info->nand_chip.dev = &pdev->dev; + init_waitqueue_head(&info->nand_chip.dma_wait_queue); + mutex_init(&info->lock); + + dev = &pdev->dev; + if (dma_supported(dev, DMA_BIT_MASK(32))) { + info->dma_mask = DMA_BIT_MASK(32); + dev->coherent_dma_mask = info->dma_mask; + } + + info->nand_chip.dma_virt_addr = + dmam_alloc_coherent(&pdev->dev, MSM_NAND_DMA_BUFFER_SIZE, + &info->nand_chip.dma_phys_addr, GFP_KERNEL); + if (!info->nand_chip.dma_virt_addr) { + pr_err("No memory for DMA buffer size %x\n", + MSM_NAND_DMA_BUFFER_SIZE); + err = -ENOMEM; + goto out; + } + err = msm_nand_bus_register(pdev, info); + if (err) + goto out; + + if (of_property_read_bool(pdev->dev.of_node, "qcom,qpic-clk-rpmh")) + info->clk_data.rpmh_clk = true; + + if (!info->clk_data.rpmh_clk) { + info->clk_data.qpic_clk = devm_clk_get(&pdev->dev, "core_clk"); + if (!IS_ERR_OR_NULL(info->clk_data.qpic_clk)) { + err = clk_set_rate(info->clk_data.qpic_clk, + MSM_NAND_BUS_VOTE_MAX_RATE); + } else { + err = PTR_ERR(info->clk_data.qpic_clk); + pr_err("Failed to get clock handle, err=%d\n", err); + } + if (err) + goto bus_unregister; + } + + err = msm_nand_setup_clocks_and_bus_bw(info, true); + if (err) + goto bus_unregister; + dev_set_drvdata(&pdev->dev, info); + err = pm_runtime_set_active(&pdev->dev); + if (err) + pr_err("pm_runtime_set_active() failed with error %d\n", err); + pm_runtime_enable(&pdev->dev); + pm_runtime_use_autosuspend(&pdev->dev); + pm_runtime_set_autosuspend_delay(&pdev->dev, MSM_NAND_IDLE_TIMEOUT); + + err = msm_nand_bam_init(info); + if (err) { + pr_err("msm_nand_bam_init() failed %d\n", err); + goto clk_rpm_disable; + } + err = msm_nand_enable_dma(info); + if (err) { + pr_err("Failed to enable DMA in NANDc\n"); + goto free_bam; + } + err = msm_nand_version_check(info, &qpic_version); + if (err) { + pr_err("Failed to read the version information\n"); + goto free_bam; + } + info->nand_chip.qpic_version = qpic_version.qpic_major; + if (info->nand_chip.qpic_version >= 2) { + mutex_lock(&info->lock); + err = msm_nand_get_device(info->nand_chip.dev); + if (err) { + pr_err("Failed to get the device err=%d\n", err); + mutex_unlock(&info->lock); + goto free_bam; + } + err = msm_nand_init_endpoint(info, + &info->sps.data_prod_stat, + SPS_DATA_PROD_STAT_PIPE_INDEX); + if (err) { + pr_err("Failed to configure read status pipe err=%d\n", + err); + msm_nand_put_device(info->nand_chip.dev); + mutex_unlock(&info->lock); + goto free_bam; + } + err = msm_nand_put_device(info->nand_chip.dev); + mutex_unlock(&info->lock); + if (err) + goto free_bam; + } + err = msm_nand_parse_smem_ptable(&nr_parts); + if (err < 0) { + pr_err("Failed to parse partition table in SMEM\n"); + goto free_bam; + } + if (msm_nand_scan(&info->mtd)) { + pr_err("No nand device found\n"); + err = -ENXIO; + goto free_bam; + } + for (i = 0; i < nr_parts; i++) { + mtd_part[i].offset *= info->mtd.erasesize; + mtd_part[i].size *= info->mtd.erasesize; + } + err = mtd_device_parse_register(&info->mtd, NULL, NULL, + &mtd_part[0], nr_parts); + if (err < 0) { + pr_err("Unable to register MTD partitions %d\n", err); + goto free_bam; + } + + pr_info("NANDc phys addr 0x%lx, BAM phys addr 0x%lx, BAM IRQ %d\n", + info->nand_phys, info->bam_phys, info->bam_irq); + pr_info("Allocated DMA buffer at virt_addr 0x%pK, phys_addr 0x%x\n", + info->nand_chip.dma_virt_addr, info->nand_chip.dma_phys_addr); + pr_info("Host capabilities:0x%08x\n", info->nand_chip.caps); + goto out; +free_bam: + msm_nand_bam_free(info); +clk_rpm_disable: + msm_nand_setup_clocks_and_bus_bw(info, false); + pm_runtime_disable(&(pdev)->dev); + pm_runtime_set_suspended(&(pdev)->dev); +bus_unregister: + msm_nand_bus_unregister(info); +out: + return err; +} + +/* + * Remove functionality that gets called when driver/device msm-nand + * is removed. + */ +static int msm_nand_remove(struct platform_device *pdev) +{ + struct msm_nand_info *info = dev_get_drvdata(&pdev->dev); + + if (pm_runtime_suspended(&(pdev)->dev)) + pm_runtime_resume(&(pdev)->dev); + + pm_runtime_disable(&(pdev)->dev); + pm_runtime_set_suspended(&(pdev)->dev); + + dev_set_drvdata(&pdev->dev, NULL); + + if (info) { + msm_nand_setup_clocks_and_bus_bw(info, false); + if (info->clk_data.client_handle) + msm_nand_bus_unregister(info); + mtd_device_unregister(&info->mtd); + msm_nand_bam_free(info); + } + return 0; +} + +#define DRIVER_NAME "msm_qpic_nand" +static const struct of_device_id msm_nand_match_table[] = { + { .compatible = "qcom,msm-nand", }, + {}, +}; + +static const struct dev_pm_ops msm_nand_pm_ops = { + .suspend = msm_nand_suspend, + .resume = msm_nand_resume, + .runtime_suspend = msm_nand_runtime_suspend, + .runtime_resume = msm_nand_runtime_resume, +}; + +static struct platform_driver msm_nand_driver = { + .probe = msm_nand_probe, + .remove = msm_nand_remove, + .driver = { + .name = DRIVER_NAME, + .of_match_table = msm_nand_match_table, + .pm = &msm_nand_pm_ops, + }, +}; + +module_platform_driver(msm_nand_driver); + +MODULE_ALIAS(DRIVER_NAME); +MODULE_LICENSE("GPL v2"); +MODULE_DESCRIPTION("MSM QPIC NAND flash driver"); diff --git a/drivers/mtd/devices/msm_qpic_nand.h b/drivers/mtd/devices/msm_qpic_nand.h new file mode 100644 index 000000000000..abb2a6ca5839 --- /dev/null +++ b/drivers/mtd/devices/msm_qpic_nand.h @@ -0,0 +1,430 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) 2007 Google, Inc. + * Copyright (c) 2012-2020 The Linux Foundation. All rights reserved. + */ + +#ifndef __QPIC_NAND_H +#define __QPIC_NAND_H + +#define pr_fmt(fmt) "%s: " fmt, __func__ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define PAGE_SIZE_2K 2048 +#define PAGE_SIZE_4K 4096 + +#undef WRITE /* To avoid redefinition in above header files */ +#undef READ /* To avoid redefinition in above header files */ +#define WRITE 1 +#define READ 0 + +#define MSM_NAND_IDLE_TIMEOUT 200 /* msecs */ +#define MSM_NAND_BUS_VOTE_MAX_RATE 100000000 /* Hz */ + +/* + * The maximum no of descriptors per transfer (page read/write) won't be more + * than 64. For more details on what those commands are, please refer to the + * page read and page write functions in the driver. + */ +#define SPS_MAX_DESC_NUM 64 +#define SPS_DATA_CONS_PIPE_INDEX 0 +#define SPS_DATA_PROD_PIPE_INDEX 1 +#define SPS_CMD_CONS_PIPE_INDEX 2 +#define SPS_DATA_PROD_STAT_PIPE_INDEX 3 + +#define msm_virt_to_dma(chip, vaddr) \ + ((chip)->dma_phys_addr + \ + ((uint8_t *)(vaddr) - (chip)->dma_virt_addr)) + +/* + * A single page read/write request would typically need DMA memory of about + * 1K memory approximately. So for a single request this memory is more than + * enough. + * + * But to accommodate multiple clients we allocate 8K of memory. Though only + * one client request can be submitted to NANDc at any time, other clients can + * still prepare the descriptors while waiting for current client request to + * be done. Thus for a total memory of 8K, the driver can currently support + * maximum clients up to 7 or 8 at a time. The client for which there is no + * free DMA memory shall wait on the wait queue until other clients free up + * the required memory. + */ +#define MSM_NAND_DMA_BUFFER_SIZE SZ_8K +/* + * This defines the granularity at which the buffer management is done. The + * total number of slots is based on the size of the atomic_t variable + * dma_buffer_busy(number of bits) within the structure msm_nand_chip. + */ +#define MSM_NAND_DMA_BUFFER_SLOT_SZ \ + (MSM_NAND_DMA_BUFFER_SIZE / (sizeof(((atomic_t *)0)->counter) * 8)) + +/* ONFI(Open NAND Flash Interface) parameters */ +#define MSM_NAND_CFG0_RAW_ONFI_IDENTIFIER 0x88000800 +#define MSM_NAND_CFG0_RAW_ONFI_PARAM_INFO 0x88040000 +#define MSM_NAND_CFG1_RAW_ONFI_IDENTIFIER 0x0005045d +#define MSM_NAND_CFG1_RAW_ONFI_PARAM_INFO 0x0005045d +#define ONFI_PARAM_INFO_LENGTH 0x0200 +#define ONFI_PARAM_PAGE_LENGTH 0x0100 +#define ONFI_PARAMETER_PAGE_SIGNATURE 0x49464E4F +#define FLASH_READ_ONFI_SIGNATURE_ADDRESS 0x20 +#define FLASH_READ_ONFI_PARAMETERS_ADDRESS 0x00 +#define FLASH_READ_DEVICE_ID_ADDRESS 0x00 + +#define MSM_NAND_RESET_FLASH_STS 0x00000020 +#define MSM_NAND_RESET_READ_STS 0x000000C0 + +/* QPIC NANDc (NAND Controller) Register Set */ +#define MSM_NAND_REG(info, off) (info->nand_phys + off) +#define MSM_NAND_REG_ADJUSTED(info, off) (info->nand_phys_adjusted + off) +#define MSM_NAND_QPIC_VERSION(info) MSM_NAND_REG_ADJUSTED(info, 0x20100) +#define MSM_NAND_FLASH_CMD(info) MSM_NAND_REG(info, 0x30000) +#define MSM_NAND_ADDR0(info) MSM_NAND_REG(info, 0x30004) +#define MSM_NAND_ADDR1(info) MSM_NAND_REG(info, 0x30008) +#define MSM_NAND_EXEC_CMD(info) MSM_NAND_REG(info, 0x30010) +#define MSM_NAND_FLASH_STATUS(info) MSM_NAND_REG(info, 0x30014) +#define FS_OP_ERR (1 << 4) +#define FS_MPU_ERR (1 << 8) +#define FS_DEVICE_STS_ERR (1 << 16) +#define FS_DEVICE_WP (1 << 23) + +#define MSM_NAND_BUFFER_STATUS(info) MSM_NAND_REG(info, 0x30018) +#define BS_UNCORRECTABLE_BIT (1 << 8) +#define BS_CORRECTABLE_ERR_MSK 0x1F + +#define MSM_NAND_DEV0_CFG0(info) MSM_NAND_REG(info, 0x30020) +#define DISABLE_STATUS_AFTER_WRITE 4 +#define CW_PER_PAGE 6 +#define UD_SIZE_BYTES 9 +#define SPARE_SIZE_BYTES 23 +#define NUM_ADDR_CYCLES 27 + +#define MSM_NAND_DEV0_CFG1(info) MSM_NAND_REG(info, 0x30024) +#define DEV0_CFG1_ECC_DISABLE 0 +#define WIDE_FLASH 1 +#define NAND_RECOVERY_CYCLES 2 +#define CS_ACTIVE_BSY 5 +#define BAD_BLOCK_BYTE_NUM 6 +#define BAD_BLOCK_IN_SPARE_AREA 16 +#define WR_RD_BSY_GAP 17 +#define ENABLE_BCH_ECC 27 + +#define BYTES_512 512 +#define BYTES_516 516 +#define BYTES_517 517 + +#define MSM_NAND_DEV0_ECC_CFG(info) MSM_NAND_REG(info, 0x30028) +#define ECC_CFG_ECC_DISABLE 0 +#define ECC_SW_RESET 1 +#define ECC_MODE 4 +#define ECC_PARITY_SIZE_BYTES 8 +#define ECC_NUM_DATA_BYTES 16 +#define ECC_FORCE_CLK_OPEN 30 + +#define MSM_NAND_READ_ID(info) MSM_NAND_REG(info, 0x30040) +#define MSM_NAND_READ_STATUS(info) MSM_NAND_REG(info, 0x30044) +#define MSM_NAND_READ_ID2(info) MSM_NAND_REG(info, 0x30048) +#define EXTENDED_FETCH_ID BIT(19) +#define MSM_NAND_DEV_CMD1(info) MSM_NAND_REG(info, 0x300A4) +#define MSM_NAND_DEV_CMD_VLD(info) MSM_NAND_REG(info, 0x300AC) +#define MSM_NAND_EBI2_ECC_BUF_CFG(info) MSM_NAND_REG(info, 0x300F0) + +#define MSM_NAND_ERASED_CW_DETECT_CFG(info) MSM_NAND_REG(info, 0x300E8) +#define ERASED_CW_ECC_MASK 1 +#define AUTO_DETECT_RES 0 +#define MASK_ECC (1 << ERASED_CW_ECC_MASK) +#define RESET_ERASED_DET (1 << AUTO_DETECT_RES) +#define ACTIVE_ERASED_DET (0 << AUTO_DETECT_RES) +#define CLR_ERASED_PAGE_DET (RESET_ERASED_DET | MASK_ECC) +#define SET_ERASED_PAGE_DET (ACTIVE_ERASED_DET | MASK_ECC) + +#define MSM_NAND_ERASED_CW_DETECT_STATUS(info) MSM_NAND_REG(info, 0x300EC) +#define PAGE_ALL_ERASED 7 +#define CODEWORD_ALL_ERASED 6 +#define PAGE_ERASED 5 +#define CODEWORD_ERASED 4 +#define ERASED_PAGE ((1 << PAGE_ALL_ERASED) | (1 << PAGE_ERASED)) +#define ERASED_CW ((1 << CODEWORD_ALL_ERASED) | (1 << CODEWORD_ERASED)) + +#define MSM_NAND_CTRL(info) MSM_NAND_REG(info, 0x30F00) +#define BAM_MODE_EN 0 +#define MSM_NAND_VERSION(info) MSM_NAND_REG_ADJUSTED(info, 0x30F08) +#define MSM_NAND_READ_LOCATION_0(info) MSM_NAND_REG(info, 0x30F20) +#define MSM_NAND_READ_LOCATION_1(info) MSM_NAND_REG(info, 0x30F24) +#define MSM_NAND_READ_LOCATION_LAST_CW_0(info) MSM_NAND_REG(info, 0x30F40) +#define MSM_NAND_READ_LOCATION_LAST_CW_1(info) MSM_NAND_REG(info, 0x30F44) +#define MSM_NAND_AUTO_STATUS_EN(info) MSM_NAND_REG(info, 0x3002c) + +#define NAND_FLASH_STATUS_EN BIT(0) +#define NANDC_BUFFER_STATUS_EN BIT(1) +#define NAND_ERASED_CW_DETECT_STATUS_EN BIT(3) +#define NAND_FLASH_STATUS_LAST_CW_EN BIT(16) +#define NANDC_BUFFER_STATUS_LAST_CW_EN BIT(17) +#define NAND_ERASED_CW_DETECT_STATUS_LAST_CW_EN BIT(19) + +/* device commands */ +#define MSM_NAND_CMD_PAGE_READ 0x32 +#define MSM_NAND_CMD_PAGE_READ_ECC 0x33 +#define MSM_NAND_CMD_PAGE_READ_ALL 0x34 +#define MSM_NAND_CMD_PAGE_READ_ONFI 0x35 +#define MSM_NAND_CMD_PRG_PAGE 0x36 +#define MSM_NAND_CMD_PRG_PAGE_ECC 0x37 +#define MSM_NAND_CMD_PRG_PAGE_ALL 0x39 +#define MSM_NAND_CMD_BLOCK_ERASE 0x3A +#define MSM_NAND_CMD_FETCH_ID 0x0B + +/* device read commands for pagescope */ + +#define MSM_NAND_CMD_PAGE_READ_ECC_PS 0x800033 +#define MSM_NAND_CMD_PAGE_READ_ALL_PS 0x800034 + +/* Version Mask */ +#define MSM_NAND_VERSION_MAJOR_MASK 0xF0000000 +#define MSM_NAND_VERSION_MAJOR_SHIFT 28 +#define MSM_NAND_VERSION_MINOR_MASK 0x0FFF0000 +#define MSM_NAND_VERSION_MINOR_SHIFT 16 + +#define CMD SPS_IOVEC_FLAG_CMD +#define CMD_LCK (CMD | SPS_IOVEC_FLAG_LOCK) +#define INT SPS_IOVEC_FLAG_INT +#define INT_UNLCK (INT | SPS_IOVEC_FLAG_UNLOCK) +#define CMD_INT_UNLCK (CMD | INT_UNLCK) +#define NWD SPS_IOVEC_FLAG_NWD + +/* Structure that defines a NAND SPS command element */ +struct msm_nand_sps_cmd { + struct sps_command_element ce; + uint32_t flags; +}; + +struct msm_nand_cmd_setup_desc { + struct sps_command_element ce[13]; + uint32_t flags; + uint32_t num_ce; +}; + +struct msm_nand_cmd_cw_desc { + struct sps_command_element ce[5]; + uint32_t flags; + uint32_t num_ce; +}; + +struct msm_nand_rw_cmd_desc { + uint32_t count; + struct msm_nand_cmd_setup_desc setup_desc; + struct msm_nand_cmd_cw_desc cw_desc[]; +}; + +/* + * Structure that holds the flash, buffer, + * erased codeword status after every codeword + * read during Pagescope read operation. + */ +struct msm_nand_read_status_desc { + uint32_t flash_status; + uint32_t buffer_status; + uint32_t erased_cw_status; +}; + +/* + * Structure that defines the NAND controller properties as per the + * NAND flash device/chip that is attached. + */ +struct msm_nand_chip { + struct device *dev; + /* + * DMA memory will be allocated only once during probe and this memory + * will be used by all NAND clients. This wait queue is needed to + * make the applications wait for DMA memory to be free'd when the + * complete memory is exhausted. + */ + wait_queue_head_t dma_wait_queue; + atomic_t dma_buffer_busy; + uint8_t *dma_virt_addr; + dma_addr_t dma_phys_addr; + uint32_t ecc_parity_bytes; + uint32_t bch_caps; /* Controller BCH ECC capabilities */ +#define MSM_NAND_CAP_4_BIT_BCH (1 << 0) +#define MSM_NAND_CAP_8_BIT_BCH (1 << 1) + uint32_t cw_size; + /* NANDc register configurations */ + uint32_t cfg0, cfg1, cfg0_raw, cfg1_raw; + uint32_t ecc_buf_cfg; + uint32_t ecc_bch_cfg; + uint32_t ecc_cfg_raw; + uint32_t qpic_version; /* To store the qpic controller version */ + uint32_t caps; /* General host capabilities */ +#define MSM_NAND_CAP_PAGE_SCOPE_READ BIT(0) +#define MSM_NAND_CAP_MULTI_PAGE_READ BIT(1) +}; + +/* Structure that defines an SPS end point for a NANDc BAM pipe. */ +struct msm_nand_sps_endpt { + struct sps_pipe *handle; + struct sps_connect config; + struct sps_register_event event; + struct completion completion; + uint32_t index; +}; + +/* + * Structure that defines NANDc SPS data - BAM handle and an end point + * for each BAM pipe. + */ +struct msm_nand_sps_info { + unsigned long bam_handle; + struct msm_nand_sps_endpt data_prod; + struct msm_nand_sps_endpt data_cons; + struct msm_nand_sps_endpt cmd_pipe; + struct msm_nand_sps_endpt data_prod_stat; +}; + +/* + * Structure that contains flash device information. This gets updated after + * the NAND flash device detection. + */ +struct flash_identification { + uint32_t flash_id; + uint64_t density; + uint32_t widebus; + uint32_t pagesize; + uint32_t blksize; + uint32_t oobsize; + uint32_t ecc_correctability; + uint32_t ecc_capability; /* Set based on the ECC capability selected. */ +}; + +struct msm_nand_clk_data { + struct clk *qpic_clk; + struct msm_bus_scale_pdata *use_cases; + uint32_t client_handle; + atomic_t clk_enabled; + atomic_t curr_vote; + bool rpmh_clk; +}; + +/* Structure that defines NANDc private data. */ +struct msm_nand_info { + struct mtd_info mtd; + struct msm_nand_chip nand_chip; + struct msm_nand_sps_info sps; + unsigned long bam_phys; + unsigned long nand_phys; + unsigned long nand_phys_adjusted; + void __iomem *bam_base; + int bam_irq; + /* + * This lock must be acquired before submitting any command or data + * descriptors to BAM pipes and must be held until all the submitted + * descriptors are processed. + * + * This is required to ensure that both command and descriptors are + * submitted atomically without interruption from other clients, + * when there are requests from more than client at any time. + * Othewise, data and command descriptors can be submitted out of + * order for a request which can cause data corruption. + */ + struct mutex lock; + struct flash_identification flash_dev; + struct msm_nand_clk_data clk_data; + u64 dma_mask; +}; + +extern struct nand_flash_dev nand_flash_ids[]; + +const struct nand_manufacturer *nand_get_manufacturer(u8 id); + +/* Structure that defines an ONFI parameter page (512B) */ +struct onfi_param_page { + uint32_t parameter_page_signature; + uint16_t revision_number; + uint16_t features_supported; + uint16_t optional_commands_supported; + uint8_t reserved0[22]; + uint8_t device_manufacturer[12]; + uint8_t device_model[20]; + uint8_t jedec_manufacturer_id; + uint16_t date_code; + uint8_t reserved1[13]; + uint32_t number_of_data_bytes_per_page; + uint16_t number_of_spare_bytes_per_page; + uint32_t number_of_data_bytes_per_partial_page; + uint16_t number_of_spare_bytes_per_partial_page; + uint32_t number_of_pages_per_block; + uint32_t number_of_blocks_per_logical_unit; + uint8_t number_of_logical_units; + uint8_t number_of_address_cycles; + uint8_t number_of_bits_per_cell; + uint16_t maximum_bad_blocks_per_logical_unit; + uint16_t block_endurance; + uint8_t guaranteed_valid_begin_blocks; + uint16_t guaranteed_valid_begin_blocks_endurance; + uint8_t number_of_programs_per_page; + uint8_t partial_program_attributes; + uint8_t number_of_bits_ecc_correctability; + uint8_t number_of_interleaved_address_bits; + uint8_t interleaved_operation_attributes; + uint8_t reserved2[13]; + uint8_t io_pin_capacitance; + uint16_t timing_mode_support; + uint16_t program_cache_timing_mode_support; + uint16_t maximum_page_programming_time; + uint16_t maximum_block_erase_time; + uint16_t maximum_page_read_time; + uint16_t maximum_change_column_setup_time; + uint8_t reserved3[23]; + uint16_t vendor_specific_revision_number; + uint8_t vendor_specific[88]; + uint16_t integrity_crc; +} __attribute__((__packed__)); + +#define FLASH_PART_MAGIC1 0x55EE73AA +#define FLASH_PART_MAGIC2 0xE35EBDDB +#define FLASH_PTABLE_V3 3 +#define FLASH_PTABLE_V4 4 +#define FLASH_PTABLE_MAX_PARTS_V3 16 +#define FLASH_PTABLE_MAX_PARTS_V4 48 +#define FLASH_PTABLE_HDR_LEN (4*sizeof(uint32_t)) +#define FLASH_PTABLE_ENTRY_NAME_SIZE 16 + +struct flash_partition_entry { + char name[FLASH_PTABLE_ENTRY_NAME_SIZE]; + u32 offset; /* Offset in blocks from beginning of device */ + u32 length; /* Length of the partition in blocks */ + u8 attr; /* Flags for this partition */ +}; + +struct flash_partition_table { + u32 magic1; + u32 magic2; + u32 version; + u32 numparts; + struct flash_partition_entry part_entry[FLASH_PTABLE_MAX_PARTS_V4]; +}; + +static struct flash_partition_table ptable; + +static struct mtd_partition mtd_part[FLASH_PTABLE_MAX_PARTS_V4]; + +static inline bool is_buffer_in_page(const void *buf, size_t len) +{ + return !(((unsigned long) buf & ~PAGE_MASK) + len > PAGE_SIZE); +} +#endif /* __QPIC_NAND_H */ diff --git a/drivers/mtd/nand/raw/nand_ids.c b/drivers/mtd/nand/raw/nand_ids.c index ba27902fc54b..1b05fdaa0813 100644 --- a/drivers/mtd/nand/raw/nand_ids.c +++ b/drivers/mtd/nand/raw/nand_ids.c @@ -52,6 +52,27 @@ struct nand_flash_dev nand_flash_ids[] = { { .id = {0xad, 0xde, 0x94, 0xda, 0x74, 0xc4} }, SZ_8K, SZ_8K, SZ_2M, NAND_NEED_SCRAMBLING, 6, 640, NAND_ECC_INFO(40, SZ_1K), 4 }, + {"NM1484KSLAXAJ-3B 4G 1.8V 8-bit", + { .id = {0x98, 0xac, 0x90, 0x26, 0x76, 0x00, 0x00, 0x00} }, + SZ_4K, SZ_512, SZ_256K, 0, 5, 256, NAND_ECC_INFO(8, SZ_512) }, + {"MT29F8G08ABBCAH4 8G 3.3V 8-bit", + { .id = {0x2c, 0xa3, 0x90, 0x26, 0x00, 0x00, 0x00, 0x00} }, + SZ_4K, SZ_1K, SZ_256K, 0, 4, 224, NAND_ECC_INFO(8, SZ_512)}, + {"TC58NYG2S0HBAI4 4G 1.8V 8-bit", + { .id = {0x98, 0xac, 0x90, 0x26, 0x76, 0x00, 0x00, 0x00} }, + SZ_4K, SZ_512, SZ_256K, 0, 5, 256, NAND_ECC_INFO(8, SZ_512) }, + {"MT29F4G08ABBFA3W 4G 1.8V 8-bit", + { .id = {0x2c, 0xac, 0x80, 0x26, 0x00, 0x00, 0x00, 0x00} }, + SZ_4K, SZ_512, SZ_256K, 0, 4, 256, NAND_ECC_INFO(8, SZ_512) }, + {"MT29F4G08ABBFAH4 4G 1.8V 8-bit", + { .id = {0x2c, 0xac, 0x80, 0x26, 0x62, 0x00, 0x00, 0x00} }, + SZ_4K, SZ_512, SZ_256K, 0, 5, 256, NAND_ECC_INFO(8, SZ_512)}, + {"MT29F8G08ADBFA 8G 1.8V 8-bit", + { .id = {0x2c, 0xa3, 0xd0, 0x26, 0x66, 0x00, 0x00, 0x00} }, + SZ_4K, SZ_1K, SZ_256K, 0, 5, 256, NAND_ECC_INFO(8, SZ_512)}, + {"XT61M2G8D2TA-B8B 2G 1.8V 8-bit", + { .id = {0x98, 0xaa, 0x90, 0x15, 0x76, 0x00, 0x00, 0x00} }, + SZ_2K, SZ_256, SZ_128K, 0, 5, 128, NAND_ECC_INFO(8, SZ_512)}, LEGACY_ID_NAND("NAND 4MiB 5V 8-bit", 0x6B, 4, SZ_8K, SP_OPTIONS), LEGACY_ID_NAND("NAND 4MiB 3,3V 8-bit", 0xE3, 4, SZ_8K, SP_OPTIONS),