diff --git a/drivers/bus/mhi/core/mhi_boot.c b/drivers/bus/mhi/core/mhi_boot.c index 816f1d9d1aaf..4601ecb0c988 100644 --- a/drivers/bus/mhi/core/mhi_boot.c +++ b/drivers/bus/mhi/core/mhi_boot.c @@ -158,13 +158,14 @@ void mhi_rddm_prepare(struct mhi_controller *mhi_cntrl, MHI_LOG("BHIe programming for RDDM\n"); - mhi_write_reg(mhi_cntrl, base, BHIE_RXVECADDR_HIGH_OFFS, + mhi_cntrl->write_reg(mhi_cntrl, base, BHIE_RXVECADDR_HIGH_OFFS, upper_32_bits(mhi_buf->dma_addr)); - mhi_write_reg(mhi_cntrl, base, BHIE_RXVECADDR_LOW_OFFS, + mhi_cntrl->write_reg(mhi_cntrl, base, BHIE_RXVECADDR_LOW_OFFS, lower_32_bits(mhi_buf->dma_addr)); - mhi_write_reg(mhi_cntrl, base, BHIE_RXVECSIZE_OFFS, mhi_buf->len); + mhi_cntrl->write_reg(mhi_cntrl, base, BHIE_RXVECSIZE_OFFS, + mhi_buf->len); sequence_id = prandom_u32() & BHIE_RXVECSTATUS_SEQNUM_BMSK; if (unlikely(!sequence_id)) @@ -234,7 +235,7 @@ static int __mhi_download_rddm_in_panic(struct mhi_controller *mhi_cntrl) /* Hardware reset; force device to enter rddm */ MHI_LOG( "Did not enter RDDM, do a host req. reset\n"); - mhi_write_reg(mhi_cntrl, mhi_cntrl->regs, + mhi_cntrl->write_reg(mhi_cntrl, mhi_cntrl->regs, MHI_SOC_RESET_REQ_OFFSET, MHI_SOC_RESET_REQ); udelay(delayus); @@ -310,13 +311,14 @@ static int mhi_fw_load_amss(struct mhi_controller *mhi_cntrl, MHI_LOG("Starting BHIe Programming\n"); - mhi_write_reg(mhi_cntrl, base, BHIE_TXVECADDR_HIGH_OFFS, + mhi_cntrl->write_reg(mhi_cntrl, base, BHIE_TXVECADDR_HIGH_OFFS, upper_32_bits(mhi_buf->dma_addr)); - mhi_write_reg(mhi_cntrl, base, BHIE_TXVECADDR_LOW_OFFS, + mhi_cntrl->write_reg(mhi_cntrl, base, BHIE_TXVECADDR_LOW_OFFS, lower_32_bits(mhi_buf->dma_addr)); - mhi_write_reg(mhi_cntrl, base, BHIE_TXVECSIZE_OFFS, mhi_buf->len); + mhi_cntrl->write_reg(mhi_cntrl, base, BHIE_TXVECSIZE_OFFS, + mhi_buf->len); mhi_cntrl->sequence_id = prandom_u32() & BHIE_TXVECSTATUS_SEQNUM_BMSK; mhi_write_reg_field(mhi_cntrl, base, BHIE_TXVECDB_OFFS, @@ -374,14 +376,15 @@ static int mhi_fw_load_sbl(struct mhi_controller *mhi_cntrl, goto invalid_pm_state; } - mhi_write_reg(mhi_cntrl, base, BHI_STATUS, 0); - mhi_write_reg(mhi_cntrl, base, BHI_IMGADDR_HIGH, + mhi_cntrl->write_reg(mhi_cntrl, base, BHI_STATUS, 0); + mhi_cntrl->write_reg(mhi_cntrl, base, BHI_IMGADDR_HIGH, upper_32_bits(dma_addr)); - mhi_write_reg(mhi_cntrl, base, BHI_IMGADDR_LOW, + mhi_cntrl->write_reg(mhi_cntrl, base, BHI_IMGADDR_LOW, lower_32_bits(dma_addr)); - mhi_write_reg(mhi_cntrl, base, BHI_IMGSIZE, size); + mhi_cntrl->write_reg(mhi_cntrl, base, BHI_IMGSIZE, size); mhi_cntrl->session_id = prandom_u32() & BHI_TXDB_SEQNUM_BMSK; - mhi_write_reg(mhi_cntrl, base, BHI_IMGTXDB, mhi_cntrl->session_id); + mhi_cntrl->write_reg(mhi_cntrl, base, BHI_IMGTXDB, + mhi_cntrl->session_id); read_unlock_bh(pm_lock); MHI_LOG("Waiting for image transfer completion\n"); diff --git a/drivers/bus/mhi/core/mhi_init.c b/drivers/bus/mhi/core/mhi_init.c index 30005f38e37c..20d8630efe3a 100644 --- a/drivers/bus/mhi/core/mhi_init.c +++ b/drivers/bus/mhi/core/mhi_init.c @@ -683,7 +683,7 @@ static int mhi_init_bw_scale(struct mhi_controller *mhi_cntrl) MHI_LOG("BW_CFG OFFSET:0x%x\n", bw_cfg_offset); /* advertise host support */ - mhi_write_reg(mhi_cntrl, mhi_cntrl->regs, bw_cfg_offset, + mhi_cntrl->write_reg(mhi_cntrl, mhi_cntrl->regs, bw_cfg_offset, MHI_BW_SCALE_SETUP(er_index)); return 0; @@ -781,8 +781,8 @@ int mhi_init_mmio(struct mhi_controller *mhi_cntrl) /* setup wake db */ mhi_cntrl->wake_db = base + val + (8 * MHI_DEV_WAKE_DB); - mhi_write_reg(mhi_cntrl, mhi_cntrl->wake_db, 4, 0); - mhi_write_reg(mhi_cntrl, mhi_cntrl->wake_db, 0, 0); + mhi_cntrl->write_reg(mhi_cntrl, mhi_cntrl->wake_db, 4, 0); + mhi_cntrl->write_reg(mhi_cntrl, mhi_cntrl->wake_db, 0, 0); mhi_cntrl->wake_set = false; /* setup bw scale db */ @@ -1405,6 +1405,8 @@ int of_register_mhi_controller(struct mhi_controller *mhi_cntrl) mhi_cntrl->unmap_single = mhi_unmap_single_no_bb; } + mhi_cntrl->write_reg = mhi_write_reg; + /* read the device info if possible */ if (mhi_cntrl->regs) { ret = mhi_read_reg(mhi_cntrl, mhi_cntrl->regs, diff --git a/drivers/bus/mhi/core/mhi_internal.h b/drivers/bus/mhi/core/mhi_internal.h index 772a7b71937e..4fc0f8e5c94f 100644 --- a/drivers/bus/mhi/core/mhi_internal.h +++ b/drivers/bus/mhi/core/mhi_internal.h @@ -1,5 +1,5 @@ /* SPDX-License-Identifier: GPL-2.0-only */ -/* Copyright (c) 2018-2019, The Linux Foundation. All rights reserved. */ +/* Copyright (c) 2018-2020, The Linux Foundation. All rights reserved. */ #include @@ -818,6 +818,8 @@ void mhi_destroy_timesync(struct mhi_controller *mhi_cntrl); int mhi_create_sysfs(struct mhi_controller *mhi_cntrl); void mhi_destroy_sysfs(struct mhi_controller *mhi_cntrl); int mhi_early_notify_device(struct device *dev, void *data); +void mhi_write_reg_offload(struct mhi_controller *mhi_cntrl, + void __iomem *base, u32 offset, u32 val); /* timesync log support */ static inline void mhi_timesync_log(struct mhi_controller *mhi_cntrl) @@ -911,6 +913,8 @@ void mhi_rddm_prepare(struct mhi_controller *mhi_cntrl, struct image_info *img_info); int mhi_prepare_channel(struct mhi_controller *mhi_cntrl, struct mhi_chan *mhi_chan); +void mhi_reset_reg_write_q(struct mhi_controller *mhi_cntrl); +void mhi_force_reg_write(struct mhi_controller *mhi_cntrl); /* isr handlers */ irqreturn_t mhi_msi_handlr(int irq_number, void *dev); @@ -918,22 +922,9 @@ irqreturn_t mhi_intvec_threaded_handlr(int irq_number, void *dev); irqreturn_t mhi_intvec_handlr(int irq_number, void *dev); void mhi_ev_task(unsigned long data); -#ifdef CONFIG_MHI_DEBUG - -#define MHI_ASSERT(cond, msg) do { \ +#define MHI_ASSERT(cond, fmt, ...) do { \ if (cond) \ - panic(msg); \ + panic(fmt); \ } while (0) -#else - -#define MHI_ASSERT(cond, msg) do { \ - if (cond) { \ - MHI_ERR(msg); \ - WARN_ON(cond); \ - } \ -} while (0) - -#endif - #endif /* _MHI_INT_H */ diff --git a/drivers/bus/mhi/core/mhi_main.c b/drivers/bus/mhi/core/mhi_main.c index 464e504a8fc6..cdae828291cb 100644 --- a/drivers/bus/mhi/core/mhi_main.c +++ b/drivers/bus/mhi/core/mhi_main.c @@ -89,6 +89,45 @@ int mhi_get_capability_offset(struct mhi_controller *mhi_cntrl, return -ENXIO; } +void mhi_force_reg_write(struct mhi_controller *mhi_cntrl) +{ + if (mhi_cntrl->offload_wq) + flush_work(&mhi_cntrl->reg_write_work); +} + +void mhi_reset_reg_write_q(struct mhi_controller *mhi_cntrl) +{ + cancel_work_sync(&mhi_cntrl->reg_write_work); + memset(mhi_cntrl->reg_write_q, 0, + sizeof(struct reg_write_info) * REG_WRITE_QUEUE_LEN); + mhi_cntrl->read_idx = 0; + atomic_set(&mhi_cntrl->write_idx, -1); +} + +static void mhi_reg_write_enqueue(struct mhi_controller *mhi_cntrl, + void __iomem *reg_addr, u32 val) +{ + u32 q_index = atomic_inc_return(&mhi_cntrl->write_idx); + + q_index = q_index & (REG_WRITE_QUEUE_LEN - 1); + + MHI_ASSERT(mhi_cntrl->reg_write_q[q_index].valid, "queue full idx %d", + q_index); + + mhi_cntrl->reg_write_q[q_index].reg_addr = reg_addr; + mhi_cntrl->reg_write_q[q_index].val = val; + mhi_cntrl->reg_write_q[q_index].valid = true; +} + +void mhi_write_reg_offload(struct mhi_controller *mhi_cntrl, + void __iomem *base, + u32 offset, + u32 val) +{ + mhi_reg_write_enqueue(mhi_cntrl, base + offset, val); + queue_work(mhi_cntrl->offload_wq, &mhi_cntrl->reg_write_work); +} + void mhi_write_reg(struct mhi_controller *mhi_cntrl, void __iomem *base, u32 offset, @@ -113,15 +152,15 @@ void mhi_write_reg_field(struct mhi_controller *mhi_cntrl, tmp &= ~mask; tmp |= (val << shift); - mhi_write_reg(mhi_cntrl, base, offset, tmp); + mhi_cntrl->write_reg(mhi_cntrl, base, offset, tmp); } void mhi_write_db(struct mhi_controller *mhi_cntrl, void __iomem *db_addr, dma_addr_t wp) { - mhi_write_reg(mhi_cntrl, db_addr, 4, upper_32_bits(wp)); - mhi_write_reg(mhi_cntrl, db_addr, 0, lower_32_bits(wp)); + mhi_cntrl->write_reg(mhi_cntrl, db_addr, 4, upper_32_bits(wp)); + mhi_cntrl->write_reg(mhi_cntrl, db_addr, 0, lower_32_bits(wp)); } void mhi_db_brstmode(struct mhi_controller *mhi_cntrl, @@ -991,7 +1030,8 @@ static int parse_xfer_event(struct mhi_controller *mhi_cntrl, mhi_cntrl->unmap_single(mhi_cntrl, buf_info); result.buf_addr = buf_info->cb_buf; - result.bytes_xferd = xfer_len; + result.bytes_xferd = min_t(u16, xfer_len, + buf_info->len); mhi_del_ring_element(mhi_cntrl, buf_ring); mhi_del_ring_element(mhi_cntrl, tre_ring); local_rp = tre_ring->rp; @@ -1072,7 +1112,12 @@ static int parse_rsc_event(struct mhi_controller *mhi_cntrl, xfer_len = MHI_TRE_GET_EV_LEN(event); /* received out of bound cookie */ - MHI_ASSERT(cookie >= buf_ring->len, "Invalid Cookie\n"); + if (cookie >= buf_ring->len) { + MHI_ERR("cookie 0x%08x bufring_len %zu", cookie, buf_ring->len); + MHI_ERR("Processing Event:0x%llx 0x%08x 0x%08x\n", + event->ptr, event->dword[0], event->dword[1]); + panic("invalid cookie"); + } buf_info = buf_ring->base + cookie; @@ -1140,6 +1185,10 @@ static void mhi_process_cmd_completion(struct mhi_controller *mhi_cntrl, complete(&mhi_tsync->completion); } else { chan = MHI_TRE_GET_CMD_CHID(cmd_pkt); + if (chan >= mhi_cntrl->max_chan) { + MHI_ERR("invalid channel id %u\n", chan); + goto del_ring_el; + } mhi_chan = &mhi_cntrl->mhi_chan[chan]; write_lock_bh(&mhi_chan->lock); mhi_chan->ccs = MHI_TRE_GET_EV_CODE(tre); @@ -1147,6 +1196,7 @@ static void mhi_process_cmd_completion(struct mhi_controller *mhi_cntrl, write_unlock_bh(&mhi_chan->lock); } +del_ring_el: mhi_del_ring_element(mhi_cntrl, mhi_ring); } @@ -1495,7 +1545,7 @@ int mhi_process_bw_scale_ev_ring(struct mhi_controller *mhi_cntrl, read_lock_bh(&mhi_cntrl->pm_lock); if (likely(MHI_DB_ACCESS_VALID(mhi_cntrl))) - mhi_write_reg(mhi_cntrl, mhi_cntrl->bw_scale_db, 0, + mhi_cntrl->write_reg(mhi_cntrl, mhi_cntrl->bw_scale_db, 0, MHI_BW_SCALE_RESULT(result, link_info.sequence_num)); diff --git a/drivers/bus/mhi/core/mhi_pm.c b/drivers/bus/mhi/core/mhi_pm.c index 7d41b9491791..04ac30009827 100644 --- a/drivers/bus/mhi/core/mhi_pm.c +++ b/drivers/bus/mhi/core/mhi_pm.c @@ -1,5 +1,5 @@ // SPDX-License-Identifier: GPL-2.0-only -/* Copyright (c) 2018-2019, The Linux Foundation. All rights reserved. */ +/* Copyright (c) 2018-2020, The Linux Foundation. All rights reserved. */ #include #include @@ -167,8 +167,8 @@ void mhi_set_mhi_state(struct mhi_controller *mhi_cntrl, mhi_write_reg_field(mhi_cntrl, mhi_cntrl->regs, MHICTRL, MHICTRL_RESET_MASK, MHICTRL_RESET_SHIFT, 1); } else { - mhi_write_reg_field(mhi_cntrl, mhi_cntrl->regs, MHICTRL, - MHICTRL_MHISTATE_MASK, MHICTRL_MHISTATE_SHIFT, state); + mhi_cntrl->write_reg(mhi_cntrl, mhi_cntrl->regs, MHICTRL, + (state << MHICTRL_MHISTATE_SHIFT)); } } @@ -481,6 +481,12 @@ static int mhi_pm_mission_mode_transition(struct mhi_controller *mhi_cntrl) wake_up_all(&mhi_cntrl->state_event); + /* offload register write if supported */ + if (mhi_cntrl->offload_wq) { + mhi_reset_reg_write_q(mhi_cntrl); + mhi_cntrl->write_reg = mhi_write_reg_offload; + } + /* force MHI to be in M0 state before continuing */ ret = __mhi_device_get_sync(mhi_cntrl); if (ret) @@ -556,6 +562,12 @@ static void mhi_pm_disable_transition(struct mhi_controller *mhi_cntrl, TO_MHI_STATE_STR(mhi_cntrl->dev_state), to_mhi_pm_state_str(transition_state)); + /* restore async write call back */ + mhi_cntrl->write_reg = mhi_write_reg; + + if (mhi_cntrl->offload_wq) + mhi_reset_reg_write_q(mhi_cntrl); + /* We must notify MHI control driver so it can clean up first */ if (transition_state == MHI_PM_SYS_ERR_PROCESS) mhi_cntrl->status_cb(mhi_cntrl, mhi_cntrl->priv_data, @@ -609,7 +621,7 @@ static void mhi_pm_disable_transition(struct mhi_controller *mhi_cntrl, * device cleares INTVEC as part of RESET processing, * re-program it */ - mhi_write_reg(mhi_cntrl, mhi_cntrl->bhi, BHI_INTVEC, 0); + mhi_cntrl->write_reg(mhi_cntrl, mhi_cntrl->bhi, BHI_INTVEC, 0); } MHI_LOG("Waiting for all pending event ring processing to complete\n"); @@ -932,7 +944,7 @@ int mhi_async_power_up(struct mhi_controller *mhi_cntrl) mhi_cntrl->bhie = mhi_cntrl->regs + val; } - mhi_write_reg(mhi_cntrl, mhi_cntrl->bhi, BHI_INTVEC, 0); + mhi_cntrl->write_reg(mhi_cntrl, mhi_cntrl->bhi, BHI_INTVEC, 0); mhi_cntrl->pm_state = MHI_PM_POR; mhi_cntrl->ee = MHI_EE_MAX; current_ee = mhi_get_exec_env(mhi_cntrl); @@ -1001,6 +1013,8 @@ void mhi_control_error(struct mhi_controller *mhi_cntrl) goto exit_control_error; } + mhi_cntrl->dev_state = MHI_STATE_SYS_ERR; + /* notify waiters to bail out early since MHI has entered ERROR state */ wake_up_all(&mhi_cntrl->state_event); @@ -1146,6 +1160,9 @@ int mhi_pm_suspend(struct mhi_controller *mhi_cntrl) write_unlock_irq(&mhi_cntrl->pm_lock); MHI_LOG("Wait for M3 completion\n"); + /* finish reg writes before D3 cold */ + mhi_force_reg_write(mhi_cntrl); + ret = wait_event_timeout(mhi_cntrl->state_event, mhi_cntrl->dev_state == MHI_STATE_M3 || MHI_PM_IN_ERROR_STATE(mhi_cntrl->pm_state), @@ -1272,6 +1289,9 @@ int mhi_pm_fast_suspend(struct mhi_controller *mhi_cntrl, bool notify_client) mhi_cntrl->M3_FAST++; write_unlock_irq(&mhi_cntrl->pm_lock); + /* finish reg writes before DRV hand-off to avoid noc err */ + mhi_force_reg_write(mhi_cntrl); + /* now safe to check ctrl event ring */ tasklet_enable(&mhi_cntrl->mhi_event->task); mhi_msi_handlr(0, mhi_cntrl->mhi_event); @@ -1477,6 +1497,8 @@ int __mhi_device_get_sync(struct mhi_controller *mhi_cntrl) mhi_trigger_resume(mhi_cntrl); read_unlock_bh(&mhi_cntrl->pm_lock); + mhi_force_reg_write(mhi_cntrl); + ret = wait_event_timeout(mhi_cntrl->state_event, mhi_cntrl->pm_state == MHI_PM_M0 || MHI_PM_IN_ERROR_STATE(mhi_cntrl->pm_state), diff --git a/include/linux/mhi.h b/include/linux/mhi.h index 6bdf731936b9..7f974e2d5991 100644 --- a/include/linux/mhi.h +++ b/include/linux/mhi.h @@ -13,6 +13,8 @@ struct bhi_vec_entry; struct mhi_timesync; struct mhi_buf_info; +#define REG_WRITE_QUEUE_LEN 1024 + /** * enum MHI_CB - MHI callback * @MHI_CB_IDLE: MHI entered idle state @@ -185,6 +187,19 @@ struct file_info { u32 rem_seg_len; }; +/** + * struct reg_write_info - offload reg write info + * @reg_addr - register address + * @val - value to be written to register + * @chan - channel number + * @valid - entry is valid or not + */ +struct reg_write_info { + void __iomem *reg_addr; + u32 val; + bool valid; +}; + /** * struct mhi_controller - Master controller structure for external modem * @dev: Device associated with this controller @@ -353,6 +368,8 @@ struct mhi_controller { void (*tsync_log)(struct mhi_controller *mhi_cntrl, u64 remote_time); int (*bw_scale)(struct mhi_controller *mhi_cntrl, struct mhi_link_info *link_info); + void (*write_reg)(struct mhi_controller *mhi_cntrl, void __iomem *base, + u32 offset, u32 val); /* channel to control DTR messaging */ struct mhi_device *dtr_dev; @@ -379,6 +396,13 @@ struct mhi_controller { void *log_buf; struct dentry *dentry; struct dentry *parent; + + /* for reg write offload */ + struct workqueue_struct *offload_wq; + struct work_struct reg_write_work; + struct reg_write_info *reg_write_q; + atomic_t write_idx; + u32 read_idx; }; /**