diff --git a/drivers/scsi/ufs/ufshcd.c b/drivers/scsi/ufs/ufshcd.c index 57a58d0b60bd..704c3a4eebe1 100644 --- a/drivers/scsi/ufs/ufshcd.c +++ b/drivers/scsi/ufs/ufshcd.c @@ -535,8 +535,8 @@ static void ufshcd_print_host_state(struct ufs_hba *hba) struct scsi_device *sdev_ufs = hba->sdev_ufs_device; dev_err(hba->dev, "UFS Host state=%d\n", hba->ufshcd_state); - dev_err(hba->dev, "outstanding reqs=0x%lx tasks=0x%lx\n", - hba->outstanding_reqs, hba->outstanding_tasks); + dev_err(hba->dev, "lrb in use=0x%lx, outstanding reqs=0x%lx tasks=0x%lx\n", + hba->lrb_in_use, hba->outstanding_reqs, hba->outstanding_tasks); dev_err(hba->dev, "saved_err=0x%x, saved_uic_err=0x%x\n", hba->saved_err, hba->saved_uic_err); dev_err(hba->dev, "Device power mode=%d, UIC link state=%d\n", @@ -1383,24 +1383,6 @@ out: return ret; } -static bool ufshcd_is_busy(struct request *req, void *priv, bool reserved) -{ - int *busy = priv; - - WARN_ON_ONCE(reserved); - (*busy)++; - return false; -} - -/* Whether or not any tag is in use by a request that is in progress. */ -static bool ufshcd_any_tag_in_use(struct ufs_hba *hba) -{ - struct request_queue *q = hba->cmd_queue; - int busy = 0; - - blk_mq_tagset_busy_iter(q->tag_set, ufshcd_is_busy, &busy); - return busy; -} static int ufshcd_devfreq_get_dev_status(struct device *dev, struct devfreq_dev_status *stat) @@ -1759,7 +1741,7 @@ static void ufshcd_gate_work(struct work_struct *work) if (hba->clk_gating.active_reqs || hba->ufshcd_state != UFSHCD_STATE_OPERATIONAL - || ufshcd_any_tag_in_use(hba) || hba->outstanding_tasks + || hba->lrb_in_use || hba->outstanding_tasks || hba->active_uic_cmd || hba->uic_async_done) goto rel_lock; @@ -1818,7 +1800,7 @@ static void __ufshcd_release(struct ufs_hba *hba) if (hba->clk_gating.active_reqs || hba->clk_gating.is_suspended || hba->ufshcd_state != UFSHCD_STATE_OPERATIONAL || - ufshcd_any_tag_in_use(hba) || hba->outstanding_tasks || + hba->lrb_in_use || hba->outstanding_tasks || hba->active_uic_cmd || hba->uic_async_done) return; @@ -2580,9 +2562,22 @@ static int ufshcd_queuecommand(struct Scsi_Host *host, struct scsi_cmnd *cmd) hba->req_abort_count = 0; + /* acquire the tag to make sure device cmds don't use it */ + if (test_and_set_bit_lock(tag, &hba->lrb_in_use)) { + /* + * Dev manage command in progress, requeue the command. + * Requeuing the command helps in cases where the request *may* + * find different tag instead of waiting for dev manage command + * completion. + */ + err = SCSI_MLQUEUE_HOST_BUSY; + goto out; + } + err = ufshcd_hold(hba, true); if (err) { err = SCSI_MLQUEUE_HOST_BUSY; + clear_bit_unlock(tag, &hba->lrb_in_use); goto out; } WARN_ON(ufshcd_is_clkgating_allowed(hba) && @@ -2602,6 +2597,7 @@ static int ufshcd_queuecommand(struct Scsi_Host *host, struct scsi_cmnd *cmd) if (err) { ufshcd_release(hba); lrbp->cmd = NULL; + clear_bit_unlock(tag, &hba->lrb_in_use); goto out; } lrbp->req_abort_skip = false; @@ -2612,6 +2608,7 @@ static int ufshcd_queuecommand(struct Scsi_Host *host, struct scsi_cmnd *cmd) if (err) { ufshcd_release(hba); lrbp->cmd = NULL; + clear_bit_unlock(tag, &hba->lrb_in_use); goto out; } /* Make sure descriptors are ready before ringing the doorbell */ @@ -2658,6 +2655,7 @@ static int ufshcd_queuecommand(struct Scsi_Host *host, struct scsi_cmnd *cmd) out_compl_cmd: scsi_dma_unmap(lrbp->cmd); lrbp->cmd = NULL; + clear_bit_unlock(tag, &hba->lrb_in_use); spin_unlock_irqrestore(hba->host->host_lock, flags); ufshcd_release(hba); if (!err) @@ -2800,6 +2798,44 @@ static int ufshcd_wait_for_dev_cmd(struct ufs_hba *hba, return err; } +/** + * ufshcd_get_dev_cmd_tag - Get device management command tag + * @hba: per-adapter instance + * @tag_out: pointer to variable with available slot value + * + * Get a free slot and lock it until device management command + * completes. + * + * Returns false if free slot is unavailable for locking, else + * return true with tag value in @tag. + */ +static bool ufshcd_get_dev_cmd_tag(struct ufs_hba *hba, int *tag_out) +{ + int tag; + bool ret = false; + unsigned long tmp; + + if (!tag_out) + goto out; + + do { + tmp = ~hba->lrb_in_use; + tag = find_last_bit(&tmp, hba->nutrs); + if (tag >= hba->nutrs) + goto out; + } while (test_and_set_bit_lock(tag, &hba->lrb_in_use)); + + *tag_out = tag; + ret = true; +out: + return ret; +} + +static inline void ufshcd_put_dev_cmd_tag(struct ufs_hba *hba, int tag) +{ + clear_bit_unlock(tag, &hba->lrb_in_use); +} + /** * ufshcd_exec_dev_cmd - API for sending device management requests * @hba: UFS hba @@ -2812,8 +2848,6 @@ static int ufshcd_wait_for_dev_cmd(struct ufs_hba *hba, static int ufshcd_exec_dev_cmd(struct ufs_hba *hba, enum dev_cmd_type cmd_type, int timeout) { - struct request_queue *q = hba->cmd_queue; - struct request *req; struct ufshcd_lrb *lrbp; int err; int tag; @@ -2827,11 +2861,7 @@ static int ufshcd_exec_dev_cmd(struct ufs_hba *hba, * Even though we use wait_event() which sleeps indefinitely, * the maximum wait time is bounded by SCSI request timeout. */ - req = blk_get_request(q, REQ_OP_DRV_OUT, 0); - if (IS_ERR(req)) - return PTR_ERR(req); - tag = req->tag; - WARN_ON_ONCE(!ufshcd_valid_tag(hba, tag)); + wait_event(hba->dev_cmd.tag_wq, ufshcd_get_dev_cmd_tag(hba, &tag)); init_completion(&wait); lrbp = &hba->lrb[tag]; @@ -2855,7 +2885,8 @@ static int ufshcd_exec_dev_cmd(struct ufs_hba *hba, err ? "query_complete_err" : "query_complete"); out_put_tag: - blk_put_request(req); + ufshcd_put_dev_cmd_tag(hba, tag); + wake_up(&hba->dev_cmd.tag_wq); up_read(&hba->clk_scaling_lock); return err; } @@ -5092,6 +5123,7 @@ static void __ufshcd_transfer_req_compl(struct ufs_hba *hba, /* Mark completed command as NULL in LRB */ lrbp->cmd = NULL; lrbp->compl_time_stamp = ktime_get(); + clear_bit_unlock(index, &hba->lrb_in_use); /* Do not touch lrbp after scsi done */ cmd->scsi_done(cmd); __ufshcd_release(hba); @@ -5112,6 +5144,9 @@ static void __ufshcd_transfer_req_compl(struct ufs_hba *hba, hba->outstanding_reqs ^= completed_reqs; ufshcd_clk_scaling_update_busy(hba); + + /* we might have free'd some tags above */ + wake_up(&hba->dev_cmd.tag_wq); } /** @@ -6399,8 +6434,6 @@ static int ufshcd_issue_devman_upiu_cmd(struct ufs_hba *hba, int cmd_type, enum query_opcode desc_op) { - struct request_queue *q = hba->cmd_queue; - struct request *req; struct ufshcd_lrb *lrbp; int err = 0; int tag; @@ -6410,11 +6443,7 @@ static int ufshcd_issue_devman_upiu_cmd(struct ufs_hba *hba, down_read(&hba->clk_scaling_lock); - req = blk_get_request(q, REQ_OP_DRV_OUT, 0); - if (IS_ERR(req)) - return PTR_ERR(req); - tag = req->tag; - WARN_ON_ONCE(!ufshcd_valid_tag(hba, tag)); + wait_event(hba->dev_cmd.tag_wq, ufshcd_get_dev_cmd_tag(hba, &tag)); init_completion(&wait); lrbp = &hba->lrb[tag]; @@ -6488,7 +6517,8 @@ static int ufshcd_issue_devman_upiu_cmd(struct ufs_hba *hba, } } - blk_put_request(req); + ufshcd_put_dev_cmd_tag(hba, tag); + wake_up(&hba->dev_cmd.tag_wq); up_read(&hba->clk_scaling_lock); return err; } @@ -8991,7 +9021,6 @@ void ufshcd_remove(struct ufs_hba *hba) { ufs_bsg_remove(hba); ufs_sysfs_remove_nodes(hba->dev); - blk_cleanup_queue(hba->cmd_queue); scsi_remove_host(hba->host); /* disable interrupts */ ufshcd_disable_intr(hba, hba->intr_mask); @@ -9167,6 +9196,9 @@ int ufshcd_init(struct ufs_hba *hba, void __iomem *mmio_base, unsigned int irq) init_rwsem(&hba->clk_scaling_lock); + /* Initialize device management tag acquire wait queue */ + init_waitqueue_head(&hba->dev_cmd.tag_wq); + ufshcd_init_clk_gating(hba); ufshcd_init_clk_scaling(hba); @@ -9200,12 +9232,6 @@ int ufshcd_init(struct ufs_hba *hba, void __iomem *mmio_base, unsigned int irq) goto exit_gating; } - hba->cmd_queue = blk_mq_init_queue(&hba->host->tag_set); - if (IS_ERR(hba->cmd_queue)) { - err = PTR_ERR(hba->cmd_queue); - goto out_remove_scsi_host; - } - /* Reset the attached device */ ufshcd_vops_device_reset(hba); @@ -9222,7 +9248,7 @@ int ufshcd_init(struct ufs_hba *hba, void __iomem *mmio_base, unsigned int irq) dev_err(hba->dev, "Host controller enable failed\n"); ufshcd_print_host_regs(hba); ufshcd_print_host_state(hba); - goto free_cmd_queue; + goto out_remove_scsi_host; } /* @@ -9262,8 +9288,6 @@ int ufshcd_init(struct ufs_hba *hba, void __iomem *mmio_base, unsigned int irq) return 0; -free_cmd_queue: - blk_cleanup_queue(hba->cmd_queue); out_remove_scsi_host: scsi_remove_host(hba->host); exit_gating: diff --git a/drivers/scsi/ufs/ufshcd.h b/drivers/scsi/ufs/ufshcd.h index 58f8290cad2c..13783dc5a0b4 100644 --- a/drivers/scsi/ufs/ufshcd.h +++ b/drivers/scsi/ufs/ufshcd.h @@ -242,11 +242,13 @@ struct ufs_query { * @type: device management command type - Query, NOP OUT * @lock: lock to allow one command at a time * @complete: internal commands completion + * @tag_wq: wait queue until free command slot is available */ struct ufs_dev_cmd { enum dev_cmd_type type; struct mutex lock; struct completion *complete; + wait_queue_head_t tag_wq; struct ufs_query query; }; @@ -558,7 +560,7 @@ struct ufs_hba_variant_params { * @host: Scsi_Host instance of the driver * @dev: device handle * @lrb: local reference block - * @cmd_queue: Used to allocate command tags from hba->host->tag_set. + * @lrb_in_use: lrb in use * @outstanding_tasks: Bits representing outstanding task requests * @outstanding_reqs: Bits representing outstanding transfer requests * @capabilities: UFS Controller Capabilities @@ -622,7 +624,6 @@ struct ufs_hba { struct Scsi_Host *host; struct device *dev; - struct request_queue *cmd_queue; /* * This field is to keep a reference to "scsi_device" corresponding to * "UFS device" W-LU. @@ -643,6 +644,7 @@ struct ufs_hba { u32 ahit; struct ufshcd_lrb *lrb; + unsigned long lrb_in_use; unsigned long outstanding_tasks; unsigned long outstanding_reqs;