diff options
Diffstat (limited to 'drivers/ufs/core/ufshcd.c')
| -rw-r--r-- | drivers/ufs/core/ufshcd.c | 148 |
1 files changed, 88 insertions, 60 deletions
diff --git a/drivers/ufs/core/ufshcd.c b/drivers/ufs/core/ufshcd.c index 96ad57c3144b..d9632d7c5f01 100644 --- a/drivers/ufs/core/ufshcd.c +++ b/drivers/ufs/core/ufshcd.c @@ -45,11 +45,6 @@ UTP_TASK_REQ_COMPL |\ UFSHCD_ERROR_MASK) -#define UFSHCD_ENABLE_MCQ_INTRS (UTP_TASK_REQ_COMPL |\ - UFSHCD_ERROR_MASK |\ - MCQ_CQ_EVENT_STATUS) - - /* UIC command timeout, unit: ms */ enum { UIC_CMD_TIMEOUT_DEFAULT = 500, @@ -316,6 +311,9 @@ static const struct ufs_dev_quirk ufs_fixups[] = { { .wmanufacturerid = UFS_VENDOR_TOSHIBA, .model = "THGLF2G9D8KBADG", .quirk = UFS_DEVICE_QUIRK_PA_TACTIVATE }, + { .wmanufacturerid = UFS_VENDOR_TOSHIBA, + .model = "THGJFJT1E45BATP", + .quirk = UFS_DEVICE_QUIRK_NO_TIMESTAMP_SUPPORT }, {} }; @@ -369,7 +367,7 @@ EXPORT_SYMBOL_GPL(ufshcd_disable_irq); * @hba: per adapter instance * @intrs: interrupt bits */ -static void ufshcd_enable_intr(struct ufs_hba *hba, u32 intrs) +void ufshcd_enable_intr(struct ufs_hba *hba, u32 intrs) { u32 old_val = ufshcd_readl(hba, REG_INTERRUPT_ENABLE); u32 new_val = old_val | intrs; @@ -606,10 +604,12 @@ void ufshcd_print_tr(struct ufs_hba *hba, int tag, bool pr_prdt) lrbp = &hba->lrb[tag]; - dev_err(hba->dev, "UPIU[%d] - issue time %lld us\n", - tag, div_u64(lrbp->issue_time_stamp_local_clock, 1000)); - dev_err(hba->dev, "UPIU[%d] - complete time %lld us\n", - tag, div_u64(lrbp->compl_time_stamp_local_clock, 1000)); + if (hba->monitor.enabled) { + dev_err(hba->dev, "UPIU[%d] - issue time %lld us\n", tag, + div_u64(lrbp->issue_time_stamp_local_clock, 1000)); + dev_err(hba->dev, "UPIU[%d] - complete time %lld us\n", tag, + div_u64(lrbp->compl_time_stamp_local_clock, 1000)); + } dev_err(hba->dev, "UPIU[%d] - Transfer Request Descriptor phys@0x%llx\n", tag, (u64)lrbp->utrd_dma_addr); @@ -1045,6 +1045,7 @@ EXPORT_SYMBOL_GPL(ufshcd_is_hba_active); */ void ufshcd_pm_qos_init(struct ufs_hba *hba) { + guard(mutex)(&hba->pm_qos_mutex); if (hba->pm_qos_enabled) return; @@ -1061,6 +1062,8 @@ void ufshcd_pm_qos_init(struct ufs_hba *hba) */ void ufshcd_pm_qos_exit(struct ufs_hba *hba) { + guard(mutex)(&hba->pm_qos_mutex); + if (!hba->pm_qos_enabled) return; @@ -1075,6 +1078,8 @@ void ufshcd_pm_qos_exit(struct ufs_hba *hba) */ static void ufshcd_pm_qos_update(struct ufs_hba *hba, bool on) { + guard(mutex)(&hba->pm_qos_mutex); + if (!hba->pm_qos_enabled) return; @@ -1303,7 +1308,7 @@ static u32 ufshcd_pending_cmds(struct ufs_hba *hba) * * Return: 0 upon success; -EBUSY upon timeout. */ -static int ufshcd_wait_for_doorbell_clr(struct ufs_hba *hba, +static int ufshcd_wait_for_pending_cmds(struct ufs_hba *hba, u64 wait_timeout_us) { int ret = 0; @@ -1431,7 +1436,7 @@ static int ufshcd_clock_scaling_prepare(struct ufs_hba *hba, u64 timeout_us) down_write(&hba->clk_scaling_lock); if (!hba->clk_scaling.is_allowed || - ufshcd_wait_for_doorbell_clr(hba, timeout_us)) { + ufshcd_wait_for_pending_cmds(hba, timeout_us)) { ret = -EBUSY; up_write(&hba->clk_scaling_lock); mutex_unlock(&hba->wb_mutex); @@ -2230,11 +2235,13 @@ static void ufshcd_exit_clk_gating(struct ufs_hba *hba) static void ufshcd_clk_scaling_start_busy(struct ufs_hba *hba) { bool queue_resume_work = false; - ktime_t curr_t = ktime_get(); + ktime_t curr_t; if (!ufshcd_is_clkscaling_supported(hba)) return; + curr_t = ktime_get(); + guard(spinlock_irqsave)(&hba->clk_scaling.lock); if (!hba->clk_scaling.active_reqs++) @@ -2354,10 +2361,12 @@ void ufshcd_send_command(struct ufs_hba *hba, unsigned int task_tag, struct ufshcd_lrb *lrbp = &hba->lrb[task_tag]; unsigned long flags; - lrbp->issue_time_stamp = ktime_get(); - lrbp->issue_time_stamp_local_clock = local_clock(); - lrbp->compl_time_stamp = ktime_set(0, 0); - lrbp->compl_time_stamp_local_clock = 0; + if (hba->monitor.enabled) { + lrbp->issue_time_stamp = ktime_get(); + lrbp->issue_time_stamp_local_clock = local_clock(); + lrbp->compl_time_stamp = ktime_set(0, 0); + lrbp->compl_time_stamp_local_clock = 0; + } ufshcd_add_command_trace(hba, task_tag, UFS_CMD_SEND); if (lrbp->cmd) ufshcd_clk_scaling_start_busy(hba); @@ -3199,7 +3208,8 @@ ufshcd_dev_cmd_completion(struct ufs_hba *hba, struct ufshcd_lrb *lrbp) } /* - * Return: 0 upon success; < 0 upon failure. + * Return: 0 upon success; > 0 in case the UFS device reported an OCS error; + * < 0 if another error occurred. */ static int ufshcd_wait_for_dev_cmd(struct ufs_hba *hba, struct ufshcd_lrb *lrbp, int max_timeout) @@ -3275,7 +3285,6 @@ retry: } } - WARN_ONCE(err > 0, "Incorrect return value %d > 0\n", err); return err; } @@ -3294,7 +3303,8 @@ static void ufshcd_dev_man_unlock(struct ufs_hba *hba) } /* - * Return: 0 upon success; < 0 upon failure. + * Return: 0 upon success; > 0 in case the UFS device reported an OCS error; + * < 0 if another error occurred. */ static int ufshcd_issue_dev_cmd(struct ufs_hba *hba, struct ufshcd_lrb *lrbp, const u32 tag, int timeout) @@ -3317,7 +3327,8 @@ static int ufshcd_issue_dev_cmd(struct ufs_hba *hba, struct ufshcd_lrb *lrbp, * @cmd_type: specifies the type (NOP, Query...) * @timeout: timeout in milliseconds * - * Return: 0 upon success; < 0 upon failure. + * Return: 0 upon success; > 0 in case the UFS device reported an OCS error; + * < 0 if another error occurred. * * NOTE: Since there is only one available tag for device management commands, * it is expected you hold the hba->dev_cmd.lock mutex. @@ -3363,6 +3374,10 @@ static inline void ufshcd_init_query(struct ufs_hba *hba, (*request)->upiu_req.selector = selector; } +/* + * Return: 0 upon success; > 0 in case the UFS device reported an OCS error; + * < 0 if another error occurred. + */ static int ufshcd_query_flag_retry(struct ufs_hba *hba, enum query_opcode opcode, enum flag_idn idn, u8 index, bool *flag_res) { @@ -3383,7 +3398,6 @@ static int ufshcd_query_flag_retry(struct ufs_hba *hba, dev_err(hba->dev, "%s: query flag, opcode %d, idn %d, failed with error %d after %d retries\n", __func__, opcode, idn, ret, retries); - WARN_ONCE(ret > 0, "Incorrect return value %d > 0\n", ret); return ret; } @@ -3395,7 +3409,8 @@ static int ufshcd_query_flag_retry(struct ufs_hba *hba, * @index: flag index to access * @flag_res: the flag value after the query request completes * - * Return: 0 for success; < 0 upon failure. + * Return: 0 upon success; > 0 in case the UFS device reported an OCS error; + * < 0 if another error occurred. */ int ufshcd_query_flag(struct ufs_hba *hba, enum query_opcode opcode, enum flag_idn idn, u8 index, bool *flag_res) @@ -3451,7 +3466,6 @@ int ufshcd_query_flag(struct ufs_hba *hba, enum query_opcode opcode, out_unlock: ufshcd_dev_man_unlock(hba); - WARN_ONCE(err > 0, "Incorrect return value %d > 0\n", err); return err; } @@ -3464,8 +3478,9 @@ out_unlock: * @selector: selector field * @attr_val: the attribute value after the query request completes * - * Return: 0 upon success; < 0 upon failure. -*/ + * Return: 0 upon success; > 0 in case the UFS device reported an OCS error; + * < 0 if another error occurred. + */ int ufshcd_query_attr(struct ufs_hba *hba, enum query_opcode opcode, enum attr_idn idn, u8 index, u8 selector, u32 *attr_val) { @@ -3513,7 +3528,6 @@ int ufshcd_query_attr(struct ufs_hba *hba, enum query_opcode opcode, out_unlock: ufshcd_dev_man_unlock(hba); - WARN_ONCE(err > 0, "Incorrect return value %d > 0\n", err); return err; } @@ -3528,8 +3542,9 @@ out_unlock: * @attr_val: the attribute value after the query request * completes * - * Return: 0 for success; < 0 upon failure. -*/ + * Return: 0 upon success; > 0 in case the UFS device reported an OCS error; + * < 0 if another error occurred. + */ int ufshcd_query_attr_retry(struct ufs_hba *hba, enum query_opcode opcode, enum attr_idn idn, u8 index, u8 selector, u32 *attr_val) @@ -3551,12 +3566,12 @@ int ufshcd_query_attr_retry(struct ufs_hba *hba, dev_err(hba->dev, "%s: query attribute, idn %d, failed with error %d after %d retries\n", __func__, idn, ret, QUERY_REQ_RETRIES); - WARN_ONCE(ret > 0, "Incorrect return value %d > 0\n", ret); return ret; } /* - * Return: 0 if successful; < 0 upon failure. + * Return: 0 upon success; > 0 in case the UFS device reported an OCS error; + * < 0 if another error occurred. */ static int __ufshcd_query_descriptor(struct ufs_hba *hba, enum query_opcode opcode, enum desc_idn idn, u8 index, @@ -3615,7 +3630,6 @@ static int __ufshcd_query_descriptor(struct ufs_hba *hba, out_unlock: hba->dev_cmd.query.descriptor = NULL; ufshcd_dev_man_unlock(hba); - WARN_ONCE(err > 0, "Incorrect return value %d > 0\n", err); return err; } @@ -3632,7 +3646,8 @@ out_unlock: * The buf_len parameter will contain, on return, the length parameter * received on the response. * - * Return: 0 for success; < 0 upon failure. + * Return: 0 upon success; > 0 in case the UFS device reported an OCS error; + * < 0 if another error occurred. */ int ufshcd_query_descriptor_retry(struct ufs_hba *hba, enum query_opcode opcode, @@ -3650,7 +3665,6 @@ int ufshcd_query_descriptor_retry(struct ufs_hba *hba, break; } - WARN_ONCE(err > 0, "Incorrect return value %d > 0\n", err); return err; } @@ -3663,7 +3677,8 @@ int ufshcd_query_descriptor_retry(struct ufs_hba *hba, * @param_read_buf: pointer to buffer where parameter would be read * @param_size: sizeof(param_read_buf) * - * Return: 0 in case of success; < 0 upon failure. + * Return: 0 upon success; > 0 in case the UFS device reported an OCS error; + * < 0 if another error occurred. */ int ufshcd_read_desc_param(struct ufs_hba *hba, enum desc_idn desc_id, @@ -3730,7 +3745,6 @@ int ufshcd_read_desc_param(struct ufs_hba *hba, out: if (is_kmalloc) kfree(desc_buf); - WARN_ONCE(ret > 0, "Incorrect return value %d > 0\n", ret); return ret; } @@ -4781,7 +4795,8 @@ EXPORT_SYMBOL_GPL(ufshcd_config_pwr_mode); * * Set fDeviceInit flag and poll until device toggles it. * - * Return: 0 upon success; < 0 upon failure. + * Return: 0 upon success; > 0 in case the UFS device reported an OCS error; + * < 0 if another error occurred. */ static int ufshcd_complete_dev_init(struct ufs_hba *hba) { @@ -5135,7 +5150,8 @@ out: * not respond with NOP IN UPIU within timeout of %NOP_OUT_TIMEOUT * and we retry sending NOP OUT for %NOP_OUT_RETRIES iterations. * - * Return: 0 upon success; < 0 upon failure. + * Return: 0 upon success; > 0 in case the UFS device reported an OCS error; + * < 0 if another error occurred. */ static int ufshcd_verify_dev_init(struct ufs_hba *hba) { @@ -5559,9 +5575,9 @@ static irqreturn_t ufshcd_uic_cmd_compl(struct ufs_hba *hba, u32 intr_status) irqreturn_t retval = IRQ_NONE; struct uic_command *cmd; - spin_lock(hba->host->host_lock); + guard(spinlock_irqsave)(hba->host->host_lock); cmd = hba->active_uic_cmd; - if (WARN_ON_ONCE(!cmd)) + if (!cmd) goto unlock; if (ufshcd_is_auto_hibern8_error(hba, intr_status)) @@ -5586,8 +5602,6 @@ static irqreturn_t ufshcd_uic_cmd_compl(struct ufs_hba *hba, u32 intr_status) ufshcd_add_uic_command_trace(hba, cmd, UFS_CMD_COMP); unlock: - spin_unlock(hba->host->host_lock); - return retval; } @@ -5617,8 +5631,10 @@ void ufshcd_compl_one_cqe(struct ufs_hba *hba, int task_tag, enum utp_ocs ocs; lrbp = &hba->lrb[task_tag]; - lrbp->compl_time_stamp = ktime_get(); - lrbp->compl_time_stamp_local_clock = local_clock(); + if (hba->monitor.enabled) { + lrbp->compl_time_stamp = ktime_get(); + lrbp->compl_time_stamp_local_clock = local_clock(); + } cmd = lrbp->cmd; if (cmd) { if (unlikely(ufshcd_should_inform_monitor(hba, lrbp))) @@ -5869,7 +5885,8 @@ static inline int ufshcd_enable_ee(struct ufs_hba *hba, u16 mask) * as the device is allowed to manage its own way of handling background * operations. * - * Return: zero on success, non-zero on failure. + * Return: 0 upon success; > 0 in case the UFS device reported an OCS error; + * < 0 if another error occurred. */ static int ufshcd_enable_auto_bkops(struct ufs_hba *hba) { @@ -5908,7 +5925,8 @@ out: * host is idle so that BKOPS are managed effectively without any negative * impacts. * - * Return: zero on success, non-zero on failure. + * Return: 0 upon success; > 0 in case the UFS device reported an OCS error; + * < 0 if another error occurred. */ static int ufshcd_disable_auto_bkops(struct ufs_hba *hba) { @@ -6058,6 +6076,10 @@ out: __func__, err); } +/* + * Return: 0 upon success; > 0 in case the UFS device reported an OCS error; + * < 0 if another error occurred. + */ int ufshcd_read_device_lvl_exception_id(struct ufs_hba *hba, u64 *exception_id) { struct utp_upiu_query_v4_0 *upiu_resp; @@ -6446,13 +6468,14 @@ void ufshcd_schedule_eh_work(struct ufs_hba *hba) } } -static void ufshcd_force_error_recovery(struct ufs_hba *hba) +void ufshcd_force_error_recovery(struct ufs_hba *hba) { spin_lock_irq(hba->host->host_lock); hba->force_reset = true; ufshcd_schedule_eh_work(hba); spin_unlock_irq(hba->host->host_lock); } +EXPORT_SYMBOL_GPL(ufshcd_force_error_recovery); static void ufshcd_clk_scaling_allow(struct ufs_hba *hba, bool allow) { @@ -6920,7 +6943,7 @@ static irqreturn_t ufshcd_check_errors(struct ufs_hba *hba, u32 intr_status) bool queue_eh_work = false; irqreturn_t retval = IRQ_NONE; - spin_lock(hba->host->host_lock); + guard(spinlock_irqsave)(hba->host->host_lock); hba->errors |= UFSHCD_ERROR_MASK & intr_status; if (hba->errors & INT_FATAL_ERRORS) { @@ -6979,7 +7002,7 @@ static irqreturn_t ufshcd_check_errors(struct ufs_hba *hba, u32 intr_status) */ hba->errors = 0; hba->uic_error = 0; - spin_unlock(hba->host->host_lock); + return retval; } @@ -7138,14 +7161,19 @@ static irqreturn_t ufshcd_threaded_intr(int irq, void *__hba) static irqreturn_t ufshcd_intr(int irq, void *__hba) { struct ufs_hba *hba = __hba; + u32 intr_status, enabled_intr_status; /* Move interrupt handling to thread when MCQ & ESI are not enabled */ if (!hba->mcq_enabled || !hba->mcq_esi_enabled) return IRQ_WAKE_THREAD; + intr_status = ufshcd_readl(hba, REG_INTERRUPT_STATUS); + enabled_intr_status = intr_status & ufshcd_readl(hba, REG_INTERRUPT_ENABLE); + + ufshcd_writel(hba, intr_status, REG_INTERRUPT_STATUS); + /* Directly handle interrupts since MCQ ESI handlers does the hard job */ - return ufshcd_sl_intr(hba, ufshcd_readl(hba, REG_INTERRUPT_STATUS) & - ufshcd_readl(hba, REG_INTERRUPT_ENABLE)); + return ufshcd_sl_intr(hba, enabled_intr_status); } static int ufshcd_clear_tm_cmd(struct ufs_hba *hba, int tag) @@ -7449,7 +7477,8 @@ int ufshcd_exec_raw_upiu_cmd(struct ufs_hba *hba, * @sg_list: Pointer to SG list when DATA IN/OUT UPIU is required in ARPMB operation * @dir: DMA direction * - * Return: zero on success, non-zero on failure. + * Return: 0 upon success; > 0 in case the UFS device reported an OCS error; + * < 0 if another error occurred. */ int ufshcd_advanced_rpmb_req_handler(struct ufs_hba *hba, struct utp_upiu_req *req_upiu, struct utp_upiu_req *rsp_upiu, struct ufs_ehs *req_ehs, @@ -8769,7 +8798,8 @@ static void ufshcd_set_timestamp_attr(struct ufs_hba *hba) struct ufs_dev_info *dev_info = &hba->dev_info; struct utp_upiu_query_v4_0 *upiu_data; - if (dev_info->wspecversion < 0x400) + if (dev_info->wspecversion < 0x400 || + hba->dev_quirks & UFS_DEVICE_QUIRK_NO_TIMESTAMP_SUPPORT) return; ufshcd_dev_man_lock(hba); @@ -8896,16 +8926,11 @@ err: static void ufshcd_config_mcq(struct ufs_hba *hba) { int ret; - u32 intrs; ret = ufshcd_mcq_vops_config_esi(hba); hba->mcq_esi_enabled = !ret; dev_info(hba->dev, "ESI %sconfigured\n", ret ? "is not " : ""); - intrs = UFSHCD_ENABLE_MCQ_INTRS; - if (hba->quirks & UFSHCD_QUIRK_MCQ_BROKEN_INTR) - intrs &= ~MCQ_CQ_EVENT_STATUS; - ufshcd_enable_intr(hba, intrs); ufshcd_mcq_make_queues_operational(hba); ufshcd_mcq_config_mac(hba, hba->nutrs); @@ -10516,8 +10541,7 @@ int ufshcd_alloc_host(struct device *dev, struct ufs_hba **hba_handle) err = devm_add_action_or_reset(dev, ufshcd_devres_release, host); if (err) - return dev_err_probe(dev, err, - "failed to add ufshcd dealloc action\n"); + return err; host->nr_maps = HCTX_TYPE_POLL + 1; hba = shost_priv(host); @@ -10740,6 +10764,10 @@ int ufshcd_init(struct ufs_hba *hba, void __iomem *mmio_base, unsigned int irq) mutex_init(&hba->ee_ctrl_mutex); mutex_init(&hba->wb_mutex); + + /* Initialize mutex for PM QoS request synchronization */ + mutex_init(&hba->pm_qos_mutex); + init_rwsem(&hba->clk_scaling_lock); ufshcd_init_clk_gating(hba); |