diff --git a/drivers/cam_req_mgr/cam_req_mgr_core.c b/drivers/cam_req_mgr/cam_req_mgr_core.c
index 208037c255f6..3b9f36ae352d 100644
--- a/drivers/cam_req_mgr/cam_req_mgr_core.c
+++ b/drivers/cam_req_mgr/cam_req_mgr_core.c
@@ -53,6 +53,7 @@ void cam_req_mgr_core_link_reset(struct cam_req_mgr_core_link *link)
 	link->prev_sof_timestamp = 0;
 	link->skip_init_frame = false;
 	link->num_sync_links = 0;
+	link->last_applied_jiffies = 0;
 	atomic_set(&link->eof_event_cnt, 0);
 
 	for (pd = 0; pd < CAM_PIPELINE_DELAY_MAX; pd++) {
@@ -1584,6 +1585,7 @@ static int __cam_req_mgr_process_req(struct cam_req_mgr_core_link *link,
 {
 	int                                  rc = 0, idx, i;
 	int                                  reset_step = 0;
+	bool                                 check_retry_cnt = false;
 	uint32_t                             trigger = trigger_data->trigger;
 	struct cam_req_mgr_slot             *slot = NULL;
 	struct cam_req_mgr_req_queue        *in_q;
@@ -1694,10 +1696,14 @@ static int __cam_req_mgr_process_req(struct cam_req_mgr_core_link *link,
 
 	rc = __cam_req_mgr_send_req(link, link->req.in_q, trigger, &dev);
 	if (rc < 0) {
-		if (in_q->last_applied_idx < in_q->rd_idx) {
-			/* Apply req failed retry at next sof */
-			slot->status = CRM_SLOT_STATUS_REQ_PENDING;
+		/* Apply req failed retry at next sof */
+		slot->status = CRM_SLOT_STATUS_REQ_PENDING;
 
+		if (jiffies_to_msecs(jiffies - link->last_applied_jiffies) >
+			MINIMUM_WORKQUEUE_SCHED_TIME_IN_MS)
+			check_retry_cnt = true;
+
+		if ((in_q->last_applied_idx < in_q->rd_idx) && check_retry_cnt) {
 			link->retry_cnt++;
 			if (link->retry_cnt == MAXIMUM_RETRY_ATTEMPTS) {
 				CAM_DBG(CAM_CRM,
@@ -1771,6 +1777,16 @@ static int __cam_req_mgr_process_req(struct cam_req_mgr_core_link *link,
 		}
 	}
 
+	/*
+	 * Only update the jiffies of last applied request
+	 * for SOF trigger, since it is used to protect from
+	 * applying fails in ISP which is triggered at SOF.
+	 * And, also don't need to do update for error case
+	 * since error case doesn't check the retry count.
+	 */
+	if (trigger == CAM_TRIGGER_POINT_SOF)
+		link->last_applied_jiffies = jiffies;
+
 	mutex_unlock(&session->lock);
 	return rc;
 error:
diff --git a/drivers/cam_req_mgr/cam_req_mgr_core.h b/drivers/cam_req_mgr/cam_req_mgr_core.h
index ebe440828ecf..b9f57a203197 100644
--- a/drivers/cam_req_mgr/cam_req_mgr_core.h
+++ b/drivers/cam_req_mgr/cam_req_mgr_core.h
@@ -36,6 +36,8 @@
 
 #define MAXIMUM_RETRY_ATTEMPTS 2
 
+#define MINIMUM_WORKQUEUE_SCHED_TIME_IN_MS 5
+
 #define VERSION_1  1
 #define VERSION_2  2
 #define CAM_REQ_MGR_MAX_TRIGGERS   2
@@ -371,6 +373,7 @@ struct cam_req_mgr_connected_device {
  * @eof_event_cnt        : Atomic variable to track the number of EOF requests
  * @skip_init_frame      : skip initial frames crm_wd_timer validation in the
  *                         case of long exposure use case
+ * @last_applied_jiffies : Record the jiffies of last applied req
  */
 struct cam_req_mgr_core_link {
 	int32_t                              link_hdl;
@@ -407,6 +410,7 @@ struct cam_req_mgr_core_link {
 	uint32_t    trigger_cnt[CAM_REQ_MGR_MAX_TRIGGERS];
 	atomic_t                             eof_event_cnt;
 	bool                                 skip_init_frame;
+	uint64_t                             last_applied_jiffies;
 };
 
 /**