diff --git a/msm/vidc/msm_vidc_clocks.c b/msm/vidc/msm_vidc_clocks.c
index 7ddf39a1ba70..168abc2f8080 100644
--- a/msm/vidc/msm_vidc_clocks.c
+++ b/msm/vidc/msm_vidc_clocks.c
@@ -1237,9 +1237,11 @@ static int msm_vidc_decide_work_mode_ar50_lt(struct msm_vidc_inst *inst)
 		return -EINVAL;
 	}
 
+	latency.enable = false;
 	hdev = inst->core->device;
 	if (inst->clk_data.low_latency_mode) {
 		pdata.video_work_mode = HFI_WORKMODE_1;
+		latency.enable = true;
 		goto decision_done;
 	}
 
@@ -1259,20 +1261,25 @@ static int msm_vidc_decide_work_mode_ar50_lt(struct msm_vidc_inst *inst)
 		}
 	} else if (inst->session_type == MSM_VIDC_ENCODER) {
 		pdata.video_work_mode = HFI_WORKMODE_1;
+		/* For WORK_MODE_1, set Low Latency mode by default */
+		latency.enable = true;
 		if (inst->rc_type == V4L2_MPEG_VIDEO_BITRATE_MODE_VBR ||
 				inst->rc_type ==
 					V4L2_MPEG_VIDEO_BITRATE_MODE_MBR ||
 				inst->rc_type ==
 					V4L2_MPEG_VIDEO_BITRATE_MODE_MBR_VFR ||
 				inst->rc_type ==
-					V4L2_MPEG_VIDEO_BITRATE_MODE_CQ)
+					V4L2_MPEG_VIDEO_BITRATE_MODE_CQ) {
 			pdata.video_work_mode = HFI_WORKMODE_2;
+			latency.enable = false;
+		}
 	} else {
 		return -EINVAL;
 	}
 
 decision_done:
-
+	s_vpr_h(inst->sid, "Configuring work mode = %u low latency = %u",
+			pdata.video_work_mode, latency.enable);
 	inst->clk_data.work_mode = pdata.video_work_mode;
 	rc = call_hfi_op(hdev, session_set_property,
 			(void *)inst->session, HFI_PROPERTY_PARAM_WORK_MODE,
@@ -1280,22 +1287,12 @@ decision_done:
 	if (rc)
 		s_vpr_e(inst->sid, "Failed to configure Work Mode\n");
 
-	/* For WORK_MODE_1, set Low Latency mode by default to HW. */
-
-	latency.enable = false;
-	if (inst->session_type == MSM_VIDC_ENCODER &&
-			inst->clk_data.work_mode == HFI_WORKMODE_1) {
-		latency.enable = true;
+	if (inst->session_type == MSM_VIDC_ENCODER && latency.enable) {
 		rc = call_hfi_op(hdev, session_set_property,
 			(void *)inst->session,
 			HFI_PROPERTY_PARAM_VENC_LOW_LATENCY_MODE,
 			(void *)&latency, sizeof(latency));
 	}
-
-	s_vpr_h(inst->sid, "Configuring work mode = %u low latency = %u",
-			pdata.video_work_mode,
-			latency.enable);
-
 	rc = msm_comm_scale_clocks_and_bus(inst, 1);
 
 	return rc;
@@ -1401,8 +1398,7 @@ int msm_vidc_decide_work_mode_iris2(struct msm_vidc_inst *inst)
 	}
 
 	s_vpr_h(inst->sid, "Configuring work mode = %u low latency = %u",
-			pdata.video_work_mode,
-			latency.enable);
+			pdata.video_work_mode, latency.enable);
 
 	if (inst->session_type == MSM_VIDC_ENCODER) {
 		rc = call_hfi_op(hdev, session_set_property,