diff --git a/msm/vidc/msm_venc.c b/msm/vidc/msm_venc.c
index 55e05e41e5a6..b13866942db4 100644
--- a/msm/vidc/msm_venc.c
+++ b/msm/vidc/msm_venc.c
@@ -3809,6 +3809,12 @@ int msm_venc_set_hp_max_layer(struct msm_vidc_inst *inst)
 			__func__);
 		return rc;
 	}
+	if (!inst->hybrid_hp && max_layer->val > 4) {
+		update_ctrl(max_layer, 0, inst->sid);
+		s_vpr_h(inst->sid,
+			"%s: Hier-P requested beyond max capability\n", __func__);
+		return 0;
+	}
 
 	/*
 	 * We send enhancement layer count to FW,
diff --git a/msm/vidc/msm_vidc_buffer_calculations.c b/msm/vidc/msm_vidc_buffer_calculations.c
index 0d63bca7395d..d19ed2b32cf1 100644
--- a/msm/vidc/msm_vidc_buffer_calculations.c
+++ b/msm/vidc/msm_vidc_buffer_calculations.c
@@ -497,13 +497,13 @@ int msm_vidc_get_num_ref_frames(struct msm_vidc_inst *inst)
 		codec = get_v4l2_codec(inst);
 		/* LTR and B - frame not supported with hybrid HP */
 		if (inst->hybrid_hp)
-			num_ref = (max_layer->val - 1);
+			num_ref = max_layer->val >> 1;
 		else if (codec == V4L2_PIX_FMT_HEVC)
 			num_ref = ((max_layer->val + 1) / 2) + ltr_count;
-		else if ((codec == V4L2_PIX_FMT_H264) && (max_layer->val <= 4))
-			num_ref = ((1 << (max_layer->val - 1)) - 1) + ltr_count;
+		else if ((codec == V4L2_PIX_FMT_H264) && (max_layer->val < 4))
+			num_ref = (max_layer->val - 1) + ltr_count;
 		else
-			num_ref = ((max_layer->val + 1) / 2) + ltr_count;
+			num_ref = max_layer->val + ltr_count;
 	}
 
 	if (is_hier_b_session(inst)) {