diff --git a/drivers/soc/qcom/rimps_memlat.c b/drivers/soc/qcom/rimps_memlat.c
index 5b4802da3d90..7e22b37fcfe3 100644
--- a/drivers/soc/qcom/rimps_memlat.c
+++ b/drivers/soc/qcom/rimps_memlat.c
@@ -824,7 +824,8 @@ static int memlat_idle_notif(struct notifier_block *nb,
 	case CPU_PM_EXIT:
 	case CPU_PM_ENTER_FAILED:
 		__this_cpu_write(cpu_is_idle, false);
-		set_pmu_cache_flag(PMU_CACHE_INVALID, cpu);
+		if (!per_cpu(cpu_is_hp, cpu))
+			set_pmu_cache_flag(PMU_CACHE_INVALID, cpu);
 		break;
 	}
 idle_exit: