diff --git a/drivers/crypto/msm/qcedev_smmu.c b/drivers/crypto/msm/qcedev_smmu.c
index 8d4844becc85..7039bce67c5c 100644
--- a/drivers/crypto/msm/qcedev_smmu.c
+++ b/drivers/crypto/msm/qcedev_smmu.c
@@ -329,10 +329,6 @@ int qcedev_check_and_map_buffer(void *handle,
 		mapped_size = binfo->ion_buf.mapped_buf_size;
 		atomic_inc(&binfo->ref_count);
 
-		/* Add buffer mapping information to regd buffer list */
-		mutex_lock(&qce_hndl->registeredbufs.lock);
-		list_add_tail(&binfo->list, &qce_hndl->registeredbufs.list);
-		mutex_unlock(&qce_hndl->registeredbufs.lock);
 	}
 
 	/* Make sure the offset is within the mapped range */
@@ -344,6 +340,13 @@ int qcedev_check_and_map_buffer(void *handle,
 		goto unmap;
 	}
 
+	if (!found) {
+		/* Add buffer mapping information to regd buffer list */
+		mutex_lock(&qce_hndl->registeredbufs.lock);
+		list_add_tail(&binfo->list, &qce_hndl->registeredbufs.list);
+		mutex_unlock(&qce_hndl->registeredbufs.lock);
+		}
+
 	/* return the mapped virtual address adjusted by offset */
 	*vaddr += offset;
 
@@ -352,9 +355,6 @@ int qcedev_check_and_map_buffer(void *handle,
 unmap:
 	if (!found) {
 		qcedev_unmap_buffer(handle, mem_client, binfo);
-		mutex_lock(&qce_hndl->registeredbufs.lock);
-		list_del(&binfo->list);
-		mutex_unlock(&qce_hndl->registeredbufs.lock);
 	}
 
 error: