diff --git a/asoc/msm-lsm-client.c b/asoc/msm-lsm-client.c
index ea62100354a2..fe461ffc5e5a 100644
--- a/asoc/msm-lsm-client.c
+++ b/asoc/msm-lsm-client.c
@@ -3117,22 +3117,22 @@ static int msm_lsm_close(struct snd_pcm_substream *substream)
 				"%s: LSM client session stopped %d\n",
 				 __func__, ret);
 
-		/*
-		 * Go Ahead and try de-register sound model,
-		 * even if stop failed
-		 */
 		prtd->lsm_client->started = false;
-
-		ret = q6lsm_deregister_sound_model(prtd->lsm_client);
-		if (ret)
-			dev_err(rtd->dev,
-				"%s: dereg_snd_model failed, err = %d\n",
-				__func__, ret);
-		else
-			dev_dbg(rtd->dev, "%s: dereg_snd_model successful\n",
-				 __func__);
 	}
 
+	/*
+	 * De-register existing sound models
+	 * to free SM and CAL buffer, even if
+	 * lsm client is not started.
+	 */
+	ret = q6lsm_deregister_sound_model(prtd->lsm_client);
+	if (ret)
+		dev_err(rtd->dev, "%s: dereg_snd_model failed, err = %d\n",
+			__func__, ret);
+	else
+		dev_dbg(rtd->dev, "%s: dereg_snd_model successful\n",
+			__func__);
+
 	msm_pcm_routing_dereg_phy_stream(rtd->dai_link->id,
 					SNDRV_PCM_STREAM_CAPTURE);
 
diff --git a/dsp/q6lsm.c b/dsp/q6lsm.c
index af0fd4326cb4..c7cb10dc55fe 100644
--- a/dsp/q6lsm.c
+++ b/dsp/q6lsm.c
@@ -1735,21 +1735,24 @@ int q6lsm_deregister_sound_model(struct lsm_client *client)
 				break;
 		}
 	} else {
-		memset(&cmd, 0, sizeof(cmd));
-		q6lsm_add_hdr(client, &cmd.hdr, sizeof(cmd.hdr), false);
-		cmd.hdr.opcode = LSM_SESSION_CMD_DEREGISTER_SOUND_MODEL;
-
-		rc = q6lsm_apr_send_pkt(client, client->apr, &cmd.hdr, true, NULL);
-		if (rc) {
-			pr_err("%s: Failed cmd opcode 0x%x, rc %d\n", __func__,
-			       cmd.hdr.opcode, rc);
-		} else {
-			pr_debug("%s: Deregister sound model succeeded\n", __func__);
-		}
-
-		p_info.param_type = LSM_DEREG_SND_MODEL;
 		sm = &client->stage_cfg[p_info.stage_idx].sound_model;
-		q6lsm_snd_model_buf_free(client, &p_info, sm);
+
+		if (sm && sm->data) {
+			memset(&cmd, 0, sizeof(cmd));
+			q6lsm_add_hdr(client, &cmd.hdr, sizeof(cmd.hdr), false);
+			cmd.hdr.opcode = LSM_SESSION_CMD_DEREGISTER_SOUND_MODEL;
+			p_info.param_type = LSM_DEREG_SND_MODEL;
+
+			rc = q6lsm_apr_send_pkt(client, client->apr, &cmd.hdr, true, NULL);
+			if (rc) {
+				pr_err("%s: Failed cmd opcode 0x%x, rc %d\n", __func__,
+				       cmd.hdr.opcode, rc);
+			} else {
+				pr_debug("%s: Deregister sound model succeeded\n", __func__);
+			}
+
+			q6lsm_snd_model_buf_free(client, &p_info, sm);
+		}
 	}
 
 	return rc;