diff --git a/drivers/platform/msm/ipa/ipa_v3/ipa.c b/drivers/platform/msm/ipa/ipa_v3/ipa.c index 14b2734381f6..4bc170e39858 100644 --- a/drivers/platform/msm/ipa/ipa_v3/ipa.c +++ b/drivers/platform/msm/ipa/ipa_v3/ipa.c @@ -3104,6 +3104,9 @@ static int ipa3_q6_clean_q6_flt_tbls(enum ipa_ip_type ip, u32 lcl_addr_mem_part; u32 lcl_hdr_sz; struct ipa_mem_buffer mem; + struct ipahal_reg_valmask valmask; + struct ipahal_imm_cmd_register_write reg_write_coal_close; + int i; IPADBG("Entry\n"); @@ -3121,13 +3124,13 @@ static int ipa3_q6_clean_q6_flt_tbls(enum ipa_ip_type ip, return retval; } - /* Up to filtering pipes we have filtering tables */ - desc = kcalloc(ipa3_ctx->ep_flt_num, sizeof(struct ipa3_desc), + /* Up to filtering pipes we have filtering tables + 1 for coal close */ + desc = kcalloc(ipa3_ctx->ep_flt_num + 1, sizeof(struct ipa3_desc), GFP_KERNEL); if (!desc) return -ENOMEM; - cmd_pyld = kcalloc(ipa3_ctx->ep_flt_num, + cmd_pyld = kcalloc(ipa3_ctx->ep_flt_num + 1, sizeof(struct ipahal_imm_cmd_pyld *), GFP_KERNEL); if (!cmd_pyld) { retval = -ENOMEM; @@ -3159,6 +3162,28 @@ static int ipa3_q6_clean_q6_flt_tbls(enum ipa_ip_type ip, goto free_cmd_pyld; } + /* IC to close the coal frame before HPS Clear if coal is enabled */ + if (ipa3_get_ep_mapping(IPA_CLIENT_APPS_WAN_COAL_CONS) != -1) { + i = ipa3_get_ep_mapping(IPA_CLIENT_APPS_WAN_COAL_CONS); + reg_write_coal_close.skip_pipeline_clear = false; + reg_write_coal_close.pipeline_clear_options = IPAHAL_HPS_CLEAR; + reg_write_coal_close.offset = ipahal_get_reg_ofst( + IPA_AGGR_FORCE_CLOSE); + ipahal_get_aggr_force_close_valmask(i, &valmask); + reg_write_coal_close.value = valmask.val; + reg_write_coal_close.value_mask = valmask.mask; + cmd_pyld[num_cmds] = ipahal_construct_imm_cmd( + IPA_IMM_CMD_REGISTER_WRITE, + ®_write_coal_close, false); + if (!cmd_pyld[num_cmds]) { + IPAERR("failed to construct coal close IC\n"); + retval = -ENOMEM; + goto free_empty_img; + } + ipa3_init_imm_cmd_desc(&desc[num_cmds], cmd_pyld[num_cmds]); + ++num_cmds; + } + for (pipe_idx = 0; pipe_idx < ipa3_ctx->ipa_num_pipes; pipe_idx++) { if (!ipa_is_ep_support_flt(pipe_idx)) continue; @@ -3224,13 +3249,17 @@ static int ipa3_q6_clean_q6_rt_tbls(enum ipa_ip_type ip, { struct ipa3_desc *desc; struct ipahal_imm_cmd_dma_shared_mem cmd = {0}; - struct ipahal_imm_cmd_pyld *cmd_pyld = NULL; + struct ipahal_imm_cmd_pyld **cmd_pyld; int retval = 0; + int num_cmds = 0; u32 modem_rt_index_lo; u32 modem_rt_index_hi; u32 lcl_addr_mem_part; u32 lcl_hdr_sz; struct ipa_mem_buffer mem; + struct ipahal_reg_valmask valmask; + struct ipahal_imm_cmd_register_write reg_write_coal_close; + int i; IPADBG("Entry\n"); @@ -3278,10 +3307,37 @@ static int ipa3_q6_clean_q6_rt_tbls(enum ipa_ip_type ip, return -ENOMEM; } - desc = kzalloc(sizeof(struct ipa3_desc), GFP_KERNEL); - if (!desc) { - IPAERR("failed to allocate memory\n"); + desc = kcalloc(2, sizeof(struct ipa3_desc), GFP_KERNEL); + if (!desc) + retval = -ENOMEM; goto free_empty_img; + + cmd_pyld = kcalloc(2, sizeof(struct ipahal_imm_cmd_pyld *), GFP_KERNEL); + if (!cmd_pyld) { + retval = -ENOMEM; + goto free_desc; + } + + /* IC to close the coal frame before HPS Clear if coal is enabled */ + if (ipa3_get_ep_mapping(IPA_CLIENT_APPS_WAN_COAL_CONS) != -1) { + i = ipa3_get_ep_mapping(IPA_CLIENT_APPS_WAN_COAL_CONS); + reg_write_coal_close.skip_pipeline_clear = false; + reg_write_coal_close.pipeline_clear_options = IPAHAL_HPS_CLEAR; + reg_write_coal_close.offset = ipahal_get_reg_ofst( + IPA_AGGR_FORCE_CLOSE); + ipahal_get_aggr_force_close_valmask(i, &valmask); + reg_write_coal_close.value = valmask.val; + reg_write_coal_close.value_mask = valmask.mask; + cmd_pyld[num_cmds] = ipahal_construct_imm_cmd( + IPA_IMM_CMD_REGISTER_WRITE, + ®_write_coal_close, false); + if (!cmd_pyld[num_cmds]) { + IPAERR("failed to construct coal close IC\n"); + retval = -ENOMEM; + goto free_cmd_pyld; + } + ipa3_init_imm_cmd_desc(&desc[num_cmds], cmd_pyld[num_cmds]); + ++num_cmds; } cmd.is_read = false; @@ -3292,23 +3348,27 @@ static int ipa3_q6_clean_q6_rt_tbls(enum ipa_ip_type ip, cmd.local_addr = ipa3_ctx->smem_restricted_bytes + lcl_addr_mem_part + modem_rt_index_lo * ipahal_get_hw_tbl_hdr_width(); - cmd_pyld = ipahal_construct_imm_cmd( + cmd_pyld[num_cmds] = ipahal_construct_imm_cmd( IPA_IMM_CMD_DMA_SHARED_MEM, &cmd, false); - if (!cmd_pyld) { + if (!cmd_pyld[num_cmds]) { IPAERR("failed to construct dma_shared_mem imm cmd\n"); retval = -ENOMEM; - goto free_desc; + goto free_cmd_pyld; } - ipa3_init_imm_cmd_desc(desc, cmd_pyld); + ipa3_init_imm_cmd_desc(&desc[num_cmds], cmd_pyld[num_cmds]); + ++num_cmds; IPADBG("Sending 1 descriptor for rt tbl clearing\n"); - retval = ipa3_send_cmd(1, desc); + retval = ipa3_send_cmd(num_cmds, desc); if (retval) { IPAERR("failed to send immediate command (err %d)\n", retval); retval = -EFAULT; } - ipahal_destroy_imm_cmd(cmd_pyld); +free_cmd_pyld: + for (i = 0; i < num_cmds; i++) + ipahal_destroy_imm_cmd(cmd_pyld[i]); + kfree(cmd_pyld); free_desc: kfree(desc); free_empty_img: @@ -3319,11 +3379,14 @@ free_empty_img: static int ipa3_q6_clean_q6_tables(void) { struct ipa3_desc *desc; - struct ipahal_imm_cmd_pyld *cmd_pyld = NULL; + struct ipahal_imm_cmd_pyld **cmd_pyld; struct ipahal_imm_cmd_register_write reg_write_cmd = {0}; int retval = 0; + int num_cmds = 0; struct ipahal_reg_fltrt_hash_flush flush; struct ipahal_reg_valmask valmask; + struct ipahal_imm_cmd_register_write reg_write_coal_close; + int i; IPADBG("Entry\n"); @@ -3369,10 +3432,38 @@ static int ipa3_q6_clean_q6_tables(void) if (ipa3_ctx->ipa_fltrt_not_hashable) return retval; /* Flush rules cache */ - desc = kzalloc(sizeof(struct ipa3_desc), GFP_KERNEL); + desc = kcalloc(2, sizeof(struct ipa3_desc), GFP_KERNEL); if (!desc) return -ENOMEM; + cmd_pyld = kcalloc(2, sizeof(struct ipahal_imm_cmd_pyld *), GFP_KERNEL); + if (!cmd_pyld) { + retval = -ENOMEM; + goto bail_desc; + } + + /* IC to close the coal frame before HPS Clear if coal is enabled */ + if (ipa3_get_ep_mapping(IPA_CLIENT_APPS_WAN_COAL_CONS) != -1) { + i = ipa3_get_ep_mapping(IPA_CLIENT_APPS_WAN_COAL_CONS); + reg_write_coal_close.skip_pipeline_clear = false; + reg_write_coal_close.pipeline_clear_options = IPAHAL_HPS_CLEAR; + reg_write_coal_close.offset = ipahal_get_reg_ofst( + IPA_AGGR_FORCE_CLOSE); + ipahal_get_aggr_force_close_valmask(i, &valmask); + reg_write_coal_close.value = valmask.val; + reg_write_coal_close.value_mask = valmask.mask; + cmd_pyld[num_cmds] = ipahal_construct_imm_cmd( + IPA_IMM_CMD_REGISTER_WRITE, + ®_write_coal_close, false); + if (!cmd_pyld[num_cmds]) { + IPAERR("failed to construct coal close IC\n"); + retval = -ENOMEM; + goto free_cmd_pyld; + } + ipa3_init_imm_cmd_desc(&desc[num_cmds], cmd_pyld[num_cmds]); + ++num_cmds; + } + flush.v4_flt = true; flush.v4_rt = true; flush.v6_flt = true; @@ -3383,24 +3474,27 @@ static int ipa3_q6_clean_q6_tables(void) reg_write_cmd.offset = ipahal_get_reg_ofst(IPA_FILT_ROUT_HASH_FLUSH); reg_write_cmd.value = valmask.val; reg_write_cmd.value_mask = valmask.mask; - cmd_pyld = ipahal_construct_imm_cmd(IPA_IMM_CMD_REGISTER_WRITE, - ®_write_cmd, false); - if (!cmd_pyld) { + cmd_pyld[num_cmds] = ipahal_construct_imm_cmd( + IPA_IMM_CMD_REGISTER_WRITE, ®_write_cmd, false); + if (!cmd_pyld[num_cmds]) { IPAERR("fail construct register_write imm cmd\n"); retval = -EFAULT; - goto bail_desc; + goto free_cmd_pyld; } - ipa3_init_imm_cmd_desc(desc, cmd_pyld); + ipa3_init_imm_cmd_desc(&desc[num_cmds], cmd_pyld[num_cmds]); + ++num_cmds; IPADBG("Sending 1 descriptor for tbls flush\n"); - retval = ipa3_send_cmd(1, desc); + retval = ipa3_send_cmd(num_cmds, desc); if (retval) { IPAERR("failed to send immediate command (err %d)\n", retval); retval = -EFAULT; } - ipahal_destroy_imm_cmd(cmd_pyld); - +free_cmd_pyld: + for (i = 0; i < num_cmds; i++) + ipahal_destroy_imm_cmd(cmd_pyld[i]); + kfree(cmd_pyld); bail_desc: kfree(desc); IPADBG("Done - retval = %d\n", retval); @@ -3417,12 +3511,39 @@ static int ipa3_q6_set_ex_path_to_apps(void) struct ipahal_imm_cmd_register_write reg_write; struct ipahal_imm_cmd_pyld *cmd_pyld; int retval; + struct ipahal_reg_valmask valmask; + struct ipahal_imm_cmd_register_write reg_write_coal_close; + int i; - desc = kcalloc(ipa3_ctx->ipa_num_pipes, sizeof(struct ipa3_desc), + desc = kcalloc(ipa3_ctx->ipa_num_pipes + 1, sizeof(struct ipa3_desc), GFP_KERNEL); if (!desc) return -ENOMEM; + /* IC to close the coal frame before HPS Clear if coal is enabled */ + if (ipa3_get_ep_mapping(IPA_CLIENT_APPS_WAN_COAL_CONS) != -1) { + i = ipa3_get_ep_mapping(IPA_CLIENT_APPS_WAN_COAL_CONS); + reg_write_coal_close.skip_pipeline_clear = false; + reg_write_coal_close.pipeline_clear_options = IPAHAL_HPS_CLEAR; + reg_write_coal_close.offset = ipahal_get_reg_ofst( + IPA_AGGR_FORCE_CLOSE); + ipahal_get_aggr_force_close_valmask(i, &valmask); + reg_write_coal_close.value = valmask.val; + reg_write_coal_close.value_mask = valmask.mask; + cmd_pyld = ipahal_construct_imm_cmd( + IPA_IMM_CMD_REGISTER_WRITE, + ®_write_coal_close, false); + if (!cmd_pyld) { + IPAERR("failed to construct coal close IC\n"); + ipa_assert(); + return -ENOMEM; + } + ipa3_init_imm_cmd_desc(&desc[num_descs], cmd_pyld); + desc[num_descs].callback = ipa3_destroy_imm; + desc[num_descs].user1 = cmd_pyld; + ++num_descs; + } + /* Set the exception path to AP */ for (client_idx = 0; client_idx < IPA_CLIENT_MAX; client_idx++) { ep_idx = ipa3_get_ep_mapping(client_idx); @@ -3750,11 +3871,17 @@ int _ipa_init_sram_v3(void) */ int _ipa_init_hdr_v3_0(void) { - struct ipa3_desc desc; + struct ipa3_desc hdr_init_desc; + struct ipa3_desc dma_cmd_desc[2]; struct ipa_mem_buffer mem; struct ipahal_imm_cmd_hdr_init_local cmd = {0}; - struct ipahal_imm_cmd_pyld *cmd_pyld; + struct ipahal_imm_cmd_pyld *hdr_init_cmd_payload; + struct ipahal_imm_cmd_pyld *cmd_pyld[2]; struct ipahal_imm_cmd_dma_shared_mem dma_cmd = { 0 }; + struct ipahal_reg_valmask valmask; + struct ipahal_imm_cmd_register_write reg_write_coal_close; + int num_cmds = 0; + int i; mem.size = IPA_MEM_PART(modem_hdr_size) + IPA_MEM_PART(apps_hdr_size); mem.base = dma_alloc_coherent(ipa3_ctx->pdev, mem.size, &mem.phys_base, @@ -3768,30 +3895,52 @@ int _ipa_init_hdr_v3_0(void) cmd.size_hdr_table = mem.size; cmd.hdr_addr = ipa3_ctx->smem_restricted_bytes + IPA_MEM_PART(modem_hdr_ofst); - cmd_pyld = ipahal_construct_imm_cmd( + hdr_init_cmd_payload = ipahal_construct_imm_cmd( IPA_IMM_CMD_HDR_INIT_LOCAL, &cmd, false); - if (!cmd_pyld) { + if (!hdr_init_cmd_payload) { IPAERR("fail to construct hdr_init_local imm cmd\n"); dma_free_coherent(ipa3_ctx->pdev, mem.size, mem.base, mem.phys_base); return -EFAULT; } - ipa3_init_imm_cmd_desc(&desc, cmd_pyld); + ipa3_init_imm_cmd_desc(&hdr_init_desc, hdr_init_cmd_payload); IPA_DUMP_BUFF(mem.base, mem.phys_base, mem.size); - if (ipa3_send_cmd(1, &desc)) { + if (ipa3_send_cmd(1, &hdr_init_desc)) { IPAERR("fail to send immediate command\n"); - ipahal_destroy_imm_cmd(cmd_pyld); + ipahal_destroy_imm_cmd(hdr_init_cmd_payload); dma_free_coherent(ipa3_ctx->pdev, mem.size, mem.base, mem.phys_base); return -EFAULT; } - ipahal_destroy_imm_cmd(cmd_pyld); + ipahal_destroy_imm_cmd(hdr_init_cmd_payload); dma_free_coherent(ipa3_ctx->pdev, mem.size, mem.base, mem.phys_base); + /* IC to close the coal frame before HPS Clear if coal is enabled */ + if (ipa3_get_ep_mapping(IPA_CLIENT_APPS_WAN_COAL_CONS) != -1) { + i = ipa3_get_ep_mapping(IPA_CLIENT_APPS_WAN_COAL_CONS); + reg_write_coal_close.skip_pipeline_clear = false; + reg_write_coal_close.pipeline_clear_options = IPAHAL_HPS_CLEAR; + reg_write_coal_close.offset = ipahal_get_reg_ofst( + IPA_AGGR_FORCE_CLOSE); + ipahal_get_aggr_force_close_valmask(i, &valmask); + reg_write_coal_close.value = valmask.val; + reg_write_coal_close.value_mask = valmask.mask; + cmd_pyld[num_cmds] = ipahal_construct_imm_cmd( + IPA_IMM_CMD_REGISTER_WRITE, + ®_write_coal_close, false); + if (!cmd_pyld[num_cmds]) { + IPAERR("failed to construct coal close IC\n"); + return -ENOMEM; + } + ipa3_init_imm_cmd_desc(&dma_cmd_desc[num_cmds], + cmd_pyld[num_cmds]); + ++num_cmds; + } + mem.size = IPA_MEM_PART(modem_hdr_proc_ctx_size) + IPA_MEM_PART(apps_hdr_proc_ctx_size); mem.base = dma_alloc_coherent(ipa3_ctx->pdev, mem.size, &mem.phys_base, @@ -3808,28 +3957,31 @@ int _ipa_init_hdr_v3_0(void) dma_cmd.local_addr = ipa3_ctx->smem_restricted_bytes + IPA_MEM_PART(modem_hdr_proc_ctx_ofst); dma_cmd.size = mem.size; - cmd_pyld = ipahal_construct_imm_cmd( + cmd_pyld[num_cmds] = ipahal_construct_imm_cmd( IPA_IMM_CMD_DMA_SHARED_MEM, &dma_cmd, false); - if (!cmd_pyld) { + if (!cmd_pyld[num_cmds]) { IPAERR("fail to construct dma_shared_mem imm\n"); dma_free_coherent(ipa3_ctx->pdev, mem.size, mem.base, mem.phys_base); return -ENOMEM; } - ipa3_init_imm_cmd_desc(&desc, cmd_pyld); + ipa3_init_imm_cmd_desc(&dma_cmd_desc[num_cmds], cmd_pyld[num_cmds]); + ++num_cmds; IPA_DUMP_BUFF(mem.base, mem.phys_base, mem.size); - if (ipa3_send_cmd(1, &desc)) { + if (ipa3_send_cmd(num_cmds, dma_cmd_desc)) { IPAERR("fail to send immediate command\n"); - ipahal_destroy_imm_cmd(cmd_pyld); + for (i = 0; i < num_cmds; i++) + ipahal_destroy_imm_cmd(cmd_pyld[i]); dma_free_coherent(ipa3_ctx->pdev, mem.size, mem.base, mem.phys_base); return -EBUSY; } - ipahal_destroy_imm_cmd(cmd_pyld); + for (i = 0; i < num_cmds; i++) + ipahal_destroy_imm_cmd(cmd_pyld[i]); ipahal_write_reg(IPA_LOCAL_PKT_PROC_CNTXT_BASE, dma_cmd.local_addr); diff --git a/drivers/platform/msm/ipa/ipa_v3/ipa_flt.c b/drivers/platform/msm/ipa/ipa_v3/ipa_flt.c index c81a0bfdf404..d8b7bf1f5153 100644 --- a/drivers/platform/msm/ipa/ipa_v3/ipa_flt.c +++ b/drivers/platform/msm/ipa/ipa_v3/ipa_flt.c @@ -491,6 +491,7 @@ int __ipa_commit_flt_v3(enum ipa_ip_type ip) u32 tbl_hdr_width; struct ipa3_flt_tbl *tbl; u16 entries; + struct ipahal_imm_cmd_register_write reg_write_coal_close; tbl_hdr_width = ipahal_get_hw_tbl_hdr_width(); memset(&alloc_params, 0, sizeof(alloc_params)); @@ -567,14 +568,38 @@ int __ipa_commit_flt_v3(enum ipa_ip_type ip) goto fail_size_valid; } - /* +3: 2 for bodies (hashable and non-hashable) and 1 for flushing */ - entries = (ipa3_ctx->ep_flt_num) * 2 + 3; + /* +4: 2 for bodies (hashable and non-hashable), 1 for flushing and 1 + * for closing the colaescing frame + */ + entries = (ipa3_ctx->ep_flt_num) * 2 + 4; if (ipa_flt_alloc_cmd_buffers(ip, entries, &desc, &cmd_pyld)) { rc = -ENOMEM; goto fail_size_valid; } + /* IC to close the coal frame before HPS Clear if coal is enabled */ + if (ipa3_get_ep_mapping(IPA_CLIENT_APPS_WAN_COAL_CONS) != -1) { + i = ipa3_get_ep_mapping(IPA_CLIENT_APPS_WAN_COAL_CONS); + reg_write_coal_close.skip_pipeline_clear = false; + reg_write_coal_close.pipeline_clear_options = IPAHAL_HPS_CLEAR; + reg_write_coal_close.offset = ipahal_get_reg_ofst( + IPA_AGGR_FORCE_CLOSE); + ipahal_get_aggr_force_close_valmask(i, &valmask); + reg_write_coal_close.value = valmask.val; + reg_write_coal_close.value_mask = valmask.mask; + cmd_pyld[num_cmd] = ipahal_construct_imm_cmd( + IPA_IMM_CMD_REGISTER_WRITE, + ®_write_coal_close, false); + if (!cmd_pyld[num_cmd]) { + IPAERR("failed to construct coal close IC\n"); + rc = -ENOMEM; + goto fail_reg_write_construct; + } + ipa3_init_imm_cmd_desc(&desc[num_cmd], cmd_pyld[num_cmd]); + ++num_cmd; + } + /* * SRAM memory not allocated to hash tables. Sending * command to hash tables(filer/routing) operation not supported. @@ -593,14 +618,14 @@ int __ipa_commit_flt_v3(enum ipa_ip_type ip) IPA_FILT_ROUT_HASH_FLUSH); reg_write_cmd.value = valmask.val; reg_write_cmd.value_mask = valmask.mask; - cmd_pyld[0] = ipahal_construct_imm_cmd( + cmd_pyld[num_cmd] = ipahal_construct_imm_cmd( IPA_IMM_CMD_REGISTER_WRITE, ®_write_cmd, false); - if (!cmd_pyld[0]) { + if (!cmd_pyld[num_cmd]) { IPAERR( "fail construct register_write imm cmd: IP %d\n", ip); rc = -EFAULT; - goto fail_reg_write_construct; + goto fail_imm_cmd_construct; } ipa3_init_imm_cmd_desc(&desc[num_cmd], cmd_pyld[num_cmd]); ++num_cmd; diff --git a/drivers/platform/msm/ipa/ipa_v3/ipa_hdr.c b/drivers/platform/msm/ipa/ipa_v3/ipa_hdr.c index e5b00bddf383..1834bdadf3bd 100644 --- a/drivers/platform/msm/ipa/ipa_v3/ipa_hdr.c +++ b/drivers/platform/msm/ipa/ipa_v3/ipa_hdr.c @@ -154,7 +154,7 @@ static int ipa3_generate_hdr_proc_ctx_hw_tbl(u64 hdr_sys_addr, */ int __ipa_commit_hdr_v3_0(void) { - struct ipa3_desc desc[2]; + struct ipa3_desc desc[3]; struct ipa_mem_buffer hdr_mem; struct ipa_mem_buffer ctx_mem; struct ipa_mem_buffer aligned_ctx_mem; @@ -164,12 +164,17 @@ int __ipa_commit_hdr_v3_0(void) struct ipahal_imm_cmd_hdr_init_system hdr_init_cmd = {0}; struct ipahal_imm_cmd_pyld *hdr_cmd_pyld = NULL; struct ipahal_imm_cmd_pyld *ctx_cmd_pyld = NULL; + struct ipahal_imm_cmd_pyld *coal_cmd_pyld = NULL; int rc = -EFAULT; + int i; + int num_cmd = 0; u32 proc_ctx_size; u32 proc_ctx_ofst; u32 proc_ctx_size_ddr; + struct ipahal_imm_cmd_register_write reg_write_coal_close; + struct ipahal_reg_valmask valmask; - memset(desc, 0, 2 * sizeof(struct ipa3_desc)); + memset(desc, 0, 3 * sizeof(struct ipa3_desc)); if (ipa3_generate_hdr_hw_tbl(&hdr_mem)) { IPAERR("fail to generate HDR HW TBL\n"); @@ -182,6 +187,27 @@ int __ipa_commit_hdr_v3_0(void) goto end; } + /* IC to close the coal frame before HPS Clear if coal is enabled */ + if (ipa3_get_ep_mapping(IPA_CLIENT_APPS_WAN_COAL_CONS) != -1) { + i = ipa3_get_ep_mapping(IPA_CLIENT_APPS_WAN_COAL_CONS); + reg_write_coal_close.skip_pipeline_clear = false; + reg_write_coal_close.pipeline_clear_options = IPAHAL_HPS_CLEAR; + reg_write_coal_close.offset = ipahal_get_reg_ofst( + IPA_AGGR_FORCE_CLOSE); + ipahal_get_aggr_force_close_valmask(i, &valmask); + reg_write_coal_close.value = valmask.val; + reg_write_coal_close.value_mask = valmask.mask; + coal_cmd_pyld = ipahal_construct_imm_cmd( + IPA_IMM_CMD_REGISTER_WRITE, + ®_write_coal_close, false); + if (!coal_cmd_pyld) { + IPAERR("failed to construct coal close IC\n"); + goto end; + } + ipa3_init_imm_cmd_desc(&desc[num_cmd], coal_cmd_pyld); + ++num_cmd; + } + if (ipa3_ctx->hdr_tbl_lcl) { if (hdr_mem.size > IPA_MEM_PART(apps_hdr_size)) { IPAERR("tbl too big needed %d avail %d\n", hdr_mem.size, @@ -220,7 +246,8 @@ int __ipa_commit_hdr_v3_0(void) } } } - ipa3_init_imm_cmd_desc(&desc[0], hdr_cmd_pyld); + ipa3_init_imm_cmd_desc(&desc[num_cmd], hdr_cmd_pyld); + ++num_cmd; IPA_DUMP_BUFF(hdr_mem.base, hdr_mem.phys_base, hdr_mem.size); proc_ctx_size = IPA_MEM_PART(apps_hdr_proc_ctx_size); @@ -274,10 +301,11 @@ int __ipa_commit_hdr_v3_0(void) } } } - ipa3_init_imm_cmd_desc(&desc[1], ctx_cmd_pyld); + ipa3_init_imm_cmd_desc(&desc[num_cmd], ctx_cmd_pyld); + ++num_cmd; IPA_DUMP_BUFF(ctx_mem.base, ctx_mem.phys_base, ctx_mem.size); - if (ipa3_send_cmd(2, desc)) + if (ipa3_send_cmd(num_cmd, desc)) IPAERR("fail to send immediate command\n"); else rc = 0; @@ -311,6 +339,9 @@ int __ipa_commit_hdr_v3_0(void) } end: + if (coal_cmd_pyld) + ipahal_destroy_imm_cmd(coal_cmd_pyld); + if (ctx_cmd_pyld) ipahal_destroy_imm_cmd(ctx_cmd_pyld); diff --git a/drivers/platform/msm/ipa/ipa_v3/ipa_hw_stats.c b/drivers/platform/msm/ipa/ipa_v3/ipa_hw_stats.c index c5c9a011585a..52034e890d59 100644 --- a/drivers/platform/msm/ipa/ipa_v3/ipa_hw_stats.c +++ b/drivers/platform/msm/ipa/ipa_v3/ipa_hw_stats.c @@ -252,6 +252,25 @@ int ipa_hw_stats_init(void) return ret; } +static void ipa_close_coal_frame(struct ipahal_imm_cmd_pyld **coal_cmd_pyld) +{ + int i; + struct ipahal_reg_valmask valmask; + struct ipahal_imm_cmd_register_write reg_write_coal_close; + + i = ipa3_get_ep_mapping(IPA_CLIENT_APPS_WAN_COAL_CONS); + reg_write_coal_close.skip_pipeline_clear = false; + reg_write_coal_close.pipeline_clear_options = IPAHAL_HPS_CLEAR; + reg_write_coal_close.offset = ipahal_get_reg_ofst( + IPA_AGGR_FORCE_CLOSE); + ipahal_get_aggr_force_close_valmask(i, &valmask); + reg_write_coal_close.value = valmask.val; + reg_write_coal_close.value_mask = valmask.mask; + *coal_cmd_pyld = ipahal_construct_imm_cmd( + IPA_IMM_CMD_REGISTER_WRITE, + ®_write_coal_close, false); +} + int ipa_init_quota_stats(u32 pipe_bitmask) { struct ipahal_stats_init_pyld *pyld; @@ -261,9 +280,11 @@ int ipa_init_quota_stats(u32 pipe_bitmask) struct ipahal_imm_cmd_pyld *quota_base_pyld; struct ipahal_imm_cmd_register_write quota_mask = {0}; struct ipahal_imm_cmd_pyld *quota_mask_pyld; - struct ipa3_desc desc[3] = { {0} }; + struct ipahal_imm_cmd_pyld *coal_cmd_pyld = NULL; + struct ipa3_desc desc[4] = { {0} }; dma_addr_t dma_address; int ret; + int num_cmd = 0; if (!ipa3_ctx->hw_stats.enabled) return 0; @@ -297,6 +318,19 @@ int ipa_init_quota_stats(u32 pipe_bitmask) goto destroy_init_pyld; } + /* IC to close the coal frame before HPS Clear if coal is enabled */ + if (ipa3_get_ep_mapping(IPA_CLIENT_APPS_WAN_COAL_CONS) != + IPA_EP_NOT_ALLOCATED) { + ipa_close_coal_frame(&coal_cmd_pyld); + if (!coal_cmd_pyld) { + IPAERR("failed to construct coal close IC\n"); + ret = -ENOMEM; + goto unmap; + } + ipa3_init_imm_cmd_desc(&desc[num_cmd], coal_cmd_pyld); + ++num_cmd; + } + /* setting the registers and init the stats pyld are done atomically */ quota_mask.skip_pipeline_clear = false; quota_mask.pipeline_clear_options = IPAHAL_FULL_PIPELINE_CLEAR; @@ -309,12 +343,13 @@ int ipa_init_quota_stats(u32 pipe_bitmask) if (!quota_mask_pyld) { IPAERR("failed to construct register_write imm cmd\n"); ret = -ENOMEM; - goto unmap; + goto destroy_coal_cmd; } - desc[0].opcode = quota_mask_pyld->opcode; - desc[0].pyld = quota_mask_pyld->data; - desc[0].len = quota_mask_pyld->len; - desc[0].type = IPA_IMM_CMD_DESC; + desc[num_cmd].opcode = quota_mask_pyld->opcode; + desc[num_cmd].pyld = quota_mask_pyld->data; + desc[num_cmd].len = quota_mask_pyld->len; + desc[num_cmd].type = IPA_IMM_CMD_DESC; + num_cmd++; quota_base.skip_pipeline_clear = false; quota_base.pipeline_clear_options = IPAHAL_FULL_PIPELINE_CLEAR; @@ -330,10 +365,11 @@ int ipa_init_quota_stats(u32 pipe_bitmask) ret = -ENOMEM; goto destroy_quota_mask; } - desc[1].opcode = quota_base_pyld->opcode; - desc[1].pyld = quota_base_pyld->data; - desc[1].len = quota_base_pyld->len; - desc[1].type = IPA_IMM_CMD_DESC; + desc[num_cmd].opcode = quota_base_pyld->opcode; + desc[num_cmd].pyld = quota_base_pyld->data; + desc[num_cmd].len = quota_base_pyld->len; + desc[num_cmd].type = IPA_IMM_CMD_DESC; + num_cmd++; cmd.is_read = false; cmd.skip_pipeline_clear = false; @@ -349,12 +385,13 @@ int ipa_init_quota_stats(u32 pipe_bitmask) ret = -ENOMEM; goto destroy_quota_base; } - desc[2].opcode = cmd_pyld->opcode; - desc[2].pyld = cmd_pyld->data; - desc[2].len = cmd_pyld->len; - desc[2].type = IPA_IMM_CMD_DESC; + desc[num_cmd].opcode = cmd_pyld->opcode; + desc[num_cmd].pyld = cmd_pyld->data; + desc[num_cmd].len = cmd_pyld->len; + desc[num_cmd].type = IPA_IMM_CMD_DESC; + num_cmd++; - ret = ipa3_send_cmd(3, desc); + ret = ipa3_send_cmd(num_cmd, desc); if (ret) { IPAERR("failed to send immediate command (error %d)\n", ret); goto destroy_imm; @@ -368,6 +405,8 @@ destroy_quota_base: ipahal_destroy_imm_cmd(quota_base_pyld); destroy_quota_mask: ipahal_destroy_imm_cmd(quota_mask_pyld); +destroy_coal_cmd: + ipahal_destroy_imm_cmd(coal_cmd_pyld); unmap: dma_unmap_single(ipa3_ctx->pdev, dma_address, pyld->len, DMA_TO_DEVICE); destroy_init_pyld: @@ -382,14 +421,18 @@ int ipa_get_quota_stats(struct ipa_quota_stats_all *out) struct ipahal_stats_get_offset_quota get_offset = { { 0 } }; struct ipahal_stats_offset offset = { 0 }; struct ipahal_imm_cmd_dma_shared_mem cmd = { 0 }; - struct ipahal_imm_cmd_pyld *cmd_pyld; + struct ipahal_imm_cmd_pyld *cmd_pyld[2]; struct ipa_mem_buffer mem; - struct ipa3_desc desc = { 0 }; + struct ipa3_desc desc[2]; struct ipahal_stats_quota_all *stats; + int num_cmd = 0; if (!ipa3_ctx->hw_stats.enabled) return 0; + memset(desc, 0, sizeof(desc)); + memset(cmd_pyld, 0, sizeof(cmd_pyld)); + get_offset.init = ipa3_ctx->hw_stats.quota.init; ret = ipahal_stats_get_offset(IPAHAL_HW_STATS_QUOTA, &get_offset, &offset); @@ -413,6 +456,19 @@ int ipa_get_quota_stats(struct ipa_quota_stats_all *out) return ret; } + /* IC to close the coal frame before HPS Clear if coal is enabled */ + if (ipa3_get_ep_mapping(IPA_CLIENT_APPS_WAN_COAL_CONS) != + IPA_EP_NOT_ALLOCATED) { + ipa_close_coal_frame(&cmd_pyld[num_cmd]); + if (!cmd_pyld[num_cmd]) { + IPAERR("failed to construct coal close IC\n"); + ret = -ENOMEM; + goto free_dma_mem; + } + ipa3_init_imm_cmd_desc(&desc[num_cmd], cmd_pyld[num_cmd]); + ++num_cmd; + } + cmd.is_read = true; cmd.clear_after_read = true; cmd.skip_pipeline_clear = false; @@ -421,19 +477,17 @@ int ipa_get_quota_stats(struct ipa_quota_stats_all *out) cmd.system_addr = mem.phys_base; cmd.local_addr = ipa3_ctx->smem_restricted_bytes + IPA_MEM_PART(stats_quota_ofst) + offset.offset; - cmd_pyld = ipahal_construct_imm_cmd( + cmd_pyld[num_cmd] = ipahal_construct_imm_cmd( IPA_IMM_CMD_DMA_SHARED_MEM, &cmd, false); - if (!cmd_pyld) { + if (!cmd_pyld[num_cmd]) { IPAERR("failed to construct dma_shared_mem imm cmd\n"); ret = -ENOMEM; goto free_dma_mem; } - desc.opcode = cmd_pyld->opcode; - desc.pyld = cmd_pyld->data; - desc.len = cmd_pyld->len; - desc.type = IPA_IMM_CMD_DESC; + ipa3_init_imm_cmd_desc(&desc[num_cmd], cmd_pyld[num_cmd]); + ++num_cmd; - ret = ipa3_send_cmd(1, &desc); + ret = ipa3_send_cmd(num_cmd, desc); if (ret) { IPAERR("failed to send immediate command (error %d)\n", ret); goto destroy_imm; @@ -483,7 +537,8 @@ int ipa_get_quota_stats(struct ipa_quota_stats_all *out) free_stats: kfree(stats); destroy_imm: - ipahal_destroy_imm_cmd(cmd_pyld); + for (i = 0; i < num_cmd; i++) + ipahal_destroy_imm_cmd(cmd_pyld[i]); free_dma_mem: dma_free_coherent(ipa3_ctx->pdev, mem.size, mem.base, mem.phys_base); return ret; @@ -546,10 +601,13 @@ int ipa_init_teth_stats(struct ipa_teth_stats_endpoints *in) struct ipahal_imm_cmd_pyld *teth_base_pyld; struct ipahal_imm_cmd_register_write teth_mask = { 0 }; struct ipahal_imm_cmd_pyld *teth_mask_pyld; - struct ipa3_desc desc[3] = { {0} }; + struct ipahal_imm_cmd_pyld *coal_cmd_pyld = NULL; + struct ipa3_desc desc[4] = { {0} }; dma_addr_t dma_address; int ret; int i; + int num_cmd = 0; + if (!ipa3_ctx->hw_stats.enabled) return 0; @@ -605,6 +663,19 @@ int ipa_init_teth_stats(struct ipa_teth_stats_endpoints *in) goto destroy_init_pyld; } + /* IC to close the coal frame before HPS Clear if coal is enabled */ + if (ipa3_get_ep_mapping(IPA_CLIENT_APPS_WAN_COAL_CONS) != + IPA_EP_NOT_ALLOCATED) { + ipa_close_coal_frame(&coal_cmd_pyld); + if (!coal_cmd_pyld) { + IPAERR("failed to construct coal close IC\n"); + ret = -ENOMEM; + goto unmap; + } + ipa3_init_imm_cmd_desc(&desc[num_cmd], coal_cmd_pyld); + ++num_cmd; + } + /* setting the registers and init the stats pyld are done atomically */ teth_mask.skip_pipeline_clear = false; teth_mask.pipeline_clear_options = IPAHAL_FULL_PIPELINE_CLEAR; @@ -617,12 +688,13 @@ int ipa_init_teth_stats(struct ipa_teth_stats_endpoints *in) if (!teth_mask_pyld) { IPAERR("failed to construct register_write imm cmd\n"); ret = -ENOMEM; - goto unmap; + goto destroy_coal_cmd; } - desc[0].opcode = teth_mask_pyld->opcode; - desc[0].pyld = teth_mask_pyld->data; - desc[0].len = teth_mask_pyld->len; - desc[0].type = IPA_IMM_CMD_DESC; + desc[num_cmd].opcode = teth_mask_pyld->opcode; + desc[num_cmd].pyld = teth_mask_pyld->data; + desc[num_cmd].len = teth_mask_pyld->len; + desc[num_cmd].type = IPA_IMM_CMD_DESC; + ++num_cmd; teth_base.skip_pipeline_clear = false; teth_base.pipeline_clear_options = IPAHAL_FULL_PIPELINE_CLEAR; @@ -638,10 +710,11 @@ int ipa_init_teth_stats(struct ipa_teth_stats_endpoints *in) ret = -ENOMEM; goto destroy_teth_mask; } - desc[1].opcode = teth_base_pyld->opcode; - desc[1].pyld = teth_base_pyld->data; - desc[1].len = teth_base_pyld->len; - desc[1].type = IPA_IMM_CMD_DESC; + desc[num_cmd].opcode = teth_base_pyld->opcode; + desc[num_cmd].pyld = teth_base_pyld->data; + desc[num_cmd].len = teth_base_pyld->len; + desc[num_cmd].type = IPA_IMM_CMD_DESC; + ++num_cmd; cmd.is_read = false; cmd.skip_pipeline_clear = false; @@ -657,12 +730,13 @@ int ipa_init_teth_stats(struct ipa_teth_stats_endpoints *in) ret = -ENOMEM; goto destroy_teth_base; } - desc[2].opcode = cmd_pyld->opcode; - desc[2].pyld = cmd_pyld->data; - desc[2].len = cmd_pyld->len; - desc[2].type = IPA_IMM_CMD_DESC; + desc[num_cmd].opcode = cmd_pyld->opcode; + desc[num_cmd].pyld = cmd_pyld->data; + desc[num_cmd].len = cmd_pyld->len; + desc[num_cmd].type = IPA_IMM_CMD_DESC; + ++num_cmd; - ret = ipa3_send_cmd(3, desc); + ret = ipa3_send_cmd(num_cmd, desc); if (ret) { IPAERR("failed to send immediate command (error %d)\n", ret); goto destroy_imm; @@ -676,6 +750,9 @@ destroy_teth_base: ipahal_destroy_imm_cmd(teth_base_pyld); destroy_teth_mask: ipahal_destroy_imm_cmd(teth_mask_pyld); +destroy_coal_cmd: + if (coal_cmd_pyld) + ipahal_destroy_imm_cmd(coal_cmd_pyld); unmap: dma_unmap_single(ipa3_ctx->pdev, dma_address, pyld->len, DMA_TO_DEVICE); destroy_init_pyld: @@ -690,9 +767,9 @@ int ipa_get_teth_stats(void) struct ipahal_stats_get_offset_tethering get_offset = { { 0 } }; struct ipahal_stats_offset offset = {0}; struct ipahal_imm_cmd_dma_shared_mem cmd = { 0 }; - struct ipahal_imm_cmd_pyld *cmd_pyld; + struct ipahal_imm_cmd_pyld *cmd_pyld[2]; struct ipa_mem_buffer mem; - struct ipa3_desc desc = { 0 }; + struct ipa3_desc desc[2]; struct ipahal_stats_tethering_all *stats_all; struct ipa_hw_stats_teth *sw_stats = &ipa3_ctx->hw_stats.teth; struct ipahal_stats_tethering *stats; @@ -700,10 +777,14 @@ int ipa_get_teth_stats(void) struct ipahal_stats_init_tethering *init = (struct ipahal_stats_init_tethering *) &ipa3_ctx->hw_stats.teth.init; + int num_cmd = 0; if (!ipa3_ctx->hw_stats.enabled) return 0; + memset(desc, 0, sizeof(desc)); + memset(cmd_pyld, 0, sizeof(cmd_pyld)); + get_offset.init = ipa3_ctx->hw_stats.teth.init; ret = ipahal_stats_get_offset(IPAHAL_HW_STATS_TETHERING, &get_offset, &offset); @@ -727,6 +808,19 @@ int ipa_get_teth_stats(void) return ret; } + /* IC to close the coal frame before HPS Clear if coal is enabled */ + if (ipa3_get_ep_mapping(IPA_CLIENT_APPS_WAN_COAL_CONS) != + IPA_EP_NOT_ALLOCATED) { + ipa_close_coal_frame(&cmd_pyld[num_cmd]); + if (!cmd_pyld[num_cmd]) { + IPAERR("failed to construct coal close IC\n"); + ret = -ENOMEM; + goto free_dma_mem; + } + ipa3_init_imm_cmd_desc(&desc[num_cmd], cmd_pyld[num_cmd]); + ++num_cmd; + } + cmd.is_read = true; cmd.clear_after_read = true; cmd.skip_pipeline_clear = false; @@ -735,19 +829,17 @@ int ipa_get_teth_stats(void) cmd.system_addr = mem.phys_base; cmd.local_addr = ipa3_ctx->smem_restricted_bytes + IPA_MEM_PART(stats_tethering_ofst) + offset.offset; - cmd_pyld = ipahal_construct_imm_cmd( + cmd_pyld[num_cmd] = ipahal_construct_imm_cmd( IPA_IMM_CMD_DMA_SHARED_MEM, &cmd, false); - if (!cmd_pyld) { + if (!cmd_pyld[num_cmd]) { IPAERR("failed to construct dma_shared_mem imm cmd\n"); ret = -ENOMEM; - goto free_dma_mem; + goto destroy_imm; } - desc.opcode = cmd_pyld->opcode; - desc.pyld = cmd_pyld->data; - desc.len = cmd_pyld->len; - desc.type = IPA_IMM_CMD_DESC; + ipa3_init_imm_cmd_desc(&desc[num_cmd], cmd_pyld[num_cmd]); + ++num_cmd; - ret = ipa3_send_cmd(1, &desc); + ret = ipa3_send_cmd(num_cmd, desc); if (ret) { IPAERR("failed to send immediate command (error %d)\n", ret); goto destroy_imm; @@ -837,7 +929,8 @@ free_stats: kfree(stats_all); stats = NULL; destroy_imm: - ipahal_destroy_imm_cmd(cmd_pyld); + for (i = 0; i < num_cmd; i++) + ipahal_destroy_imm_cmd(cmd_pyld[i]); free_dma_mem: dma_free_coherent(ipa3_ctx->pdev, mem.size, mem.base, mem.phys_base); return ret; @@ -963,9 +1056,11 @@ int ipa_init_flt_rt_stats(void) struct ipahal_imm_cmd_pyld *rt_v4_base_pyld; struct ipahal_imm_cmd_register_write rt_v6_base = {0}; struct ipahal_imm_cmd_pyld *rt_v6_base_pyld; - struct ipa3_desc desc[5] = { {0} }; + struct ipahal_imm_cmd_pyld *coal_cmd_pyld = NULL; + struct ipa3_desc desc[6] = { {0} }; dma_addr_t dma_address; int ret; + int num_cmd = 0; if (!ipa3_ctx->hw_stats.enabled) return 0; @@ -997,6 +1092,19 @@ int ipa_init_flt_rt_stats(void) goto destroy_init_pyld; } + /* IC to close the coal frame before HPS Clear if coal is enabled */ + if (ipa3_get_ep_mapping(IPA_CLIENT_APPS_WAN_COAL_CONS) != + IPA_EP_NOT_ALLOCATED) { + ipa_close_coal_frame(&coal_cmd_pyld); + if (!coal_cmd_pyld) { + IPAERR("failed to construct coal close IC\n"); + ret = -ENOMEM; + goto unmap; + } + ipa3_init_imm_cmd_desc(&desc[num_cmd], coal_cmd_pyld); + ++num_cmd; + } + stats_base_flt_v4 = ipahal_get_reg_ofst(IPA_STAT_FILTER_IPV4_BASE); stats_base_flt_v6 = ipahal_get_reg_ofst(IPA_STAT_FILTER_IPV6_BASE); stats_base_rt_v4 = ipahal_get_reg_ofst(IPA_STAT_ROUTER_IPV4_BASE); @@ -1015,12 +1123,13 @@ int ipa_init_flt_rt_stats(void) if (!flt_v4_base_pyld) { IPAERR("failed to construct register_write imm cmd\n"); ret = -ENOMEM; - goto unmap; + goto destroy_coal_cmd; } - desc[0].opcode = flt_v4_base_pyld->opcode; - desc[0].pyld = flt_v4_base_pyld->data; - desc[0].len = flt_v4_base_pyld->len; - desc[0].type = IPA_IMM_CMD_DESC; + desc[num_cmd].opcode = flt_v4_base_pyld->opcode; + desc[num_cmd].pyld = flt_v4_base_pyld->data; + desc[num_cmd].len = flt_v4_base_pyld->len; + desc[num_cmd].type = IPA_IMM_CMD_DESC; + +num_cmd; /* set IPA_STAT_FILTER_IPV6_BASE */ flt_v6_base.skip_pipeline_clear = false; @@ -1036,11 +1145,11 @@ int ipa_init_flt_rt_stats(void) ret = -ENOMEM; goto destroy_flt_v4_base; } - desc[1].opcode = flt_v6_base_pyld->opcode; - desc[1].pyld = flt_v6_base_pyld->data; - desc[1].len = flt_v6_base_pyld->len; - desc[1].type = IPA_IMM_CMD_DESC; - + desc[num_cmd].opcode = flt_v6_base_pyld->opcode; + desc[num_cmd].pyld = flt_v6_base_pyld->data; + desc[num_cmd].len = flt_v6_base_pyld->len; + desc[num_cmd].type = IPA_IMM_CMD_DESC; + ++num_cmd; /* set IPA_STAT_ROUTER_IPV4_BASE */ rt_v4_base.skip_pipeline_clear = false; @@ -1056,10 +1165,11 @@ int ipa_init_flt_rt_stats(void) ret = -ENOMEM; goto destroy_flt_v6_base; } - desc[2].opcode = rt_v4_base_pyld->opcode; - desc[2].pyld = rt_v4_base_pyld->data; - desc[2].len = rt_v4_base_pyld->len; - desc[2].type = IPA_IMM_CMD_DESC; + desc[num_cmd].opcode = rt_v4_base_pyld->opcode; + desc[num_cmd].pyld = rt_v4_base_pyld->data; + desc[num_cmd].len = rt_v4_base_pyld->len; + desc[num_cmd].type = IPA_IMM_CMD_DESC; + ++num_cmd; /* set IPA_STAT_ROUTER_IPV6_BASE */ rt_v6_base.skip_pipeline_clear = false; @@ -1075,10 +1185,11 @@ int ipa_init_flt_rt_stats(void) ret = -ENOMEM; goto destroy_rt_v4_base; } - desc[3].opcode = rt_v6_base_pyld->opcode; - desc[3].pyld = rt_v6_base_pyld->data; - desc[3].len = rt_v6_base_pyld->len; - desc[3].type = IPA_IMM_CMD_DESC; + desc[num_cmd].opcode = rt_v6_base_pyld->opcode; + desc[num_cmd].pyld = rt_v6_base_pyld->data; + desc[num_cmd].len = rt_v6_base_pyld->len; + desc[num_cmd].type = IPA_IMM_CMD_DESC; + ++num_cmd; cmd.is_read = false; cmd.skip_pipeline_clear = false; @@ -1094,12 +1205,13 @@ int ipa_init_flt_rt_stats(void) ret = -ENOMEM; goto destroy_rt_v6_base; } - desc[4].opcode = cmd_pyld->opcode; - desc[4].pyld = cmd_pyld->data; - desc[4].len = cmd_pyld->len; - desc[4].type = IPA_IMM_CMD_DESC; + desc[num_cmd].opcode = cmd_pyld->opcode; + desc[num_cmd].pyld = cmd_pyld->data; + desc[num_cmd].len = cmd_pyld->len; + desc[num_cmd].type = IPA_IMM_CMD_DESC; + ++num_cmd; - ret = ipa3_send_cmd(5, desc); + ret = ipa3_send_cmd(num_cmd, desc); if (ret) { IPAERR("failed to send immediate command (error %d)\n", ret); goto destroy_imm; @@ -1117,6 +1229,9 @@ destroy_flt_v6_base: ipahal_destroy_imm_cmd(flt_v6_base_pyld); destroy_flt_v4_base: ipahal_destroy_imm_cmd(flt_v4_base_pyld); +destroy_coal_cmd: + if (coal_cmd_pyld) + ipahal_destroy_imm_cmd(coal_cmd_pyld); unmap: dma_unmap_single(ipa3_ctx->pdev, dma_address, pyld->len, DMA_TO_DEVICE); destroy_init_pyld: @@ -1132,9 +1247,14 @@ static int __ipa_get_flt_rt_stats(struct ipa_ioc_flt_rt_query *query) struct ipahal_stats_get_offset_flt_rt_v4_5 *get_offset; struct ipahal_stats_offset offset = { 0 }; struct ipahal_imm_cmd_dma_shared_mem cmd = { 0 }; - struct ipahal_imm_cmd_pyld *cmd_pyld; + struct ipahal_imm_cmd_pyld *cmd_pyld[2]; struct ipa_mem_buffer mem; - struct ipa3_desc desc = { 0 }; + struct ipa3_desc desc[2]; + int num_cmd = 0; + int i; + + memset(desc, 0, sizeof(desc)); + memset(cmd_pyld, 0, sizeof(cmd_pyld)); get_offset = kzalloc(sizeof(*get_offset), GFP_KERNEL); if (!get_offset) { @@ -1171,6 +1291,19 @@ static int __ipa_get_flt_rt_stats(struct ipa_ioc_flt_rt_query *query) goto free_offset; } + /* IC to close the coal frame before HPS Clear if coal is enabled */ + if (ipa3_get_ep_mapping(IPA_CLIENT_APPS_WAN_COAL_CONS) != + IPA_EP_NOT_ALLOCATED) { + ipa_close_coal_frame(&cmd_pyld[num_cmd]); + if (!cmd_pyld[num_cmd]) { + IPAERR("failed to construct coal close IC\n"); + ret = -ENOMEM; + goto free_dma_mem; + } + ipa3_init_imm_cmd_desc(&desc[num_cmd], cmd_pyld[num_cmd]); + ++num_cmd; + } + cmd.is_read = true; cmd.clear_after_read = clear; cmd.skip_pipeline_clear = false; @@ -1179,19 +1312,17 @@ static int __ipa_get_flt_rt_stats(struct ipa_ioc_flt_rt_query *query) cmd.system_addr = mem.phys_base; cmd.local_addr = ipa3_ctx->smem_restricted_bytes + smem_ofst + offset.offset; - cmd_pyld = ipahal_construct_imm_cmd( + cmd_pyld[num_cmd] = ipahal_construct_imm_cmd( IPA_IMM_CMD_DMA_SHARED_MEM, &cmd, false); - if (!cmd_pyld) { + if (!cmd_pyld[num_cmd]) { IPAERR("failed to construct dma_shared_mem imm cmd\n"); ret = -ENOMEM; - goto free_dma_mem; + goto destroy_imm; } - desc.opcode = cmd_pyld->opcode; - desc.pyld = cmd_pyld->data; - desc.len = cmd_pyld->len; - desc.type = IPA_IMM_CMD_DESC; + ipa3_init_imm_cmd_desc(&desc[num_cmd], cmd_pyld[num_cmd]); + ++num_cmd; - ret = ipa3_send_cmd(1, &desc); + ret = ipa3_send_cmd(num_cmd, desc); if (ret) { IPAERR("failed to send immediate command (error %d)\n", ret); goto destroy_imm; @@ -1206,7 +1337,8 @@ static int __ipa_get_flt_rt_stats(struct ipa_ioc_flt_rt_query *query) ret = 0; destroy_imm: - ipahal_destroy_imm_cmd(cmd_pyld); + for (i = 0; i < num_cmd; i++) + ipahal_destroy_imm_cmd(cmd_pyld[i]); free_dma_mem: dma_free_coherent(ipa3_ctx->pdev, mem.size, mem.base, mem.phys_base); free_offset: @@ -1379,9 +1511,11 @@ int ipa_init_drop_stats(u32 pipe_bitmask) struct ipahal_imm_cmd_pyld *drop_base_pyld; struct ipahal_imm_cmd_register_write drop_mask = {0}; struct ipahal_imm_cmd_pyld *drop_mask_pyld; - struct ipa3_desc desc[3] = { {0} }; + struct ipahal_imm_cmd_pyld *coal_cmd_pyld = NULL; + struct ipa3_desc desc[4] = { {0} }; dma_addr_t dma_address; int ret; + int num_cmd = 0; if (!ipa3_ctx->hw_stats.enabled) return 0; @@ -1415,6 +1549,19 @@ int ipa_init_drop_stats(u32 pipe_bitmask) goto destroy_init_pyld; } + /* IC to close the coal frame before HPS Clear if coal is enabled */ + if (ipa3_get_ep_mapping(IPA_CLIENT_APPS_WAN_COAL_CONS) != + IPA_EP_NOT_ALLOCATED) { + ipa_close_coal_frame(&coal_cmd_pyld); + if (!coal_cmd_pyld) { + IPAERR("failed to construct coal close IC\n"); + ret = -ENOMEM; + goto unmap; + } + ipa3_init_imm_cmd_desc(&desc[num_cmd], coal_cmd_pyld); + ++num_cmd; + } + /* setting the registers and init the stats pyld are done atomically */ drop_mask.skip_pipeline_clear = false; drop_mask.pipeline_clear_options = IPAHAL_FULL_PIPELINE_CLEAR; @@ -1427,12 +1574,13 @@ int ipa_init_drop_stats(u32 pipe_bitmask) if (!drop_mask_pyld) { IPAERR("failed to construct register_write imm cmd\n"); ret = -ENOMEM; - goto unmap; + goto destroy_coal_cmd; } - desc[0].opcode = drop_mask_pyld->opcode; - desc[0].pyld = drop_mask_pyld->data; - desc[0].len = drop_mask_pyld->len; - desc[0].type = IPA_IMM_CMD_DESC; + desc[num_cmd].opcode = drop_mask_pyld->opcode; + desc[num_cmd].pyld = drop_mask_pyld->data; + desc[num_cmd].len = drop_mask_pyld->len; + desc[num_cmd].type = IPA_IMM_CMD_DESC; + ++num_cmd; drop_base.skip_pipeline_clear = false; drop_base.pipeline_clear_options = IPAHAL_FULL_PIPELINE_CLEAR; @@ -1448,10 +1596,11 @@ int ipa_init_drop_stats(u32 pipe_bitmask) ret = -ENOMEM; goto destroy_drop_mask; } - desc[1].opcode = drop_base_pyld->opcode; - desc[1].pyld = drop_base_pyld->data; - desc[1].len = drop_base_pyld->len; - desc[1].type = IPA_IMM_CMD_DESC; + desc[num_cmd].opcode = drop_base_pyld->opcode; + desc[num_cmd].pyld = drop_base_pyld->data; + desc[num_cmd].len = drop_base_pyld->len; + desc[num_cmd].type = IPA_IMM_CMD_DESC; + ++num_cmd; cmd.is_read = false; cmd.skip_pipeline_clear = false; @@ -1467,12 +1616,13 @@ int ipa_init_drop_stats(u32 pipe_bitmask) ret = -ENOMEM; goto destroy_drop_base; } - desc[2].opcode = cmd_pyld->opcode; - desc[2].pyld = cmd_pyld->data; - desc[2].len = cmd_pyld->len; - desc[2].type = IPA_IMM_CMD_DESC; + desc[num_cmd].opcode = cmd_pyld->opcode; + desc[num_cmd].pyld = cmd_pyld->data; + desc[num_cmd].len = cmd_pyld->len; + desc[num_cmd].type = IPA_IMM_CMD_DESC; + ++num_cmd; - ret = ipa3_send_cmd(3, desc); + ret = ipa3_send_cmd(num_cmd, desc); if (ret) { IPAERR("failed to send immediate command (error %d)\n", ret); goto destroy_imm; @@ -1486,6 +1636,9 @@ destroy_drop_base: ipahal_destroy_imm_cmd(drop_base_pyld); destroy_drop_mask: ipahal_destroy_imm_cmd(drop_mask_pyld); +destroy_coal_cmd: + if (coal_cmd_pyld) + ipahal_destroy_imm_cmd(coal_cmd_pyld); unmap: dma_unmap_single(ipa3_ctx->pdev, dma_address, pyld->len, DMA_TO_DEVICE); destroy_init_pyld: @@ -1500,14 +1653,18 @@ int ipa_get_drop_stats(struct ipa_drop_stats_all *out) struct ipahal_stats_get_offset_drop get_offset = { { 0 } }; struct ipahal_stats_offset offset = { 0 }; struct ipahal_imm_cmd_dma_shared_mem cmd = { 0 }; - struct ipahal_imm_cmd_pyld *cmd_pyld; + struct ipahal_imm_cmd_pyld *cmd_pyld[2]; struct ipa_mem_buffer mem; - struct ipa3_desc desc = { 0 }; + struct ipa3_desc desc[2]; struct ipahal_stats_drop_all *stats; + int num_cmd = 0; if (!ipa3_ctx->hw_stats.enabled) return 0; + memset(desc, 0, sizeof(desc)); + memset(cmd_pyld, 0, sizeof(cmd_pyld)); + get_offset.init = ipa3_ctx->hw_stats.drop.init; ret = ipahal_stats_get_offset(IPAHAL_HW_STATS_DROP, &get_offset, &offset); @@ -1531,6 +1688,19 @@ int ipa_get_drop_stats(struct ipa_drop_stats_all *out) return ret; } + /* IC to close the coal frame before HPS Clear if coal is enabled */ + if (ipa3_get_ep_mapping(IPA_CLIENT_APPS_WAN_COAL_CONS) != + IPA_EP_NOT_ALLOCATED) { + ipa_close_coal_frame(&cmd_pyld[num_cmd]); + if (!cmd_pyld[num_cmd]) { + IPAERR("failed to construct coal close IC\n"); + ret = -ENOMEM; + goto free_dma_mem; + } + ipa3_init_imm_cmd_desc(&desc[num_cmd], cmd_pyld[num_cmd]); + ++num_cmd; + } + cmd.is_read = true; cmd.clear_after_read = true; cmd.skip_pipeline_clear = false; @@ -1539,19 +1709,17 @@ int ipa_get_drop_stats(struct ipa_drop_stats_all *out) cmd.system_addr = mem.phys_base; cmd.local_addr = ipa3_ctx->smem_restricted_bytes + IPA_MEM_PART(stats_drop_ofst) + offset.offset; - cmd_pyld = ipahal_construct_imm_cmd( + cmd_pyld[num_cmd] = ipahal_construct_imm_cmd( IPA_IMM_CMD_DMA_SHARED_MEM, &cmd, false); - if (!cmd_pyld) { + if (!cmd_pyld[num_cmd]) { IPAERR("failed to construct dma_shared_mem imm cmd\n"); ret = -ENOMEM; - goto free_dma_mem; + goto destroy_imm; } - desc.opcode = cmd_pyld->opcode; - desc.pyld = cmd_pyld->data; - desc.len = cmd_pyld->len; - desc.type = IPA_IMM_CMD_DESC; + ipa3_init_imm_cmd_desc(&desc[num_cmd], cmd_pyld[num_cmd]); + ++num_cmd; - ret = ipa3_send_cmd(1, &desc); + ret = ipa3_send_cmd(num_cmd, desc); if (ret) { IPAERR("failed to send immediate command (error %d)\n", ret); goto destroy_imm; @@ -1603,7 +1771,8 @@ int ipa_get_drop_stats(struct ipa_drop_stats_all *out) free_stats: kfree(stats); destroy_imm: - ipahal_destroy_imm_cmd(cmd_pyld); + for (i = 0; i < num_cmd; i++) + ipahal_destroy_imm_cmd(cmd_pyld[i]); free_dma_mem: dma_free_coherent(ipa3_ctx->pdev, mem.size, mem.base, mem.phys_base); return ret; diff --git a/drivers/platform/msm/ipa/ipa_v3/ipa_nat.c b/drivers/platform/msm/ipa/ipa_v3/ipa_nat.c index 8a0b2e996aad..91715e5ddff0 100644 --- a/drivers/platform/msm/ipa/ipa_v3/ipa_nat.c +++ b/drivers/platform/msm/ipa/ipa_v3/ipa_nat.c @@ -27,9 +27,9 @@ #define IPA_NAT_IPV6CT_TEMP_MEM_SIZE 128 -#define IPA_NAT_MAX_NUM_OF_INIT_CMD_DESC 3 -#define IPA_IPV6CT_MAX_NUM_OF_INIT_CMD_DESC 2 -#define IPA_MAX_NUM_OF_TABLE_DMA_CMD_DESC 4 +#define IPA_NAT_MAX_NUM_OF_INIT_CMD_DESC 4 +#define IPA_IPV6CT_MAX_NUM_OF_INIT_CMD_DESC 3 +#define IPA_MAX_NUM_OF_TABLE_DMA_CMD_DESC 5 /* * The base table max entries is limited by index into table 13 bits number. @@ -728,15 +728,43 @@ static int ipa3_nat_send_init_cmd(struct ipahal_imm_cmd_ip_v4_nat_init *cmd, struct ipa3_desc desc[IPA_NAT_MAX_NUM_OF_INIT_CMD_DESC]; struct ipahal_imm_cmd_pyld *cmd_pyld[IPA_NAT_MAX_NUM_OF_INIT_CMD_DESC]; int i, num_cmd = 0, result; + struct ipahal_reg_valmask valmask; + struct ipahal_imm_cmd_register_write reg_write_coal_close; IPADBG("\n"); + memset(desc, 0, sizeof(desc)); + memset(cmd_pyld, 0, sizeof(cmd_pyld)); + + /* IC to close the coal frame before HPS Clear if coal is enabled */ + if (ipa3_get_ep_mapping(IPA_CLIENT_APPS_WAN_COAL_CONS) != -1) { + i = ipa3_get_ep_mapping(IPA_CLIENT_APPS_WAN_COAL_CONS); + reg_write_coal_close.skip_pipeline_clear = false; + reg_write_coal_close.pipeline_clear_options = IPAHAL_HPS_CLEAR; + reg_write_coal_close.offset = ipahal_get_reg_ofst( + IPA_AGGR_FORCE_CLOSE); + ipahal_get_aggr_force_close_valmask(i, &valmask); + reg_write_coal_close.value = valmask.val; + reg_write_coal_close.value_mask = valmask.mask; + cmd_pyld[num_cmd] = ipahal_construct_imm_cmd( + IPA_IMM_CMD_REGISTER_WRITE, + ®_write_coal_close, false); + if (!cmd_pyld[num_cmd]) { + IPAERR("failed to construct coal close IC\n"); + result = -ENOMEM; + goto destroy_imm_cmd; + } + ipa3_init_imm_cmd_desc(&desc[num_cmd], cmd_pyld[num_cmd]); + ++num_cmd; + } + /* NO-OP IC for ensuring that IPA pipeline is empty */ cmd_pyld[num_cmd] = ipahal_construct_nop_imm_cmd(false, IPAHAL_HPS_CLEAR, false); if (!cmd_pyld[num_cmd]) { IPAERR("failed to construct NOP imm cmd\n"); - return -ENOMEM; + result = -ENOMEM; + goto destroy_imm_cmd; } ipa3_init_imm_cmd_desc(&desc[num_cmd], cmd_pyld[num_cmd]); @@ -798,15 +826,43 @@ static int ipa3_ipv6ct_send_init_cmd(struct ipahal_imm_cmd_ip_v6_ct_init *cmd) struct ipahal_imm_cmd_pyld *cmd_pyld[IPA_IPV6CT_MAX_NUM_OF_INIT_CMD_DESC]; int i, num_cmd = 0, result; + struct ipahal_reg_valmask valmask; + struct ipahal_imm_cmd_register_write reg_write_coal_close; IPADBG("\n"); + memset(desc, 0, sizeof(desc)); + memset(cmd_pyld, 0, sizeof(cmd_pyld)); + + /* IC to close the coal frame before HPS Clear if coal is enabled */ + if (ipa3_get_ep_mapping(IPA_CLIENT_APPS_WAN_COAL_CONS) != -1) { + i = ipa3_get_ep_mapping(IPA_CLIENT_APPS_WAN_COAL_CONS); + reg_write_coal_close.skip_pipeline_clear = false; + reg_write_coal_close.pipeline_clear_options = IPAHAL_HPS_CLEAR; + reg_write_coal_close.offset = ipahal_get_reg_ofst( + IPA_AGGR_FORCE_CLOSE); + ipahal_get_aggr_force_close_valmask(i, &valmask); + reg_write_coal_close.value = valmask.val; + reg_write_coal_close.value_mask = valmask.mask; + cmd_pyld[num_cmd] = ipahal_construct_imm_cmd( + IPA_IMM_CMD_REGISTER_WRITE, + ®_write_coal_close, false); + if (!cmd_pyld[num_cmd]) { + IPAERR("failed to construct coal close IC\n"); + result = -ENOMEM; + goto destroy_imm_cmd; + } + ipa3_init_imm_cmd_desc(&desc[num_cmd], cmd_pyld[num_cmd]); + ++num_cmd; + } + /* NO-OP IC for ensuring that IPA pipeline is empty */ cmd_pyld[num_cmd] = ipahal_construct_nop_imm_cmd(false, IPAHAL_HPS_CLEAR, false); if (!cmd_pyld[num_cmd]) { IPAERR("failed to construct NOP imm cmd\n"); - return -ENOMEM; + result = -ENOMEM; + goto destroy_imm_cmd; } ipa3_init_imm_cmd_desc(&desc[num_cmd], cmd_pyld[num_cmd]); @@ -1306,8 +1362,15 @@ int ipa3_table_dma_cmd(struct ipa_ioc_nat_dma_cmd *dma) struct ipa3_desc desc[IPA_MAX_NUM_OF_TABLE_DMA_CMD_DESC]; uint8_t cnt, num_cmd = 0; int result = 0; + int i; + struct ipahal_reg_valmask valmask; + struct ipahal_imm_cmd_register_write reg_write_coal_close; IPADBG("\n"); + + memset(desc, 0, sizeof(desc)); + memset(cmd_pyld, 0, sizeof(cmd_pyld)); + if (!dma->entries || dma->entries >= IPA_MAX_NUM_OF_TABLE_DMA_CMD_DESC) { IPAERR_RL("Invalid number of entries %d\n", @@ -1330,6 +1393,28 @@ int ipa3_table_dma_cmd(struct ipa_ioc_nat_dma_cmd *dma) } } + /* IC to close the coal frame before HPS Clear if coal is enabled */ + if (ipa3_get_ep_mapping(IPA_CLIENT_APPS_WAN_COAL_CONS) != -1) { + i = ipa3_get_ep_mapping(IPA_CLIENT_APPS_WAN_COAL_CONS); + reg_write_coal_close.skip_pipeline_clear = false; + reg_write_coal_close.pipeline_clear_options = IPAHAL_HPS_CLEAR; + reg_write_coal_close.offset = ipahal_get_reg_ofst( + IPA_AGGR_FORCE_CLOSE); + ipahal_get_aggr_force_close_valmask(i, &valmask); + reg_write_coal_close.value = valmask.val; + reg_write_coal_close.value_mask = valmask.mask; + cmd_pyld[num_cmd] = ipahal_construct_imm_cmd( + IPA_IMM_CMD_REGISTER_WRITE, + ®_write_coal_close, false); + if (!cmd_pyld[num_cmd]) { + IPAERR("failed to construct coal close IC\n"); + result = -ENOMEM; + goto destroy_imm_cmd; + } + ipa3_init_imm_cmd_desc(&desc[num_cmd], cmd_pyld[num_cmd]); + ++num_cmd; + } + /* NO-OP IC for ensuring that IPA pipeline is empty */ cmd_pyld[num_cmd] = ipahal_construct_nop_imm_cmd(false, IPAHAL_HPS_CLEAR, false); diff --git a/drivers/platform/msm/ipa/ipa_v3/ipa_rt.c b/drivers/platform/msm/ipa/ipa_v3/ipa_rt.c index 896db484a1af..7c6f123febc0 100644 --- a/drivers/platform/msm/ipa/ipa_v3/ipa_rt.c +++ b/drivers/platform/msm/ipa/ipa_v3/ipa_rt.c @@ -14,7 +14,7 @@ #define IPA_RT_STATUS_OF_DEL_FAILED (-1) #define IPA_RT_STATUS_OF_MDFY_FAILED (-1) -#define IPA_RT_MAX_NUM_OF_COMMIT_TABLES_CMD_DESC 5 +#define IPA_RT_MAX_NUM_OF_COMMIT_TABLES_CMD_DESC 6 #define IPA_RT_GET_RULE_TYPE(__entry) \ ( \ @@ -475,6 +475,7 @@ int __ipa_commit_rt_v3(enum ipa_ip_type ip) struct ipa3_rt_tbl_set *set; struct ipa3_rt_tbl *tbl; u32 tbl_hdr_width; + struct ipahal_imm_cmd_register_write reg_write_coal_close; tbl_hdr_width = ipahal_get_hw_tbl_hdr_width(); memset(desc, 0, sizeof(desc)); @@ -565,6 +566,27 @@ int __ipa_commit_rt_v3(enum ipa_ip_type ip) goto fail_size_valid; } + /* IC to close the coal frame before HPS Clear if coal is enabled */ + if (ipa3_get_ep_mapping(IPA_CLIENT_APPS_WAN_COAL_CONS) != -1) { + i = ipa3_get_ep_mapping(IPA_CLIENT_APPS_WAN_COAL_CONS); + reg_write_coal_close.skip_pipeline_clear = false; + reg_write_coal_close.pipeline_clear_options = IPAHAL_HPS_CLEAR; + reg_write_coal_close.offset = ipahal_get_reg_ofst( + IPA_AGGR_FORCE_CLOSE); + ipahal_get_aggr_force_close_valmask(i, &valmask); + reg_write_coal_close.value = valmask.val; + reg_write_coal_close.value_mask = valmask.mask; + cmd_pyld[num_cmd] = ipahal_construct_imm_cmd( + IPA_IMM_CMD_REGISTER_WRITE, + ®_write_coal_close, false); + if (!cmd_pyld[num_cmd]) { + IPAERR("failed to construct coal close IC\n"); + goto fail_size_valid; + } + ipa3_init_imm_cmd_desc(&desc[num_cmd], cmd_pyld[num_cmd]); + ++num_cmd; + } + /* * SRAM memory not allocated to hash tables. Sending * command to hash tables(filer/routing) operation not supported. @@ -589,7 +611,7 @@ int __ipa_commit_rt_v3(enum ipa_ip_type ip) if (!cmd_pyld[num_cmd]) { IPAERR( "fail construct register_write imm cmd. IP %d\n", ip); - goto fail_size_valid; + goto fail_imm_cmd_construct; } ipa3_init_imm_cmd_desc(&desc[num_cmd], cmd_pyld[num_cmd]); num_cmd++; diff --git a/drivers/platform/msm/ipa/ipa_v3/ipa_utils.c b/drivers/platform/msm/ipa/ipa_v3/ipa_utils.c index 491c15263612..a38d20e852d3 100644 --- a/drivers/platform/msm/ipa/ipa_v3/ipa_utils.c +++ b/drivers/platform/msm/ipa/ipa_v3/ipa_utils.c @@ -6627,6 +6627,8 @@ int ipa3_tag_process(struct ipa3_desc desc[], struct ipa3_tag_completion *comp; int ep_idx; u32 retry_cnt = 0; + struct ipahal_reg_valmask valmask; + struct ipahal_imm_cmd_register_write reg_write_coal_close; /* Not enough room for the required descriptors for the tag process */ if (IPA_TAG_MAX_DESC - descs_num < REQUIRED_TAG_PROCESS_DESCRIPTORS) { @@ -6655,6 +6657,31 @@ int ipa3_tag_process(struct ipa3_desc desc[], memcpy(&(tag_desc[0]), desc, descs_num * sizeof(tag_desc[0])); desc_idx += descs_num; + } else + goto fail_free_tag_desc; + + /* IC to close the coal frame before HPS Clear if coal is enabled */ + if (ipa3_get_ep_mapping(IPA_CLIENT_APPS_WAN_COAL_CONS) != -1) { + ep_idx = ipa3_get_ep_mapping(IPA_CLIENT_APPS_WAN_COAL_CONS); + reg_write_coal_close.skip_pipeline_clear = false; + reg_write_coal_close.pipeline_clear_options = IPAHAL_HPS_CLEAR; + reg_write_coal_close.offset = ipahal_get_reg_ofst( + IPA_AGGR_FORCE_CLOSE); + ipahal_get_aggr_force_close_valmask(ep_idx, &valmask); + reg_write_coal_close.value = valmask.val; + reg_write_coal_close.value_mask = valmask.mask; + cmd_pyld = ipahal_construct_imm_cmd( + IPA_IMM_CMD_REGISTER_WRITE, + ®_write_coal_close, false); + if (!cmd_pyld) { + IPAERR("failed to construct coal close IC\n"); + res = -ENOMEM; + goto fail_free_tag_desc; + } + ipa3_init_imm_cmd_desc(&tag_desc[desc_idx], cmd_pyld); + desc[desc_idx].callback = ipa3_tag_destroy_imm; + desc[desc_idx].user1 = cmd_pyld; + ++desc_idx; } /* NO-OP IC for ensuring that IPA pipeline is empty */ @@ -6663,7 +6690,7 @@ int ipa3_tag_process(struct ipa3_desc desc[], if (!cmd_pyld) { IPAERR("failed to construct NOP imm cmd\n"); res = -ENOMEM; - goto fail_free_tag_desc; + goto fail_free_desc; } ipa3_init_imm_cmd_desc(&tag_desc[desc_idx], cmd_pyld); tag_desc[desc_idx].callback = ipa3_tag_destroy_imm;