diff --git a/drivers/platform/msm/ipa/ipa_v3/ipa.c b/drivers/platform/msm/ipa/ipa_v3/ipa.c index c4141001bb30..09f5b1cc2fa5 100644 --- a/drivers/platform/msm/ipa/ipa_v3/ipa.c +++ b/drivers/platform/msm/ipa/ipa_v3/ipa.c @@ -7361,6 +7361,11 @@ static int ipa3_pre_init(const struct ipa3_plat_drv_res *resource_p, ipa3_ctx->stats.page_recycle_stats[1].tmp_alloc = 0; memset(ipa3_ctx->stats.page_recycle_cnt, 0, sizeof(ipa3_ctx->stats.page_recycle_cnt)); + ipa3_ctx->stats.num_sort_tasklet_sched[0] = 0; + ipa3_ctx->stats.num_sort_tasklet_sched[1] = 0; + ipa3_ctx->stats.num_sort_tasklet_sched[2] = 0; + ipa3_ctx->stats.num_of_times_wq_reschd = 0; + ipa3_ctx->stats.page_recycle_cnt_in_tasklet = 0; ipa3_ctx->skip_uc_pipe_reset = resource_p->skip_uc_pipe_reset; ipa3_ctx->tethered_flow_control = resource_p->tethered_flow_control; ipa3_ctx->ee = resource_p->ee; @@ -7554,6 +7559,10 @@ static int ipa3_pre_init(const struct ipa3_plat_drv_res *resource_p, /* Initialize Page poll threshold. */ ipa3_ctx->page_poll_threshold = IPA_PAGE_POLL_DEFAULT_THRESHOLD; + /*Initialize number napi without prealloc buff*/ + ipa3_ctx->ipa_max_napi_sort_page_thrshld = IPA_MAX_NAPI_SORT_PAGE_THRSHLD; + ipa3_ctx->page_wq_reschd_time = IPA_MAX_PAGE_WQ_RESCHED_TIME; + /* Use common page pool for Def/Coal pipe. */ if (ipa3_ctx->ipa_hw_type >= IPA_HW_v4_9) ipa3_ctx->wan_common_page_pool = true; diff --git a/drivers/platform/msm/ipa/ipa_v3/ipa_debugfs.c b/drivers/platform/msm/ipa/ipa_v3/ipa_debugfs.c index aad988894dbc..426cbbea09fa 100644 --- a/drivers/platform/msm/ipa/ipa_v3/ipa_debugfs.c +++ b/drivers/platform/msm/ipa/ipa_v3/ipa_debugfs.c @@ -1434,15 +1434,23 @@ static ssize_t ipa3_read_page_recycle_stats(struct file *file, "COAL : Total number of packets replenished =%llu\n" "COAL : Number of page recycled packets =%llu\n" "COAL : Number of tmp alloc packets =%llu\n" + "COAL : Number of times tasklet scheduled =%llu\n" "DEF : Total number of packets replenished =%llu\n" "DEF : Number of page recycled packets =%llu\n" - "DEF : Number of tmp alloc packets =%llu\n", + "DEF : Number of tmp alloc packets =%llu\n" + "DEF : Number of times tasklet scheduled =%llu\n" + "COMMON : Number of page recycled in tasklet =%llu\n" + "COMMON : Number of times free pages not found in tasklet =%llu\n", ipa3_ctx->stats.page_recycle_stats[0].total_replenished, ipa3_ctx->stats.page_recycle_stats[0].page_recycled, ipa3_ctx->stats.page_recycle_stats[0].tmp_alloc, + ipa3_ctx->stats.num_sort_tasklet_sched[0], ipa3_ctx->stats.page_recycle_stats[1].total_replenished, ipa3_ctx->stats.page_recycle_stats[1].page_recycled, - ipa3_ctx->stats.page_recycle_stats[1].tmp_alloc); + ipa3_ctx->stats.page_recycle_stats[1].tmp_alloc, + ipa3_ctx->stats.num_sort_tasklet_sched[1], + ipa3_ctx->stats.page_recycle_cnt_in_tasklet, + ipa3_ctx->stats.num_of_times_wq_reschd); cnt += nbytes; @@ -2788,6 +2796,68 @@ static ssize_t ipa3_enable_ipc_low(struct file *file, return count; } +static ssize_t ipa3_read_ipa_max_napi_sort_page_thrshld(struct file *file, + char __user *buf, size_t count, loff_t *ppos) { + + int nbytes; + nbytes = scnprintf(dbg_buff, IPA_MAX_MSG_LEN, + "page max napi without free page = %d\n", + ipa3_ctx->ipa_max_napi_sort_page_thrshld); + return simple_read_from_buffer(buf, count, ppos, dbg_buff, nbytes); + +} + +static ssize_t ipa3_write_ipa_max_napi_sort_page_thrshld(struct file *file, + const char __user *buf, size_t count, loff_t *ppos) { + + int ret; + u8 ipa_max_napi_sort_page_thrshld = 0; + + if (count >= sizeof(dbg_buff)) + return -EFAULT; + + ret = kstrtou8_from_user(buf, count, 0, &ipa_max_napi_sort_page_thrshld); + if(ret) + return ret; + + ipa3_ctx->ipa_max_napi_sort_page_thrshld = ipa_max_napi_sort_page_thrshld; + + IPADBG("napi cnt without prealloc pages = %d", ipa3_ctx->ipa_max_napi_sort_page_thrshld); + + return count; +} + +static ssize_t ipa3_read_page_wq_reschd_time(struct file *file, + char __user *buf, size_t count, loff_t *ppos) { + + int nbytes; + nbytes = scnprintf(dbg_buff, IPA_MAX_MSG_LEN, + "Page WQ reschduule time = %d\n", + ipa3_ctx->page_wq_reschd_time); + return simple_read_from_buffer(buf, count, ppos, dbg_buff, nbytes); + +} + +static ssize_t ipa3_write_page_wq_reschd_time(struct file *file, + const char __user *buf, size_t count, loff_t *ppos) { + + int ret; + u8 page_wq_reschd_time = 0; + + if (count >= sizeof(dbg_buff)) + return -EFAULT; + + ret = kstrtou8_from_user(buf, count, 0, &page_wq_reschd_time); + if(ret) + return ret; + + ipa3_ctx->page_wq_reschd_time = page_wq_reschd_time; + + IPADBG("Updated page WQ reschedule time = %d", ipa3_ctx->page_wq_reschd_time); + + return count; +} + static ssize_t ipa3_read_gsi_wdi3_db_polling(struct file *file, char __user *buf, size_t count, loff_t *ppos) { @@ -3042,6 +3112,16 @@ static const struct ipa3_debugfs_file debugfs_files[] = { .read = ipa3_read_page_poll_threshold, .write = ipa3_write_page_poll_threshold, } + }, { + "page_wq_reschd_time", IPA_READ_WRITE_MODE, NULL, { + .read = ipa3_read_page_wq_reschd_time, + .write = ipa3_write_page_wq_reschd_time, + } + }, { + "ipa_max_napi_sort_page_thrshld", IPA_READ_WRITE_MODE, NULL, { + .read = ipa3_read_ipa_max_napi_sort_page_thrshld, + .write = ipa3_write_ipa_max_napi_sort_page_thrshld, + } }, }; diff --git a/drivers/platform/msm/ipa/ipa_v3/ipa_dp.c b/drivers/platform/msm/ipa/ipa_v3/ipa_dp.c index 42eaf09dbed7..2714e1920a47 100644 --- a/drivers/platform/msm/ipa/ipa_v3/ipa_dp.c +++ b/drivers/platform/msm/ipa/ipa_v3/ipa_dp.c @@ -137,6 +137,7 @@ static int ipa_poll_gsi_n_pkt(struct ipa3_sys_context *sys, static unsigned long tag_to_pointer_wa(uint64_t tag); static uint64_t pointer_to_tag_wa(struct ipa3_tx_pkt_wrapper *tx_pkt); static void ipa3_tasklet_rx_notify(unsigned long data); +static void ipa3_tasklet_find_freepage(unsigned long data); static u32 ipa_adjust_ra_buff_base_sz(u32 aggr_byte_limit); @@ -1048,6 +1049,60 @@ static void ipa_pm_sys_pipe_cb(void *p, enum ipa_pm_cb_event event) } } +static void ipa3_schd_freepage_work(struct work_struct *work) +{ + struct delayed_work *dwork; + struct ipa3_sys_context *sys; + + dwork = container_of(work, struct delayed_work, work); + sys = container_of(dwork, struct ipa3_sys_context, freepage_work); + + IPADBG_LOW("WQ scheduled, reschedule sort tasklet\n"); + + tasklet_schedule(&sys->tasklet_find_freepage); +} + +static void ipa3_tasklet_find_freepage(unsigned long data) +{ + struct ipa3_sys_context *sys; + struct ipa3_rx_pkt_wrapper *rx_pkt = NULL; + struct ipa3_rx_pkt_wrapper *tmp = NULL; + struct page *cur_page; + int found_free_page = 0; + struct list_head temp_head; + + sys = (struct ipa3_sys_context *)data; + + INIT_LIST_HEAD(&temp_head); + spin_lock_bh(&sys->common_sys->spinlock); + list_for_each_entry_safe(rx_pkt, tmp, + &sys->page_recycle_repl->page_repl_head, link) { + cur_page = rx_pkt->page_data.page; + if (page_ref_count(cur_page) == 1) { + /* Found a free page. */ + list_del_init(&rx_pkt->link); + list_add(&rx_pkt->link, &temp_head); + found_free_page++; + } + } + if (!found_free_page) { + /*Not found free page rescheduling tasklet after 2msec*/ + IPADBG_LOW("Scheduling WQ not found free pages\n"); + ++ipa3_ctx->stats.num_of_times_wq_reschd; + queue_delayed_work(sys->freepage_wq, + &sys->freepage_work, + msecs_to_jiffies(ipa3_ctx->page_wq_reschd_time)); + } else { + /*Allow to use pre-allocated buffers*/ + list_splice(&temp_head, &sys->page_recycle_repl->page_repl_head); + ipa3_ctx->stats.page_recycle_cnt_in_tasklet += found_free_page; + IPADBG_LOW("found free pages count = %d\n", found_free_page); + atomic_set(&sys->common_sys->page_avilable, 1); + } + spin_unlock_bh(&sys->common_sys->spinlock); + +} + /** * ipa3_setup_sys_pipe() - Setup an IPA GPI pipe and perform * IPA EP configuration @@ -1139,6 +1194,9 @@ int ipa3_setup_sys_pipe(struct ipa_sys_connect_params *sys_in, u32 *clnt_hdl) goto fail_wq2; } + snprintf(buff, IPA_RESOURCE_NAME_MAX, "ipafreepagewq%d", + sys_in->client); + INIT_LIST_HEAD(&ep->sys->head_desc_list); INIT_LIST_HEAD(&ep->sys->rcycl_list); INIT_LIST_HEAD(&ep->sys->avail_tx_wrapper_list); @@ -1155,6 +1213,16 @@ int ipa3_setup_sys_pipe(struct ipa_sys_connect_params *sys_in, u32 *clnt_hdl) /* Use coalescing pipe PM handle for default pipe also*/ ep->sys->pm_hdl = ipa3_ctx->ep[coal_ep_id].sys->pm_hdl; } else if (IPA_CLIENT_IS_CONS(sys_in->client)) { + ep->sys->freepage_wq = alloc_workqueue(buff, + WQ_MEM_RECLAIM | WQ_UNBOUND | WQ_SYSFS | + WQ_HIGHPRI, 1); + if (!ep->sys->freepage_wq) { + IPAERR("failed to create freepage wq for client %d\n", + sys_in->client); + result = -EFAULT; + goto fail_wq3; + } + pm_reg.name = ipa_clients_strings[sys_in->client]; pm_reg.callback = ipa_pm_sys_pipe_cb; pm_reg.user_data = ep->sys; @@ -1255,6 +1323,7 @@ int ipa3_setup_sys_pipe(struct ipa_sys_connect_params *sys_in, u32 *clnt_hdl) } *clnt_hdl = ipa_ep_idx; + ep->sys->common_sys = ipa3_ctx->ep[ipa_ep_idx].sys; if (ep->sys->repl_hdlr == ipa3_fast_replenish_rx_cache) { ep->sys->repl = kzalloc(sizeof(*ep->sys->repl), GFP_KERNEL); @@ -1330,6 +1399,10 @@ int ipa3_setup_sys_pipe(struct ipa_sys_connect_params *sys_in, u32 *clnt_hdl) atomic_set(&ep->sys->repl->head_idx, 0); atomic_set(&ep->sys->repl->tail_idx, 0); + tasklet_init(&ep->sys->tasklet_find_freepage, + ipa3_tasklet_find_freepage, (unsigned long) ep->sys); + INIT_DELAYED_WORK(&ep->sys->freepage_work, ipa3_schd_freepage_work); + ep->sys->napi_sort_page_thrshld_cnt = 0; ipa3_replenish_rx_page_cache(ep->sys); ipa3_wq_page_repl(&ep->sys->repl_work); } else { @@ -1431,6 +1504,8 @@ fail_napi: fail_gen2: ipa_pm_deregister(ep->sys->pm_hdl); fail_pm: + destroy_workqueue(ep->sys->freepage_wq); +fail_wq3: destroy_workqueue(ep->sys->repl_wq); fail_wq2: destroy_workqueue(ep->sys->wq); @@ -2157,6 +2232,7 @@ static void ipa3_replenish_rx_page_cache(struct ipa3_sys_context *sys) list_add_tail(&rx_pkt->link, &sys->page_recycle_repl->page_repl_head); } + atomic_set(&sys->common_sys->page_avilable, 1); return; @@ -2235,6 +2311,7 @@ static struct ipa3_rx_pkt_wrapper * ipa3_get_free_page int i = 0; u8 LOOP_THRESHOLD = ipa3_ctx->page_poll_threshold; + spin_lock_bh(&sys->common_sys->spinlock); list_for_each_entry_safe(rx_pkt, tmp, &sys->page_recycle_repl->page_repl_head, link) { if (i == LOOP_THRESHOLD) @@ -2245,10 +2322,23 @@ static struct ipa3_rx_pkt_wrapper * ipa3_get_free_page page_ref_inc(cur_page); list_del_init(&rx_pkt->link); ++ipa3_ctx->stats.page_recycle_cnt[stats_i][i]; + sys->common_sys->napi_sort_page_thrshld_cnt = 0; + spin_unlock_bh(&sys->common_sys->spinlock); return rx_pkt; } i++; } + spin_unlock_bh(&sys->common_sys->spinlock); + IPADBG_LOW("napi_sort_page_thrshld_cnt = %d ipa_max_napi_sort_page_thrshld = %d\n", + sys->common_sys->napi_sort_page_thrshld_cnt, + ipa3_ctx->ipa_max_napi_sort_page_thrshld); + /*Scheduling tasklet to find the free page*/ + if (sys->common_sys->napi_sort_page_thrshld_cnt >= + ipa3_ctx->ipa_max_napi_sort_page_thrshld) { + atomic_set(&sys->common_sys->page_avilable, 0); + tasklet_schedule(&sys->common_sys->tasklet_find_freepage); + ++ipa3_ctx->stats.num_sort_tasklet_sched[stats_i]; + } return NULL; } @@ -2272,7 +2362,8 @@ static void ipa3_replenish_rx_page_recycle(struct ipa3_sys_context *sys) while (rx_len_cached < sys->rx_pool_sz) { /* check for an idle page that can be used */ - if ((rx_pkt = ipa3_get_free_page(sys,stats_i)) != NULL) { + if (atomic_read(&sys->common_sys->page_avilable) && + ((rx_pkt = ipa3_get_free_page(sys,stats_i)) != NULL)) { ipa3_ctx->stats.page_recycle_stats[stats_i].page_recycled++; } else { @@ -3712,11 +3803,11 @@ static struct sk_buff *handle_page_completion(struct gsi_chan_xfer_notify IPAERR("notify->veid > GSI_VEID_MAX\n"); if (!rx_page.is_tmp_alloc) { init_page_count(rx_page.page); - spin_lock_bh(&rx_pkt->sys->spinlock); + spin_lock_bh(&rx_pkt->sys->common_sys->spinlock); /* Add the element to head. */ list_add(&rx_pkt->link, &rx_pkt->sys->page_recycle_repl->page_repl_head); - spin_unlock_bh(&rx_pkt->sys->spinlock); + spin_unlock_bh(&rx_pkt->sys->common_sys->spinlock); } else { dma_unmap_page(ipa3_ctx->pdev, rx_page.dma_addr, rx_pkt->len, DMA_FROM_DEVICE); @@ -3737,19 +3828,24 @@ static struct sk_buff *handle_page_completion(struct gsi_chan_xfer_notify sys->ep->client == IPA_CLIENT_APPS_LAN_CONS) { rx_skb = alloc_skb(0, GFP_ATOMIC); if (unlikely(!rx_skb)) { - IPAERR("skb alloc failure\n"); - list_del_init(&rx_pkt->link); - if (!rx_page.is_tmp_alloc) { - init_page_count(rx_page.page); - spin_lock_bh(&rx_pkt->sys->spinlock); - /* Add the element to head. */ - list_add(&rx_pkt->link, - &rx_pkt->sys->page_recycle_repl->page_repl_head); - spin_unlock_bh(&rx_pkt->sys->spinlock); - } else { - dma_unmap_page(ipa3_ctx->pdev, rx_page.dma_addr, - rx_pkt->len, DMA_FROM_DEVICE); - __free_pages(rx_pkt->page_data.page, rx_pkt->page_data.page_order); + IPAERR("skb alloc failure, free all pending pages\n"); + list_for_each_entry_safe(rx_pkt, tmp, head, link) { + rx_page = rx_pkt->page_data; + size = rx_pkt->data_len; + list_del_init(&rx_pkt->link); + if (!rx_page.is_tmp_alloc) { + init_page_count(rx_page.page); + spin_lock_bh(&rx_pkt->sys->common_sys->spinlock); + /* Add the element to head. */ + list_add(&rx_pkt->link, + &rx_pkt->sys->page_recycle_repl->page_repl_head); + spin_unlock_bh(&rx_pkt->sys->common_sys->spinlock); + } else { + dma_unmap_page(ipa3_ctx->pdev, rx_page.dma_addr, + rx_pkt->len, DMA_FROM_DEVICE); + __free_pages(rx_pkt->page_data.page, rx_pkt->page_data.page_order); + } + rx_pkt->sys->free_rx_wrapper(rx_pkt); } IPA_STATS_INC_CNT(ipa3_ctx->stats.rx_page_drop_cnt); return NULL; @@ -3763,11 +3859,11 @@ static struct sk_buff *handle_page_completion(struct gsi_chan_xfer_notify dma_unmap_page(ipa3_ctx->pdev, rx_page.dma_addr, rx_pkt->len, DMA_FROM_DEVICE); } else { - spin_lock_bh(&rx_pkt->sys->spinlock); + spin_lock_bh(&rx_pkt->sys->common_sys->spinlock); /* Add the element back to tail. */ list_add_tail(&rx_pkt->link, &rx_pkt->sys->page_recycle_repl->page_repl_head); - spin_unlock_bh(&rx_pkt->sys->spinlock); + spin_unlock_bh(&rx_pkt->sys->common_sys->spinlock); dma_sync_single_for_cpu(ipa3_ctx->pdev, rx_page.dma_addr, rx_pkt->len, DMA_FROM_DEVICE); @@ -5446,6 +5542,7 @@ int ipa3_rx_poll(u32 clnt_hdl, int weight) return -EINVAL; } + ep->sys->common_sys->napi_sort_page_thrshld_cnt++; start_poll: /* * it is guaranteed we already have clock here. diff --git a/drivers/platform/msm/ipa/ipa_v3/ipa_i.h b/drivers/platform/msm/ipa/ipa_v3/ipa_i.h index 3e5138cc6c80..9c10e4233043 100644 --- a/drivers/platform/msm/ipa/ipa_v3/ipa_i.h +++ b/drivers/platform/msm/ipa/ipa_v3/ipa_i.h @@ -91,6 +91,9 @@ #define IPA_PAGE_POLL_DEFAULT_THRESHOLD 15 #define IPA_PAGE_POLL_THRESHOLD_MAX 30 +#define IPA_MAX_NAPI_SORT_PAGE_THRSHLD 3 +#define IPA_MAX_PAGE_WQ_RESCHED_TIME 2 + #define IPADBG(fmt, args...) \ do { \ @@ -1126,6 +1129,10 @@ struct ipa3_sys_context { atomic_t in_napi_context; bool common_buff_pool; struct ipa3_sys_context *common_sys; + struct tasklet_struct tasklet_find_freepage; + atomic_t page_avilable; + struct delayed_work freepage_work; + u32 napi_sort_page_thrshld_cnt; /* ordering is important - mutable fields go above */ struct ipa3_ep_context *ep; @@ -1141,6 +1148,7 @@ struct ipa3_sys_context { u32 pm_hdl; unsigned int napi_sch_cnt; unsigned int napi_comp_cnt; + struct workqueue_struct *freepage_wq; /* ordering is important - other immutable fields go below */ }; @@ -1479,6 +1487,9 @@ struct ipa3_stats { u64 lower_order; u32 pipe_setup_fail_cnt; u64 page_recycle_cnt[2][IPA_PAGE_POLL_THRESHOLD_MAX]; + u64 num_sort_tasklet_sched[3]; + u64 num_of_times_wq_reschd; + u64 page_recycle_cnt_in_tasklet; }; /* offset for each stats */ @@ -2196,6 +2207,8 @@ struct ipa3_context { int ipa_pil_load; u8 page_poll_threshold; bool wan_common_page_pool; + u32 ipa_max_napi_sort_page_thrshld; + u32 page_wq_reschd_time; }; struct ipa3_plat_drv_res {