diff --git a/drivers/platform/msm/ipa/ipa_v3/ipa_dp.c b/drivers/platform/msm/ipa/ipa_v3/ipa_dp.c index ac0d6f0f2777..e1020ae55e37 100644 --- a/drivers/platform/msm/ipa/ipa_v3/ipa_dp.c +++ b/drivers/platform/msm/ipa/ipa_v3/ipa_dp.c @@ -91,6 +91,7 @@ static void ipa3_replenish_rx_cache(struct ipa3_sys_context *sys); static void ipa3_replenish_rx_work_func(struct work_struct *work); static void ipa3_fast_replenish_rx_cache(struct ipa3_sys_context *sys); static void ipa3_replenish_rx_page_cache(struct ipa3_sys_context *sys); +static void ipa3_wq_page_repl(struct work_struct *work); static void ipa3_replenish_rx_page_recycle(struct ipa3_sys_context *sys); static struct ipa3_rx_pkt_wrapper *ipa3_alloc_rx_pkt_page(gfp_t flag, bool is_tmp_alloc); @@ -1112,21 +1113,40 @@ int ipa3_setup_sys_pipe(struct ipa_sys_connect_params *sys_in, u32 *clnt_hdl) } if (ep->sys->repl_hdlr == ipa3_replenish_rx_page_recycle) { - ep->sys->repl = kzalloc(sizeof(*ep->sys->repl), GFP_KERNEL); - if (!ep->sys->repl) { + ep->sys->page_recycle_repl = kzalloc( + sizeof(*ep->sys->page_recycle_repl), GFP_KERNEL); + if (!ep->sys->page_recycle_repl) { IPAERR("failed to alloc repl for client %d\n", sys_in->client); result = -ENOMEM; goto fail_gen2; } - atomic_set(&ep->sys->repl->pending, 0); - ep->sys->repl->capacity = (ep->sys->rx_pool_sz + 1) * 2; + atomic_set(&ep->sys->page_recycle_repl->pending, 0); + ep->sys->page_recycle_repl->capacity = + (ep->sys->rx_pool_sz + 1) * 2; + ep->sys->page_recycle_repl->cache = + kcalloc(ep->sys->page_recycle_repl->capacity, + sizeof(void *), GFP_KERNEL); + atomic_set(&ep->sys->page_recycle_repl->head_idx, 0); + atomic_set(&ep->sys->page_recycle_repl->tail_idx, 0); + ep->sys->repl = kzalloc(sizeof(*ep->sys->repl), GFP_KERNEL); + if (!ep->sys->repl) { + IPAERR("failed to alloc repl for client %d\n", + sys_in->client); + result = -ENOMEM; + goto fail_page_recycle_repl; + } + ep->sys->repl->capacity = (ep->sys->rx_pool_sz + 1); + + atomic_set(&ep->sys->repl->pending, 0); ep->sys->repl->cache = kcalloc(ep->sys->repl->capacity, sizeof(void *), GFP_KERNEL); atomic_set(&ep->sys->repl->head_idx, 0); atomic_set(&ep->sys->repl->tail_idx, 0); + ipa3_replenish_rx_page_cache(ep->sys); + ipa3_wq_page_repl(&ep->sys->repl_work); } if (IPA_CLIENT_IS_CONS(sys_in->client)) { @@ -1199,6 +1219,11 @@ fail_repl: ep->sys->repl_hdlr = ipa3_replenish_rx_cache; ep->sys->repl->capacity = 0; kfree(ep->sys->repl); +fail_page_recycle_repl: + if (ep->sys->page_recycle_repl) { + ep->sys->page_recycle_repl->capacity = 0; + kfree(ep->sys->page_recycle_repl); + } fail_gen2: ipa_pm_deregister(ep->sys->pm_hdl); fail_pm: @@ -1802,13 +1827,12 @@ static struct ipa3_rx_pkt_wrapper *ipa3_alloc_rx_pkt_page( flag |= __GFP_NOMEMALLOC; rx_pkt = kmem_cache_zalloc(ipa3_ctx->rx_pkt_wrapper_cache, flag); - if (!rx_pkt) + if (unlikely(!rx_pkt)) return NULL; rx_pkt->len = PAGE_SIZE << IPA_WAN_PAGE_ORDER; rx_pkt->page_data.page = __dev_alloc_pages(flag, IPA_WAN_PAGE_ORDER); - - if (!rx_pkt->page_data.page) + if (unlikely(!rx_pkt->page_data.page)) goto fail_page_alloc; rx_pkt->page_data.dma_addr = dma_map_page(ipa3_ctx->pdev, @@ -1839,7 +1863,7 @@ static void ipa3_replenish_rx_page_cache(struct ipa3_sys_context *sys) struct ipa3_rx_pkt_wrapper *rx_pkt; u32 curr; - for (curr = 0; curr < sys->repl->capacity; curr++) { + for (curr = 0; curr < sys->page_recycle_repl->capacity; curr++) { rx_pkt = ipa3_alloc_rx_pkt_page(GFP_KERNEL, false); if (!rx_pkt) { IPAERR("ipa3_alloc_rx_pkt_page fails\n"); @@ -1847,13 +1871,75 @@ static void ipa3_replenish_rx_page_cache(struct ipa3_sys_context *sys) break; } rx_pkt->sys = sys; - sys->repl->cache[curr] = rx_pkt; + sys->page_recycle_repl->cache[curr] = rx_pkt; } return; } +static void ipa3_wq_page_repl(struct work_struct *work) +{ + struct ipa3_sys_context *sys; + struct ipa3_rx_pkt_wrapper *rx_pkt; + u32 next; + u32 curr; + + sys = container_of(work, struct ipa3_sys_context, repl_work); + atomic_set(&sys->repl->pending, 0); + curr = atomic_read(&sys->repl->tail_idx); + +begin: + while (1) { + next = (curr + 1) % sys->repl->capacity; + if (unlikely(next == atomic_read(&sys->repl->head_idx))) + goto fail_kmem_cache_alloc; + rx_pkt = ipa3_alloc_rx_pkt_page(GFP_KERNEL, true); + if (unlikely(!rx_pkt)) { + IPAERR("ipa3_alloc_rx_pkt_page fails\n"); + break; + } + rx_pkt->sys = sys; + sys->repl->cache[curr] = rx_pkt; + curr = next; + /* ensure write is done before setting tail index */ + mb(); + atomic_set(&sys->repl->tail_idx, next); + } + + return; + +fail_kmem_cache_alloc: + if (atomic_read(&sys->repl->tail_idx) == + atomic_read(&sys->repl->head_idx)) { + if (sys->ep->client == IPA_CLIENT_APPS_WAN_CONS || + sys->ep->client == IPA_CLIENT_APPS_WAN_COAL_CONS) + IPA_STATS_INC_CNT(ipa3_ctx->stats.wan_repl_rx_empty); + pr_err_ratelimited("%s sys=%pK wq_repl ring empty\n", + __func__, sys); + goto begin; + } + +} + +static inline void __trigger_repl_work(struct ipa3_sys_context *sys) +{ + int tail, head, avail; + + if (atomic_read(&sys->repl->pending)) + return; + + tail = atomic_read(&sys->repl->tail_idx); + head = atomic_read(&sys->repl->head_idx); + avail = (tail - head) % sys->repl->capacity; + + if (avail < sys->repl->capacity / 4) { + atomic_set(&sys->repl->pending, 1); + queue_work(sys->repl_wq, &sys->repl_work); + } +} + + static void ipa3_replenish_rx_page_recycle(struct ipa3_sys_context *sys) { struct ipa3_rx_pkt_wrapper *rx_pkt; @@ -1861,37 +1947,38 @@ static void ipa3_replenish_rx_page_recycle(struct ipa3_sys_context *sys) int rx_len_cached = 0; struct gsi_xfer_elem gsi_xfer_elem_array[IPA_REPL_XFER_MAX]; u32 curr; + u32 curr_wq; int idx = 0; struct page *cur_page; - gfp_t flag; /* start replenish only when buffers go lower than the threshold */ if (sys->rx_pool_sz - sys->len < IPA_REPL_XFER_THRESH) return; - flag = gfp_any(); spin_lock_bh(&sys->spinlock); rx_len_cached = sys->len; - curr = atomic_read(&sys->repl->head_idx); + curr = atomic_read(&sys->page_recycle_repl->head_idx); + curr_wq = atomic_read(&sys->repl->head_idx); while (rx_len_cached < sys->rx_pool_sz) { - cur_page = sys->repl->cache[curr]->page_data.page; + cur_page = sys->page_recycle_repl->cache[curr]->page_data.page; /* Found an idle page that can be used */ if (page_ref_count(cur_page) == 1) { page_ref_inc(cur_page); - rx_pkt = sys->repl->cache[curr]; - curr = (++curr == sys->repl->capacity) ? 0 : curr; + rx_pkt = sys->page_recycle_repl->cache[curr]; + curr = (++curr == sys->page_recycle_repl->capacity) ? + 0 : curr; } else { /* * Could not find idle page at curr index. * Allocate a new one. */ - rx_pkt = ipa3_alloc_rx_pkt_page(flag, true); - if (!rx_pkt && flag == GFP_ATOMIC) + if (curr_wq == atomic_read(&sys->repl->tail_idx)) break; - else if (!rx_pkt) - goto fail_kmem_cache_alloc; - rx_pkt->sys = sys; + rx_pkt = sys->repl->cache[curr_wq]; + curr_wq = (++curr_wq == sys->repl->capacity) ? + 0 : curr_wq; + atomic_set(&sys->repl->head_idx, curr_wq); } dma_sync_single_for_device(ipa3_ctx->pdev, @@ -1928,7 +2015,7 @@ static void ipa3_replenish_rx_page_recycle(struct ipa3_sys_context *sys) if (ret == GSI_STATUS_SUCCESS) { /* ensure write is done before setting head index */ mb(); - atomic_set(&sys->repl->head_idx, curr); + atomic_set(&sys->page_recycle_repl->head_idx, curr); sys->len = rx_len_cached; } else { /* we don't expect this will happen */ @@ -1936,15 +2023,21 @@ static void ipa3_replenish_rx_page_recycle(struct ipa3_sys_context *sys) ipa_assert(); } spin_unlock_bh(&sys->spinlock); + __trigger_repl_work(sys); - if (rx_len_cached < sys->rx_pool_sz) { + if (rx_len_cached <= IPA_DEFAULT_SYS_YELLOW_WM) { + if (sys->ep->client == IPA_CLIENT_APPS_WAN_CONS || + sys->ep->client == IPA_CLIENT_APPS_WAN_COAL_CONS) + IPA_STATS_INC_CNT(ipa3_ctx->stats.wan_rx_empty); + else if (sys->ep->client == IPA_CLIENT_APPS_LAN_CONS) + IPA_STATS_INC_CNT(ipa3_ctx->stats.lan_rx_empty); + else + WARN_ON(1); queue_delayed_work(sys->wq, &sys->replenish_rx_work, - msecs_to_jiffies(1)); + msecs_to_jiffies(1)); } + return; -fail_kmem_cache_alloc: - ipa_assert(); - spin_unlock_bh(&sys->spinlock); } static void ipa3_replenish_wlan_rx_cache(struct ipa3_sys_context *sys) @@ -2308,23 +2401,6 @@ done: } } -static inline void __trigger_repl_work(struct ipa3_sys_context *sys) -{ - int tail, head, avail; - - if (atomic_read(&sys->repl->pending)) - return; - - tail = atomic_read(&sys->repl->tail_idx); - head = atomic_read(&sys->repl->head_idx); - avail = (tail - head) % sys->repl->capacity; - - if (avail < sys->repl->capacity / 4) { - atomic_set(&sys->repl->pending, 1); - queue_work(sys->repl_wq, &sys->repl_work); - } -} - static void ipa3_fast_replenish_rx_cache(struct ipa3_sys_context *sys) { struct ipa3_rx_pkt_wrapper *rx_pkt; @@ -2450,12 +2526,12 @@ static void free_rx_page(void *chan_user_data, void *xfer_user_data) struct ipa3_sys_context *sys = rx_pkt->sys; int i; - for (i = 0; i < sys->repl->capacity; i++) - if (sys->repl->cache[i] == rx_pkt) + for (i = 0; i < sys->page_recycle_repl->capacity; i++) + if (sys->page_recycle_repl->cache[i] == rx_pkt) break; - if (i < sys->repl->capacity) { + if (i < sys->page_recycle_repl->capacity) { page_ref_dec(rx_pkt->page_data.page); - sys->repl->cache[i] = NULL; + sys->page_recycle_repl->cache[i] = NULL; } dma_unmap_page(ipa3_ctx->pdev, rx_pkt->page_data.dma_addr, rx_pkt->len, DMA_FROM_DEVICE); @@ -2493,39 +2569,50 @@ static void ipa3_cleanup_rx(struct ipa3_sys_context *sys) spin_unlock_bh(&sys->spinlock); if (sys->repl) { - if (!ipa3_ctx->ipa_wan_skb_page) { - head = atomic_read(&sys->repl->head_idx); - tail = atomic_read(&sys->repl->tail_idx); - while (head != tail) { - rx_pkt = sys->repl->cache[head]; + head = atomic_read(&sys->repl->head_idx); + tail = atomic_read(&sys->repl->tail_idx); + while (head != tail) { + rx_pkt = sys->repl->cache[head]; + if (!ipa3_ctx->ipa_wan_skb_page) { dma_unmap_single(ipa3_ctx->pdev, rx_pkt->data.dma_addr, sys->rx_buff_sz, DMA_FROM_DEVICE); sys->free_skb(rx_pkt->data.skb); - kmem_cache_free(ipa3_ctx->rx_pkt_wrapper_cache, - rx_pkt); - head = (head + 1) % sys->repl->capacity; - } - } else { - for (i = 0; i < sys->repl->capacity; i++) { - rx_pkt = sys->repl->cache[i]; - if (rx_pkt) { - dma_unmap_page(ipa3_ctx->pdev, - rx_pkt->page_data.dma_addr, - rx_pkt->len, - DMA_FROM_DEVICE); - __free_pages(rx_pkt->page_data.page, - IPA_WAN_PAGE_ORDER); - kmem_cache_free( - ipa3_ctx->rx_pkt_wrapper_cache, - rx_pkt); - } + } else { + dma_unmap_page(ipa3_ctx->pdev, + rx_pkt->page_data.dma_addr, + rx_pkt->len, + DMA_FROM_DEVICE); + __free_pages(rx_pkt->page_data.page, + IPA_WAN_PAGE_ORDER); } + kmem_cache_free(ipa3_ctx->rx_pkt_wrapper_cache, + rx_pkt); + head = (head + 1) % sys->repl->capacity; } + kfree(sys->repl->cache); kfree(sys->repl); } + if (sys->page_recycle_repl) { + for (i = 0; i < sys->page_recycle_repl->capacity; i++) { + rx_pkt = sys->page_recycle_repl->cache[i]; + if (rx_pkt) { + dma_unmap_page(ipa3_ctx->pdev, + rx_pkt->page_data.dma_addr, + rx_pkt->len, + DMA_FROM_DEVICE); + __free_pages(rx_pkt->page_data.page, + IPA_WAN_PAGE_ORDER); + kmem_cache_free( + ipa3_ctx->rx_pkt_wrapper_cache, + rx_pkt); + } + } + kfree(sys->page_recycle_repl->cache); + kfree(sys->page_recycle_repl); + } } static struct sk_buff *ipa3_skb_copy_for_client(struct sk_buff *skb, int len) @@ -3570,7 +3657,6 @@ static int ipa3_assign_policy(struct ipa_sys_connect_params *in, ipa3_switch_to_intr_rx_work_func); INIT_DELAYED_WORK(&sys->replenish_rx_work, ipa3_replenish_rx_work_func); - INIT_WORK(&sys->repl_work, ipa3_wq_repl_rx); atomic_set(&sys->curr_polling_state, 0); sys->rx_buff_sz = IPA_GENERIC_RX_BUFF_SZ( IPA_GENERIC_RX_BUFF_BASE_SZ); @@ -3584,6 +3670,7 @@ static int ipa3_assign_policy(struct ipa_sys_connect_params *in, in->ipa_ep_cfg.aggr.aggr_time_limit = IPA_GENERIC_AGGR_TIME_LIMIT; if (in->client == IPA_CLIENT_APPS_LAN_CONS) { + INIT_WORK(&sys->repl_work, ipa3_wq_repl_rx); sys->pyld_hdlr = ipa3_lan_rx_pyld_hdlr; sys->repl_hdlr = ipa3_replenish_rx_cache_recycle; @@ -3599,6 +3686,8 @@ static int ipa3_assign_policy(struct ipa_sys_connect_params *in, in->client == IPA_CLIENT_APPS_WAN_COAL_CONS) { if (ipa3_ctx->ipa_wan_skb_page && in->napi_obj) { + INIT_WORK(&sys->repl_work, + ipa3_wq_page_repl); sys->pyld_hdlr = ipa3_wan_rx_pyld_hdlr; sys->free_rx_wrapper = ipa3_recycle_rx_page_wrapper; @@ -3607,6 +3696,8 @@ static int ipa3_assign_policy(struct ipa_sys_connect_params *in, sys->rx_pool_sz = ipa3_ctx->wan_rx_ring_size; } else { + INIT_WORK(&sys->repl_work, + ipa3_wq_repl_rx); sys->pyld_hdlr = ipa3_wan_rx_pyld_hdlr; sys->free_rx_wrapper = ipa3_free_rx_wrapper; diff --git a/drivers/platform/msm/ipa/ipa_v3/ipa_i.h b/drivers/platform/msm/ipa/ipa_v3/ipa_i.h index 95841d619069..36344c98411e 100644 --- a/drivers/platform/msm/ipa/ipa_v3/ipa_i.h +++ b/drivers/platform/msm/ipa/ipa_v3/ipa_i.h @@ -1009,6 +1009,7 @@ struct ipa3_sys_context { struct work_struct repl_work; void (*repl_hdlr)(struct ipa3_sys_context *sys); struct ipa3_repl_ctx *repl; + struct ipa3_repl_ctx *page_recycle_repl; u32 pkt_sent; struct napi_struct *napi_obj; struct list_head pending_pkts[GSI_VEID_MAX];