From d45037d3caf3194e11ad058a5bc66c4289291af0 Mon Sep 17 00:00:00 2001 From: Bojun Pan Date: Thu, 21 Feb 2019 21:06:55 -0800 Subject: [PATCH] msm: ipa: Enable NAPI on IPA LAN CONS pipe LAN RX optimization, change LAN RX data path bottom half to run in softirq using NAPI. Also added an API for other clients to see if we are using NAPI. Change-Id: Iecd47f7412975b58786d81a5df3460de9aeffa21 Acked-by: Tal Gelbard Signed-off-by: Bojun Pan Signed-off-by: Amir Levy --- drivers/platform/msm/ipa/ipa_api.c | 13 +++ drivers/platform/msm/ipa/ipa_api.h | 2 + .../platform/msm/ipa/ipa_clients/ecm_ipa.c | 17 ++- .../platform/msm/ipa/ipa_clients/rndis_ipa.c | 16 ++- drivers/platform/msm/ipa/ipa_v3/ipa.c | 46 +++++++- drivers/platform/msm/ipa/ipa_v3/ipa_dp.c | 107 ++++++++++++++++-- drivers/platform/msm/ipa/ipa_v3/ipa_i.h | 12 ++ drivers/platform/msm/ipa/ipa_v3/ipa_utils.c | 1 + include/linux/ipa.h | 9 ++ 9 files changed, 206 insertions(+), 17 deletions(-) diff --git a/drivers/platform/msm/ipa/ipa_api.c b/drivers/platform/msm/ipa/ipa_api.c index 4c6e4046620f..2d38c8aba77b 100644 --- a/drivers/platform/msm/ipa/ipa_api.c +++ b/drivers/platform/msm/ipa/ipa_api.c @@ -3662,6 +3662,19 @@ int ipa_disable_wigig_pipe_i(enum ipa_client_type client) } EXPORT_SYMBOL(ipa_disable_wigig_pipe_i); +/** + * ipa_get_lan_rx_napi() - returns if NAPI is enabled in LAN RX + */ +bool ipa_get_lan_rx_napi(void) +{ + bool ret; + + IPA_API_DISPATCH_RETURN_BOOL(ipa_get_lan_rx_napi); + + return ret; +} +EXPORT_SYMBOL(ipa_get_lan_rx_napi); + /** * ipa_tz_unlock_reg() - Allow AP access to memory regions controlled by TZ */ diff --git a/drivers/platform/msm/ipa/ipa_api.h b/drivers/platform/msm/ipa/ipa_api.h index 2e136ff0318f..113cedfe72b8 100644 --- a/drivers/platform/msm/ipa/ipa_api.h +++ b/drivers/platform/msm/ipa/ipa_api.h @@ -470,6 +470,8 @@ struct ipa_api_controller { bool (*teth_port_state)(void), enum ipa_client_type client); void (*ipa_deregister_client_callback)(enum ipa_client_type client); + bool (*ipa_get_lan_rx_napi)(void); + }; #ifdef CONFIG_IPA3 diff --git a/drivers/platform/msm/ipa/ipa_clients/ecm_ipa.c b/drivers/platform/msm/ipa/ipa_clients/ecm_ipa.c index ff70d6494bb8..ffd4a9b7f6bc 100644 --- a/drivers/platform/msm/ipa/ipa_clients/ecm_ipa.c +++ b/drivers/platform/msm/ipa/ipa_clients/ecm_ipa.c @@ -149,6 +149,7 @@ enum ecm_ipa_operation { * @usb_to_ipa_client: producer client * @pm_hdl: handle for IPA PM * @is_vlan_mode: does the driver need to work in VLAN mode? + * @netif_rx_function: holds the correct network stack API, needed for NAPI */ struct ecm_ipa_dev { struct net_device *net; @@ -166,6 +167,7 @@ struct ecm_ipa_dev { enum ipa_client_type usb_to_ipa_client; u32 pm_hdl; bool is_vlan_mode; + int (*netif_rx_function)(struct sk_buff *skb); }; static int ecm_ipa_open(struct net_device *net); @@ -286,6 +288,13 @@ int ecm_ipa_init(struct ecm_ipa_params *params) snprintf(net->name, sizeof(net->name), "%s%%d", "ecm"); net->netdev_ops = &ecm_ipa_netdev_ops; net->watchdog_timeo = TX_TIMEOUT; + if (ipa_get_lan_rx_napi()) { + ecm_ipa_ctx->netif_rx_function = netif_receive_skb; + ECM_IPA_DEBUG("LAN RX NAPI enabled = True"); + } else { + ecm_ipa_ctx->netif_rx_function = netif_rx_ni; + ECM_IPA_DEBUG("LAN RX NAPI enabled = False"); + } ECM_IPA_DEBUG("internal data structures were initialized\n"); if (!params->device_ready_notify) @@ -655,7 +664,7 @@ static void ecm_ipa_packet_receive_notify return; } - if (evt != IPA_RECEIVE) { + if (unlikely(evt != IPA_RECEIVE)) { ECM_IPA_ERROR("A none IPA_RECEIVE event in ecm_ipa_receive\n"); return; } @@ -663,9 +672,9 @@ static void ecm_ipa_packet_receive_notify skb->dev = ecm_ipa_ctx->net; skb->protocol = eth_type_trans(skb, ecm_ipa_ctx->net); - result = netif_rx(skb); - if (result) - ECM_IPA_ERROR("fail on netif_rx\n"); + result = ecm_ipa_ctx->netif_rx_function(skb); + if (unlikely(result)) + ECM_IPA_ERROR("fail on netif_rx_function\n"); ecm_ipa_ctx->net->stats.rx_packets++; ecm_ipa_ctx->net->stats.rx_bytes += packet_len; } diff --git a/drivers/platform/msm/ipa/ipa_clients/rndis_ipa.c b/drivers/platform/msm/ipa/ipa_clients/rndis_ipa.c index fd2eab5a421b..5f37a77a3740 100644 --- a/drivers/platform/msm/ipa/ipa_clients/rndis_ipa.c +++ b/drivers/platform/msm/ipa/ipa_clients/rndis_ipa.c @@ -183,6 +183,7 @@ enum rndis_ipa_operation { * @state_lock: used to protect the state variable. * @pm_hdl: handle for IPA PM framework * @is_vlan_mode: should driver work in vlan mode? + * @netif_rx_function: holds the correct network stack API, needed for NAPI */ struct rndis_ipa_dev { struct net_device *net; @@ -212,6 +213,7 @@ struct rndis_ipa_dev { spinlock_t state_lock; /* Spinlock for the state variable.*/ u32 pm_hdl; bool is_vlan_mode; + int (*netif_rx_function)(struct sk_buff *skb); }; /** @@ -623,6 +625,14 @@ int rndis_ipa_init(struct ipa_usb_init_params *params) ("netdev:%s registration succeeded, index=%d\n", net->name, net->ifindex); + if (ipa_get_lan_rx_napi()) { + rndis_ipa_ctx->netif_rx_function = netif_receive_skb; + RNDIS_IPA_DEBUG("LAN RX NAPI enabled = True"); + } else { + rndis_ipa_ctx->netif_rx_function = netif_rx_ni; + RNDIS_IPA_DEBUG("LAN RX NAPI enabled = False"); + } + rndis_ipa = rndis_ipa_ctx; params->ipa_rx_notify = rndis_ipa_packet_receive_notify; params->ipa_tx_notify = rndis_ipa_tx_complete_notify; @@ -1139,9 +1149,9 @@ static void rndis_ipa_packet_receive_notify( } trace_rndis_netif_ni(skb->protocol); - result = netif_rx_ni(skb); - if (result) - RNDIS_IPA_ERROR("fail on netif_rx_ni\n"); + result = rndis_ipa_ctx->netif_rx_function(skb); + if (unlikely(result)) + RNDIS_IPA_ERROR("fail on netif_rx_function\n"); rndis_ipa_ctx->net->stats.rx_packets++; rndis_ipa_ctx->net->stats.rx_bytes += packet_len; } diff --git a/drivers/platform/msm/ipa/ipa_v3/ipa.c b/drivers/platform/msm/ipa/ipa_v3/ipa.c index 5802d755ff47..7a2b911715a4 100644 --- a/drivers/platform/msm/ipa/ipa_v3/ipa.c +++ b/drivers/platform/msm/ipa/ipa_v3/ipa.c @@ -4268,6 +4268,8 @@ static int ipa3_setup_apps_pipes(void) sys_in.desc_fifo_sz = IPA_SYS_DESC_FIFO_SZ; sys_in.notify = ipa3_lan_rx_cb; sys_in.priv = NULL; + if (ipa3_ctx->lan_rx_napi_enable) + sys_in.napi_obj = &ipa3_ctx->napi_lan_rx; sys_in.ipa_ep_cfg.hdr.hdr_len = IPA_LAN_RX_HEADER_LENGTH; sys_in.ipa_ep_cfg.hdr_ext.hdr_little_endian = false; sys_in.ipa_ep_cfg.hdr_ext.hdr_total_len_or_pad_valid = true; @@ -5512,6 +5514,13 @@ static int ipa3_alloc_gsi_channel(void) } return ret; } + +static inline void ipa3_enable_napi_lan_rx(void) +{ + if (ipa3_ctx->lan_rx_napi_enable) + napi_enable(&ipa3_ctx->napi_lan_rx); +} + /** * ipa3_post_init() - Initialize the IPA Driver (Part II). * This part contains all initialization which requires interaction with @@ -5803,7 +5812,7 @@ static int ipa3_post_init(const struct ipa3_plat_drv_res *resource_p, mutex_lock(&ipa3_ctx->lock); ipa3_ctx->ipa_initialization_complete = true; mutex_unlock(&ipa3_ctx->lock); - + ipa3_enable_napi_lan_rx(); ipa3_trigger_ipa_ready_cbs(); complete_all(&ipa3_ctx->init_completion_obj); pr_info("IPA driver initialization was successful.\n"); @@ -6196,6 +6205,24 @@ static bool ipa_is_mem_dump_allowed(void) return (desc.ret[0] == 1); } +static int ipa3_lan_poll(struct napi_struct *napi, int budget) +{ + int rcvd_pkts = 0; + + rcvd_pkts = ipa3_lan_rx_poll(ipa3_ctx->clnt_hdl_data_in, + NAPI_WEIGHT); + return rcvd_pkts; +} + +static inline void ipa3_enable_napi_netdev(void) +{ + if (ipa3_ctx->lan_rx_napi_enable) { + init_dummy_netdev(&ipa3_ctx->lan_ndev); + netif_napi_add(&ipa3_ctx->lan_ndev, &ipa3_ctx->napi_lan_rx, + ipa3_lan_poll, NAPI_WEIGHT); + } +} + /** * ipa3_pre_init() - Initialize the IPA Driver. * This part contains all initialization which doesn't require IPA HW, such @@ -6302,6 +6329,7 @@ static int ipa3_pre_init(const struct ipa3_plat_drv_res *resource_p, resource_p->secure_debug_check_action; ipa3_ctx->do_ram_collection_on_crash = resource_p->do_ram_collection_on_crash; + ipa3_ctx->lan_rx_napi_enable = resource_p->lan_rx_napi_enable; if (ipa3_ctx->secure_debug_check_action == USE_SCM) { if (ipa_is_mem_dump_allowed()) @@ -6707,6 +6735,10 @@ static int ipa3_pre_init(const struct ipa3_plat_drv_res *resource_p, /* proxy vote for modem is added in ipa3_post_init() phase */ if (ipa3_ctx->ipa_hw_type != IPA_HW_v4_0) ipa3_proxy_clk_unvote(); + + /* Create the dummy netdev for LAN RX NAPI*/ + ipa3_enable_napi_netdev(); + return 0; fail_cdev_add: fail_gsi_pre_fw_load_init: @@ -7049,6 +7081,13 @@ static int get_ipa_dts_configuration(struct platform_device *pdev, ipa_drv_res->tethered_flow_control ? "True" : "False"); + ipa_drv_res->lan_rx_napi_enable = + of_property_read_bool(pdev->dev.of_node, + "qcom,lan-rx-napi"); + IPADBG(": Enable LAN rx NAPI = %s\n", + ipa_drv_res->lan_rx_napi_enable + ? "True" : "False"); + /* Get IPA wrapper address */ resource = platform_get_resource_byname(pdev, IORESOURCE_MEM, "ipa-base"); @@ -7961,6 +8000,11 @@ struct ipa3_context *ipa3_get_ctx(void) return ipa3_ctx; } +bool ipa3_get_lan_rx_napi(void) +{ + return ipa3_ctx->lan_rx_napi_enable; +} + static void ipa_gsi_notify_cb(struct gsi_per_notify *notify) { /* diff --git a/drivers/platform/msm/ipa/ipa_v3/ipa_dp.c b/drivers/platform/msm/ipa/ipa_v3/ipa_dp.c index 059e6cade2ed..dcf0b6a006a6 100644 --- a/drivers/platform/msm/ipa/ipa_v3/ipa_dp.c +++ b/drivers/platform/msm/ipa/ipa_v3/ipa_dp.c @@ -18,6 +18,8 @@ #define IPA_WAN_AGGR_PKT_CNT 5 #define IPA_WAN_NAPI_MAX_FRAMES (NAPI_WEIGHT / IPA_WAN_AGGR_PKT_CNT) #define IPA_WAN_PAGE_ORDER 3 +#define IPA_LAN_AGGR_PKT_CNT 5 +#define IPA_LAN_NAPI_MAX_FRAMES (NAPI_WEIGHT / IPA_LAN_AGGR_PKT_CNT) #define IPA_LAST_DESC_CNT 0xFFFF #define POLLING_INACTIVITY_RX 40 #define POLLING_MIN_SLEEP_RX 1010 @@ -2626,7 +2628,13 @@ static struct sk_buff *ipa3_skb_copy_for_client(struct sk_buff *skb, int len) { struct sk_buff *skb2 = NULL; - skb2 = __dev_alloc_skb(len + IPA_RX_BUFF_CLIENT_HEADROOM, GFP_KERNEL); + if (!ipa3_ctx->lan_rx_napi_enable) + skb2 = __dev_alloc_skb(len + IPA_RX_BUFF_CLIENT_HEADROOM, + GFP_KERNEL); + else + skb2 = __dev_alloc_skb(len + IPA_RX_BUFF_CLIENT_HEADROOM, + GFP_ATOMIC); + if (likely(skb2)) { /* Set the data pointer */ skb_reserve(skb2, IPA_RX_BUFF_CLIENT_HEADROOM); @@ -2679,8 +2687,12 @@ static int ipa3_lan_rx_pyld_hdlr(struct sk_buff *skb, sys->len_pad); if (sys->len_rem <= skb->len) { if (sys->prev_skb) { - skb2 = skb_copy_expand(sys->prev_skb, 0, - sys->len_rem, GFP_KERNEL); + if (!ipa3_ctx->lan_rx_napi_enable) + skb2 = skb_copy_expand(sys->prev_skb, + 0, sys->len_rem, GFP_KERNEL); + else + skb2 = skb_copy_expand(sys->prev_skb, + 0, sys->len_rem, GFP_ATOMIC); if (likely(skb2)) { memcpy(skb_put(skb2, sys->len_rem), skb->data, sys->len_rem); @@ -2706,8 +2718,12 @@ static int ipa3_lan_rx_pyld_hdlr(struct sk_buff *skb, sys->len_pad = 0; } else { if (sys->prev_skb) { - skb2 = skb_copy_expand(sys->prev_skb, 0, - skb->len, GFP_KERNEL); + if (!ipa3_ctx->lan_rx_napi_enable) + skb2 = skb_copy_expand(sys->prev_skb, 0, + skb->len, GFP_KERNEL); + else + skb2 = skb_copy_expand(sys->prev_skb, 0, + skb->len, GFP_ATOMIC); if (likely(skb2)) { memcpy(skb_put(skb2, skb->len), skb->data, skb->len); @@ -2731,7 +2747,10 @@ begin: if (skb->len < pkt_status_sz) { WARN_ON(sys->prev_skb != NULL); IPADBG_LOW("status straddles buffer\n"); - sys->prev_skb = skb_copy(skb, GFP_KERNEL); + if (!ipa3_ctx->lan_rx_napi_enable) + sys->prev_skb = skb_copy(skb, GFP_KERNEL); + else + sys->prev_skb = skb_copy(skb, GFP_ATOMIC); sys->len_partial = skb->len; goto out; } @@ -2824,14 +2843,18 @@ begin: IPAHAL_PKT_STATUS_EXCEPTION_NONE) { WARN_ON(sys->prev_skb != NULL); IPADBG_LOW("Ins header in next buffer\n"); - sys->prev_skb = skb_copy(skb, GFP_KERNEL); + if (!ipa3_ctx->lan_rx_napi_enable) + sys->prev_skb = skb_copy(skb, + GFP_KERNEL); + else + sys->prev_skb = skb_copy(skb, + GFP_ATOMIC); sys->len_partial = skb->len; goto out; } pad_len_byte = ((status.pkt_len + 3) & ~3) - status.pkt_len; - len = status.pkt_len + pad_len_byte; IPADBG_LOW("pad %d pkt_len %d len %d\n", pad_len_byte, status.pkt_len, len); @@ -4782,9 +4805,75 @@ static int ipa_poll_gsi_n_pkt(struct ipa3_sys_context *sys, *actual_num = idx + poll_num; return ret; } +/** + * ipa3_lan_rx_poll() - Poll the LAN rx packets from IPA HW. + * This function is executed in the softirq context + * + * if input budget is zero, the driver switches back to + * interrupt mode. + * + * return number of polled packets, on error 0(zero) + */ +int ipa3_lan_rx_poll(u32 clnt_hdl, int weight) +{ + struct ipa3_ep_context *ep; + int ret; + int cnt = 0; + int remain_aggr_weight; + struct gsi_chan_xfer_notify notify; + + if (unlikely(clnt_hdl >= ipa3_ctx->ipa_num_pipes || + ipa3_ctx->ep[clnt_hdl].valid == 0)) { + IPAERR("bad param 0x%x\n", clnt_hdl); + return cnt; + } + remain_aggr_weight = weight / IPA_LAN_AGGR_PKT_CNT; + if (unlikely(remain_aggr_weight > IPA_LAN_NAPI_MAX_FRAMES)) { + IPAERR("NAPI weight is higher than expected\n"); + IPAERR("expected %d got %d\n", + IPA_LAN_NAPI_MAX_FRAMES, remain_aggr_weight); + return cnt; + } + ep = &ipa3_ctx->ep[clnt_hdl]; + +start_poll: + while (remain_aggr_weight > 0 && + atomic_read(&ep->sys->curr_polling_state)) { + atomic_set(&ipa3_ctx->transport_pm.eot_activity, 1); + ret = ipa_poll_gsi_pkt(ep->sys, ¬ify); + if (ret) + break; + + if (IPA_CLIENT_IS_MEMCPY_DMA_CONS(ep->client)) + ipa3_dma_memcpy_notify(ep->sys); + else if (IPA_CLIENT_IS_WLAN_CONS(ep->client)) + ipa3_wlan_wq_rx_common(ep->sys, ¬ify); + else + ipa3_wq_rx_common(ep->sys, ¬ify); + + remain_aggr_weight--; + if (ep->sys->len == 0) { + if (remain_aggr_weight == 0) + cnt--; + break; + } + } + cnt += weight - remain_aggr_weight * IPA_LAN_AGGR_PKT_CNT; + if (cnt < weight) { + napi_complete(ep->sys->napi_obj); + ret = ipa3_rx_switch_to_intr_mode(ep->sys); + if (ret == -GSI_STATUS_PENDING_IRQ && + napi_reschedule(ep->sys->napi_obj)) + goto start_poll; + + ipa_pm_deferred_deactivate(ep->sys->pm_hdl); + } + + return cnt; +} /** - * ipa3_rx_poll() - Poll the rx packets from IPA HW. This + * ipa3_rx_poll() - Poll the WAN rx packets from IPA HW. This * function is exectued in the softirq context * * if input budget is zero, the driver switches back to diff --git a/drivers/platform/msm/ipa/ipa_v3/ipa_i.h b/drivers/platform/msm/ipa/ipa_v3/ipa_i.h index 2e2e8db0684a..73a0ad33e07d 100644 --- a/drivers/platform/msm/ipa/ipa_v3/ipa_i.h +++ b/drivers/platform/msm/ipa/ipa_v3/ipa_i.h @@ -16,6 +16,7 @@ #include #include #include +#include #include #include #include @@ -1763,6 +1764,9 @@ struct ipa3_pc_mbox_data { * @wdi3_ctx: IPA wdi3 context * @gsi_info: channel/protocol info for GSI offloading uC stats * IPA context - holds all relevant info about IPA driver and its state + * @lan_rx_napi_enable: flag if NAPI is enabled on the LAN dp + * @lan_ndev: dummy netdev for LAN rx NAPI + * @napi_lan_rx: NAPI object for LAN rx */ struct ipa3_context { struct ipa3_char_device_context cdev; @@ -1932,6 +1936,10 @@ struct ipa3_context { gsi_info[IPA_HW_PROTOCOL_MAX]; bool ipa_wan_skb_page; struct ipacm_fnr_info fnr_info; + /* dummy netdev for lan RX NAPI */ + bool lan_rx_napi_enable; + struct net_device lan_ndev; + struct napi_struct napi_lan_rx; }; struct ipa3_plat_drv_res { @@ -1964,6 +1972,7 @@ struct ipa3_plat_drv_res { bool apply_rg10_wa; bool gsi_ch20_wa; bool tethered_flow_control; + bool lan_rx_napi_enable; u32 mhi_evid_limits[2]; /* start and end values */ bool ipa_mhi_dynamic_config; u32 ipa_tz_unlock_reg_num; @@ -2703,6 +2712,8 @@ u8 ipa3_get_qmb_master_sel(enum ipa_client_type client); int ipa3_get_smmu_params(struct ipa_smmu_in_params *in, struct ipa_smmu_out_params *out); +bool ipa3_get_lan_rx_napi(void); + /* internal functions */ int ipa3_bind_api_controller(enum ipa_hw_type ipa_hw_type, @@ -2996,6 +3007,7 @@ int ipa3_register_ipa_ready_cb(void (*ipa_ready_cb)(void *), void *user_data); const char *ipa_hw_error_str(enum ipa3_hw_errors err_type); int ipa_gsi_ch20_wa(void); int ipa3_rx_poll(u32 clnt_hdl, int budget); +int ipa3_lan_rx_poll(u32 clnt_hdl, int weight); int ipa3_smmu_map_peer_reg(phys_addr_t phys_addr, bool map, enum ipa_smmu_cb_type cb_type); int ipa3_smmu_map_peer_buff(u64 iova, u32 size, bool map, struct sg_table *sgt, diff --git a/drivers/platform/msm/ipa/ipa_v3/ipa_utils.c b/drivers/platform/msm/ipa/ipa_v3/ipa_utils.c index d83cb46c2f5b..a36b94f92050 100644 --- a/drivers/platform/msm/ipa/ipa_v3/ipa_utils.c +++ b/drivers/platform/msm/ipa/ipa_v3/ipa_utils.c @@ -7316,6 +7316,7 @@ int ipa3_bind_api_controller(enum ipa_hw_type ipa_hw_type, ipa3_register_client_callback; api_ctrl->ipa_deregister_client_callback = ipa3_deregister_client_callback; + api_ctrl->ipa_get_lan_rx_napi = ipa3_get_lan_rx_napi; return 0; } diff --git a/include/linux/ipa.h b/include/linux/ipa.h index d384ef899a77..a0b319348713 100644 --- a/include/linux/ipa.h +++ b/include/linux/ipa.h @@ -1671,6 +1671,10 @@ int ipa_get_smmu_params(struct ipa_smmu_in_params *in, * Returns: 0 on success, negative on failure */ int ipa_is_vlan_mode(enum ipa_vlan_ifaces iface, bool *res); +/** + * ipa_get_lan_rx_napi - returns true if NAPI is enabled in the LAN RX dp + */ +bool ipa_get_lan_rx_napi(void); #else /* (CONFIG_IPA || CONFIG_IPA3) */ /* @@ -2524,6 +2528,11 @@ static inline int ipa_is_vlan_mode(enum ipa_vlan_ifaces iface, bool *res) { return -EPERM; } +static inline bool ipa_get_lan_rx_napi(void) +{ + return false; +} + #endif /* (CONFIG_IPA || CONFIG_IPA3) */ #endif /* _IPA_H_ */