diff --git a/drivers/platform/msm/ipa/ipa_api.c b/drivers/platform/msm/ipa/ipa_api.c index 4c6e4046620f..2d38c8aba77b 100644 --- a/drivers/platform/msm/ipa/ipa_api.c +++ b/drivers/platform/msm/ipa/ipa_api.c @@ -3662,6 +3662,19 @@ int ipa_disable_wigig_pipe_i(enum ipa_client_type client) } EXPORT_SYMBOL(ipa_disable_wigig_pipe_i); +/** + * ipa_get_lan_rx_napi() - returns if NAPI is enabled in LAN RX + */ +bool ipa_get_lan_rx_napi(void) +{ + bool ret; + + IPA_API_DISPATCH_RETURN_BOOL(ipa_get_lan_rx_napi); + + return ret; +} +EXPORT_SYMBOL(ipa_get_lan_rx_napi); + /** * ipa_tz_unlock_reg() - Allow AP access to memory regions controlled by TZ */ diff --git a/drivers/platform/msm/ipa/ipa_api.h b/drivers/platform/msm/ipa/ipa_api.h index 2e136ff0318f..113cedfe72b8 100644 --- a/drivers/platform/msm/ipa/ipa_api.h +++ b/drivers/platform/msm/ipa/ipa_api.h @@ -470,6 +470,8 @@ struct ipa_api_controller { bool (*teth_port_state)(void), enum ipa_client_type client); void (*ipa_deregister_client_callback)(enum ipa_client_type client); + bool (*ipa_get_lan_rx_napi)(void); + }; #ifdef CONFIG_IPA3 diff --git a/drivers/platform/msm/ipa/ipa_clients/ecm_ipa.c b/drivers/platform/msm/ipa/ipa_clients/ecm_ipa.c index ff70d6494bb8..ffd4a9b7f6bc 100644 --- a/drivers/platform/msm/ipa/ipa_clients/ecm_ipa.c +++ b/drivers/platform/msm/ipa/ipa_clients/ecm_ipa.c @@ -149,6 +149,7 @@ enum ecm_ipa_operation { * @usb_to_ipa_client: producer client * @pm_hdl: handle for IPA PM * @is_vlan_mode: does the driver need to work in VLAN mode? + * @netif_rx_function: holds the correct network stack API, needed for NAPI */ struct ecm_ipa_dev { struct net_device *net; @@ -166,6 +167,7 @@ struct ecm_ipa_dev { enum ipa_client_type usb_to_ipa_client; u32 pm_hdl; bool is_vlan_mode; + int (*netif_rx_function)(struct sk_buff *skb); }; static int ecm_ipa_open(struct net_device *net); @@ -286,6 +288,13 @@ int ecm_ipa_init(struct ecm_ipa_params *params) snprintf(net->name, sizeof(net->name), "%s%%d", "ecm"); net->netdev_ops = &ecm_ipa_netdev_ops; net->watchdog_timeo = TX_TIMEOUT; + if (ipa_get_lan_rx_napi()) { + ecm_ipa_ctx->netif_rx_function = netif_receive_skb; + ECM_IPA_DEBUG("LAN RX NAPI enabled = True"); + } else { + ecm_ipa_ctx->netif_rx_function = netif_rx_ni; + ECM_IPA_DEBUG("LAN RX NAPI enabled = False"); + } ECM_IPA_DEBUG("internal data structures were initialized\n"); if (!params->device_ready_notify) @@ -655,7 +664,7 @@ static void ecm_ipa_packet_receive_notify return; } - if (evt != IPA_RECEIVE) { + if (unlikely(evt != IPA_RECEIVE)) { ECM_IPA_ERROR("A none IPA_RECEIVE event in ecm_ipa_receive\n"); return; } @@ -663,9 +672,9 @@ static void ecm_ipa_packet_receive_notify skb->dev = ecm_ipa_ctx->net; skb->protocol = eth_type_trans(skb, ecm_ipa_ctx->net); - result = netif_rx(skb); - if (result) - ECM_IPA_ERROR("fail on netif_rx\n"); + result = ecm_ipa_ctx->netif_rx_function(skb); + if (unlikely(result)) + ECM_IPA_ERROR("fail on netif_rx_function\n"); ecm_ipa_ctx->net->stats.rx_packets++; ecm_ipa_ctx->net->stats.rx_bytes += packet_len; } diff --git a/drivers/platform/msm/ipa/ipa_clients/rndis_ipa.c b/drivers/platform/msm/ipa/ipa_clients/rndis_ipa.c index fd2eab5a421b..5f37a77a3740 100644 --- a/drivers/platform/msm/ipa/ipa_clients/rndis_ipa.c +++ b/drivers/platform/msm/ipa/ipa_clients/rndis_ipa.c @@ -183,6 +183,7 @@ enum rndis_ipa_operation { * @state_lock: used to protect the state variable. * @pm_hdl: handle for IPA PM framework * @is_vlan_mode: should driver work in vlan mode? + * @netif_rx_function: holds the correct network stack API, needed for NAPI */ struct rndis_ipa_dev { struct net_device *net; @@ -212,6 +213,7 @@ struct rndis_ipa_dev { spinlock_t state_lock; /* Spinlock for the state variable.*/ u32 pm_hdl; bool is_vlan_mode; + int (*netif_rx_function)(struct sk_buff *skb); }; /** @@ -623,6 +625,14 @@ int rndis_ipa_init(struct ipa_usb_init_params *params) ("netdev:%s registration succeeded, index=%d\n", net->name, net->ifindex); + if (ipa_get_lan_rx_napi()) { + rndis_ipa_ctx->netif_rx_function = netif_receive_skb; + RNDIS_IPA_DEBUG("LAN RX NAPI enabled = True"); + } else { + rndis_ipa_ctx->netif_rx_function = netif_rx_ni; + RNDIS_IPA_DEBUG("LAN RX NAPI enabled = False"); + } + rndis_ipa = rndis_ipa_ctx; params->ipa_rx_notify = rndis_ipa_packet_receive_notify; params->ipa_tx_notify = rndis_ipa_tx_complete_notify; @@ -1139,9 +1149,9 @@ static void rndis_ipa_packet_receive_notify( } trace_rndis_netif_ni(skb->protocol); - result = netif_rx_ni(skb); - if (result) - RNDIS_IPA_ERROR("fail on netif_rx_ni\n"); + result = rndis_ipa_ctx->netif_rx_function(skb); + if (unlikely(result)) + RNDIS_IPA_ERROR("fail on netif_rx_function\n"); rndis_ipa_ctx->net->stats.rx_packets++; rndis_ipa_ctx->net->stats.rx_bytes += packet_len; } diff --git a/drivers/platform/msm/ipa/ipa_v3/ipa.c b/drivers/platform/msm/ipa/ipa_v3/ipa.c index 5802d755ff47..7a2b911715a4 100644 --- a/drivers/platform/msm/ipa/ipa_v3/ipa.c +++ b/drivers/platform/msm/ipa/ipa_v3/ipa.c @@ -4268,6 +4268,8 @@ static int ipa3_setup_apps_pipes(void) sys_in.desc_fifo_sz = IPA_SYS_DESC_FIFO_SZ; sys_in.notify = ipa3_lan_rx_cb; sys_in.priv = NULL; + if (ipa3_ctx->lan_rx_napi_enable) + sys_in.napi_obj = &ipa3_ctx->napi_lan_rx; sys_in.ipa_ep_cfg.hdr.hdr_len = IPA_LAN_RX_HEADER_LENGTH; sys_in.ipa_ep_cfg.hdr_ext.hdr_little_endian = false; sys_in.ipa_ep_cfg.hdr_ext.hdr_total_len_or_pad_valid = true; @@ -5512,6 +5514,13 @@ static int ipa3_alloc_gsi_channel(void) } return ret; } + +static inline void ipa3_enable_napi_lan_rx(void) +{ + if (ipa3_ctx->lan_rx_napi_enable) + napi_enable(&ipa3_ctx->napi_lan_rx); +} + /** * ipa3_post_init() - Initialize the IPA Driver (Part II). * This part contains all initialization which requires interaction with @@ -5803,7 +5812,7 @@ static int ipa3_post_init(const struct ipa3_plat_drv_res *resource_p, mutex_lock(&ipa3_ctx->lock); ipa3_ctx->ipa_initialization_complete = true; mutex_unlock(&ipa3_ctx->lock); - + ipa3_enable_napi_lan_rx(); ipa3_trigger_ipa_ready_cbs(); complete_all(&ipa3_ctx->init_completion_obj); pr_info("IPA driver initialization was successful.\n"); @@ -6196,6 +6205,24 @@ static bool ipa_is_mem_dump_allowed(void) return (desc.ret[0] == 1); } +static int ipa3_lan_poll(struct napi_struct *napi, int budget) +{ + int rcvd_pkts = 0; + + rcvd_pkts = ipa3_lan_rx_poll(ipa3_ctx->clnt_hdl_data_in, + NAPI_WEIGHT); + return rcvd_pkts; +} + +static inline void ipa3_enable_napi_netdev(void) +{ + if (ipa3_ctx->lan_rx_napi_enable) { + init_dummy_netdev(&ipa3_ctx->lan_ndev); + netif_napi_add(&ipa3_ctx->lan_ndev, &ipa3_ctx->napi_lan_rx, + ipa3_lan_poll, NAPI_WEIGHT); + } +} + /** * ipa3_pre_init() - Initialize the IPA Driver. * This part contains all initialization which doesn't require IPA HW, such @@ -6302,6 +6329,7 @@ static int ipa3_pre_init(const struct ipa3_plat_drv_res *resource_p, resource_p->secure_debug_check_action; ipa3_ctx->do_ram_collection_on_crash = resource_p->do_ram_collection_on_crash; + ipa3_ctx->lan_rx_napi_enable = resource_p->lan_rx_napi_enable; if (ipa3_ctx->secure_debug_check_action == USE_SCM) { if (ipa_is_mem_dump_allowed()) @@ -6707,6 +6735,10 @@ static int ipa3_pre_init(const struct ipa3_plat_drv_res *resource_p, /* proxy vote for modem is added in ipa3_post_init() phase */ if (ipa3_ctx->ipa_hw_type != IPA_HW_v4_0) ipa3_proxy_clk_unvote(); + + /* Create the dummy netdev for LAN RX NAPI*/ + ipa3_enable_napi_netdev(); + return 0; fail_cdev_add: fail_gsi_pre_fw_load_init: @@ -7049,6 +7081,13 @@ static int get_ipa_dts_configuration(struct platform_device *pdev, ipa_drv_res->tethered_flow_control ? "True" : "False"); + ipa_drv_res->lan_rx_napi_enable = + of_property_read_bool(pdev->dev.of_node, + "qcom,lan-rx-napi"); + IPADBG(": Enable LAN rx NAPI = %s\n", + ipa_drv_res->lan_rx_napi_enable + ? "True" : "False"); + /* Get IPA wrapper address */ resource = platform_get_resource_byname(pdev, IORESOURCE_MEM, "ipa-base"); @@ -7961,6 +8000,11 @@ struct ipa3_context *ipa3_get_ctx(void) return ipa3_ctx; } +bool ipa3_get_lan_rx_napi(void) +{ + return ipa3_ctx->lan_rx_napi_enable; +} + static void ipa_gsi_notify_cb(struct gsi_per_notify *notify) { /* diff --git a/drivers/platform/msm/ipa/ipa_v3/ipa_dp.c b/drivers/platform/msm/ipa/ipa_v3/ipa_dp.c index 059e6cade2ed..dcf0b6a006a6 100644 --- a/drivers/platform/msm/ipa/ipa_v3/ipa_dp.c +++ b/drivers/platform/msm/ipa/ipa_v3/ipa_dp.c @@ -18,6 +18,8 @@ #define IPA_WAN_AGGR_PKT_CNT 5 #define IPA_WAN_NAPI_MAX_FRAMES (NAPI_WEIGHT / IPA_WAN_AGGR_PKT_CNT) #define IPA_WAN_PAGE_ORDER 3 +#define IPA_LAN_AGGR_PKT_CNT 5 +#define IPA_LAN_NAPI_MAX_FRAMES (NAPI_WEIGHT / IPA_LAN_AGGR_PKT_CNT) #define IPA_LAST_DESC_CNT 0xFFFF #define POLLING_INACTIVITY_RX 40 #define POLLING_MIN_SLEEP_RX 1010 @@ -2626,7 +2628,13 @@ static struct sk_buff *ipa3_skb_copy_for_client(struct sk_buff *skb, int len) { struct sk_buff *skb2 = NULL; - skb2 = __dev_alloc_skb(len + IPA_RX_BUFF_CLIENT_HEADROOM, GFP_KERNEL); + if (!ipa3_ctx->lan_rx_napi_enable) + skb2 = __dev_alloc_skb(len + IPA_RX_BUFF_CLIENT_HEADROOM, + GFP_KERNEL); + else + skb2 = __dev_alloc_skb(len + IPA_RX_BUFF_CLIENT_HEADROOM, + GFP_ATOMIC); + if (likely(skb2)) { /* Set the data pointer */ skb_reserve(skb2, IPA_RX_BUFF_CLIENT_HEADROOM); @@ -2679,8 +2687,12 @@ static int ipa3_lan_rx_pyld_hdlr(struct sk_buff *skb, sys->len_pad); if (sys->len_rem <= skb->len) { if (sys->prev_skb) { - skb2 = skb_copy_expand(sys->prev_skb, 0, - sys->len_rem, GFP_KERNEL); + if (!ipa3_ctx->lan_rx_napi_enable) + skb2 = skb_copy_expand(sys->prev_skb, + 0, sys->len_rem, GFP_KERNEL); + else + skb2 = skb_copy_expand(sys->prev_skb, + 0, sys->len_rem, GFP_ATOMIC); if (likely(skb2)) { memcpy(skb_put(skb2, sys->len_rem), skb->data, sys->len_rem); @@ -2706,8 +2718,12 @@ static int ipa3_lan_rx_pyld_hdlr(struct sk_buff *skb, sys->len_pad = 0; } else { if (sys->prev_skb) { - skb2 = skb_copy_expand(sys->prev_skb, 0, - skb->len, GFP_KERNEL); + if (!ipa3_ctx->lan_rx_napi_enable) + skb2 = skb_copy_expand(sys->prev_skb, 0, + skb->len, GFP_KERNEL); + else + skb2 = skb_copy_expand(sys->prev_skb, 0, + skb->len, GFP_ATOMIC); if (likely(skb2)) { memcpy(skb_put(skb2, skb->len), skb->data, skb->len); @@ -2731,7 +2747,10 @@ begin: if (skb->len < pkt_status_sz) { WARN_ON(sys->prev_skb != NULL); IPADBG_LOW("status straddles buffer\n"); - sys->prev_skb = skb_copy(skb, GFP_KERNEL); + if (!ipa3_ctx->lan_rx_napi_enable) + sys->prev_skb = skb_copy(skb, GFP_KERNEL); + else + sys->prev_skb = skb_copy(skb, GFP_ATOMIC); sys->len_partial = skb->len; goto out; } @@ -2824,14 +2843,18 @@ begin: IPAHAL_PKT_STATUS_EXCEPTION_NONE) { WARN_ON(sys->prev_skb != NULL); IPADBG_LOW("Ins header in next buffer\n"); - sys->prev_skb = skb_copy(skb, GFP_KERNEL); + if (!ipa3_ctx->lan_rx_napi_enable) + sys->prev_skb = skb_copy(skb, + GFP_KERNEL); + else + sys->prev_skb = skb_copy(skb, + GFP_ATOMIC); sys->len_partial = skb->len; goto out; } pad_len_byte = ((status.pkt_len + 3) & ~3) - status.pkt_len; - len = status.pkt_len + pad_len_byte; IPADBG_LOW("pad %d pkt_len %d len %d\n", pad_len_byte, status.pkt_len, len); @@ -4782,9 +4805,75 @@ static int ipa_poll_gsi_n_pkt(struct ipa3_sys_context *sys, *actual_num = idx + poll_num; return ret; } +/** + * ipa3_lan_rx_poll() - Poll the LAN rx packets from IPA HW. + * This function is executed in the softirq context + * + * if input budget is zero, the driver switches back to + * interrupt mode. + * + * return number of polled packets, on error 0(zero) + */ +int ipa3_lan_rx_poll(u32 clnt_hdl, int weight) +{ + struct ipa3_ep_context *ep; + int ret; + int cnt = 0; + int remain_aggr_weight; + struct gsi_chan_xfer_notify notify; + + if (unlikely(clnt_hdl >= ipa3_ctx->ipa_num_pipes || + ipa3_ctx->ep[clnt_hdl].valid == 0)) { + IPAERR("bad param 0x%x\n", clnt_hdl); + return cnt; + } + remain_aggr_weight = weight / IPA_LAN_AGGR_PKT_CNT; + if (unlikely(remain_aggr_weight > IPA_LAN_NAPI_MAX_FRAMES)) { + IPAERR("NAPI weight is higher than expected\n"); + IPAERR("expected %d got %d\n", + IPA_LAN_NAPI_MAX_FRAMES, remain_aggr_weight); + return cnt; + } + ep = &ipa3_ctx->ep[clnt_hdl]; + +start_poll: + while (remain_aggr_weight > 0 && + atomic_read(&ep->sys->curr_polling_state)) { + atomic_set(&ipa3_ctx->transport_pm.eot_activity, 1); + ret = ipa_poll_gsi_pkt(ep->sys, ¬ify); + if (ret) + break; + + if (IPA_CLIENT_IS_MEMCPY_DMA_CONS(ep->client)) + ipa3_dma_memcpy_notify(ep->sys); + else if (IPA_CLIENT_IS_WLAN_CONS(ep->client)) + ipa3_wlan_wq_rx_common(ep->sys, ¬ify); + else + ipa3_wq_rx_common(ep->sys, ¬ify); + + remain_aggr_weight--; + if (ep->sys->len == 0) { + if (remain_aggr_weight == 0) + cnt--; + break; + } + } + cnt += weight - remain_aggr_weight * IPA_LAN_AGGR_PKT_CNT; + if (cnt < weight) { + napi_complete(ep->sys->napi_obj); + ret = ipa3_rx_switch_to_intr_mode(ep->sys); + if (ret == -GSI_STATUS_PENDING_IRQ && + napi_reschedule(ep->sys->napi_obj)) + goto start_poll; + + ipa_pm_deferred_deactivate(ep->sys->pm_hdl); + } + + return cnt; +} /** - * ipa3_rx_poll() - Poll the rx packets from IPA HW. This + * ipa3_rx_poll() - Poll the WAN rx packets from IPA HW. This * function is exectued in the softirq context * * if input budget is zero, the driver switches back to diff --git a/drivers/platform/msm/ipa/ipa_v3/ipa_i.h b/drivers/platform/msm/ipa/ipa_v3/ipa_i.h index 2e2e8db0684a..73a0ad33e07d 100644 --- a/drivers/platform/msm/ipa/ipa_v3/ipa_i.h +++ b/drivers/platform/msm/ipa/ipa_v3/ipa_i.h @@ -16,6 +16,7 @@ #include #include #include +#include #include #include #include @@ -1763,6 +1764,9 @@ struct ipa3_pc_mbox_data { * @wdi3_ctx: IPA wdi3 context * @gsi_info: channel/protocol info for GSI offloading uC stats * IPA context - holds all relevant info about IPA driver and its state + * @lan_rx_napi_enable: flag if NAPI is enabled on the LAN dp + * @lan_ndev: dummy netdev for LAN rx NAPI + * @napi_lan_rx: NAPI object for LAN rx */ struct ipa3_context { struct ipa3_char_device_context cdev; @@ -1932,6 +1936,10 @@ struct ipa3_context { gsi_info[IPA_HW_PROTOCOL_MAX]; bool ipa_wan_skb_page; struct ipacm_fnr_info fnr_info; + /* dummy netdev for lan RX NAPI */ + bool lan_rx_napi_enable; + struct net_device lan_ndev; + struct napi_struct napi_lan_rx; }; struct ipa3_plat_drv_res { @@ -1964,6 +1972,7 @@ struct ipa3_plat_drv_res { bool apply_rg10_wa; bool gsi_ch20_wa; bool tethered_flow_control; + bool lan_rx_napi_enable; u32 mhi_evid_limits[2]; /* start and end values */ bool ipa_mhi_dynamic_config; u32 ipa_tz_unlock_reg_num; @@ -2703,6 +2712,8 @@ u8 ipa3_get_qmb_master_sel(enum ipa_client_type client); int ipa3_get_smmu_params(struct ipa_smmu_in_params *in, struct ipa_smmu_out_params *out); +bool ipa3_get_lan_rx_napi(void); + /* internal functions */ int ipa3_bind_api_controller(enum ipa_hw_type ipa_hw_type, @@ -2996,6 +3007,7 @@ int ipa3_register_ipa_ready_cb(void (*ipa_ready_cb)(void *), void *user_data); const char *ipa_hw_error_str(enum ipa3_hw_errors err_type); int ipa_gsi_ch20_wa(void); int ipa3_rx_poll(u32 clnt_hdl, int budget); +int ipa3_lan_rx_poll(u32 clnt_hdl, int weight); int ipa3_smmu_map_peer_reg(phys_addr_t phys_addr, bool map, enum ipa_smmu_cb_type cb_type); int ipa3_smmu_map_peer_buff(u64 iova, u32 size, bool map, struct sg_table *sgt, diff --git a/drivers/platform/msm/ipa/ipa_v3/ipa_utils.c b/drivers/platform/msm/ipa/ipa_v3/ipa_utils.c index d83cb46c2f5b..a36b94f92050 100644 --- a/drivers/platform/msm/ipa/ipa_v3/ipa_utils.c +++ b/drivers/platform/msm/ipa/ipa_v3/ipa_utils.c @@ -7316,6 +7316,7 @@ int ipa3_bind_api_controller(enum ipa_hw_type ipa_hw_type, ipa3_register_client_callback; api_ctrl->ipa_deregister_client_callback = ipa3_deregister_client_callback; + api_ctrl->ipa_get_lan_rx_napi = ipa3_get_lan_rx_napi; return 0; } diff --git a/include/linux/ipa.h b/include/linux/ipa.h index d384ef899a77..a0b319348713 100644 --- a/include/linux/ipa.h +++ b/include/linux/ipa.h @@ -1671,6 +1671,10 @@ int ipa_get_smmu_params(struct ipa_smmu_in_params *in, * Returns: 0 on success, negative on failure */ int ipa_is_vlan_mode(enum ipa_vlan_ifaces iface, bool *res); +/** + * ipa_get_lan_rx_napi - returns true if NAPI is enabled in the LAN RX dp + */ +bool ipa_get_lan_rx_napi(void); #else /* (CONFIG_IPA || CONFIG_IPA3) */ /* @@ -2524,6 +2528,11 @@ static inline int ipa_is_vlan_mode(enum ipa_vlan_ifaces iface, bool *res) { return -EPERM; } +static inline bool ipa_get_lan_rx_napi(void) +{ + return false; +} + #endif /* (CONFIG_IPA || CONFIG_IPA3) */ #endif /* _IPA_H_ */