From 48503e340f66a4afbf7586db548dd1e0624d5c7e Mon Sep 17 00:00:00 2001 From: Lakshit Tyagi Date: Wed, 18 Nov 2020 00:47:42 +0530 Subject: [PATCH] msm: ipa: IPA offload subsystem for ethernet devices Porting IPA offload subsystem from 4.14. Change-Id: I71363e4958c3d11f5c6144eeef06ce4d373952ab Signed-off-by: Lakshit Tyagi --- drivers/platform/msm/Kconfig | 28 + drivers/platform/msm/Makefile | 1 + drivers/platform/msm/ethernet/Makefile | 21 + drivers/platform/msm/ethernet/ipa_eth.c | 1135 ++++++++++++ drivers/platform/msm/ethernet/ipa_eth_bus.c | 136 ++ .../platform/msm/ethernet/ipa_eth_debugfs.c | 307 ++++ .../platform/msm/ethernet/ipa_eth_debugfs.h | 20 + drivers/platform/msm/ethernet/ipa_eth_ep.c | 680 ++++++++ drivers/platform/msm/ethernet/ipa_eth_gsi.c | 371 ++++ drivers/platform/msm/ethernet/ipa_eth_i.h | 272 +++ drivers/platform/msm/ethernet/ipa_eth_net.c | 1551 +++++++++++++++++ .../platform/msm/ethernet/ipa_eth_offload.c | 260 +++ drivers/platform/msm/ethernet/ipa_eth_pci.c | 612 +++++++ drivers/platform/msm/ethernet/ipa_eth_pm.c | 131 ++ drivers/platform/msm/ethernet/ipa_eth_trace.c | 7 + drivers/platform/msm/ethernet/ipa_eth_trace.h | 85 + drivers/platform/msm/ethernet/ipa_eth_uc.c | 224 +++ drivers/platform/msm/ethernet/ipa_eth_utils.c | 154 ++ include/linux/ipa_eth.h | 1098 ++++++++++++ 19 files changed, 7093 insertions(+) create mode 100644 drivers/platform/msm/ethernet/Makefile create mode 100644 drivers/platform/msm/ethernet/ipa_eth.c create mode 100644 drivers/platform/msm/ethernet/ipa_eth_bus.c create mode 100644 drivers/platform/msm/ethernet/ipa_eth_debugfs.c create mode 100644 drivers/platform/msm/ethernet/ipa_eth_debugfs.h create mode 100644 drivers/platform/msm/ethernet/ipa_eth_ep.c create mode 100644 drivers/platform/msm/ethernet/ipa_eth_gsi.c create mode 100644 drivers/platform/msm/ethernet/ipa_eth_i.h create mode 100644 drivers/platform/msm/ethernet/ipa_eth_net.c create mode 100644 drivers/platform/msm/ethernet/ipa_eth_offload.c create mode 100644 drivers/platform/msm/ethernet/ipa_eth_pci.c create mode 100644 drivers/platform/msm/ethernet/ipa_eth_pm.c create mode 100644 drivers/platform/msm/ethernet/ipa_eth_trace.c create mode 100644 drivers/platform/msm/ethernet/ipa_eth_trace.h create mode 100644 drivers/platform/msm/ethernet/ipa_eth_uc.c create mode 100644 drivers/platform/msm/ethernet/ipa_eth_utils.c diff --git a/drivers/platform/msm/Kconfig b/drivers/platform/msm/Kconfig index a03a41691c32..ed7816e60655 100644 --- a/drivers/platform/msm/Kconfig +++ b/drivers/platform/msm/Kconfig @@ -143,4 +143,32 @@ config R8125 To compile this driver as a module, choose M here: the module will be called r8125. +config IPA_ETH + bool "IPA Ethernet Offload Sub-system support" + depends on IPA3 + help + Enables IPA Ethernet Offload Subsystem for offloading PCI based + ethernet devices to IPA. The offload subsystem still require a + compatible network driver to register with it and a corresponding + offload driver to manage one of more offload data paths that uses + the network device. + +config IPA_ETH_NOAUTO + bool "Disable automatic offload initialization of interfaces" + depends on IPA_ETH + help + Enabling this option prevents automatic initialization of offload on + ethernet interfaces. Debugfs control interface will instead be used + to enable offloading. This feature is meant only for debugging. + If unsure, say N. + +config IPA_ETH_DEBUG + bool "Enable addtional debug logging in the offload sub-system" + depends on IPA_ETH + help + Enabling this option increases the number of messages logged by the offload + sub-system for aiding debugging. Additional debug messages are logged into + IPC log as well as kernel log using dynamic debug. + If unsure, say N. + endmenu diff --git a/drivers/platform/msm/Makefile b/drivers/platform/msm/Makefile index 2164415c7c5e..89eb8be8f12a 100644 --- a/drivers/platform/msm/Makefile +++ b/drivers/platform/msm/Makefile @@ -13,3 +13,4 @@ obj-$(CONFIG_SPS) += sps/ obj-$(CONFIG_MSM_11AD) += msm_11ad/ obj-$(CONFIG_EP_PCIE) += ep_pcie/ obj-$(CONFIG_MSM_MHI_DEV) += mhi_dev/ +obj-$(CONFIG_IPA_ETH) += ethernet/ diff --git a/drivers/platform/msm/ethernet/Makefile b/drivers/platform/msm/ethernet/Makefile new file mode 100644 index 000000000000..b6d45027ab9a --- /dev/null +++ b/drivers/platform/msm/ethernet/Makefile @@ -0,0 +1,21 @@ +# SPDX-License-Identifier: GPL-2.0-only + +# +# Makefile for the MSM specific device drivers. +# + +obj-$(CONFIG_IPA_ETH) += ipa-eth.o + +ipa-eth-y := \ + ipa_eth_bus.o \ + ipa_eth.o \ + ipa_eth_debugfs.o \ + ipa_eth_ep.o \ + ipa_eth_gsi.o \ + ipa_eth_net.o \ + ipa_eth_offload.o \ + ipa_eth_pci.o \ + ipa_eth_pm.o \ + ipa_eth_trace.o \ + ipa_eth_uc.o \ + ipa_eth_utils.o diff --git a/drivers/platform/msm/ethernet/ipa_eth.c b/drivers/platform/msm/ethernet/ipa_eth.c new file mode 100644 index 000000000000..fe1184c23aa0 --- /dev/null +++ b/drivers/platform/msm/ethernet/ipa_eth.c @@ -0,0 +1,1135 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2019-2020, The Linux Foundation. All rights reserved + */ + +#include +#include + +#include "ipa_eth_i.h" + +unsigned long ipa_eth_state; + +static LIST_HEAD(ipa_eth_devices); +static DEFINE_MUTEX(ipa_eth_devices_lock); + +#ifdef CONFIG_IPA_ETH_NOAUTO +bool ipa_eth_noauto = true; +#else +bool ipa_eth_noauto; +#endif + +module_param(ipa_eth_noauto, bool, 0444); +MODULE_PARM_DESC(ipa_eth_noauto, + "Disable automatic offload initialization of interfaces"); + +static struct workqueue_struct *ipa_eth_wq; + +bool ipa_eth_is_ready(void) +{ + return test_bit(IPA_ETH_ST_READY, &ipa_eth_state); +} + +bool ipa_eth_all_ready(void) +{ + return ipa_eth_is_ready() && + test_bit(IPA_ETH_ST_API_READY, &ipa_eth_state); +} + +static inline bool present(struct ipa_eth_device *eth_dev) +{ + return + !test_bit(IPA_ETH_DEV_F_REMOVING, ð_dev->flags); +} + +static inline bool reachable(struct ipa_eth_device *eth_dev) +{ + return + present(eth_dev) && + !test_bit(IPA_ETH_DEV_F_RESETTING, ð_dev->flags); +} + +static inline bool offloadable(struct ipa_eth_device *eth_dev) +{ + return + ipa_eth_all_ready() && + reachable(eth_dev) && + !test_bit(IPA_ETH_DEV_F_UNPAIRING, ð_dev->flags); +} + +static inline bool initable(struct ipa_eth_device *eth_dev) +{ + return + offloadable(eth_dev) && + test_bit(IPA_ETH_IF_ST_UP, ð_dev->if_state) && + eth_dev->init; +} + +static inline bool startable(struct ipa_eth_device *eth_dev) +{ + return + initable(eth_dev) && + test_bit(IPA_ETH_IF_ST_LOWER_UP, ð_dev->if_state) && + eth_dev->start; +} + +static int ipa_eth_init_device(struct ipa_eth_device *eth_dev) +{ + int rc; + + if (eth_dev->of_state == IPA_ETH_OF_ST_INITED) + return 0; + + if (eth_dev->of_state != IPA_ETH_OF_ST_DEINITED) + return -EFAULT; + + rc = ipa_eth_ep_init_headers(eth_dev); + if (rc) { + ipa_eth_dev_err(eth_dev, "Failed to init EP headers"); + eth_dev->of_state = IPA_ETH_OF_ST_ERROR; + return rc; + } + + rc = ipa_eth_pm_register(eth_dev); + if (rc) { + ipa_eth_dev_err(eth_dev, "Failed to register with IPA PM"); + eth_dev->of_state = IPA_ETH_OF_ST_ERROR; + return rc; + } + + rc = ipa_eth_offload_init(eth_dev); + if (rc) { + ipa_eth_dev_err(eth_dev, "Failed to init offload"); + eth_dev->of_state = IPA_ETH_OF_ST_ERROR; + return rc; + } + + rc = ipa_eth_uc_stats_init(eth_dev); + if (rc) + ipa_eth_dev_err(eth_dev, + "Failed to init uC stats monitor, continuing."); + + ipa_eth_dev_log(eth_dev, "Initialized device"); + + eth_dev->of_state = IPA_ETH_OF_ST_INITED; + + return 0; +} + +static int ipa_eth_deinit_device(struct ipa_eth_device *eth_dev) +{ + int rc; + + if (eth_dev->of_state == IPA_ETH_OF_ST_DEINITED) + return 0; + + if (eth_dev->of_state != IPA_ETH_OF_ST_INITED) + return -EFAULT; + + rc = ipa_eth_uc_stats_deinit(eth_dev); + if (rc) + ipa_eth_dev_err(eth_dev, + "Failed to deinit uC stats monitor, continuing."); + + rc = ipa_eth_offload_deinit(eth_dev); + if (rc) { + ipa_eth_dev_err(eth_dev, "Failed to deinit offload"); + eth_dev->of_state = IPA_ETH_OF_ST_ERROR; + return rc; + } + + rc = ipa_eth_pm_unregister(eth_dev); + if (rc) { + ipa_eth_dev_err(eth_dev, "Failed to unregister with IPA PM"); + eth_dev->of_state = IPA_ETH_OF_ST_ERROR; + return rc; + } + + rc = ipa_eth_ep_deinit_headers(eth_dev); + if (rc) { + ipa_eth_dev_err(eth_dev, "Failed to deinit EP headers"); + eth_dev->of_state = IPA_ETH_OF_ST_ERROR; + return rc; + } + + ipa_eth_dev_log(eth_dev, "Deinitialized device"); + + eth_dev->of_state = IPA_ETH_OF_ST_DEINITED; + + return 0; +} + +static int ipa_eth_start_device(struct ipa_eth_device *eth_dev) +{ + int rc; + + if (eth_dev->of_state == IPA_ETH_OF_ST_STARTED) + return 0; + + if (eth_dev->of_state != IPA_ETH_OF_ST_INITED) + return -EFAULT; + + rc = ipa_eth_pm_activate(eth_dev); + if (rc) { + ipa_eth_dev_err(eth_dev, "Failed to activate device PM"); + eth_dev->of_state = IPA_ETH_OF_ST_ERROR; + return rc; + } + + rc = ipa_eth_offload_start(eth_dev); + if (rc) { + ipa_eth_dev_err(eth_dev, "Failed to start offload"); + eth_dev->of_state = IPA_ETH_OF_ST_ERROR; + return rc; + } + + rc = ipa_eth_bus_disable_pc(eth_dev); + if (rc) { + ipa_eth_dev_err(eth_dev, + "Failed to disable bus power collapse"); + eth_dev->of_state = IPA_ETH_OF_ST_ERROR; + return rc; + } + + /* We cannot register the interface during offload init phase because + * it will cause IPACM to install IPA filter rules when it receives a + * link-up netdev event, even if offload path is not started and/or no + * ECM_CONNECT event is received from the driver. Since installing IPA + * filter rules while offload path is stopped can cause DL data stall, + * register the interface only after the offload path is started. + */ + rc = ipa_eth_ep_register_interface(eth_dev); + if (rc) { + ipa_eth_dev_err(eth_dev, "Failed to register EP interface"); + eth_dev->of_state = IPA_ETH_OF_ST_ERROR; + return rc; + } + + rc = ipa_eth_uc_stats_start(eth_dev); + if (rc) + ipa_eth_dev_err(eth_dev, + "Failed to start uC stats monitor, continuing."); + + ipa_eth_dev_log(eth_dev, "Started device"); + + eth_dev->of_state = IPA_ETH_OF_ST_STARTED; + + return 0; +} + +static int ipa_eth_stop_device(struct ipa_eth_device *eth_dev) +{ + int rc; + + if (eth_dev->of_state == IPA_ETH_OF_ST_DEINITED) + return 0; + + if (eth_dev->of_state != IPA_ETH_OF_ST_STARTED) + return -EFAULT; + + rc = ipa_eth_uc_stats_stop(eth_dev); + if (rc) + ipa_eth_dev_err(eth_dev, + "Failed to stop uC stats monitor, continuing."); + + rc = ipa_eth_ep_unregister_interface(eth_dev); + if (rc) { + ipa_eth_dev_err(eth_dev, "Failed to unregister IPA interface"); + eth_dev->of_state = IPA_ETH_OF_ST_ERROR; + return rc; + } + + rc = ipa_eth_bus_enable_pc(eth_dev); + if (rc) { + ipa_eth_dev_err(eth_dev, + "Failed to enable bus power collapse"); + eth_dev->of_state = IPA_ETH_OF_ST_ERROR; + return rc; + } + + rc = ipa_eth_offload_stop(eth_dev); + if (rc) { + ipa_eth_dev_err(eth_dev, "Failed to stop offload"); + eth_dev->of_state = IPA_ETH_OF_ST_ERROR; + return rc; + } + + rc = ipa_eth_pm_deactivate(eth_dev); + if (rc) { + ipa_eth_dev_err(eth_dev, "Failed to deactivate device PM"); + eth_dev->of_state = IPA_ETH_OF_ST_ERROR; + return rc; + } + + ipa_eth_dev_log(eth_dev, "Stopped device"); + + eth_dev->of_state = IPA_ETH_OF_ST_INITED; + + return 0; +} + +static void ipa_eth_device_refresh(struct ipa_eth_device *eth_dev) +{ + ipa_eth_dev_log(eth_dev, "Refreshing offload state for device"); + + if (!ipa_eth_offload_device_paired(eth_dev)) { + ipa_eth_dev_log(eth_dev, "Device is not paired. Skipping."); + return; + } + + if (eth_dev->of_state == IPA_ETH_OF_ST_ERROR) { + ipa_eth_dev_err(eth_dev, + "Device in ERROR state, skipping refresh"); + return; + } + + if (initable(eth_dev)) { + if (eth_dev->of_state == IPA_ETH_OF_ST_DEINITED) { + IPA_ACTIVE_CLIENTS_INC_SIMPLE(); + (void) ipa_eth_init_device(eth_dev); + IPA_ACTIVE_CLIENTS_DEC_SIMPLE(); + + if (eth_dev->of_state != IPA_ETH_OF_ST_INITED) { + ipa_eth_dev_err(eth_dev, + "Failed to init device"); + return; + } + } + } + + if (startable(eth_dev)) { + IPA_ACTIVE_CLIENTS_INC_SIMPLE(); + (void) ipa_eth_start_device(eth_dev); + IPA_ACTIVE_CLIENTS_DEC_SIMPLE(); + + if (eth_dev->of_state != IPA_ETH_OF_ST_STARTED) { + ipa_eth_dev_err(eth_dev, "Failed to start device"); + return; + } + + if (ipa_eth_net_register_upper(eth_dev)) { + ipa_eth_dev_err(eth_dev, + "Failed to register upper interfaces"); + eth_dev->of_state = IPA_ETH_OF_ST_ERROR; + } + + if (ipa_eth_pm_vote_bw(eth_dev)) + ipa_eth_dev_err(eth_dev, + "Failed to vote for required BW"); + } else { + ipa_eth_dev_log(eth_dev, "Start is disallowed for the device"); + + if (ipa_eth_net_unregister_upper(eth_dev)) { + ipa_eth_dev_err(eth_dev, + "Failed to unregister upper interfaces"); + eth_dev->of_state = IPA_ETH_OF_ST_ERROR; + } + + if (eth_dev->of_state == IPA_ETH_OF_ST_STARTED) { + IPA_ACTIVE_CLIENTS_INC_SIMPLE(); + ipa_eth_stop_device(eth_dev); + IPA_ACTIVE_CLIENTS_DEC_SIMPLE(); + + if (eth_dev->of_state != IPA_ETH_OF_ST_INITED) { + ipa_eth_dev_err(eth_dev, + "Failed to stop device"); + return; + } + } + } + + if (!initable(eth_dev)) { + ipa_eth_dev_log(eth_dev, "Init is disallowed for the device"); + + IPA_ACTIVE_CLIENTS_INC_SIMPLE(); + ipa_eth_deinit_device(eth_dev); + IPA_ACTIVE_CLIENTS_DEC_SIMPLE(); + + if (eth_dev->of_state != IPA_ETH_OF_ST_DEINITED) { + ipa_eth_dev_err(eth_dev, "Failed to deinit device"); + return; + } + } +} + +static int ipa_eth_init_device_skip_ipa(struct ipa_eth_device *eth_dev) +{ + int rc; + + if (eth_dev->of_state == IPA_ETH_OF_ST_INITED) + return 0; + + if (eth_dev->of_state != IPA_ETH_OF_ST_DEINITED) + return -EFAULT; + + rc = ipa_eth_offload_init(eth_dev); + if (rc) { + ipa_eth_dev_err(eth_dev, "Failed to init offload"); + eth_dev->of_state = IPA_ETH_OF_ST_ERROR; + return rc; + } + + ipa_eth_dev_log(eth_dev, "Initialized device"); + + eth_dev->of_state = IPA_ETH_OF_ST_INITED; + + return 0; +} + +static int ipa_eth_deinit_device_skip_ipa(struct ipa_eth_device *eth_dev) +{ + int rc; + + if (eth_dev->of_state == IPA_ETH_OF_ST_DEINITED) + return 0; + + if (eth_dev->of_state != IPA_ETH_OF_ST_INITED) + return -EFAULT; + + rc = ipa_eth_offload_deinit(eth_dev); + if (rc) { + ipa_eth_dev_err(eth_dev, "Failed to deinit offload"); + eth_dev->of_state = IPA_ETH_OF_ST_ERROR; + return rc; + } + + ipa_eth_dev_log(eth_dev, "Deinitialized device"); + + eth_dev->of_state = IPA_ETH_OF_ST_DEINITED; + + return 0; +} + +static int ipa_eth_start_device_skip_ipa(struct ipa_eth_device *eth_dev) +{ + int rc; + + if (eth_dev->of_state == IPA_ETH_OF_ST_STARTED) + return 0; + + if (eth_dev->of_state != IPA_ETH_OF_ST_INITED) + return -EFAULT; + + rc = ipa_eth_offload_start(eth_dev); + if (rc) { + ipa_eth_dev_err(eth_dev, "Failed to start offload"); + eth_dev->of_state = IPA_ETH_OF_ST_ERROR; + return rc; + } + + ipa_eth_dev_log(eth_dev, "Started device"); + + eth_dev->of_state = IPA_ETH_OF_ST_STARTED; + + return 0; +} + +static int ipa_eth_stop_device_skip_ipa(struct ipa_eth_device *eth_dev) +{ + int rc; + + if (eth_dev->of_state == IPA_ETH_OF_ST_DEINITED) + return 0; + + if (eth_dev->of_state != IPA_ETH_OF_ST_STARTED) + return -EFAULT; + + rc = ipa_eth_offload_stop(eth_dev); + if (rc) { + ipa_eth_dev_err(eth_dev, "Failed to stop offload"); + eth_dev->of_state = IPA_ETH_OF_ST_ERROR; + return rc; + } + + ipa_eth_dev_log(eth_dev, "Stopped device"); + + eth_dev->of_state = IPA_ETH_OF_ST_INITED; + + return 0; +} + +static void ipa_eth_device_refresh_skip_ipa(struct ipa_eth_device *eth_dev) +{ + ipa_eth_dev_log(eth_dev, "Refreshing offload state for device"); + + if (!ipa_eth_offload_device_paired(eth_dev)) { + ipa_eth_dev_log(eth_dev, "Device is not paired. Skipping."); + return; + } + + if (eth_dev->of_state == IPA_ETH_OF_ST_ERROR) { + ipa_eth_dev_err(eth_dev, + "Device in ERROR state, skipping refresh"); + return; + } + + if (initable(eth_dev)) { + if (eth_dev->of_state == IPA_ETH_OF_ST_DEINITED) { + (void) ipa_eth_init_device_skip_ipa(eth_dev); + + if (eth_dev->of_state != IPA_ETH_OF_ST_INITED) { + ipa_eth_dev_err(eth_dev, + "Failed to init device"); + return; + } + } + } + + if (startable(eth_dev)) { + (void) ipa_eth_start_device_skip_ipa(eth_dev); + + if (eth_dev->of_state != IPA_ETH_OF_ST_STARTED) { + ipa_eth_dev_err(eth_dev, "Failed to start device"); + return; + } + } else { + ipa_eth_dev_log(eth_dev, "Start is disallowed for the device"); + + if (eth_dev->of_state == IPA_ETH_OF_ST_STARTED) { + ipa_eth_stop_device_skip_ipa(eth_dev); + + if (eth_dev->of_state != IPA_ETH_OF_ST_INITED) { + ipa_eth_dev_err(eth_dev, + "Failed to stop device"); + return; + } + } + } + + if (!initable(eth_dev)) { + ipa_eth_dev_log(eth_dev, "Init is disallowed for the device"); + + ipa_eth_deinit_device_skip_ipa(eth_dev); + + if (eth_dev->of_state != IPA_ETH_OF_ST_DEINITED) { + ipa_eth_dev_err(eth_dev, "Failed to deinit device"); + return; + } + } +} + +static void ipa_eth_device_refresh_work(struct work_struct *work) +{ + struct ipa_eth_device *eth_dev = container_of(work, + struct ipa_eth_device, refresh); + + if (unlikely(eth_dev->skip_ipa)) + ipa_eth_device_refresh_skip_ipa(eth_dev); + else + ipa_eth_device_refresh(eth_dev); +} + +void ipa_eth_device_refresh_sched(struct ipa_eth_device *eth_dev) +{ + queue_work(ipa_eth_wq, ð_dev->refresh); +} + +void ipa_eth_device_refresh_sync(struct ipa_eth_device *eth_dev) +{ + ipa_eth_device_refresh_sched(eth_dev); + flush_work(ð_dev->refresh); +} + +static void ipa_eth_global_refresh_work(struct work_struct *work) +{ + struct ipa_eth_device *eth_dev; + + ipa_eth_log("Performing global refresh"); + + mutex_lock(&ipa_eth_devices_lock); + + if (ipa_eth_all_ready()) { + list_for_each_entry(eth_dev, &ipa_eth_devices, device_list) { + ipa_eth_device_refresh_sched(eth_dev); + } + } + + mutex_unlock(&ipa_eth_devices_lock); +} + +static DECLARE_WORK(ipa_eth_global_refresh, ipa_eth_global_refresh_work); + +void ipa_eth_global_refresh_sched(void) +{ + queue_work(ipa_eth_wq, &ipa_eth_global_refresh); +} + +static void ipa_eth_global_refresh_sync(void) +{ + ipa_eth_global_refresh_sched(); + flush_workqueue(ipa_eth_wq); +} + +static int ipa_eth_device_prepare_reset( + struct ipa_eth_device *eth_dev, void *data) +{ + int rc = 0; + + /* Set the bit so that any in-progress operation can possibly + * return early. + */ + set_bit(IPA_ETH_DEV_F_RESETTING, ð_dev->flags); + + IPA_ACTIVE_CLIENTS_INC_SIMPLE(); + rc = ipa_eth_offload_prepare_reset(eth_dev, data); + IPA_ACTIVE_CLIENTS_DEC_SIMPLE(); + + return rc; +} + +static int ipa_eth_device_complete_reset( + struct ipa_eth_device *eth_dev, void *data) +{ + int rc = 0; + + IPA_ACTIVE_CLIENTS_INC_SIMPLE(); + rc = ipa_eth_offload_complete_reset(eth_dev, data); + IPA_ACTIVE_CLIENTS_DEC_SIMPLE(); + + /* Clear the flag before unlocking the mutex so that blocked threads + * can resume with the updated value. + */ + clear_bit(IPA_ETH_DEV_F_RESETTING, ð_dev->flags); + + return rc; +} + +static int ipa_eth_device_event_add_macsec( + struct ipa_eth_device *eth_dev, void *data) +{ + struct net_device *net_dev = data; + + ipa_eth_dev_log(eth_dev, + "Network device added MACSec interface %s", net_dev->name); + + return 0; +} + +static int ipa_eth_device_event_del_macsec( + struct ipa_eth_device *eth_dev, void *data) +{ + struct net_device *net_dev = data; + + ipa_eth_dev_log(eth_dev, + "Network device removed MACSec interface %s", net_dev->name); + + return 0; +} + +/** + * ipa_eth_device_notify() - Notifies a device event to the offload sub-system + * @eth_dev: Device for which the event is generated + * @event: Device event + * @data: Event specific data, if required + * + * Return: 0 on success, non-zero otherwise + */ +int ipa_eth_device_notify(struct ipa_eth_device *eth_dev, + enum ipa_eth_device_event event, void *data) +{ + int rc = -EINVAL; + + ipa_eth_dev_log(eth_dev, + "Received notificaiton %s", ipa_eth_device_event_name(event)); + + switch (event) { + case IPA_ETH_DEV_RESET_PREPARE: + rc = ipa_eth_device_prepare_reset(eth_dev, data); + break; + case IPA_ETH_DEV_RESET_COMPLETE: + rc = ipa_eth_device_complete_reset(eth_dev, data); + break; + case IPA_ETH_DEV_ADD_MACSEC_IF: + rc = ipa_eth_device_event_add_macsec(eth_dev, data); + break; + case IPA_ETH_DEV_DEL_MACSEC_IF: + rc = ipa_eth_device_event_del_macsec(eth_dev, data); + break; + default: + ipa_eth_dev_log(eth_dev, "Unknown event %d", event); + break; + } + + if (rc) + ipa_eth_dev_err(eth_dev, "Failed to handle notification"); + + return rc; +} +EXPORT_SYMBOL(ipa_eth_device_notify); + +static void ipa_eth_dev_start_timer_cb(unsigned long data) +{ + struct ipa_eth_device *eth_dev = (struct ipa_eth_device *)data; + + ipa_eth_dev_log(eth_dev, "Start timer has fired"); + + /* Do not start offload if user disabled timer in between */ + if (present(eth_dev) && eth_dev->start_on_timeout) { + eth_dev->start = true; + + ipa_eth_device_refresh_sched(eth_dev); + } +} + +static int ipa_eth_panic_notifier(struct notifier_block *nb, + unsigned long event, void *ptr) +{ + struct ipa_eth_device_private *ipa_priv = container_of(nb, + struct ipa_eth_device_private, panic_nb); + struct ipa_eth_device *eth_dev = ipa_priv->eth_dev; + + if (!reachable(eth_dev)) + return -ENODEV; + + ipa_eth_net_save_regs(eth_dev); + ipa_eth_offload_save_regs(eth_dev); + + return NOTIFY_DONE; +} + +/* During a system suspend, suspend-prepare callbacks are called first by the + * PM framework before freezing processes. This gives us an early opportunity + * to abort the suspend and reduces the chances for device resumes at a later + * stage. + */ +static int ipa_eth_pm_notifier_event_suspend_prepare( + struct ipa_eth_device *eth_dev) +{ + /* We look for any ethernet rx activity since previous attempt to + * suspend, and if such an activity is found, we hold a wake lock + * for WAKE_TIME_MS time. Any Rx packets received beyond this point + * should cause a wake up due to the Rx interrupt. In rare cases + * where Rx interrupt was already processed and NAPI poll is yet to + * complete, we perform a second check in the suspend late handler + * and reverts the device suspension by aborting the system suspend. + */ + if (ipa_eth_net_check_active(eth_dev)) { + pr_info("%s: %s is active, preventing suspend for %u ms\n", + IPA_ETH_SUBSYS, eth_dev->net_dev->name, + IPA_ETH_WAKE_TIME_MS); + pm_wakeup_dev_event(eth_dev->dev, IPA_ETH_WAKE_TIME_MS, false); + return NOTIFY_BAD; + } + + return NOTIFY_OK; +} + +static int ipa_eth_pm_notifier_cb(struct notifier_block *nb, + unsigned long pm_event, void *unused) +{ + struct ipa_eth_device_private *ipa_priv = container_of(nb, + struct ipa_eth_device_private, pm_nb); + struct ipa_eth_device *eth_dev = ipa_priv->eth_dev; + + ipa_eth_dbg("PM notifier called for event %s (0x%04lx)", + ipa_eth_pm_notifier_event_name(pm_event), pm_event); + + switch (pm_event) { + case PM_SUSPEND_PREPARE: + return ipa_eth_pm_notifier_event_suspend_prepare(eth_dev); + default: + break; + } + + return NOTIFY_DONE; +} + +/* + * ipa_eth_alloc_device() - Allocate an ipa_eth_device structure and initialize + * all common fields + * @dev: struct device pointer of the net device + * @nd: Offload sub-system net driver structure + * + * This API is meant to be called by the bus layer (ex. ipa_eth_pci.c) to + * allocate an ipa_eth_device object, initialize it with bus specific values + * and register the discovered device using ipa_eth_register_device() API. + * + * Return: 0 on success, non-zero otherwise + */ +struct ipa_eth_device *ipa_eth_alloc_device( + struct device *dev, + struct ipa_eth_net_driver *nd) +{ + struct ipa_eth_device *eth_dev; + struct ipa_eth_device_private *ipa_priv; + + if (!dev || !nd) { + ipa_eth_err("Invalid device or net driver"); + return NULL; + } + + eth_dev = kzalloc(sizeof(*eth_dev), GFP_KERNEL); + if (!eth_dev) + return NULL; + + ipa_priv = kzalloc(sizeof(*ipa_priv), GFP_KERNEL); + if (!ipa_priv) { + kfree(eth_dev); + return NULL; + } + + eth_dev->dev = dev; + eth_dev->nd = nd; + + /* Network/offload driver supports direct call to IPA driver. Skip IPA + * driver calls for the device. + */ + if (nd->features & IPA_ETH_DEV_F_IPA_API) { + ipa_eth_dev_log(eth_dev, "Device requests for skipping IPA"); + eth_dev->skip_ipa = true; + } + + eth_dev->of_state = IPA_ETH_OF_ST_DEINITED; + eth_dev->pm_handle = IPA_PM_MAX_CLIENTS; + INIT_WORK(ð_dev->refresh, ipa_eth_device_refresh_work); + + INIT_LIST_HEAD(ð_dev->rx_channels); + INIT_LIST_HEAD(ð_dev->tx_channels); + + init_timer(ð_dev->start_timer); + + eth_dev->start_timer.function = ipa_eth_dev_start_timer_cb; + eth_dev->start_timer.data = (unsigned long)eth_dev; + + eth_dev->init = eth_dev->start = !ipa_eth_noauto; + + /* Initialize private data */ + + ipa_priv->eth_dev = eth_dev; + + INIT_LIST_HEAD(&ipa_priv->upper_devices); + + ipa_priv->pm_nb.notifier_call = ipa_eth_pm_notifier_cb; + ipa_priv->panic_nb.notifier_call = ipa_eth_panic_notifier; + + eth_dev->ipa_priv = ipa_priv; + + return eth_dev; +} + +/* + * ipa_eth_free_device() - Free an ipa_eth_device object previously allocated + * using ipa_eth_alloc_device() + * @eth_dev: ipa_eth_device object to be freed + */ +void ipa_eth_free_device(struct ipa_eth_device *eth_dev) +{ + kzfree(eth_dev->ipa_priv); + kzfree(eth_dev); +} + +/* + * ipa_eth_register_device() - Register a new device with offload sub-system + * @eth_dev: Eth device to register + * + * This API is meant to be called by the bus layer (ex. ipa_eth_pci.c) to + * register an ipa_eth_device object linked to a newly discovered device. The + * @eth_dev object is expected to be allocated using ipa_eth_alloc_device() API. + * + * Return: 0 on success, non-zero otherwise + */ +int ipa_eth_register_device(struct ipa_eth_device *eth_dev) +{ + int rc; + struct ipa_eth_device_private *ipa_priv = eth_dev->ipa_priv; + + ipa_eth_dev_log(eth_dev, "Registering new device"); + + (void) atomic_notifier_chain_register( + &panic_notifier_list, &ipa_priv->panic_nb); + + rc = ipa_eth_net_open_device(eth_dev); + if (rc) { + ipa_eth_dev_err(eth_dev, "Failed to open network device"); + return rc; + } + + /* Add new device to the devices list before pairing with offload + * driver in order to synchronize with offload driver deregistration. + */ + mutex_lock(&ipa_eth_devices_lock); + list_add(ð_dev->device_list, &ipa_eth_devices); + mutex_unlock(&ipa_eth_devices_lock); + + (void) register_pm_notifier(&ipa_priv->pm_nb); + + ipa_eth_dev_log(eth_dev, "Registered new device"); + + rc = ipa_eth_offload_pair_device(eth_dev); + if (rc) + ipa_eth_dev_log(eth_dev, "Failed to pair device. Deferring."); + + ipa_eth_debugfs_add_device(eth_dev); + + return 0; +} + +/* + * ipa_eth_unpair_device() - Unpair a device from its offload driver + * @eth_dev: Device to unpair + * + * It is safe to call this function in parallel from ipa_eth_unregister_device() + * and ipa_eth_unregister_offload_driver() as the necessary locking is placed + * in ipa_eth_offload_unpair_device(). + * + * Return: 0 on success, non-zero otherwise + */ + +static void ipa_eth_unpair_device(struct ipa_eth_device *eth_dev) +{ + ipa_eth_dev_log(eth_dev, "Unpairing device"); + + /* Set UNPAIRING flag to prevent offload init or start */ + set_bit(IPA_ETH_DEV_F_UNPAIRING, ð_dev->flags); + + /* Wait for next refresh to tear down offload path */ + ipa_eth_device_refresh_sync(eth_dev); + + ipa_eth_offload_unpair_device(eth_dev); + + clear_bit(IPA_ETH_DEV_F_UNPAIRING, ð_dev->flags); + + ipa_eth_dev_log(eth_dev, "Unpaired device"); +} + +/* + * ipa_eth_unregister_device() - Unregister a device from the offload sub-system + * + * This API is meant to be called by the bus layer when a device is removed from + * the system. The @eth_dev object itself need to be freed separately by calling + * the ipa_eth_free_device() API. + */ +void ipa_eth_unregister_device(struct ipa_eth_device *eth_dev) +{ + struct ipa_eth_device_private *ipa_priv = eth_dev->ipa_priv; + + ipa_eth_dev_log(eth_dev, "Unregistering device"); + + /* Set REMOVING flag so that device refreshes do not happen */ + set_bit(IPA_ETH_DEV_F_REMOVING, ð_dev->flags); + + /* Remove debugfs node to prevent any new 'start_timer' to + * be started. + */ + ipa_eth_debugfs_remove_device(eth_dev); + + del_timer_sync(ð_dev->start_timer); + + /* Unpair the device before removing from devices list so that + * unregister_offload_driver() does not skip this device. + */ + ipa_eth_unpair_device(eth_dev); + + (void) unregister_pm_notifier(&ipa_priv->pm_nb); + + /* Remove from devices list so that no new global refreshes are + * scheduled. + */ + mutex_lock(&ipa_eth_devices_lock); + list_del(ð_dev->device_list); + mutex_unlock(&ipa_eth_devices_lock); + + /* Closing device from ipa_eth_net should prevent further events to be + * received from either network driver via ipa_eth_device_notify() or + * the network sub-system via netdevice notifier. + */ + ipa_eth_net_close_device(eth_dev); + + /* No more refresh work would be queued beyond this point. Flush out + * any pending refresh work items. + */ + cancel_work_sync(ð_dev->refresh); + + /* By this time we would have processed all pending events for the + * device and removed/disabled all event sources. + */ + clear_bit(IPA_ETH_DEV_F_REMOVING, ð_dev->flags); + + (void) atomic_notifier_chain_unregister( + &panic_notifier_list, &ipa_priv->panic_nb); + + ipa_eth_dev_log(eth_dev, "Device unregistered"); +} + +/** + * ipa_eth_register_net_driver() - Register a network driver with the offload + * subsystem + * @nd: Network driver to register + * + * Return: 0 on success, negative errno otherwise + */ +int ipa_eth_register_net_driver(struct ipa_eth_net_driver *nd) +{ + return ipa_eth_net_register_driver(nd); +} +EXPORT_SYMBOL(ipa_eth_register_net_driver); + +/** + * ipa_eth_unregister_net_driver() - Unregister a network driver + * @nd: Network driver to unregister + */ +void ipa_eth_unregister_net_driver(struct ipa_eth_net_driver *nd) +{ + ipa_eth_net_unregister_driver(nd); +} +EXPORT_SYMBOL(ipa_eth_unregister_net_driver); + +/** + * ipa_eth_register_offload_driver - Register an offload driver with the offload + * subsystem + * @nd: Offload driver to register + * + * Return: 0 on success, negative errno otherwise + */ +int ipa_eth_register_offload_driver(struct ipa_eth_offload_driver *od) +{ + int rc; + struct ipa_eth_device *eth_dev; + + rc = ipa_eth_offload_register_driver(od); + if (rc) { + ipa_eth_err("Failed to register offload driver %s", od->name); + return rc; + } + + ipa_eth_log("Registered offload driver %s", od->name); + + mutex_lock(&ipa_eth_devices_lock); + + list_for_each_entry(eth_dev, &ipa_eth_devices, device_list) { + if (!ipa_eth_offload_pair_device(eth_dev)) + ipa_eth_device_refresh_sched(eth_dev); + } + + mutex_unlock(&ipa_eth_devices_lock); + + return 0; +} +EXPORT_SYMBOL(ipa_eth_register_offload_driver); + +/** + * ipa_eth_unregister_offload_driver() - Unregister an offload driver + * @nd: Offload driver to unregister + */ +void ipa_eth_unregister_offload_driver(struct ipa_eth_offload_driver *od) +{ + struct ipa_eth_device *eth_dev; + + /* Unregister offload driver first from ipa_eth_offload so that any + * new device registration will not be able to pair with the driver. + */ + ipa_eth_offload_unregister_driver(od); + + mutex_lock(&ipa_eth_devices_lock); + + list_for_each_entry(eth_dev, &ipa_eth_devices, device_list) + if (eth_dev->od == od) + ipa_eth_unpair_device(eth_dev); + + mutex_unlock(&ipa_eth_devices_lock); +} +EXPORT_SYMBOL(ipa_eth_unregister_offload_driver); + +static void ipa_eth_api_ready_cb(void *user_data) +{ + ipa_eth_log("IPA API is ready"); + + set_bit(IPA_ETH_ST_API_READY, &ipa_eth_state); + ipa_eth_global_refresh_sched(); +} + +static struct ipa_eth_ready eth_api_rdy = { + .notify = ipa_eth_api_ready_cb, +}; + +int ipa_eth_init(void) +{ + int rc; + unsigned int wq_flags = WQ_UNBOUND | WQ_MEM_RECLAIM; + + /* Freeze the workqueue so that a refresh will not happen while the + * device is suspended as the suspend operation itself can generate + * Netlink events. + */ + wq_flags |= WQ_FREEZABLE; + + rc = ipa_eth_ipc_log_init(); + if (rc) { + ipa_eth_err("Failed to initialize IPC logging"); + goto err_ipclog; + } + + ipa_eth_dbg("Initializing IPA Ethernet Offload Sub-System"); + + ipa_eth_wq = alloc_workqueue("ipa_eth", wq_flags, 0); + if (!ipa_eth_wq) { + ipa_eth_err("Failed to alloc workqueue"); + goto err_wq; + } + + rc = ipa_eth_bus_modinit(); + if (rc) { + ipa_eth_err("Failed to initialize bus"); + goto err_bus; + } + + rc = ipa_eth_debugfs_init(); + if (rc) { + ipa_eth_err("Failed to initialize debugfs"); + goto err_dbgfs; + } + + rc = ipa_eth_register_ready_cb(ð_api_rdy); + if (rc) { + ipa_eth_err("Failed to register ready cb with IPA driver"); + goto err_reg_rdy; + } + + set_bit(IPA_ETH_ST_READY, &ipa_eth_state); + + ipa_eth_log("Offload sub-system init is complete"); + + ipa_eth_global_refresh_sched(); + + return 0; + +err_reg_rdy: + ipa_eth_debugfs_cleanup(); +err_dbgfs: + ipa_eth_bus_modexit(); +err_bus: + destroy_workqueue(ipa_eth_wq); + ipa_eth_wq = NULL; +err_wq: + ipa_eth_ipc_log_cleanup(); +err_ipclog: + return rc; +} + +void ipa_eth_exit(void) +{ + ipa_eth_dbg("De-initializing IPA Ethernet Offload Sub-System"); + + /* Clear READY bit so that all active offload paths can start + * deinitializing. + */ + clear_bit(IPA_ETH_ST_READY, &ipa_eth_state); + + ipa_eth_unregister_ready_cb(ð_api_rdy); + + ipa_eth_debugfs_cleanup(); + + /* Wait for all offload paths to deinit. Although the chances for any + * such path to exist is quite low when IPA is a platform device, we + * still need to make sure we wait here before destroying the work + * queue. + */ + ipa_eth_global_refresh_sync(); + + ipa_eth_bus_modexit(); + + destroy_workqueue(ipa_eth_wq); + ipa_eth_wq = NULL; + + ipa_eth_ipc_log_cleanup(); +} diff --git a/drivers/platform/msm/ethernet/ipa_eth_bus.c b/drivers/platform/msm/ethernet/ipa_eth_bus.c new file mode 100644 index 000000000000..fa95f8f21d88 --- /dev/null +++ b/drivers/platform/msm/ethernet/ipa_eth_bus.c @@ -0,0 +1,136 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2019-2020, The Linux Foundation. All rights reserved + */ + +#include + +#include "ipa_eth_i.h" + +static bool ipa_eth_bus_is_ready; + +struct ipa_eth_bus_map { + struct bus_type *bus; + struct ipa_eth_bus *eth_bus; + int (*modinit)(void); + void (*modexit)(void); +} static const bus_map[] = { + { + &pci_bus_type, + &ipa_eth_pci_bus, + &ipa_eth_pci_modinit, + &ipa_eth_pci_modexit + }, + {}, +}; + +static struct ipa_eth_bus *lookup_eth_bus(struct bus_type *bus) +{ + const struct ipa_eth_bus_map *map; + + for (map = bus_map; map->bus != NULL; map++) { + if (map->bus == bus) + return map->eth_bus; + } + + return NULL; +} + +int ipa_eth_bus_register_driver(struct ipa_eth_net_driver *nd) +{ + struct ipa_eth_bus *eth_bus; + + eth_bus = lookup_eth_bus(nd->bus); + if (!eth_bus) { + ipa_eth_err("Unsupported bus %s", nd->bus->name); + return -EOPNOTSUPP; + } + + return eth_bus->register_driver(nd); +} + +void ipa_eth_bus_unregister_driver(struct ipa_eth_net_driver *nd) +{ + struct ipa_eth_bus *eth_bus = lookup_eth_bus(nd->bus); + + if (!eth_bus) { + ipa_eth_bug("Failed to lookup eth_bus for %s", nd->bus->name); + return; + } + + eth_bus->unregister_driver(nd); +} + +int ipa_eth_bus_enable_pc(struct ipa_eth_device *eth_dev) +{ + struct ipa_eth_net_driver *nd = eth_dev->nd; + struct ipa_eth_bus *eth_bus = lookup_eth_bus(nd->bus); + + if (!eth_bus) { + ipa_eth_dev_bug(eth_dev, "Failed to lookup eth_bus"); + return -EFAULT; + } + + if (eth_bus->enable_pc) + return eth_bus->enable_pc(eth_dev); + + return -EFAULT; +} + +int ipa_eth_bus_disable_pc(struct ipa_eth_device *eth_dev) +{ + struct ipa_eth_net_driver *nd = eth_dev->nd; + struct ipa_eth_bus *eth_bus = lookup_eth_bus(nd->bus); + + if (!eth_bus) { + ipa_eth_dev_bug(eth_dev, "Failed to lookup eth_bus"); + return -EFAULT; + } + + if (eth_bus->disable_pc) + return eth_bus->disable_pc(eth_dev); + + return -EFAULT; +} + +int ipa_eth_bus_modinit(void) +{ + int rc; + const struct ipa_eth_bus_map *map; + + /* initialize all registered busses */ + for (rc = 0, map = bus_map; map->bus != NULL; map++) + rc |= map->modinit(); + + if (rc) { + ipa_eth_err("Failed to initialize one or more busses"); + goto err_init; + } + + ipa_eth_bus_is_ready = true; + + ipa_eth_log("Offload sub-system bus module init is complete"); + + return 0; + +err_init: + for (map = bus_map; map->bus != NULL; map++) + map->modexit(); + + return rc; +} + +void ipa_eth_bus_modexit(void) +{ + const struct ipa_eth_bus_map *map; + + ipa_eth_log("De-initing offload sub-system bus module"); + + if (!ipa_eth_bus_is_ready) + return; + + ipa_eth_bus_is_ready = false; + + for (map = bus_map; map->bus != NULL; map++) + map->modexit(); +} diff --git a/drivers/platform/msm/ethernet/ipa_eth_debugfs.c b/drivers/platform/msm/ethernet/ipa_eth_debugfs.c new file mode 100644 index 000000000000..9e3f65a45353 --- /dev/null +++ b/drivers/platform/msm/ethernet/ipa_eth_debugfs.c @@ -0,0 +1,307 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2019-2020, The Linux Foundation. All rights reserved + */ + +#include + +#include "ipa_eth_debugfs.h" + +static struct dentry *ipa_eth_debugfs; + +static struct dentry *ipa_eth_devices_debugfs; +static struct dentry *ipa_eth_drivers_debugfs; + +static struct dentry *ipa_eth_bus_debugfs; +static struct dentry *ipa_eth_pci_debugfs; +static struct dentry *ipa_eth_offload_debugfs; + +static ssize_t ipa_eth_dev_write_init(struct file *file, + const char __user *user_buf, + size_t count, loff_t *ppos) +{ + ssize_t ret = debugfs_write_file_bool(file, user_buf, count, ppos); + struct ipa_eth_device *eth_dev = container_of(file->private_data, + struct ipa_eth_device, + init); + + ipa_eth_device_refresh_sync(eth_dev); + + return ret; +} + +static const struct file_operations fops_eth_dev_init = { + .read = debugfs_read_file_bool, + .write = ipa_eth_dev_write_init, + .open = simple_open, + .llseek = default_llseek, +}; + +static ssize_t ipa_eth_dev_write_start(struct file *file, + const char __user *user_buf, + size_t count, loff_t *ppos) +{ + ssize_t ret = debugfs_write_file_bool(file, user_buf, count, ppos); + struct ipa_eth_device *eth_dev = container_of(file->private_data, + struct ipa_eth_device, + start); + + /* Set/reset timer to automatically start offload after the timeout + * specified in eth_dev->start_on_timeout (milliseconds) expires. + */ + if (!eth_dev->start && eth_dev->start_on_timeout) + mod_timer(ð_dev->start_timer, + jiffies + msecs_to_jiffies(eth_dev->start_on_timeout)); + + ipa_eth_device_refresh_sync(eth_dev); + + return ret; +} + +static const struct file_operations fops_eth_dev_start = { + .read = debugfs_read_file_bool, + .write = ipa_eth_dev_write_start, + .open = simple_open, + .llseek = default_llseek, +}; + +static ssize_t eth_dev_stats_print_one(char *buf, const size_t size, + const char *dir, const char *link, + struct ipa_eth_offload_link_stats *stats) +{ + return scnprintf(buf, size, + "%10s%10s%10s%10llu%10llu%10llu%10llu\n", + dir, link, (stats->valid ? "yes" : "no"), + stats->events, stats->frames, + stats->packets, stats->octets); +} + +static ssize_t eth_dev_stats_print(char *buf, const size_t size, + struct ipa_eth_device *eth_dev) +{ + ssize_t n = 0; + struct ipa_eth_offload_stats stats; + + if (!eth_dev->od->ops->get_stats) + return scnprintf(buf, size - n, "Not supported\n"); + + memset(&stats, 0, sizeof(stats)); + + if (eth_dev->od->ops->get_stats(eth_dev, &stats)) + return scnprintf(buf, size - n, "Operation failed\n"); + + n += scnprintf(&buf[n], size - n, + "%10s%10s%10s%10s%10s%10s%10s\n", + "Dir", "Link", "Valid", + "Events", "Frames", "Packets", "Octets"); + + n += eth_dev_stats_print_one(&buf[n], size - n, + "rx", "ndev", &stats.rx.ndev); + n += eth_dev_stats_print_one(&buf[n], size - n, + "rx", "host", &stats.rx.host); + n += eth_dev_stats_print_one(&buf[n], size - n, + "rx", "uc", &stats.rx.uc); + n += eth_dev_stats_print_one(&buf[n], size - n, + "rx", "gsi", &stats.rx.gsi); + n += eth_dev_stats_print_one(&buf[n], size - n, + "rx", "ipa", &stats.rx.ipa); + + n += scnprintf(&buf[n], size - n, "\n"); + + n += eth_dev_stats_print_one(&buf[n], size - n, + "tx", "ndev", &stats.tx.ndev); + n += eth_dev_stats_print_one(&buf[n], size - n, + "tx", "host", &stats.tx.host); + n += eth_dev_stats_print_one(&buf[n], size - n, + "tx", "uc", &stats.tx.uc); + n += eth_dev_stats_print_one(&buf[n], size - n, + "tx", "gsi", &stats.tx.gsi); + n += eth_dev_stats_print_one(&buf[n], size - n, + "tx", "ipa", &stats.tx.ipa); + + return n; +} + +static ssize_t eth_dev_stats_read(struct file *file, char __user *user_buf, + size_t count, loff_t *ppos) +{ + ssize_t n = 0, size = 2048; + char *buf = NULL; + struct ipa_eth_device *eth_dev = file->private_data; + + buf = kzalloc(size, GFP_KERNEL); + if (buf == NULL) + return 0; + + n = eth_dev_stats_print(buf, size, eth_dev); + n = simple_read_from_buffer(user_buf, count, ppos, buf, n); + + kfree(buf); + + return n; +} + +static const struct file_operations fops_eth_dev_stats = { + .read = eth_dev_stats_read, + .open = simple_open, + .llseek = default_llseek, +}; + +int ipa_eth_debugfs_add_device(struct ipa_eth_device *eth_dev) +{ + eth_dev->debugfs = debugfs_create_dir(eth_dev->net_dev->name, + ipa_eth_devices_debugfs); + if (IS_ERR_OR_NULL(eth_dev->debugfs)) { + ipa_eth_dev_err(eth_dev, "Failed to create debugfs root"); + return -EFAULT; + } + + debugfs_create_file("init", 0644, eth_dev->debugfs, ð_dev->init, + &fops_eth_dev_init); + + debugfs_create_file("start", 0644, eth_dev->debugfs, ð_dev->start, + &fops_eth_dev_start); + + debugfs_create_bool("start_on_wakeup", 0644, + eth_dev->debugfs, ð_dev->start_on_wakeup); + + debugfs_create_bool("start_on_resume", 0644, + eth_dev->debugfs, ð_dev->start_on_resume); + + debugfs_create_u32("start_on_timeout", 0644, eth_dev->debugfs, + ð_dev->start_on_timeout); + + debugfs_create_file("stats", 0644, eth_dev->debugfs, eth_dev, + &fops_eth_dev_stats); + + return 0; +} + +void ipa_eth_debugfs_remove_device(struct ipa_eth_device *eth_dev) +{ + debugfs_remove_recursive(eth_dev->debugfs); + eth_dev->debugfs = NULL; +} + +int ipa_eth_debugfs_add_offload_driver(struct ipa_eth_offload_driver *od) +{ + if (!od->debugfs && ipa_eth_offload_debugfs) { + od->debugfs = + debugfs_create_dir(od->name, ipa_eth_offload_debugfs); + if (!IS_ERR_OR_NULL(od->debugfs)) + return 0; + } + + return -EFAULT; +} + +void ipa_eth_debugfs_remove_offload_driver(struct ipa_eth_offload_driver *od) +{ + debugfs_remove_recursive(od->debugfs); + od->debugfs = NULL; +} + +static ssize_t eth_dev_ready_read(struct file *file, char __user *user_buf, + size_t count, loff_t *ppos) +{ + char *buf; + ssize_t n = 0, size = 128; + + buf = kzalloc(size, GFP_KERNEL); + if (buf == NULL) + return 0; + + n += scnprintf(&buf[n], size - n, "Offload Sub-system: %s\n", + test_bit(IPA_ETH_ST_READY, &ipa_eth_state) ? + "Ready" : "Not Ready"); + + n += scnprintf(&buf[n], size - n, "IPA API: %s\n", + test_bit(IPA_ETH_ST_API_READY, &ipa_eth_state) ? + "Ready" : "Not Ready"); + + n += scnprintf(&buf[n], size - n, "ALL: %s\n", + ipa_eth_all_ready() ? "Ready" : "Not Ready"); + + n = simple_read_from_buffer(user_buf, count, ppos, buf, n); + + kfree(buf); + + return n; +} + +static const struct file_operations fops_eth_dev_ready = { + .read = eth_dev_ready_read, + .open = simple_open, + .llseek = default_llseek, +}; + +int ipa_eth_debugfs_init(void) +{ + struct dentry *ipa_debugfs = ipa_debugfs_get_root(); + + if (IS_ERR_OR_NULL(ipa_debugfs)) + return -EFAULT; + + ipa_eth_debugfs = + debugfs_create_dir("ethernet", ipa_debugfs); + if (IS_ERR_OR_NULL(ipa_eth_debugfs)) { + ipa_eth_log("Unable to create debugfs root"); + goto err_exit; + } + + (void) debugfs_create_file("ready", 0644, ipa_eth_debugfs, NULL, + &fops_eth_dev_ready); + + (void) debugfs_create_bool("no_auto", 0644, + ipa_eth_debugfs, &ipa_eth_noauto); + + (void) debugfs_create_bool("ipc_logdbg", 0644, + ipa_eth_debugfs, &ipa_eth_ipc_logdbg); + + ipa_eth_devices_debugfs = + debugfs_create_dir("devices", ipa_eth_debugfs); + if (IS_ERR_OR_NULL(ipa_eth_devices_debugfs)) { + ipa_eth_log("Unable to create debugfs root for devices"); + goto err_exit; + } + + ipa_eth_drivers_debugfs = + debugfs_create_dir("drivers", ipa_eth_debugfs); + if (IS_ERR_OR_NULL(ipa_eth_drivers_debugfs)) { + ipa_eth_log("Unable to create debugfs root for drivers"); + goto err_exit; + } + + ipa_eth_bus_debugfs = + debugfs_create_dir("bus", ipa_eth_drivers_debugfs); + if (IS_ERR_OR_NULL(ipa_eth_bus_debugfs)) { + ipa_eth_log("Unable to create debugfs root for bus"); + goto err_exit; + } + + ipa_eth_pci_debugfs = debugfs_create_dir("pci", ipa_eth_bus_debugfs); + if (IS_ERR_OR_NULL(ipa_eth_pci_debugfs)) { + ipa_eth_log("Unable to create debugfs root for pci"); + goto err_exit; + } + + ipa_eth_offload_debugfs = + debugfs_create_dir("offload", ipa_eth_drivers_debugfs); + if (IS_ERR_OR_NULL(ipa_eth_offload_debugfs)) { + ipa_eth_log("Unable to create debugfs root for offload"); + goto err_exit; + } + + ipa_eth_log("Debugfs root is initialized"); + + return 0; + +err_exit: + ipa_eth_debugfs_cleanup(); + return -EFAULT; +} + +void ipa_eth_debugfs_cleanup(void) +{ + debugfs_remove_recursive(ipa_eth_debugfs); +} diff --git a/drivers/platform/msm/ethernet/ipa_eth_debugfs.h b/drivers/platform/msm/ethernet/ipa_eth_debugfs.h new file mode 100644 index 000000000000..b7d65efc7005 --- /dev/null +++ b/drivers/platform/msm/ethernet/ipa_eth_debugfs.h @@ -0,0 +1,20 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (c) 2019-2020, The Linux Foundation. All rights reserved + */ + +#ifndef _IPA_ETH_DEBUGFS_H_ +#define _IPA_ETH_DEBUGFS_H_ + +#include "ipa_eth_i.h" + +int ipa_eth_debugfs_add_device(struct ipa_eth_device *eth_dev); +void ipa_eth_debugfs_remove_device(struct ipa_eth_device *eth_dev); + +int ipa_eth_debugfs_add_offload_driver(struct ipa_eth_offload_driver *od); +void ipa_eth_debugfs_remove_offload_driver(struct ipa_eth_offload_driver *od); + +int ipa_eth_debugfs_init(void); +void ipa_eth_debugfs_cleanup(void); + +#endif /* _IPA_ETH_DEBUGFS_H_ */ diff --git a/drivers/platform/msm/ethernet/ipa_eth_ep.c b/drivers/platform/msm/ethernet/ipa_eth_ep.c new file mode 100644 index 000000000000..34c9cf0e7fca --- /dev/null +++ b/drivers/platform/msm/ethernet/ipa_eth_ep.c @@ -0,0 +1,680 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2019-2020, The Linux Foundation. All rights reserved + */ + +#include + +#include "ipa_eth_i.h" +#include "ipa_eth_trace.h" + +#ifndef IPA_ETH_EP_LOOPBACK +static int __handle_ipa_receive(struct ipa_eth_channel *ch, + struct sk_buff *skb) +{ + if (!ch->process_skb) { + dev_kfree_skb_any(skb); + return NET_RX_DROP; + } + + return ch->process_skb(ch, skb); +} +#else +static int __handle_ipa_receive(struct ipa_eth_channel *ch, + struct sk_buff *skb) +{ + if (ipa_tx_dp(IPA_CLIENT_AQC_ETHERNET_CONS, skb, NULL)) { + dev_kfree_skb_any(skb); + return NET_RX_DROP; + } + + ch->exception_loopback++; + + return NET_RX_SUCCESS; +} +#endif + +static void handle_ipa_receive(struct ipa_eth_channel *ch, + unsigned long data) +{ + bool tracing = trace_lan_rx_skb_enabled(); + struct sk_buff *skb = (struct sk_buff *) data; + + ch->exception_total++; + + /* Keep skb from being freed until tracing is completed */ + if (tracing) + skb_get(skb); + + if (__handle_ipa_receive(ch, skb) == NET_RX_DROP) + ch->exception_drops++; + + if (tracing) { + trace_lan_rx_skb(ch, skb); + dev_kfree_skb_any(skb); + } +} + +static void ipa_ep_client_notifier(void *priv, enum ipa_dp_evt_type evt, + unsigned long data) +{ + struct ipa_eth_channel *ch = (struct ipa_eth_channel *) priv; + + if (evt == IPA_RECEIVE) + handle_ipa_receive(ch, data); +} + +enum ipa_eth_phdr_type { + IPA_ETH_PHDR_V4, + IPA_ETH_PHDR_V6, + IPA_ETH_PHDR_MAX = IPA_ETH_PHDR_V6, +}; + +#define IPA_ETH_PHDR_NUM (IPA_ETH_PHDR_MAX + 1) + +struct ipa_eth_phdr_add_ioc { + struct ipa_ioc_add_hdr ioc; + struct ipa_hdr_add hdrs[IPA_ETH_PHDR_NUM]; +}; + +static const struct ipa_eth_phdr_add_ioc ADD_HDR_TEMPLATE = { + .ioc = { + .commit = 1, + .num_hdrs = IPA_ETH_PHDR_NUM, + }, + .hdrs = { + [IPA_ETH_PHDR_V4] = { + .is_partial = 1, + .is_eth2_ofst_valid = 1, + .eth2_ofst = 0, + }, + [IPA_ETH_PHDR_V6] = { + .is_partial = 1, + .is_eth2_ofst_valid = 1, + .eth2_ofst = 0, + }, + }, +}; + +static inline __be16 phdr_type_to_proto(enum ipa_eth_phdr_type type) +{ + return (type == IPA_ETH_PHDR_V4) ? + htons(ETH_P_IP) : htons(ETH_P_IPV6); +} + +static u8 ipa_eth_init_ethhdr(struct ethhdr *eth_hdr, + enum ipa_eth_phdr_type type, + struct net_device *net_dev) +{ + memset(eth_hdr, 0, sizeof(*eth_hdr)); + + eth_hdr->h_proto = phdr_type_to_proto(type); + memcpy(ð_hdr->h_source, net_dev->dev_addr, ETH_ALEN); + + return ETH_HLEN; +} + +static u8 ipa_eth_init_vlan_ethhdr(struct vlan_ethhdr *eth_hdr, + enum ipa_eth_phdr_type type, + struct net_device *net_dev) +{ + memset(eth_hdr, 0, sizeof(*eth_hdr)); + + eth_hdr->h_vlan_proto = htons(ETH_P_8021Q); + eth_hdr->h_vlan_encapsulated_proto = phdr_type_to_proto(type); + memcpy(ð_hdr->h_source, net_dev->dev_addr, ETH_ALEN); + + return VLAN_ETH_HLEN; +} + +static void ipa_eth_init_hdr_add(struct ipa_hdr_add *hdr_add, + bool vlan_mode, + enum ipa_eth_phdr_type type, + struct net_device *net_dev) +{ + const char *fmt_str = (type == IPA_ETH_PHDR_V4) ? "%s_ipv4" : "%s_ipv6"; + + snprintf(hdr_add->name, sizeof(hdr_add->name), fmt_str, net_dev->name); + + hdr_add->hdr_len = vlan_mode ? + ipa_eth_init_vlan_ethhdr((void *)hdr_add->hdr, type, net_dev) : + ipa_eth_init_ethhdr((void *)hdr_add->hdr, type, net_dev); + + hdr_add->type = vlan_mode ? IPA_HDR_L2_802_1Q : IPA_HDR_L2_ETHERNET_II; +} + +/** + * ipa_eth_ep_init_headers() - Install partial headers + * @eth_dev: Offload device + * + * Installs partial headers in IPA for IPv4 and IPv6 traffic. + * + * Return: 0 on success, negative errno code otherwise + */ +int ipa_eth_ep_init_headers(struct ipa_eth_device *eth_dev) +{ + int rc; + bool vlan_mode; + struct net_device *net_dev = eth_dev->net_dev; + struct ipa_eth_phdr_add_ioc phdr_add = ADD_HDR_TEMPLATE; + struct ipa_hdr_add *hdr_v4 = &phdr_add.hdrs[IPA_ETH_PHDR_V4]; + struct ipa_hdr_add *hdr_v6 = &phdr_add.hdrs[IPA_ETH_PHDR_V6]; + + rc = ipa3_is_vlan_mode(IPA_VLAN_IF_ETH, &vlan_mode); + if (rc) { + ipa_eth_dev_err(eth_dev, "Could not determine IPA VLAN mode"); + return rc; + } + + ipa_eth_init_hdr_add(hdr_v4, vlan_mode, IPA_ETH_PHDR_V4, net_dev); + ipa_eth_init_hdr_add(hdr_v6, vlan_mode, IPA_ETH_PHDR_V6, net_dev); + + rc = ipa_add_hdr(&phdr_add.ioc); + if (rc) { + ipa_eth_dev_err(eth_dev, "Failed to install partial headers"); + } else { + eth_dev->phdr_v4_handle = hdr_v4->hdr_hdl; + eth_dev->phdr_v6_handle = hdr_v6->hdr_hdl; + } + + return rc; +} + +struct ipa_eth_phdr_del_ioc { + struct ipa_ioc_del_hdr ioc; + struct ipa_hdr_del hdrs[IPA_ETH_PHDR_NUM]; +}; + +static const struct ipa_eth_phdr_del_ioc DEL_HDR_TEMPLATE = { + .ioc = { + .commit = 1, + .num_hdls = IPA_ETH_PHDR_NUM, + }, +}; + +int ipa_eth_ep_deinit_headers(struct ipa_eth_device *eth_dev) +{ + int rc; + struct ipa_eth_phdr_del_ioc phdr_del = DEL_HDR_TEMPLATE; + + phdr_del.hdrs[IPA_ETH_PHDR_V4].hdl = eth_dev->phdr_v4_handle; + phdr_del.hdrs[IPA_ETH_PHDR_V6].hdl = eth_dev->phdr_v6_handle; + + rc = ipa_del_hdr(&phdr_del.ioc); + if (rc) + ipa_eth_dev_err(eth_dev, "Failed to remove partial headers"); + + return rc; +} + +static inline size_t __list_size(struct list_head *head) +{ + size_t count = 0; + struct list_head *l; + + list_for_each(l, head) + count++; + + return count; +} + +static void ipa_eth_ep_init_rx_props( + struct ipa_ioc_rx_intf_prop *props, + struct ipa_eth_channel *ch, + bool is_ipv4, bool vlan_mode) +{ + props->ip = is_ipv4 ? IPA_IP_v4 : IPA_IP_v6; + props->src_pipe = ch->ipa_client; + props->hdr_l2_type = vlan_mode ? + IPA_HDR_L2_802_1Q : IPA_HDR_L2_ETHERNET_II; +} + +static void ipa_eth_ep_init_tx_props( + struct ipa_ioc_tx_intf_prop *props, + struct ipa_eth_channel *ch, + bool is_ipv4, bool vlan_mode) +{ + const char *fmt_str = is_ipv4 ? "%s_ipv4" : "%s_ipv6"; + + props->ip = is_ipv4 ? IPA_IP_v4 : IPA_IP_v6; + props->dst_pipe = ch->ipa_client; + props->hdr_l2_type = vlan_mode ? + IPA_HDR_L2_802_1Q : IPA_HDR_L2_ETHERNET_II; + + snprintf(props->hdr_name, sizeof(props->hdr_name), fmt_str, + ch->eth_dev->net_dev->name); +} + +static int ipa_eth_ep_init_tx_intf( + struct ipa_eth_device *eth_dev, + bool vlan_mode) +{ + u32 num_props; + struct ipa_eth_channel *ch; + struct ipa_tx_intf *tx_intf = ð_dev->ipa_tx_intf; + + if (tx_intf->prop) { + ipa_eth_dev_bug(eth_dev, "IPA interface Tx prop is not empty"); + return -EFAULT; + } + + /* one each for IPv4 and IPv6 */ + num_props = __list_size(ð_dev->tx_channels) * 2; + + tx_intf->prop = kcalloc(num_props, sizeof(*tx_intf->prop), GFP_KERNEL); + if (!tx_intf->prop) { + ipa_eth_dev_err(eth_dev, "Failed to alloc tx props"); + return -ENOMEM; + } + + tx_intf->num_props = 0; + list_for_each_entry(ch, ð_dev->tx_channels, channel_list) { + ipa_eth_ep_init_tx_props(&tx_intf->prop[tx_intf->num_props++], + ch, true, vlan_mode); + + ipa_eth_ep_init_tx_props(&tx_intf->prop[tx_intf->num_props++], + ch, false, vlan_mode); + } + + return 0; +} + +static int ipa_eth_ep_init_rx_intf( + struct ipa_eth_device *eth_dev, + bool vlan_mode) +{ + u32 num_props; + struct ipa_eth_channel *ch; + struct ipa_rx_intf *rx_intf = ð_dev->ipa_rx_intf; + + if (rx_intf->prop) { + ipa_eth_dev_bug(eth_dev, "IPA interface Rx prop is not empty"); + return -EFAULT; + } + + /* one each for IPv4 and IPv6 */ + num_props = __list_size(ð_dev->rx_channels) * 2; + + rx_intf->prop = kcalloc(num_props, sizeof(*rx_intf->prop), GFP_KERNEL); + if (!rx_intf->prop) { + ipa_eth_dev_err(eth_dev, "Failed to alloc rx props"); + return -ENOMEM; + } + + rx_intf->num_props = 0; + list_for_each_entry(ch, ð_dev->rx_channels, channel_list) { + ipa_eth_ep_init_rx_props(&rx_intf->prop[rx_intf->num_props++], + ch, true, vlan_mode); + + ipa_eth_ep_init_rx_props(&rx_intf->prop[rx_intf->num_props++], + ch, false, vlan_mode); + } + + return 0; +} + +static void ipa_eth_ep_deinit_interfaces(struct ipa_eth_device *eth_dev) +{ + kzfree(eth_dev->ipa_tx_intf.prop); + memset(ð_dev->ipa_tx_intf, 0, sizeof(eth_dev->ipa_tx_intf)); + + kzfree(eth_dev->ipa_rx_intf.prop); + memset(ð_dev->ipa_rx_intf, 0, sizeof(eth_dev->ipa_rx_intf)); + +} + +static int ipa_eth_ep_init_interfaces(struct ipa_eth_device *eth_dev) +{ + int rc; + bool vlan_mode; + + if (eth_dev->ipa_rx_intf.num_props || eth_dev->ipa_tx_intf.num_props) { + ipa_eth_dev_err(eth_dev, "Interface properties already exist"); + return -EFAULT; + } + + rc = ipa3_is_vlan_mode(IPA_VLAN_IF_ETH, &vlan_mode); + if (rc) { + ipa_eth_dev_err(eth_dev, "Could not determine IPA VLAN mode"); + return rc; + } + + rc = ipa_eth_ep_init_tx_intf(eth_dev, vlan_mode); + if (rc) + goto free_and_exit; + + rc = ipa_eth_ep_init_rx_intf(eth_dev, vlan_mode); + if (rc) + goto free_and_exit; + + return 0; + +free_and_exit: + ipa_eth_ep_deinit_interfaces(eth_dev); + + return rc; +} + +static int ipa_eth_ep_deregister_ipa_intf( + struct net_device *net_dev) +{ + int rc; + + ipa_eth_log("Deregistering IPA intf %s", net_dev->name); + + rc = ipa_deregister_intf(net_dev->name); + if (rc) { + ipa_eth_err("Failed to deregister IPA intf %s", net_dev->name); + return rc; + } + + rc = ipa_eth_send_msg_disconnect(net_dev); + if (rc) + ipa_eth_err("Failed to send disconnect message", net_dev->name); + + return rc; +} + +static int ipa_eth_ep_register_ipa_intf( + struct net_device *net_dev, + struct ipa_tx_intf *tx_intf, + struct ipa_rx_intf *rx_intf) +{ + int rc; + + ipa_eth_log("Registering IPA intf %s", net_dev->name); + + rc = ipa_register_intf(net_dev->name, tx_intf, rx_intf); + if (rc) { + ipa_eth_err("Failed to register IPA intf %s", net_dev->name); + return rc; + } + + rc = ipa_eth_send_msg_connect(net_dev); + if (rc) { + ipa_eth_err("Failed to send connect message", net_dev->name); + (void) ipa_eth_ep_deregister_ipa_intf(net_dev); + } + + return rc; +} + +static int ipa_eth_ep_register_alt_interface( + struct ipa_eth_device *eth_dev, + struct net_device *net_dev) +{ + struct ipa_tx_intf *tx_intf = ð_dev->ipa_tx_intf; + struct ipa_rx_intf *rx_intf = ð_dev->ipa_rx_intf; + + return ipa_eth_ep_register_ipa_intf(net_dev, tx_intf, rx_intf); +} + +static int ipa_eth_ep_deregister_alt_interface( + struct ipa_eth_device *eth_dev, + struct net_device *net_dev) +{ + return ipa_eth_ep_deregister_ipa_intf(net_dev); +} + +static int __ipa_eth_ep_register_interface( + struct ipa_eth_device *eth_dev) +{ + int rc; + struct net_device *net_dev = eth_dev->net_dev; + struct ipa_tx_intf *tx_intf = ð_dev->ipa_tx_intf; + struct ipa_rx_intf *rx_intf = ð_dev->ipa_rx_intf; + + ipa_eth_dev_log(eth_dev, "Registering interface %s", net_dev->name); + + rc = ipa_eth_ep_register_ipa_intf(net_dev, tx_intf, rx_intf); + if (rc) { + ipa_eth_err("Failed to register IPA interface"); + return rc; + } + + return 0; +} + +/** + * ipa_eth_ep_register_interface() - Set Rx and Tx properties and register the + * interface with IPA + * @eth_dev: Offload device to register + * + * Register a logical interface with IPA. The API expects netdev and channels + * allocated prior to being called. + * + * Return: 0 on success, negative errno code otherwise + */ +int ipa_eth_ep_register_interface(struct ipa_eth_device *eth_dev) +{ + int rc; + + ipa_eth_dev_log(eth_dev, "Registering interface with IPA driver"); + + rc = ipa_eth_ep_init_interfaces(eth_dev); + if (rc) + return rc; + + rc = __ipa_eth_ep_register_interface(eth_dev); + if (rc) { + ipa_eth_dev_err(eth_dev, "Failed to register IPA interface"); + ipa_eth_ep_deinit_interfaces(eth_dev); + return rc; + } + + return 0; +} + +/** + * ipa_eth_ep_unregister_interface() - Unregister a previously registered + * interface + * @eth_dev: Offload device to unregister + */ +int ipa_eth_ep_unregister_interface(struct ipa_eth_device *eth_dev) +{ + int rc; + + ipa_eth_dev_log(eth_dev, "Unregistering interface from IPA driver"); + + rc = ipa_eth_ep_deregister_ipa_intf(eth_dev->net_dev); + if (rc) { + ipa_eth_dev_err(eth_dev, + "Failed to de-register one or more interfaces"); + return rc; + } + + ipa_eth_ep_deinit_interfaces(eth_dev); + + return 0; +} + +int ipa_eth_ep_register_upper_interface( + struct ipa_eth_upper_device *upper_eth_dev) +{ + int rc; + struct net_device *net_dev = upper_eth_dev->net_dev; + struct ipa_eth_device *eth_dev = upper_eth_dev->eth_dev; + + if (upper_eth_dev->registered) + return 0; + + ipa_eth_dev_log(eth_dev, + "Registering upper interface %s", net_dev->name); + + rc = ipa_eth_ep_register_alt_interface(eth_dev, net_dev); + if (rc) { + ipa_eth_dev_err(eth_dev, + "Failed to register upper interface %s", net_dev->name); + return rc; + } + + upper_eth_dev->registered = true; + + return 0; +} + +int ipa_eth_ep_unregister_upper_interface( + struct ipa_eth_upper_device *upper_eth_dev) +{ + int rc; + struct net_device *net_dev = upper_eth_dev->net_dev; + struct ipa_eth_device *eth_dev = upper_eth_dev->eth_dev; + + if (!upper_eth_dev->registered) + return 0; + + ipa_eth_dev_log(eth_dev, + "Unegistering upper interface %s", net_dev->name); + + rc = ipa_eth_ep_deregister_alt_interface(eth_dev, net_dev); + if (rc) { + ipa_eth_dev_err(eth_dev, + "Failed to unregister upper interface %s", + net_dev->name); + return rc; + } + + upper_eth_dev->registered = false; + + return 0; +} + +/** + * ipa_eth_ep_init_ctx - Initialize IPA endpoint context for a channel + * @ch: Channel for which EP ctx need to be initialized + * @vlan_mode: true if VLAN mode is enabled for the EP + */ +void ipa_eth_ep_init_ctx(struct ipa_eth_channel *ch, bool vlan_mode) +{ + struct ipa3_ep_context *ep_ctx = &ipa3_ctx->ep[ch->ipa_ep_num]; + + if (ep_ctx->valid) + return; + + memset(ep_ctx, 0, offsetof(typeof(*ep_ctx), sys)); + + ep_ctx->valid = 1; + ep_ctx->client = ch->ipa_client; + ep_ctx->client_notify = ipa_ep_client_notifier; + ep_ctx->priv = ch; + + ep_ctx->cfg.nat.nat_en = IPA_CLIENT_IS_PROD(ch->ipa_client) ? + IPA_SRC_NAT : IPA_BYPASS_NAT; + ep_ctx->cfg.hdr.hdr_len = vlan_mode ? VLAN_ETH_HLEN : ETH_HLEN; + ep_ctx->cfg.mode.mode = IPA_BASIC; +} + +/** + * ipa_eth_ep_deinit_ctx - Deinitialize IPA endpoint context for a channel + * @ch: Channel for which EP ctx need to be deinitialized + */ +void ipa_eth_ep_deinit_ctx(struct ipa_eth_channel *ch) +{ + struct ipa3_ep_context *ep_ctx = &ipa3_ctx->ep[ch->ipa_ep_num]; + + if (!ep_ctx->valid) + return; + + ep_ctx->valid = false; + + memset(ep_ctx, 0, offsetof(typeof(*ep_ctx), sys)); +} + +static int __ipa_eth_ep_init(struct ipa_eth_channel *ch, + struct ipa3_ep_context *ep_ctx) +{ + int rc; + + rc = ipa3_cfg_ep(ch->ipa_ep_num, &ep_ctx->cfg); + if (rc) { + ipa_eth_dev_err(ch->eth_dev, + "Failed to configure EP %d", ch->ipa_ep_num); + goto err_exit; + } + + if (IPA_CLIENT_IS_PROD(ch->ipa_client)) + ipa3_install_dflt_flt_rules(ch->ipa_ep_num); + +err_exit: + return rc; +} + +/** + * ipa_eth_ep_init() - Initialize IPA endpoint for a channel + * @ch: Channel for which EP need to be initialized + * + * Return: 0 on success, negative errno otherwise + */ +int ipa_eth_ep_init(struct ipa_eth_channel *ch) +{ + int rc; + struct ipa3_ep_context *ep_ctx; + + ep_ctx = &ipa3_ctx->ep[ch->ipa_ep_num]; + if (!ep_ctx->valid) { + ipa_eth_dev_bug(ch->eth_dev, "EP context is not initialiazed"); + return -EFAULT; + } + + IPA_ACTIVE_CLIENTS_INC_SIMPLE(); + rc = __ipa_eth_ep_init(ch, ep_ctx); + IPA_ACTIVE_CLIENTS_DEC_SIMPLE(); + + return rc; +} +EXPORT_SYMBOL(ipa_eth_ep_init); + +/** + * ipa_eth_ep_deinit() - Deinitialize IPA endpoint for a channel + * @ch: Channel for which EP need to be deinitialized + * + * Return: 0 on success, negative errno otherwise + */ +int ipa_eth_ep_deinit(struct ipa_eth_channel *ch) +{ + IPA_ACTIVE_CLIENTS_INC_SIMPLE(); + + if (IPA_CLIENT_IS_PROD(ch->ipa_client)) + ipa3_delete_dflt_flt_rules(ch->ipa_ep_num); + + IPA_ACTIVE_CLIENTS_DEC_SIMPLE(); + + return 0; +} +EXPORT_SYMBOL(ipa_eth_ep_deinit); + +/** + * ipa_eth_ep_start() - Start an IPA endpoint + * @ch: Channel for which the IPA EP need to be started + * + * Return: 0 on success, negative errno otherwise + */ +int ipa_eth_ep_start(struct ipa_eth_channel *ch) +{ + int rc = ipa3_enable_data_path(ch->ipa_ep_num); + + if (rc) + ipa_eth_dev_err(ch->eth_dev, + "Failed to start EP %d", ch->ipa_ep_num); + + return rc; +} +EXPORT_SYMBOL(ipa_eth_ep_start); + +/** + * ipa_eth_ep_stop() - Stop an IPA endpoint + * @ch: Channel for which the IPA EP need to be stopped + * + * Return: 0 on success, negative errno otherwise + */ +int ipa_eth_ep_stop(struct ipa_eth_channel *ch) +{ + int rc = ipa3_disable_data_path(ch->ipa_ep_num); + + if (rc) + ipa_eth_dev_err(ch->eth_dev, + "Failed to stop EP %d", ch->ipa_ep_num); + + return rc; +} +EXPORT_SYMBOL(ipa_eth_ep_stop); diff --git a/drivers/platform/msm/ethernet/ipa_eth_gsi.c b/drivers/platform/msm/ethernet/ipa_eth_gsi.c new file mode 100644 index 000000000000..118eeae4aa6a --- /dev/null +++ b/drivers/platform/msm/ethernet/ipa_eth_gsi.c @@ -0,0 +1,371 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2019-2020, The Linux Foundation. All rights reserved + */ + +#include + +#include "ipa_eth_i.h" + +static void ipa_eth_gsi_ev_err(struct gsi_evt_err_notify *notify) +{ + struct ipa_eth_channel *ch = notify->user_data; + struct ipa3_ep_context *ep_ctx = &ipa3_ctx->ep[ch->ipa_ep_num]; + + ipa_eth_dev_err(ch->eth_dev, + "Error (id=%d, edesc=%04x) in GSI event ring %u", + notify->evt_id, notify->err_desc, + ep_ctx->gsi_evt_ring_hdl); +} + +static void ipa_eth_gsi_ch_err(struct gsi_chan_err_notify *notify) +{ + struct ipa_eth_channel *ch = notify->chan_user_data; + struct ipa3_ep_context *ep_ctx = &ipa3_ctx->ep[ch->ipa_ep_num]; + + ipa_eth_dev_err(ch->eth_dev, + "Error (id=%d, edesc=%04x) in GSI channel %u", + notify->evt_id, notify->err_desc, + ep_ctx->gsi_chan_hdl); +} + +/* + * ipa_eth_gsi_alloc() - Allocate GSI channel and event ring for an offload + * channel, optionally writing to the ring scratch + * register and fetch the ring doorbell address + * @ch: Offload channel + * @gsi_ev_props: Properties of the GSI event ring to be allocated + * @gsi_ev_scratch: Optional. Points to the value to be written to GSI + * event ring scratch register + * @gsi_ev_db: Optional. Writes event ring doorbell LSB address to the location + * pointed to by the argument + * @gsi_ch_props: Properties of the GSI channel to be allocated + * @gsi_ch_scratch: Optional. Points to the value to be written to GSI + * event ring scratch register + * @gsi_ch_db: Optional. Writes channel doorbell LSB address to the location + * pointed to by the argument + * + * Return: 0 on success, negative errno otherwise + */ +int ipa_eth_gsi_alloc(struct ipa_eth_channel *ch, + struct gsi_evt_ring_props *gsi_ev_props, + union gsi_evt_scratch *gsi_ev_scratch, + phys_addr_t *gsi_ev_db, + struct gsi_chan_props *gsi_ch_props, + union gsi_channel_scratch *gsi_ch_scratch, + phys_addr_t *gsi_ch_db) +{ + enum gsi_status gsi_rc = GSI_STATUS_SUCCESS; + const struct ipa_gsi_ep_config *gsi_ep_cfg; + struct ipa3_ep_context *ep_ctx = &ipa3_ctx->ep[ch->ipa_ep_num]; + + if (!ep_ctx->valid) { + ipa_eth_dev_err(ch->eth_dev, "EP context is not initialized"); + return -EFAULT; + } + + gsi_ep_cfg = ipa3_get_gsi_ep_info(ep_ctx->client); + if (!gsi_ep_cfg) { + ipa_eth_dev_err(ch->eth_dev, "Failed to obtain GSI EP info"); + return -EFAULT; + } + + if (!gsi_ev_props->err_cb) { + gsi_ev_props->err_cb = ipa_eth_gsi_ev_err; + gsi_ev_props->user_data = ch; + } + + gsi_rc = gsi_alloc_evt_ring(gsi_ev_props, ipa3_ctx->gsi_dev_hdl, + &ep_ctx->gsi_evt_ring_hdl); + if (gsi_rc != GSI_STATUS_SUCCESS) { + ipa_eth_dev_err(ch->eth_dev, "Failed to alloc GSI event ring"); + return -EFAULT; + } + + ipa_eth_dev_dbg(ch->eth_dev, "GSI event ring handle is %lu", + ep_ctx->gsi_evt_ring_hdl); + + if (gsi_ev_db) { + u32 db_addr_lsb = 0; + u32 db_addr_msb = 0; + + gsi_rc = gsi_query_evt_ring_db_addr(ep_ctx->gsi_evt_ring_hdl, + &db_addr_lsb, &db_addr_msb); + if (gsi_rc != GSI_STATUS_SUCCESS) { + ipa_eth_dev_err(ch->eth_dev, + "Failed to get DB address for event ring %lu", + ep_ctx->gsi_evt_ring_hdl); + goto err_free_ev; + } + + ipa_eth_dev_dbg(ch->eth_dev, + "GSI event ring %lu DB address LSB is 0x%08x", + ep_ctx->gsi_evt_ring_hdl, db_addr_lsb); + ipa_eth_dev_dbg(ch->eth_dev, + "GSI event ring %lu DB address MSB is 0x%08x", + ep_ctx->gsi_evt_ring_hdl, db_addr_msb); + + *gsi_ev_db = db_addr_lsb; + } + + if (gsi_ev_scratch) { + gsi_rc = gsi_write_evt_ring_scratch(ep_ctx->gsi_evt_ring_hdl, + *gsi_ev_scratch); + if (gsi_rc != GSI_STATUS_SUCCESS) { + ipa_eth_dev_err(ch->eth_dev, + "Failed to write scratch for event ring %lu", + ep_ctx->gsi_evt_ring_hdl); + goto err_free_ev; + } + } + + gsi_ch_props->ch_id = gsi_ep_cfg->ipa_gsi_chan_num; + gsi_ch_props->evt_ring_hdl = ep_ctx->gsi_evt_ring_hdl; + + gsi_ch_props->prefetch_mode = gsi_ep_cfg->prefetch_mode; + gsi_ch_props->empty_lvl_threshold = gsi_ep_cfg->prefetch_threshold; + + if (!gsi_ch_props->err_cb) { + gsi_ch_props->err_cb = ipa_eth_gsi_ch_err; + gsi_ch_props->chan_user_data = ch; + } + + gsi_rc = gsi_alloc_channel(gsi_ch_props, ipa3_ctx->gsi_dev_hdl, + &ep_ctx->gsi_chan_hdl); + if (gsi_rc != GSI_STATUS_SUCCESS) { + ipa_eth_dev_err(ch->eth_dev, "Failed to alloc GSI channel"); + goto err_free_ev; + } + + ipa_eth_dev_dbg(ch->eth_dev, "GSI channel handle is %lu", + ep_ctx->gsi_chan_hdl); + + if (gsi_ch_db) { + u32 db_addr_lsb = 0; + u32 db_addr_msb = 0; + + gsi_rc = gsi_query_channel_db_addr(ep_ctx->gsi_chan_hdl, + &db_addr_lsb, &db_addr_msb); + if (gsi_rc != GSI_STATUS_SUCCESS) { + ipa_eth_dev_err(ch->eth_dev, + "Failed to get DB address for channel %lu", + ep_ctx->gsi_chan_hdl); + goto err_free_ch; + } + + ipa_eth_dev_dbg(ch->eth_dev, + "GSI channel %lu DB address LSB is 0x%08x", + ep_ctx->gsi_chan_hdl, db_addr_lsb); + ipa_eth_dev_dbg(ch->eth_dev, + "GSI channel %lu DB address MSB is 0x%08x", + ep_ctx->gsi_chan_hdl, db_addr_msb); + + *gsi_ch_db = db_addr_lsb; + } + + if (gsi_ch_scratch) { + gsi_rc = gsi_write_channel_scratch(ep_ctx->gsi_chan_hdl, + *gsi_ch_scratch); + if (gsi_rc != GSI_STATUS_SUCCESS) { + ipa_eth_dev_err(ch->eth_dev, + "Failed to write scratch for channel %lu", + ep_ctx->gsi_chan_hdl); + goto err_free_ch; + } + } + + return 0; + +err_free_ch: + if (gsi_dealloc_channel(ep_ctx->gsi_chan_hdl)) + ipa_eth_dev_err(ch->eth_dev, + "Failed to dealloc GSI channel %lu", + ep_ctx->gsi_chan_hdl); + + ep_ctx->gsi_chan_hdl = ~0; + +err_free_ev: + if (gsi_dealloc_evt_ring(ep_ctx->gsi_evt_ring_hdl)) + ipa_eth_dev_err(ch->eth_dev, + "Failed to dealloc GSI event ring %lu", + ep_ctx->gsi_evt_ring_hdl); + + ep_ctx->gsi_evt_ring_hdl = ~0; + + return gsi_rc; +} +EXPORT_SYMBOL(ipa_eth_gsi_alloc); + +/** + * ipa_eth_gsi_dealloc() - De-allocate GSI event ring and channel associated + * with an offload channel, previously allocated with + * ipa_eth_gsi_alloc() + * @ch: Offload channel + * + * Return: 0 on success, negative errno otherwise + */ +int ipa_eth_gsi_dealloc(struct ipa_eth_channel *ch) +{ + enum gsi_status gsi_rc = GSI_STATUS_SUCCESS; + struct ipa3_ep_context *ep_ctx = &ipa3_ctx->ep[ch->ipa_ep_num]; + + if (!ep_ctx->valid) { + ipa_eth_dev_err(ch->eth_dev, "EP context is not initialized"); + return -EFAULT; + } + + if (ep_ctx->gsi_chan_hdl != ~0) { + gsi_rc = gsi_reset_channel(ep_ctx->gsi_chan_hdl); + if (gsi_rc != GSI_STATUS_SUCCESS) { + ipa_eth_dev_err(ch->eth_dev, + "Failed to reset channel %u", + ep_ctx->gsi_chan_hdl); + return gsi_rc; + } + + gsi_rc = gsi_dealloc_channel(ep_ctx->gsi_chan_hdl); + if (gsi_rc != GSI_STATUS_SUCCESS) { + ipa_eth_dev_err(ch->eth_dev, + "Failed to dealloc channel %lu", + ep_ctx->gsi_chan_hdl); + return gsi_rc; + } + + ep_ctx->gsi_chan_hdl = ~0; + } + + if (ep_ctx->gsi_evt_ring_hdl != ~0) { + gsi_rc = gsi_reset_evt_ring(ep_ctx->gsi_evt_ring_hdl); + if (gsi_rc != GSI_STATUS_SUCCESS) { + ipa_eth_dev_err(ch->eth_dev, + "Failed to reset event ring %lu", + ep_ctx->gsi_evt_ring_hdl); + return gsi_rc; + } + + gsi_rc = gsi_dealloc_evt_ring(ep_ctx->gsi_evt_ring_hdl); + if (gsi_rc != GSI_STATUS_SUCCESS) { + ipa_eth_dev_err(ch->eth_dev, + "Failed to dealloc event ring %lu", + ep_ctx->gsi_evt_ring_hdl); + return gsi_rc; + } + + ep_ctx->gsi_evt_ring_hdl = ~0; + } + + return 0; +} +EXPORT_SYMBOL(ipa_eth_gsi_dealloc); + +/* + * ipa_eth_gsi_ring_evtring() - Ring an offload channel event ring doorbell + * @ch: Offload channel associated with the event ring + * @value: Value to write to the doorbell + * + * Return: 0 on success, negative errno otherwise + */ +int ipa_eth_gsi_ring_evtring(struct ipa_eth_channel *ch, u64 value) +{ + enum gsi_status gsi_rc = GSI_STATUS_SUCCESS; + struct ipa3_ep_context *ep_ctx = &ipa3_ctx->ep[ch->ipa_ep_num]; + + if (!ep_ctx->valid) { + ipa_eth_dev_err(ch->eth_dev, "EP context is not initialized"); + return -EFAULT; + } + + gsi_rc = gsi_ring_evt_ring_db(ep_ctx->gsi_evt_ring_hdl, value); + if (gsi_rc != GSI_STATUS_SUCCESS) { + ipa_eth_dev_err(ch->eth_dev, + "Failed to ring DB for event ring %lu", + ep_ctx->gsi_evt_ring_hdl); + return gsi_rc; + } + + return 0; +} +EXPORT_SYMBOL(ipa_eth_gsi_ring_evtring); + +/* + * ipa_eth_gsi_ring_channel() - Ring an offload channel GSI channel doorbell + * @ch: Offload channel associated with the GSI channel + * @value: Value to write to the doorbell + * + * Return: 0 on success, negative errno otherwise + */ +int ipa_eth_gsi_ring_channel(struct ipa_eth_channel *ch, u64 value) +{ + enum gsi_status gsi_rc = GSI_STATUS_SUCCESS; + struct ipa3_ep_context *ep_ctx = &ipa3_ctx->ep[ch->ipa_ep_num]; + + if (!ep_ctx->valid) { + ipa_eth_dev_err(ch->eth_dev, "EP context is not initialized"); + return -EFAULT; + } + + gsi_rc = gsi_ring_ch_ring_db(ep_ctx->gsi_chan_hdl, value); + if (gsi_rc != GSI_STATUS_SUCCESS) { + ipa_eth_dev_err(ch->eth_dev, + "Failed to ring DB for channel %lu", + ep_ctx->gsi_chan_hdl); + return gsi_rc; + } + + return 0; +} +EXPORT_SYMBOL(ipa_eth_gsi_ring_channel); + +/** + * ipa_eth_gsi_start() - Start GSI channel associated with offload channel + * @ch: Offload channel + * + * Return: 0 on success, negative errno otherwise + */ +int ipa_eth_gsi_start(struct ipa_eth_channel *ch) +{ + enum gsi_status gsi_rc = GSI_STATUS_SUCCESS; + struct ipa3_ep_context *ep_ctx = &ipa3_ctx->ep[ch->ipa_ep_num]; + + if (!ep_ctx->valid) { + ipa_eth_dev_err(ch->eth_dev, "EP context is not initialized"); + return -EFAULT; + } + + gsi_rc = gsi_start_channel(ep_ctx->gsi_chan_hdl); + if (gsi_rc != GSI_STATUS_SUCCESS) { + ipa_eth_dev_err(ch->eth_dev, "Failed to start GSI channel %lu", + ep_ctx->gsi_chan_hdl); + return gsi_rc; + } + + return 0; +} +EXPORT_SYMBOL(ipa_eth_gsi_start); + +/** + * ipa_eth_gsi_stop() - Stop GSI channel associated with offload channel + * @ch: Offload channel + * + * Return: 0 on success, negative errno otherwise + */ +int ipa_eth_gsi_stop(struct ipa_eth_channel *ch) +{ + enum gsi_status gsi_rc = GSI_STATUS_SUCCESS; + struct ipa3_ep_context *ep_ctx = &ipa3_ctx->ep[ch->ipa_ep_num]; + + if (!ep_ctx->valid) { + ipa_eth_dev_err(ch->eth_dev, "EP context is not initialized"); + return -EFAULT; + } + + gsi_rc = gsi_stop_channel(ep_ctx->gsi_chan_hdl); + if (gsi_rc != GSI_STATUS_SUCCESS) { + ipa_eth_dev_err(ch->eth_dev, "Failed to stop GSI channel %lu", + ep_ctx->gsi_chan_hdl); + return gsi_rc; + } + + return 0; +} +EXPORT_SYMBOL(ipa_eth_gsi_stop); diff --git a/drivers/platform/msm/ethernet/ipa_eth_i.h b/drivers/platform/msm/ethernet/ipa_eth_i.h new file mode 100644 index 000000000000..97cfa94d820a --- /dev/null +++ b/drivers/platform/msm/ethernet/ipa_eth_i.h @@ -0,0 +1,272 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (c) 2019-2020, The Linux Foundation. All rights reserved + */ + +#ifndef _IPA_ETH_I_H_ +#define _IPA_ETH_I_H_ + +#include + +#define IPA_ETH_NET_DRIVER +#define IPA_ETH_OFFLOAD_DRIVER +#include + +#include "ipa_eth_debugfs.h" + +#include "../ipa_i.h" + +/* Time to remain awake after a suspend abort due to NIC activity */ +#define IPA_ETH_WAKE_TIME_MS 500 + +/* Time for NIC HW to settle down (ex. receive link interrupt) after a resume */ +#define IPA_ETH_RESUME_SETTLE_MS 2000 + +#define IPA_ETH_PFDEV (ipa3_ctx ? ipa3_ctx->pdev : NULL) +#define IPA_ETH_SUBSYS "ipa_eth" + +#define IPA_ETH_NET_DEVICE_MAX_EVENTS (NETDEV_CHANGE_TX_QUEUE_LEN + 1) +#define IPA_ETH_PM_NOTIFIER_MAX_EVENTS (PM_POST_RESTORE + 1) + +enum ipa_eth_states { + IPA_ETH_ST_READY, + IPA_ETH_ST_API_READY, + IPA_ETH_ST_MAX, +}; + +enum ipa_eth_dev_flags { + IPA_ETH_DEV_F_REMOVING, + IPA_ETH_DEV_F_UNPAIRING, + IPA_ETH_DEV_F_RESETTING, + IPA_ETH_DEV_F_COUNT, +}; + +#define ipa_eth_err(fmt, args...) \ + do { \ + dev_err(IPA_ETH_PFDEV, \ + IPA_ETH_SUBSYS " %s:%d ERROR: " fmt "\n", \ + __func__, __LINE__, ## args); \ + ipa_eth_ipc_log("ERROR: " fmt, ## args); \ + } while (0) + +#define ipa_eth_bug(fmt, args...) \ + do { \ + ipa_eth_err("BUG: " fmt, ## args); \ + dump_stack(); \ + } while (0) + +#define ipa_eth_log(fmt, args...) \ + do { \ + dev_dbg(IPA_ETH_PFDEV, \ + IPA_ETH_SUBSYS " %s:%d " fmt "\n", \ + __func__, __LINE__, ## args); \ + ipa_eth_ipc_log(fmt, ## args); \ + } while (0) + +#define ipa_eth_dbg(fmt, args...) \ + do { \ + dev_dbg(IPA_ETH_PFDEV, \ + IPA_ETH_SUBSYS " %s:%d " fmt "\n", \ + __func__, __LINE__, ## args); \ + ipa_eth_ipc_dbg("DEBUG: " fmt, ## args); \ + } while (0) + +#define ipa_eth_dev_err(edev, fmt, args...) \ + ipa_eth_err("(%s) " fmt, ((edev && edev->net_dev) ? \ + edev->net_dev->name : ""), \ + ## args) + +#define ipa_eth_dev_bug(edev, fmt, args...) \ + ipa_eth_bug("(%s) " fmt, ((edev && edev->net_dev) ? \ + edev->net_dev->name : ""), \ + ## args) + +#define ipa_eth_dev_dbg(edev, fmt, args...) \ + ipa_eth_dbg("(%s) " fmt, ((edev && edev->net_dev) ? \ + edev->net_dev->name : ""), \ + ## args) + +#define ipa_eth_dev_log(edev, fmt, args...) \ + ipa_eth_log("(%s) " fmt, ((edev && edev->net_dev) ? \ + edev->net_dev->name : ""), \ + ## args) + +struct ipa_eth_upper_device { + struct list_head upper_list; + + struct net_device *net_dev; + struct ipa_eth_device *eth_dev; + + bool up; /* interface is up */ + bool registered; /* registered with IPA */ +}; + +struct ipa_eth_device_private { + struct ipa_eth_device *eth_dev; + + struct list_head upper_devices; + + unsigned long assume_active; + struct rtnl_link_stats64 last_rtnl_stats; + + struct notifier_block pm_nb; + struct notifier_block panic_nb; +}; + +#define eth_dev_priv(eth_dev) \ + ((struct ipa_eth_device_private *)((eth_dev)->ipa_priv)) + +struct ipa_eth_bus { + struct list_head bus_list; + + struct bus_type *bus; + + int (*register_driver)(struct ipa_eth_net_driver *nd); + void (*unregister_driver)(struct ipa_eth_net_driver *nd); + + int (*enable_pc)(struct ipa_eth_device *eth_dev); + int (*disable_pc)(struct ipa_eth_device *eth_dev); +}; + +extern struct ipa_eth_bus ipa_eth_pci_bus; + +struct ipa_eth_cb_map_param { + bool map; + bool sym; + int iommu_prot; + enum dma_data_direction dma_dir; + const struct ipa_smmu_cb_ctx *cb_ctx; +}; + +extern unsigned long ipa_eth_state; +extern bool ipa_eth_noauto; +extern bool ipa_eth_ipc_logdbg; + +bool ipa_eth_is_ready(void); +bool ipa_eth_all_ready(void); + +static inline void ipa_eth_dev_assume_active_ms( + struct ipa_eth_device *eth_dev, + unsigned int msec) +{ + eth_dev_priv(eth_dev)->assume_active += + DIV_ROUND_UP(msec, IPA_ETH_WAKE_TIME_MS); + pm_system_wakeup(); +} + +static inline void ipa_eth_dev_assume_active_inc( + struct ipa_eth_device *eth_dev, + unsigned int count) +{ + eth_dev_priv(eth_dev)->assume_active += count; + pm_system_wakeup(); +} + +static inline void ipa_eth_dev_assume_active_dec( + struct ipa_eth_device *eth_dev, + unsigned int count) +{ + if (eth_dev_priv(eth_dev)->assume_active > count) + eth_dev_priv(eth_dev)->assume_active -= count; + else + eth_dev_priv(eth_dev)->assume_active = 0; +} + +struct ipa_eth_device *ipa_eth_alloc_device( + struct device *dev, + struct ipa_eth_net_driver *nd); +void ipa_eth_free_device(struct ipa_eth_device *eth_dev); +int ipa_eth_register_device(struct ipa_eth_device *eth_dev); +void ipa_eth_unregister_device(struct ipa_eth_device *eth_dev); + +void ipa_eth_device_refresh_sync(struct ipa_eth_device *eth_dev); +void ipa_eth_device_refresh_sched(struct ipa_eth_device *eth_dev); +void ipa_eth_global_refresh_sched(void); + +int ipa_eth_pci_modinit(void); +void ipa_eth_pci_modexit(void); + +int ipa_eth_bus_modinit(void); +void ipa_eth_bus_modexit(void); + +int ipa_eth_bus_register_driver(struct ipa_eth_net_driver *nd); +void ipa_eth_bus_unregister_driver(struct ipa_eth_net_driver *nd); + +int ipa_eth_bus_enable_pc(struct ipa_eth_device *eth_dev); +int ipa_eth_bus_disable_pc(struct ipa_eth_device *eth_dev); + +int ipa_eth_offload_register_driver(struct ipa_eth_offload_driver *od); +void ipa_eth_offload_unregister_driver(struct ipa_eth_offload_driver *od); + +static inline bool ipa_eth_offload_device_paired(struct ipa_eth_device *eth_dev) +{ + return eth_dev->od != NULL; +} + +int ipa_eth_offload_pair_device(struct ipa_eth_device *eth_dev); +void ipa_eth_offload_unpair_device(struct ipa_eth_device *eth_dev); + +int ipa_eth_offload_init(struct ipa_eth_device *eth_dev); +int ipa_eth_offload_deinit(struct ipa_eth_device *eth_dev); +int ipa_eth_offload_start(struct ipa_eth_device *eth_dev); +int ipa_eth_offload_stop(struct ipa_eth_device *eth_dev); + +int ipa_eth_offload_save_regs(struct ipa_eth_device *eth_dev); +int ipa_eth_offload_prepare_reset(struct ipa_eth_device *eth_dev, void *data); +int ipa_eth_offload_complete_reset(struct ipa_eth_device *eth_dev, void *data); + +bool ipa_eth_net_check_active(struct ipa_eth_device *eth_dev); + +int ipa_eth_net_register_driver(struct ipa_eth_net_driver *nd); +void ipa_eth_net_unregister_driver(struct ipa_eth_net_driver *nd); +int ipa_eth_net_register_upper(struct ipa_eth_device *eth_dev); +int ipa_eth_net_unregister_upper(struct ipa_eth_device *eth_dev); + +int ipa_eth_net_open_device(struct ipa_eth_device *eth_dev); +void ipa_eth_net_close_device(struct ipa_eth_device *eth_dev); + +int ipa_eth_net_save_regs(struct ipa_eth_device *eth_dev); + +int ipa_eth_ep_init_headers(struct ipa_eth_device *eth_dev); +int ipa_eth_ep_deinit_headers(struct ipa_eth_device *eth_dev); + +int ipa_eth_ep_register_interface(struct ipa_eth_device *eth_dev); +int ipa_eth_ep_unregister_interface(struct ipa_eth_device *eth_dev); + +int ipa_eth_ep_register_upper_interface( + struct ipa_eth_upper_device *upper_eth_dev); +int ipa_eth_ep_unregister_upper_interface( + struct ipa_eth_upper_device *upper_eth_dev); + +void ipa_eth_ep_init_ctx(struct ipa_eth_channel *ch, bool vlan_mode); +void ipa_eth_ep_deinit_ctx(struct ipa_eth_channel *ch); + +int ipa_eth_pm_register(struct ipa_eth_device *eth_dev); +int ipa_eth_pm_unregister(struct ipa_eth_device *eth_dev); + +int ipa_eth_pm_activate(struct ipa_eth_device *eth_dev); +int ipa_eth_pm_deactivate(struct ipa_eth_device *eth_dev); + +int ipa_eth_pm_vote_bw(struct ipa_eth_device *eth_dev); + +int ipa_eth_uc_stats_init(struct ipa_eth_device *eth_dev); +int ipa_eth_uc_stats_deinit(struct ipa_eth_device *eth_dev); +int ipa_eth_uc_stats_start(struct ipa_eth_device *eth_dev); +int ipa_eth_uc_stats_stop(struct ipa_eth_device *eth_dev); + +/* ipa_eth_utils.c APIs */ + +const char *ipa_eth_device_event_name(enum ipa_eth_device_event event); +const char *ipa_eth_net_device_event_name(unsigned long event); +const char *ipa_eth_pm_notifier_event_name(unsigned long event); + +int ipa_eth_send_msg_connect(struct net_device *net_dev); +int ipa_eth_send_msg_disconnect(struct net_device *net_dev); + +void *ipa_eth_get_ipc_logbuf(void); +void *ipa_eth_get_ipc_logbuf_dbg(void); + +int ipa_eth_ipc_log_init(void); +void ipa_eth_ipc_log_cleanup(void); + +#endif // _IPA_ETH_I_H_ diff --git a/drivers/platform/msm/ethernet/ipa_eth_net.c b/drivers/platform/msm/ethernet/ipa_eth_net.c new file mode 100644 index 000000000000..566d366f0386 --- /dev/null +++ b/drivers/platform/msm/ethernet/ipa_eth_net.c @@ -0,0 +1,1551 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2019-2020, The Linux Foundation. All rights reserved + */ + +#include + +#include "ipa_eth_i.h" +#include "ipa_eth_trace.h" + +#define ipa_eth_nd_op(eth_dev, op, args...) (eth_dev->nd->ops->op(args)) + +static LIST_HEAD(ipa_eth_net_drivers); +static DEFINE_MUTEX(ipa_eth_net_drivers_mutex); + +static bool ipa_eth_net_check_ops(struct ipa_eth_net_ops *ops) +{ + /* Following call backs are optional: + * - save_regs() + */ + return + ops && + ops->open_device && + ops->close_device && + ops->request_channel && + ops->release_channel && + ops->enable_channel && + ops->disable_channel && + ops->request_event && + ops->release_event && + ops->enable_event && + ops->disable_event && + ops->moderate_event && + ops->receive_skb && + ops->transmit_skb; +} + +static bool ipa_eth_net_check_driver(struct ipa_eth_net_driver *nd) +{ + return + nd && + nd->bus && + nd->name && + nd->driver && + ipa_eth_net_check_ops(nd->ops); +} + +static int __ipa_eth_net_register_driver(struct ipa_eth_net_driver *nd) +{ + int rc; + + rc = ipa_eth_bus_register_driver(nd); + if (rc) { + ipa_eth_err("Failed to register network driver %s", nd->name); + return rc; + } + + mutex_lock(&ipa_eth_net_drivers_mutex); + list_add(&nd->driver_list, &ipa_eth_net_drivers); + mutex_unlock(&ipa_eth_net_drivers_mutex); + + ipa_eth_log("Registered network driver %s", nd->name); + + return 0; +} + +int ipa_eth_net_register_driver(struct ipa_eth_net_driver *nd) +{ + if (!ipa_eth_net_check_driver(nd)) { + ipa_eth_err("Net driver validation failed"); + return -EINVAL; + } + + return __ipa_eth_net_register_driver(nd); +} + +void ipa_eth_net_unregister_driver(struct ipa_eth_net_driver *nd) +{ + mutex_lock(&ipa_eth_net_drivers_mutex); + list_del(&nd->driver_list); + mutex_unlock(&ipa_eth_net_drivers_mutex); + + ipa_eth_bus_unregister_driver(nd); +} + +static inline bool __is_netdev_link_up(struct ipa_eth_device *eth_dev) +{ + return netif_carrier_ok(eth_dev->net_dev); +} + +static inline bool __is_netdev_iface_up(struct ipa_eth_device *eth_dev) +{ + return !!(eth_dev->net_dev->flags & IFF_UP); +} + +bool ipa_eth_net_check_active(struct ipa_eth_device *eth_dev) +{ + bool active = false; + struct rtnl_link_stats64 curr_rtnl_stats; + struct ipa_eth_device_private *ipa_priv = eth_dev_priv(eth_dev); + struct rtnl_link_stats64 *last_rtnl_stats = &ipa_priv->last_rtnl_stats; + + dev_get_stats(eth_dev->net_dev, &curr_rtnl_stats); + + trace_net_check_active(eth_dev, + last_rtnl_stats, &curr_rtnl_stats, ipa_priv->assume_active); + + if (ipa_priv->assume_active) { + ipa_priv->assume_active--; + active = true; + } + + if (curr_rtnl_stats.rx_packets != last_rtnl_stats->rx_packets) + active = true; + + *last_rtnl_stats = curr_rtnl_stats; + + return active; +} + +/* Event handler for netdevice events from upper interfaces */ +static int ipa_eth_net_upper_event( + struct ipa_eth_upper_device *upper_dev, + unsigned long event, void *ptr) +{ + struct ipa_eth_device *eth_dev = upper_dev->eth_dev; + + ASSERT_RTNL(); + + ipa_eth_dev_log(eth_dev, + "Received netdev event %s (0x%04lx) for %s", + ipa_eth_net_device_event_name(event), event, + upper_dev->net_dev->name); + + switch (event) { + case NETDEV_UP: + upper_dev->up = true; + break; + case NETDEV_DOWN: + upper_dev->up = false; + break; + default: + return NOTIFY_DONE; + } + + /* Register/unregister the upper interface from refresh wq to avoid + * race conditions. + */ + ipa_eth_device_refresh_sched(eth_dev); + + return NOTIFY_OK; +} + +static inline struct ipa_eth_upper_device *ipa_eth_search_upper( + struct ipa_eth_device *eth_dev, + struct net_device *net_dev) +{ + struct ipa_eth_upper_device *upper_dev = NULL; + struct ipa_eth_device_private *dev_priv = eth_dev->ipa_priv; + + ASSERT_RTNL(); + + list_for_each_entry(upper_dev, &dev_priv->upper_devices, upper_list) { + if (upper_dev->net_dev == net_dev) + return upper_dev; + } + + return NULL; +} + +/* Register all active upper interfaces, unregistering ones that are down. */ +int ipa_eth_net_register_upper(struct ipa_eth_device *eth_dev) +{ + int rc = 0; + struct ipa_eth_device_private *dev_priv = eth_dev->ipa_priv; + struct ipa_eth_upper_device *upper_dev = NULL; + + ipa_eth_dev_log(eth_dev, "Registering all active upper devices"); + + /* The list and its entries are updated by events from netdevice + * notifier which will hold rtnl mutex; use same mutex to synchronize. + */ + rtnl_lock(); + + list_for_each_entry(upper_dev, &dev_priv->upper_devices, upper_list) { + + /* Register upper interfaces that are up, unregister ones that + * are down. + */ + if (upper_dev->up && !upper_dev->registered) + rc |= ipa_eth_ep_register_upper_interface(upper_dev); + else if (upper_dev->registered && !upper_dev->up) + rc |= ipa_eth_ep_unregister_upper_interface(upper_dev); + } + + rtnl_unlock(); + + if (rc) + ipa_eth_dev_err(eth_dev, + "Failed to {un}register one or more upper devices"); + + return rc; +} + +/* Unregister all upper interfaces */ +int ipa_eth_net_unregister_upper(struct ipa_eth_device *eth_dev) +{ + int rc = 0; + struct ipa_eth_device_private *dev_priv = eth_dev->ipa_priv; + struct ipa_eth_upper_device *upper_dev = NULL; + + ipa_eth_dev_log(eth_dev, "Unregistering all upper devices"); + + rtnl_lock(); + + list_for_each_entry(upper_dev, &dev_priv->upper_devices, upper_list) { + if (likely(upper_dev->registered)) + rc |= ipa_eth_ep_unregister_upper_interface(upper_dev); + + } + + rtnl_unlock(); + + if (rc) + ipa_eth_dev_err(eth_dev, + "Failed to unregister one or more upper devices"); + + return rc; +} + +static int ipa_eth_net_link_upper(struct ipa_eth_device *eth_dev, + struct net_device *net_dev) +{ + struct ipa_eth_upper_device *upper_dev; + struct ipa_eth_device_private *dev_priv = eth_dev->ipa_priv; + + ASSERT_RTNL(); + + ipa_eth_dev_log(eth_dev, + "Linking upper interface %s", net_dev->name); + + upper_dev = kzalloc(sizeof(*upper_dev), GFP_KERNEL); + if (!upper_dev) { + ipa_eth_dev_err(eth_dev, "Failed to alloc upper dev"); + return -ENOMEM; + } + + upper_dev->eth_dev = eth_dev; + upper_dev->net_dev = net_dev; + + /* Fetch link status for unlike scenarios where the upper interface is + * being linked from ipa_eth_net_event_register(). + */ + upper_dev->up = __is_netdev_iface_up(eth_dev); + + list_add(&upper_dev->upper_list, &dev_priv->upper_devices); + + return 0; +} + +static int __ipa_eth_net_unlink_upper(struct ipa_eth_upper_device *upper_dev) +{ + ASSERT_RTNL(); + + ipa_eth_dev_log(upper_dev->eth_dev, + "Unlinking upper interface %s", upper_dev->net_dev->name); + + /* Even though we would have received a NETDEV_DOWN event prior to + * receiving a PRECHANGEUPPER(unlink) event, a device refresh may not + * have completed by this time; explicitly unregister the upper device. + */ + ipa_eth_ep_unregister_upper_interface(upper_dev); + + list_del(&upper_dev->upper_list); + kzfree(upper_dev); + + return 0; +} + +static int ipa_eth_net_unlink_upper(struct ipa_eth_device *eth_dev, + struct net_device *net_dev) +{ + struct ipa_eth_upper_device *upper_dev = + ipa_eth_search_upper(eth_dev, net_dev); + + if (!upper_dev) { + ipa_eth_dev_bug(eth_dev, + "Failed to find upper dev %s", net_dev->name); + return -ENODEV; + } + + return __ipa_eth_net_unlink_upper(upper_dev); +} + +static bool ipa_eth_net_update_link(struct ipa_eth_device *eth_dev) +{ + return __is_netdev_link_up(eth_dev) ? + !test_and_set_bit(IPA_ETH_IF_ST_LOWER_UP, ð_dev->if_state) : + test_and_clear_bit(IPA_ETH_IF_ST_LOWER_UP, ð_dev->if_state); + +} + +/* Use the register event to detect any previously registered upper devices. */ +static bool ipa_eth_net_event_register(struct ipa_eth_device *eth_dev, + unsigned long event, void *ptr) +{ + struct net_device *udev; + struct list_head *iter; + + bool refresh_needed = false; + struct net_device *net_dev = eth_dev->net_dev; + + /* In the unlikely scenario where an upper interface was created before + * we registered with netdevice notifier, manually link the upper dev to + * eth_dev. + */ + netdev_for_each_upper_dev_rcu(net_dev, udev, iter) { + if (!ipa_eth_net_link_upper(eth_dev, udev)) + refresh_needed = true; + } + + return refresh_needed; +} + +static bool ipa_eth_net_event_unregister(struct ipa_eth_device *eth_dev, + unsigned long event, void *ptr) +{ + struct ipa_eth_device_private *dev_priv = eth_dev->ipa_priv; + struct ipa_eth_upper_device *upper_dev = NULL; + struct ipa_eth_upper_device *tmp = NULL; + + /* Any upper interfaces discovered previously are expected to have been + * unlinked before the real dev deregisters with network sub-system. + * Since offload sub-system never deregisters with netdevice notifier + * until real dev is removed from the bus, and as the network driver is + * expected to keep registered with network sub-system until a bus level + * removal happens for the device, we really should not see any upper + * interfaces in upper_devices list at this point. + */ + list_for_each_entry_safe(upper_dev, tmp, + &dev_priv->upper_devices, upper_list) { + ipa_eth_dev_bug(eth_dev, + "Upper interface %s is unexpected", + upper_dev->net_dev->name); + + __ipa_eth_net_unlink_upper(upper_dev); + } + + return false; +} + +static bool ipa_eth_net_event_up(struct ipa_eth_device *eth_dev, + unsigned long event, void *ptr) +{ + return !test_and_set_bit(IPA_ETH_IF_ST_UP, ð_dev->if_state); +} + +static bool ipa_eth_net_event_down(struct ipa_eth_device *eth_dev, + unsigned long event, void *ptr) +{ + return test_and_clear_bit(IPA_ETH_IF_ST_UP, ð_dev->if_state); +} + +static bool ipa_eth_net_event_change(struct ipa_eth_device *eth_dev, + unsigned long event, void *ptr) +{ + return ipa_eth_net_update_link(eth_dev); +} + +static bool ipa_eth_net_event_pre_change_upper(struct ipa_eth_device *eth_dev, + unsigned long event, void *ptr) +{ + struct netdev_notifier_changeupper_info *upper_info = ptr; + + if (!upper_info->linking) + ipa_eth_net_unlink_upper(eth_dev, upper_info->upper_dev); + + return true; +} + +static bool ipa_eth_net_event_change_upper(struct ipa_eth_device *eth_dev, + unsigned long event, void *ptr) +{ + struct netdev_notifier_changeupper_info *upper_info = ptr; + + if (upper_info->linking) + ipa_eth_net_link_upper(eth_dev, upper_info->upper_dev); + + return true; +} + +typedef bool (*ipa_eth_net_event_handler)(struct ipa_eth_device *eth_dev, + unsigned long event, void *ptr); + +/* Event handlers for netdevice events from real interface */ +static ipa_eth_net_event_handler + ipa_eth_net_event_handlers[IPA_ETH_NET_DEVICE_MAX_EVENTS] = { + [NETDEV_REGISTER] = ipa_eth_net_event_register, + [NETDEV_UNREGISTER] = ipa_eth_net_event_unregister, + [NETDEV_UP] = ipa_eth_net_event_up, + [NETDEV_DOWN] = ipa_eth_net_event_down, + [NETDEV_CHANGE] = ipa_eth_net_event_change, + [NETDEV_CHANGELOWERSTATE] = ipa_eth_net_event_change, + [NETDEV_PRECHANGEUPPER] = ipa_eth_net_event_pre_change_upper, + [NETDEV_CHANGEUPPER] = ipa_eth_net_event_change_upper, +}; + +static int ipa_eth_net_device_event(struct notifier_block *nb, + unsigned long event, void *ptr) +{ + struct ipa_eth_upper_device *upper_dev; + struct net_device *net_dev = netdev_notifier_info_to_dev(ptr); + struct ipa_eth_device *eth_dev = container_of(nb, + struct ipa_eth_device, netdevice_nb); + + /* We re-use the notifier registered for real dev to also monitor for + * events from its upper interfaces. Once an upper interface is detected + * through CHANGEUPPER(linking) event from real dev and added to the + * upper_devices list, we let ipa_eth_net_upper_event() receive future + * events for the interface. The upper interface is similarly removed + * from the list on receiving PRECHANGEUPPER(!linking) event. Although + * this method will prevent us from handling REGISTER/UNREGISTER events, + * those can be deduced from {PRE}CHANGEUPPER if needed. + */ + upper_dev = ipa_eth_search_upper(eth_dev, net_dev); + if (upper_dev) + return ipa_eth_net_upper_event(upper_dev, event, ptr); + + if (net_dev != eth_dev->net_dev) + return NOTIFY_DONE; + + ipa_eth_dev_log(eth_dev, "Received netdev event %s (0x%04lx)", + ipa_eth_net_device_event_name(event), event); + + if (event < IPA_ETH_NET_DEVICE_MAX_EVENTS) { + ipa_eth_net_event_handler handler = + ipa_eth_net_event_handlers[event]; + bool refresh_needed = handler && handler(eth_dev, event, ptr); + + /* We can not wait for refresh to complete as we are holding + * the rtnl mutex. + */ + if (refresh_needed) + ipa_eth_device_refresh_sched(eth_dev); + } else { + ipa_eth_dev_err(eth_dev, "Event number out of bounds"); + return NOTIFY_DONE; + } + + return NOTIFY_OK; +} + +int ipa_eth_net_open_device(struct ipa_eth_device *eth_dev) +{ + int rc; + + rc = ipa_eth_nd_op(eth_dev, open_device, eth_dev); + if (rc) { + ipa_eth_dev_err(eth_dev, "Failed to open net device"); + goto err_open; + } + + if (!eth_dev->net_dev) { + rc = -EFAULT; + ipa_eth_dev_err(eth_dev, + "Network driver failed to fill net_dev"); + goto err_net_dev; + } + + /* We need to register with netdevice notifier early in the real dev + * lifetime as we need to also use the same registration to discover all + * its upper devices using CHANGEUPPER events, and further accept any + * events from those upper devices too. + */ + eth_dev->netdevice_nb.notifier_call = ipa_eth_net_device_event; + rc = register_netdevice_notifier(ð_dev->netdevice_nb); + if (rc) { + ipa_eth_dev_err(eth_dev, "Failed to register netdev notifier"); + goto err_register; + } + + /* When registering with netdevice notifier, only REGISTER and UP events + * are replayed. Fetch current link state; future link state updates + * will be processed through CHANGE event. + */ + if (ipa_eth_net_update_link(eth_dev)) + ipa_eth_device_refresh_sched(eth_dev); + + return 0; + +err_register: + eth_dev->netdevice_nb.notifier_call = NULL; +err_net_dev: + ipa_eth_nd_op(eth_dev, close_device, eth_dev); +err_open: + return rc; +} + +void ipa_eth_net_close_device(struct ipa_eth_device *eth_dev) +{ + unregister_netdevice_notifier(ð_dev->netdevice_nb); + eth_dev->netdevice_nb.notifier_call = NULL; + + ipa_eth_nd_op(eth_dev, close_device, eth_dev); +} + +static phys_addr_t ipa_eth_dma_pgaddr(struct ipa_eth_device *eth_dev, + const void *vaddr) +{ + return page_to_phys(vmalloc_to_page(vaddr)); +} + +static phys_addr_t ipa_eth_dma_paddr(struct ipa_eth_device *eth_dev, + const void *vaddr) +{ + return ipa_eth_dma_pgaddr(eth_dev, vaddr) | + ((phys_addr_t)vaddr & ~PAGE_MASK); +} + +static int ipa_eth_dma_alloc(struct ipa_eth_device *eth_dev, + size_t size, gfp_t gfp, struct ipa_eth_resource *mem) +{ + if (!eth_dev || !eth_dev->dev) { + ipa_eth_dev_err(eth_dev, "eth_dev is invalid"); + return -EFAULT; + } + + if (!mem) { + ipa_eth_dev_err(eth_dev, "Missing mem parameter"); + return -EINVAL; + } + + memset(mem, 0, sizeof(*mem)); + + mem->vaddr = dma_alloc_coherent(eth_dev->dev, size, &mem->daddr, gfp); + if (!mem->vaddr) { + ipa_eth_dev_err(eth_dev, + "Failed to allocate memory of size %zu", size); + return -ENOMEM; + } + + mem->size = size; + mem->paddr = ipa_eth_dma_paddr(eth_dev, mem->vaddr); + + ipa_eth_dev_log(eth_dev, + "Allocated memory of size %zu at [%pK,%pad,%pap]", + mem->size, mem->vaddr, &mem->daddr, &mem->paddr); + + return 0; +} + +static void ipa_eth_dma_free(struct ipa_eth_device *eth_dev, + struct ipa_eth_resource *mem) +{ + dma_free_coherent(eth_dev->dev, mem->size, mem->vaddr, mem->daddr); +} + +static size_t ipa_eth_dma_walk(struct ipa_eth_device *eth_dev, + const struct ipa_eth_resource *mem, + ipa_eth_mem_it_t it, void *arg) +{ + struct ipa_eth_resource cmem = { + .size = PAGE_SIZE, + .vaddr = (void *) rounddown((unsigned long) mem->vaddr, + PAGE_SIZE), + .daddr = rounddown(mem->daddr, PAGE_SIZE), + }; + + if ((mem->daddr - cmem.daddr) != (mem->vaddr - cmem.vaddr)) { + ipa_eth_dev_err(eth_dev, + "Alignment mismatch between daddr and addr"); + return 0; + } + + if (cmem.daddr != mem->daddr) + ipa_eth_dev_dbg(eth_dev, "Daddr %pad is realigned to %pad", + mem->daddr, &cmem.daddr); + + if (cmem.vaddr != mem->vaddr) + ipa_eth_dev_dbg(eth_dev, "Vaddr %pK is realigned to %pK", + mem->vaddr, cmem.vaddr); + + while (cmem.vaddr < (mem->vaddr + mem->size)) { + cmem.paddr = ipa_eth_dma_paddr(eth_dev, cmem.vaddr); + + if (it(eth_dev, &cmem, arg)) { + ipa_eth_dev_err(eth_dev, + "Remap failed for page at [%pK,%pad,%pap]", + cmem.vaddr, &cmem.daddr, &cmem.paddr); + break; + } + + cmem.vaddr += PAGE_SIZE; + cmem.daddr += PAGE_SIZE; + } + + return clamp_val(cmem.vaddr, mem->vaddr, mem->vaddr + mem->size) - + mem->vaddr; +} + +static struct ipa_eth_dma_allocator default_dma_allocator = { + .name = "ipa_eth_dma_alloc_coherent", + .paddr = ipa_eth_dma_paddr, + .alloc = ipa_eth_dma_alloc, + .free = ipa_eth_dma_free, + .walk = ipa_eth_dma_walk, +}; + +static int ipa_eth_net_process_skb(struct ipa_eth_channel *ch, + struct sk_buff *skb) +{ + if (likely(IPA_ETH_CH_IS_RX(ch))) + return ipa_eth_net_receive_skb(ch->eth_dev, skb); + else + return ipa_eth_net_transmit_skb(ch->eth_dev, skb); +} + +static enum ipa_smmu_cb_type ipa_eth_hw_to_cb_type( + struct ipa_eth_channel *ch, + enum ipa_eth_hw_type hw_type) +{ + enum ipa_smmu_cb_type cb_type = IPA_SMMU_CB_MAX; + + switch (hw_type) { + case IPA_ETH_HW_UC: + cb_type = IPA_SMMU_CB_UC; + break; + + case IPA_ETH_HW_GSI: + case IPA_ETH_HW_IPA: /* All current ethernet client EPs uses GSI CB */ + cb_type = IPA_SMMU_CB_AP; + break; + + default: + ipa_eth_bug("Unknown Eth CB type %d", hw_type); + break; + } + + return cb_type; +} + +static struct ipa_smmu_cb_ctx *ipa_eth_get_smmu_ctx( + enum ipa_smmu_cb_type cb_type) +{ + struct ipa_smmu_cb_ctx *cb; + + if (cb_type >= IPA_SMMU_CB_MAX) + return NULL; + + cb = ipa3_get_smmu_ctx(cb_type); + + if (!cb || !cb->valid) + return NULL; + + return cb; +} + +static int ipa_eth_hw_to_cb_map_one(struct ipa_eth_channel *ch, + enum ipa_eth_hw_type hw_type, + struct ipa_eth_hw_map_param *hw_map_param, + enum ipa_smmu_cb_type cb_type, + struct ipa_eth_cb_map_param *cb_map_param) +{ + struct ipa_smmu_cb_ctx *cb_ctx = ipa_eth_get_smmu_ctx(cb_type); + + if (!cb_ctx) { + ipa_eth_dev_err(ch->eth_dev, + "Failed to get smmu ctx for hw type %d", + hw_type); + return -EFAULT; + } + + /* One CB can be shared between multiple HW. Make sure the parameters + * for CB are aggregated to a common denominator. + */ + cb_map_param->map = cb_map_param->map || hw_map_param->map; + cb_map_param->sym = cb_map_param->sym || hw_map_param->sym; + cb_map_param->iommu_prot |= hw_map_param->read ? IOMMU_READ : 0; + cb_map_param->iommu_prot |= hw_map_param->write ? IOMMU_WRITE : 0; + + switch (cb_map_param->iommu_prot) { + case IOMMU_READ: + cb_map_param->dma_dir = DMA_TO_DEVICE; + break; + case IOMMU_WRITE: + cb_map_param->dma_dir = DMA_FROM_DEVICE; + break; + case 0: + cb_map_param->dma_dir = DMA_NONE; + break; + default: + cb_map_param->dma_dir = DMA_BIDIRECTIONAL; + break; + } + + cb_map_param->cb_ctx = cb_ctx; + + return 0; +} + +/** + * ipa_eth_hw_to_cb_map() - Translate memory mapping parameters from IPA + * hardware types to context bank types + * @ch: Channel to which the parameters belong + * @hw_map_params: Mapping parameters for various hardware types + * @cb_map_params: Translated mapping parameters output for context banks + * + * Return: 0 if successful, non-zero otherwise + */ +static int ipa_eth_hw_to_cb_map(struct ipa_eth_channel *ch, + struct ipa_eth_hw_map_param *hw_map_params, + struct ipa_eth_cb_map_param *cb_map_params) +{ + int rc; + enum ipa_eth_hw_type hw_type; + + for (hw_type = 0; hw_type < IPA_ETH_HW_MAX; hw_type++) { + enum ipa_smmu_cb_type cb_type = + ipa_eth_hw_to_cb_type(ch, hw_type); + + rc = ipa_eth_hw_to_cb_map_one(ch, + hw_type, &hw_map_params[hw_type], + cb_type, &cb_map_params[cb_type]); + if (rc) { + ipa_eth_dev_err(ch->eth_dev, + "Failed to convert from hw to cb map params"); + return rc; + } + } + + return 0; +} + +static int ipa_eth_cb_mapper_sym(struct ipa_eth_device *eth_dev, + const struct ipa_eth_resource *cmem, void *arg) +{ + int rc; + struct ipa_eth_cb_map_param *cb_map_param = arg; + const struct ipa_smmu_cb_ctx *cb_ctx = cb_map_param->cb_ctx; + + if (!cb_map_param->map) { + ipa_eth_dev_bug(eth_dev, "CB map is not enabled"); + return -EFAULT; + } + + if (!cmem->size) { + ipa_eth_dev_bug(eth_dev, "Requested CB mapping of size 0"); + return -EINVAL; + } + + rc = ipa3_iommu_map(cb_ctx->mapping->domain, + cmem->daddr, cmem->paddr, cmem->size, cb_map_param->iommu_prot); + if (rc) { + ipa_eth_dev_err(eth_dev, + "Failed to map %zu bytes into CB %s at [%pK,%pad,%pap]", + cmem->size, cb_ctx->iommu->name, + cmem->vaddr, &cmem->daddr, &cmem->paddr); + } + + return rc; +} + +static int ipa_eth_cb_unmapper_sym(struct ipa_eth_device *eth_dev, + const struct ipa_eth_resource *cmem, void *arg) +{ + size_t size; + struct ipa_eth_cb_map_param *cb_map_param = arg; + const struct ipa_smmu_cb_ctx *cb_ctx = cb_map_param->cb_ctx; + + if (!cmem->size) { + ipa_eth_dev_err(eth_dev, "Requested CB unmapping of size 0"); + return -EINVAL; + } + + size = iommu_unmap(cb_ctx->mapping->domain, cmem->daddr, cmem->size); + if (size != cmem->size) { + ipa_eth_dev_err(eth_dev, + "Failed to unmap %zu bytes in domain %s at daddr %pad", + cmem->size, dev_name(cb_ctx->dev), &cmem->daddr); + return -EFAULT; + } + + return 0; +} + +static size_t ipa_eth_net_remap(struct ipa_eth_device *eth_dev, + const struct ipa_eth_resource *mem, + struct ipa_eth_dma_allocator *allocator, + ipa_eth_mem_it_t map_op, + struct ipa_eth_cb_map_param *cb_map_param) +{ + return allocator->walk(eth_dev, mem, map_op, cb_map_param); +} + +/** + * ipa_eth_net_cb_map_sym() - Symmetrically map a channel memory to a given IPA + * SMMU context bank + * @eth_dev: Device to which the channel memory belong + * @ch_mem: Channel memory that need to be mapped + * @allocator: Allocator used for allocating the channel memory + * @cb_type: IPA SMMU context bank to which to map the memory + * @cb_map_param: Parameters for mapping memory to the given context bank + * + * Symmetric mapping creates the same DADDR->PADDR memory mapping in the + * given IPA context bank as in the original channel memory @ch_mem. Use this + * API if any of the IPA hardware need to use the same IO virtual address as + * the network device. + * + * Return: 0 on successful mapping to the given @cb_type, non-zero otherwise. + */ +static int ipa_eth_net_cb_map_sym(struct ipa_eth_device *eth_dev, + struct ipa_eth_channel_mem *ch_mem, + struct ipa_eth_dma_allocator *allocator, + enum ipa_smmu_cb_type cb_type, + struct ipa_eth_cb_map_param *cb_map_param) +{ + size_t size; + struct ipa_eth_resource *cb_mem = &ch_mem->cb_mem[cb_type]; + const struct ipa_smmu_cb_ctx *cb_ctx = cb_map_param->cb_ctx; + + if (!ch_mem->mem.daddr) { + ipa_eth_dev_err(eth_dev, + "Symmetric mapping requires a valid DMA address"); + return -EFAULT; + } + + ipa_eth_dev_log(eth_dev, + "Mapping %zu bytes into domain %s at [%pK,%pad,%pap]", + ch_mem->mem.size, dev_name(cb_ctx->dev), + ch_mem->mem.vaddr, &ch_mem->mem.daddr, &ch_mem->mem.paddr); + + size = ipa_eth_net_remap(eth_dev, &ch_mem->mem, allocator, + ipa_eth_cb_mapper_sym, cb_map_param); + if (size != ch_mem->mem.size) { + /* Unmap any partially mapped memory */ + struct ipa_eth_resource unmap_mem = { + .size = size, + .vaddr = ch_mem->mem.vaddr, + .daddr = ch_mem->mem.daddr, + .paddr = ch_mem->mem.paddr, + }; + + ipa_eth_dev_err(eth_dev, + "Failed to map %zu bytes to domain %s, undo mapping", + ch_mem->mem.size - size, dev_name(cb_ctx->dev)); + + (void) ipa_eth_net_remap(eth_dev, &unmap_mem, allocator, + ipa_eth_cb_unmapper_sym, cb_map_param); + + return -EFAULT; + } + + *cb_mem = ch_mem->mem; + + ipa_eth_dev_log(eth_dev, + "Mapped %zu bytes into domain %s at [%pK,%pad,%pap]", + cb_mem->size, dev_name(cb_ctx->dev), + cb_mem->vaddr, &cb_mem->daddr, &cb_mem->paddr); + + return 0; +} + +/** + * ipa_eth_net_cb_map_asym() - Map a channel memory to a context bank, not + * necessarily using the same IO virtual address + * @eth_dev: Device to which the channel memory belong + * @ch_mem: Channel memory that need to be mapped + * @allocator: Allocator used for allocating the channel memory + * @cb_type: IPA SMMU context bank to which to map the memory + * @cb_map_param: Parameters for mapping memory to the given context bank + * + * Use this API if symmetric mapping is not required for a channel memory. See + * ipa_eth_net_cb_map_sym() for more details on symmetric mapping. + * + * Note that since assymmetric mapping uses dma_map_single(), it can only be + * used to map memory that was allocated using kmalloc(). Memory allocated from + * vmalloc region (like dma_alloc_coherent() on ARM targets) can not be mapped + * again using this API. + * + * Return: 0 on success, non-zero otherwise + */ +static int ipa_eth_net_cb_map_asym(struct ipa_eth_device *eth_dev, + struct ipa_eth_channel_mem *ch_mem, + struct ipa_eth_dma_allocator *allocator, + enum ipa_smmu_cb_type cb_type, + struct ipa_eth_cb_map_param *cb_map_param) +{ + struct ipa_eth_resource *cb_mem = &ch_mem->cb_mem[cb_type]; + const struct ipa_smmu_cb_ctx *cb_ctx = cb_map_param->cb_ctx; + + if (is_vmalloc_addr(ch_mem->mem.vaddr)) { + ipa_eth_dev_err(eth_dev, + "Asymmetric mapping cannot use vmalloc address"); + return -EFAULT; + } + + ipa_eth_dev_log(eth_dev, + "Mapping %zu bytes into device %s from [%pK,%pad,%pap]", + ch_mem->mem.size, dev_name(cb_ctx->dev), + ch_mem->mem.vaddr, &ch_mem->mem.daddr, &ch_mem->mem.paddr); + + *cb_mem = ch_mem->mem; + + cb_mem->daddr = dma_map_single(cb_ctx->dev, + ch_mem->mem.vaddr, ch_mem->mem.size, + cb_map_param->dma_dir); + if (dma_mapping_error(cb_ctx->dev, cb_mem->daddr)) { + cb_mem->size = 0; + ipa_eth_dev_err(eth_dev, "Failed to map buffer to device %s", + dev_name(cb_ctx->dev)); + return -EFAULT; + } + + ipa_eth_dev_log(eth_dev, + "Mapped %zu bytes into device %s at [%pK,%pad,%pap]", + cb_mem->size, dev_name(cb_ctx->dev), + cb_mem->vaddr, &cb_mem->daddr, &cb_mem->paddr); + + return 0; +} + +static int ipa_eth_net_cb_map_one(struct ipa_eth_device *eth_dev, + struct ipa_eth_channel_mem *ch_mem, + struct ipa_eth_dma_allocator *allocator, + enum ipa_smmu_cb_type cb_type, + struct ipa_eth_cb_map_param *cb_map_param) +{ + if (cb_map_param->sym) + return ipa_eth_net_cb_map_sym( + eth_dev, ch_mem, allocator, cb_type, cb_map_param); + else + return ipa_eth_net_cb_map_asym( + eth_dev, ch_mem, allocator, cb_type, cb_map_param); +} + +static void ipa_eth_net_cb_unmap_sym(struct ipa_eth_device *eth_dev, + struct ipa_eth_channel_mem *ch_mem, + struct ipa_eth_dma_allocator *allocator, + enum ipa_smmu_cb_type cb_type, + struct ipa_eth_cb_map_param *cb_map_param) +{ + struct ipa_eth_resource *cb_mem = &ch_mem->cb_mem[cb_type]; + const struct ipa_smmu_cb_ctx *cb_ctx = cb_map_param->cb_ctx; + + if (!cb_mem->size) + return; + + ipa_eth_dev_log(eth_dev, + "Unmapping %zu bytes in domain %s from daddr %pad", + cb_mem->size, dev_name(cb_ctx->dev), &cb_mem->daddr); + + (void) ipa_eth_net_remap(eth_dev, &ch_mem->mem, allocator, + ipa_eth_cb_unmapper_sym, cb_map_param); + + ipa_eth_dev_log(eth_dev, + "Unmapped %zu bytes in domain %s from daddr %pad", + cb_mem->size, dev_name(cb_ctx->dev), &cb_mem->daddr); + + cb_mem->size = 0; +} + +static void ipa_eth_net_cb_unmap_asym(struct ipa_eth_device *eth_dev, + struct ipa_eth_channel_mem *ch_mem, + struct ipa_eth_dma_allocator *allocator, + enum ipa_smmu_cb_type cb_type, + struct ipa_eth_cb_map_param *cb_map_param) +{ + struct ipa_eth_resource *cb_mem = &ch_mem->cb_mem[cb_type]; + const struct ipa_smmu_cb_ctx *cb_ctx = cb_map_param->cb_ctx; + + if (!cb_mem->size) + return; + + ipa_eth_dev_log(eth_dev, + "Unmapping %zu bytes in device %s from daddr %pad", + cb_mem->size, dev_name(cb_ctx->dev), &cb_mem->daddr); + + dma_unmap_single(cb_ctx->dev, cb_mem->daddr, cb_mem->size, + cb_map_param->dma_dir); + + ipa_eth_dev_log(eth_dev, + "Unmapped %zu bytes in device %s from daddr %pad", + cb_mem->size, dev_name(cb_ctx->dev), &cb_mem->daddr); + + cb_mem->size = 0; +} + +static void ipa_eth_net_cb_unmap_one(struct ipa_eth_device *eth_dev, + struct ipa_eth_channel_mem *ch_mem, + struct ipa_eth_dma_allocator *allocator, + enum ipa_smmu_cb_type cb_type, + struct ipa_eth_cb_map_param *cb_map_param) +{ + if (cb_map_param->sym) + return ipa_eth_net_cb_unmap_sym( + eth_dev, ch_mem, allocator, cb_type, cb_map_param); + else + return ipa_eth_net_cb_unmap_asym( + eth_dev, ch_mem, allocator, cb_type, cb_map_param); +} + +/** + * ipa_eth_net_cb_unmap_ch_mem() - Unmap one channel memory from one of more + * IPA context banks + * @eth_dev: Device to which the channel memory belong + * @ch_mem: Channel memory that need to be unmapped + * @allocator: Allocator used for allocating the channel memory + * @cb_map_params: Mapping parameters for all known IPA context banks + * + * Use this API to unmap a channel memory from various IPA context banks that + * was previously mapped using ipa_eth_net_cb_map_ch_mem(). + */ +static void ipa_eth_net_cb_unmap_ch_mem(struct ipa_eth_device *eth_dev, + struct ipa_eth_channel_mem *ch_mem, + struct ipa_eth_dma_allocator *allocator, + struct ipa_eth_cb_map_param *cb_map_params) +{ + enum ipa_smmu_cb_type cb_type; + + if (!ch_mem->cb_mem) + return; + + for (cb_type = 0; cb_type < IPA_SMMU_CB_MAX; cb_type++) { + if (!cb_map_params[cb_type].map) + continue; + + ipa_eth_net_cb_unmap_one(eth_dev, ch_mem, allocator, + cb_type, &cb_map_params[cb_type]); + } + + kfree(ch_mem->cb_mem); + ch_mem->cb_mem = NULL; +} + +/** + * ipa_eth_net_cb_map_ch_mem() - Map one channel memory to one or more IPA + * context banks + * @eth_dev: Device to which the channel memory belong + * @ch_mem: Channel memory that need to be mapped + * @allocator: Allocator used for allocating the channel memory + * @cb_map_params: Mapping parameters for all known IPA context banks + * + * Return: 0 on success, non-zero otherwise + */ +static int ipa_eth_net_cb_map_ch_mem(struct ipa_eth_device *eth_dev, + struct ipa_eth_channel_mem *ch_mem, + struct ipa_eth_dma_allocator *allocator, + struct ipa_eth_cb_map_param *cb_map_params) +{ + int rc; + enum ipa_smmu_cb_type cb_type; + + if (ch_mem->cb_mem) { + ipa_eth_dev_err(eth_dev, "CB mem is already initialized"); + return -EEXIST; + } + + ch_mem->cb_mem = kzalloc(sizeof(*ch_mem->cb_mem) * IPA_SMMU_CB_MAX, + GFP_KERNEL); + if (!ch_mem->cb_mem) { + ipa_eth_dev_err(eth_dev, "Failed to alloc CB mem resource"); + return -ENOMEM; + } + + for (cb_type = 0; cb_type < IPA_SMMU_CB_MAX; cb_type++) { + if (!cb_map_params[cb_type].map) + continue; + + rc = ipa_eth_net_cb_map_one(eth_dev, ch_mem, allocator, + cb_type, &cb_map_params[cb_type]); + if (rc) + goto err_map; + } + + return 0; + +err_map: + ipa_eth_net_cb_unmap_ch_mem(eth_dev, ch_mem, allocator, cb_map_params); + return -ENOMEM; +} + +/** + * ipa_eth_net_hw_unmap_desc_mem() - Unmap all descriptor memory from various + * IPA hardware types + * @eth_dev: Device to which the channel belong + * @ch: Channel whose descriptor memory need to be unmapped + * + * Use this API to unmap any descriptor memory previously mapped to any of the + * IPA hardware types. + */ +static void ipa_eth_net_hw_unmap_desc_mem(struct ipa_eth_device *eth_dev, + struct ipa_eth_channel *ch) +{ + int rc; + struct ipa_eth_channel_mem *ch_mem; + struct ipa_eth_cb_map_param cb_map_params[IPA_SMMU_CB_MAX]; + + memset(cb_map_params, 0, sizeof(cb_map_params)); + + rc = ipa_eth_hw_to_cb_map(ch, + ch->mem_params.desc.hw_map_params, cb_map_params); + if (rc) { + ipa_eth_dev_err(eth_dev, + "Failed to convert map params from hw to cb"); + return; + } + + list_for_each_entry(ch_mem, &ch->desc_mem, mem_list_entry) { + ipa_eth_net_cb_unmap_ch_mem(eth_dev, ch_mem, + ch->mem_params.desc.allocator, cb_map_params); + } +} + +/** + * ipa_eth_net_hw_map_desc_mem() - Map all descriptor memory to various IPA + * hadware types + * @eth_dev: Device to which the channel belong + * @ch: Channel from which the descriptor memory need to be mapped + * + * The API uses hardware mapping parameters listed in hw_map_params[] of struct + * ipa_eth_desc_params to determine how the mapping need to be perfomed. The + * actual SMMU context banks used by each hardware type is determined by using + * the ipa_eth_hw_to_cb_map() API. + * + * Return: 0 on success, non-zero otherwise + */ +static int ipa_eth_net_hw_map_desc_mem(struct ipa_eth_device *eth_dev, + struct ipa_eth_channel *ch) +{ + int rc; + struct ipa_eth_channel_mem *ch_mem; + struct ipa_eth_cb_map_param cb_map_params[IPA_SMMU_CB_MAX]; + + memset(cb_map_params, 0, sizeof(cb_map_params)); + + rc = ipa_eth_hw_to_cb_map(ch, + ch->mem_params.desc.hw_map_params, cb_map_params); + if (rc) { + ipa_eth_dev_err(eth_dev, + "Failed to convert map params from hw to cb"); + return rc; + } + + list_for_each_entry(ch_mem, &ch->desc_mem, mem_list_entry) { + rc = ipa_eth_net_cb_map_ch_mem(eth_dev, ch_mem, + ch->mem_params.desc.allocator, cb_map_params); + if (rc) + goto err_map; + } + + return 0; + +err_map: + ipa_eth_net_hw_unmap_desc_mem(eth_dev, ch); + return rc; +} + +/** + * ipa_eth_net_hw_unmap_buff_mem() - Unmap all buffer memory from various IPA + * hardware types + * @eth_dev: Device to which the channel belong + * @ch: Channel whose buffer memory need to be unmapped + * + * Use this API to unmap any buffer memory previously mapped to any of the IPA + * hardware types. + */ +static void ipa_eth_net_hw_unmap_buff_mem(struct ipa_eth_device *eth_dev, + struct ipa_eth_channel *ch) +{ + int rc; + struct ipa_eth_channel_mem *ch_mem; + struct ipa_eth_cb_map_param cb_map_params[IPA_SMMU_CB_MAX]; + + memset(cb_map_params, 0, sizeof(cb_map_params)); + + rc = ipa_eth_hw_to_cb_map(ch, + ch->mem_params.buff.hw_map_params, cb_map_params); + if (rc) { + ipa_eth_dev_err(eth_dev, + "Failed to convert map params from hw to cb"); + return; + } + + list_for_each_entry(ch_mem, &ch->buff_mem, mem_list_entry) { + ipa_eth_net_cb_unmap_ch_mem(eth_dev, ch_mem, + ch->mem_params.buff.allocator, cb_map_params); + } +} + +/** + * ipa_eth_net_hw_map_buff_mem() - Map all buffer memory to various IPA hadware + * types + * @eth_dev: Device to which the channel belong + * @ch: Channel from which the buffer memory need to be mapped + * + * The API uses hardware mapping parameters listed in hw_map_params[] of struct + * ipa_eth_buff_params to determine how the mapping need to be perfomed. The + * actual SMMU context banks used by each hardware type is determined by using + * the ipa_eth_hw_to_cb_map() API. + * + * Return: 0 on success, non-zero otherwise + */ +static int ipa_eth_net_hw_map_buff_mem(struct ipa_eth_device *eth_dev, + struct ipa_eth_channel *ch) +{ + int rc; + struct ipa_eth_channel_mem *ch_mem; + struct ipa_eth_cb_map_param cb_map_params[IPA_SMMU_CB_MAX]; + + memset(cb_map_params, 0, sizeof(cb_map_params)); + + rc = ipa_eth_hw_to_cb_map(ch, + ch->mem_params.buff.hw_map_params, cb_map_params); + if (rc) { + ipa_eth_dev_err(eth_dev, + "Failed to convert map params from hw to cb"); + return rc; + } + + list_for_each_entry(ch_mem, &ch->buff_mem, mem_list_entry) { + rc = ipa_eth_net_cb_map_ch_mem(eth_dev, ch_mem, + ch->mem_params.buff.allocator, cb_map_params); + if (rc) + goto err_map; + } + + return 0; + +err_map: + ipa_eth_net_hw_unmap_buff_mem(eth_dev, ch); + return rc; +} + +/** + * ipa_eth_net_hw_map_channel() - Map all the channel memory to various IPA + * hardware types + * @eth_dev: Device to which the channel belong + * @ch: Channel whose memory need to be mapped + * + * Return: 0 if successful, non-zero otherwise + */ +static int ipa_eth_net_hw_map_channel(struct ipa_eth_device *eth_dev, + struct ipa_eth_channel *ch) +{ + int rc; + + rc = ipa_eth_net_hw_map_desc_mem(eth_dev, ch); + if (rc) + return rc; + + rc = ipa_eth_net_hw_map_buff_mem(eth_dev, ch); + if (rc) { + ipa_eth_net_hw_unmap_desc_mem(eth_dev, ch); + return rc; + } + + return 0; +} + +/** + * ipa_eth_net_cb_unmap_channel() - Unmap channel descriptor and buffer memory + * from IPA CBs + * @eth_dev: Ethernet device + * @ch: Ethernet device channel + */ +static void ipa_eth_net_hw_unmap_channel(struct ipa_eth_device *eth_dev, + struct ipa_eth_channel *ch) +{ + ipa_eth_net_hw_unmap_desc_mem(eth_dev, ch); + ipa_eth_net_hw_unmap_buff_mem(eth_dev, ch); +} + +/** + * ipa_eth_net_alloc_channel() - Allocate and initialize ipa_eth_channel + * @eth_dev: Ethernet device + * @dir: Channel direction + * @events: Supported events + * @featues: Supported features + * @mem_params: Channel memory allocation parameters + * + * This API is expected to be called by network driver .request_channel() + * callback implementation. + * + * Return: Pointer to the allocated ipa_eth_channel, NULL if the allocation + * fails + */ +struct ipa_eth_channel *ipa_eth_net_alloc_channel( + struct ipa_eth_device *eth_dev, enum ipa_eth_channel_dir dir, + unsigned long events, unsigned long features, + const struct ipa_eth_channel_mem_params *mem_params) +{ + struct ipa_eth_channel *channel; + + if (!mem_params) { + ipa_eth_dev_err(eth_dev, "Missing channel mem params"); + return NULL; + } + + channel = kzalloc(sizeof(*channel), GFP_KERNEL); + if (!channel) + return NULL; + + channel->eth_dev = eth_dev; + channel->direction = dir; + channel->events = events; + channel->features = features; + + channel->mem_params = *mem_params; + + INIT_LIST_HEAD(&channel->desc_mem); + INIT_LIST_HEAD(&channel->buff_mem); + + return channel; +} +EXPORT_SYMBOL(ipa_eth_net_alloc_channel); + +/** + * ipa_eth_net_free_channel() - Deallocate an ipa_eth_channel previously + * allocated by ipa_eth_net_alloc_channel() + * @channel: Channel to be deallocated + */ +void ipa_eth_net_free_channel(struct ipa_eth_channel *channel) +{ + struct ipa_eth_device *eth_dev = channel->eth_dev; + + if (!list_empty(&channel->desc_mem)) + ipa_eth_dev_bug(eth_dev, "Descriptor memory still in use"); + + if (!list_empty(&channel->desc_mem)) + ipa_eth_dev_bug(eth_dev, "Buffer memory still in use"); + + kzfree(channel); +} +EXPORT_SYMBOL(ipa_eth_net_free_channel); + + +/** + * ipa_eth_net_request_channel() - Request a channel from network device to be + * used for a specific end-point + * @eth_dev: Ethernet device + * @ipa_client: IPA EP client enum that also determines the channel direction + * @events: Refer documentation of ipa_eth_net_ops.request_channel() + * @features: Refer documentation of ipa_eth_net_ops.request_channel() + * @mem_params: Refer documentation of ipa_eth_net_ops.request_channel() + * + * Offload drivers should use this API in order to invoke the network driver API + * ipa_eth_net_ops.request_channel(). The function also initializes EP context + * and maps channel memory to various IPA SMMU context banks. + * + * Return: Allocated channel if the allocation succeeds. NULL otherwise. + */ +struct ipa_eth_channel *ipa_eth_net_request_channel( + struct ipa_eth_device *eth_dev, enum ipa_client_type ipa_client, + unsigned long events, unsigned long features, + const struct ipa_eth_channel_mem_params *mem_params) +{ + int rc; + bool vlan_mode; + int ipa_ep_num; + struct ipa_eth_channel *ch; + enum ipa_eth_channel_dir dir; + struct ipa_eth_channel_mem_params params; + struct ipa3_ep_context *ep_ctx = NULL; + + if (!mem_params) { + ipa_eth_dev_err(eth_dev, "Missing channel mem params"); + return NULL; + } + + params = *mem_params; + + if (!params.desc.allocator) + params.desc.allocator = &default_dma_allocator; + + if (!params.buff.allocator) + params.buff.allocator = &default_dma_allocator; + + dir = IPA_CLIENT_IS_PROD(ipa_client) ? IPA_ETH_DIR_RX : IPA_ETH_DIR_TX; + + ipa_ep_num = ipa_get_ep_mapping(ipa_client); + if (ipa_ep_num == IPA_EP_NOT_ALLOCATED) { + ipa_eth_dev_err(eth_dev, + "Could not determine EP number for client %d", + ipa_client); + return NULL; + } + + ep_ctx = &ipa3_ctx->ep[ipa_ep_num]; + if (ep_ctx->valid) { + ipa_eth_dev_err(eth_dev, + "EP context is already initialiazed"); + return NULL; + } + + rc = ipa3_is_vlan_mode(IPA_VLAN_IF_ETH, &vlan_mode); + if (rc) { + ipa_eth_dev_err(eth_dev, + "Could not determine IPA VLAN mode"); + return NULL; + } + + ch = ipa_eth_nd_op(eth_dev, request_channel, + eth_dev, dir, events, features, ¶ms); + if (IS_ERR_OR_NULL(ch)) { + ipa_eth_dev_err(eth_dev, + "Failed to request channel from net driver %s", + eth_dev->nd->name); + return ch; + } + + ch->ipa_ep_num = ipa_ep_num; + ch->ipa_client = ipa_client; + ch->process_skb = ipa_eth_net_process_skb; + ch->eth_dev = eth_dev; + + if (ipa_eth_net_hw_map_channel(eth_dev, ch)) { + ipa_eth_dev_err(eth_dev, + "Failed to map channel memory to IPA CBs"); + ipa_eth_nd_op(eth_dev, release_channel, ch); + return NULL; + } + + ipa_eth_ep_init_ctx(ch, vlan_mode); + + if (dir == IPA_ETH_CH_DIR_RX) + list_add(&ch->channel_list, ð_dev->rx_channels); + else + list_add(&ch->channel_list, ð_dev->tx_channels); + + return ch; +} +EXPORT_SYMBOL(ipa_eth_net_request_channel); + +/** + * ipa_eth_net_release_channel() - Releases a channel presiously allocated using + * ipa_eth_net_request_channel() + * @ch: Channel to be released + */ +void ipa_eth_net_release_channel(struct ipa_eth_channel *ch) +{ + list_del(&ch->channel_list); + ipa_eth_ep_deinit_ctx(ch); + ipa_eth_net_hw_unmap_channel(ch->eth_dev, ch); + return ipa_eth_nd_op(ch->eth_dev, release_channel, ch); +} +EXPORT_SYMBOL(ipa_eth_net_release_channel); + +int ipa_eth_net_enable_channel(struct ipa_eth_channel *ch) +{ + return ipa_eth_nd_op(ch->eth_dev, enable_channel, ch); +} +EXPORT_SYMBOL(ipa_eth_net_enable_channel); + +int ipa_eth_net_disable_channel(struct ipa_eth_channel *ch) +{ + return ipa_eth_nd_op(ch->eth_dev, disable_channel, ch); +} +EXPORT_SYMBOL(ipa_eth_net_disable_channel); + +int ipa_eth_net_request_event(struct ipa_eth_channel *ch, unsigned long event, + phys_addr_t addr, u64 data) +{ + return ipa_eth_nd_op(ch->eth_dev, request_event, ch, event, addr, data); +} +EXPORT_SYMBOL(ipa_eth_net_request_event); + +void ipa_eth_net_release_event(struct ipa_eth_channel *ch, unsigned long event) +{ + return ipa_eth_nd_op(ch->eth_dev, release_event, ch, event); +} +EXPORT_SYMBOL(ipa_eth_net_release_event); + +int ipa_eth_net_enable_event(struct ipa_eth_channel *ch, unsigned long event) +{ + return ipa_eth_nd_op(ch->eth_dev, enable_event, ch, event); +} +EXPORT_SYMBOL(ipa_eth_net_enable_event); + +int ipa_eth_net_disable_event(struct ipa_eth_channel *ch, unsigned long event) +{ + return ipa_eth_nd_op(ch->eth_dev, disable_event, ch, event); +} +EXPORT_SYMBOL(ipa_eth_net_disable_event); + +int ipa_eth_net_moderate_event(struct ipa_eth_channel *ch, unsigned long event, + u64 min_count, u64 max_count, + u64 min_usecs, u64 max_usecs) +{ + return ipa_eth_nd_op(ch->eth_dev, moderate_event, ch, event, + min_count, max_count, min_usecs, max_usecs); +} +EXPORT_SYMBOL(ipa_eth_net_moderate_event); + +int ipa_eth_net_receive_skb(struct ipa_eth_device *eth_dev, + struct sk_buff *skb) +{ + return ipa_eth_nd_op(eth_dev, + receive_skb, eth_dev, skb, ipa_get_lan_rx_napi()); +} +EXPORT_SYMBOL(ipa_eth_net_receive_skb); + +int ipa_eth_net_transmit_skb(struct ipa_eth_device *eth_dev, + struct sk_buff *skb) +{ + return ipa_eth_nd_op(eth_dev, transmit_skb, eth_dev, skb); +} +EXPORT_SYMBOL(ipa_eth_net_transmit_skb); + +/** + * ipa_eth_net_ch_to_cb_mem() - Provides memory mapping of a specific channel + * memory on an IPA hardware type + * @ch: Channel to which the memory belong + * @ch_mem: Channel memory whose hardware mapping need to be found out + * @hw_type: Hardware for which the mapping need to be determined + * + * The SMMU context bank used by each hw_type could vary based on channel. Use + * this API to correctly identify the context bank and the mapping made for the + * channel memory to it. + * + * Return: Memory mapping info for the given @hw_type if a mapping was + * previously made via one of ipa_eth_net_*() APIs. NULL if no mapping + * was made before to the context bank associated with a @hw_type. + */ +struct ipa_eth_resource *ipa_eth_net_ch_to_cb_mem( + struct ipa_eth_channel *ch, + struct ipa_eth_channel_mem *ch_mem, + enum ipa_eth_hw_type hw_type) +{ + struct ipa_eth_resource *cb_mem; + enum ipa_smmu_cb_type cb_type = ipa_eth_hw_to_cb_type(ch, hw_type); + + if (ch_mem->cb_mem == NULL || cb_type >= IPA_SMMU_CB_MAX) + return NULL; + + cb_mem = &ch_mem->cb_mem[cb_type]; + if (!cb_mem->size) + return NULL; + + return cb_mem; +} +EXPORT_SYMBOL(ipa_eth_net_ch_to_cb_mem); + +int ipa_eth_net_save_regs(struct ipa_eth_device *eth_dev) +{ + struct ipa_eth_net_driver *nd = eth_dev->nd; + + if (nd && nd->ops->save_regs) + return ipa_eth_nd_op(eth_dev, save_regs, eth_dev, NULL, NULL); + + return 0; +} diff --git a/drivers/platform/msm/ethernet/ipa_eth_offload.c b/drivers/platform/msm/ethernet/ipa_eth_offload.c new file mode 100644 index 000000000000..75492b9dde7f --- /dev/null +++ b/drivers/platform/msm/ethernet/ipa_eth_offload.c @@ -0,0 +1,260 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2019-2020, The Linux Foundation. All rights reserved + */ + +#include "ipa_eth_i.h" + +static LIST_HEAD(ipa_eth_offload_drivers); +static DEFINE_MUTEX(ipa_eth_offload_drivers_lock); + +int ipa_eth_offload_init(struct ipa_eth_device *eth_dev) +{ + int rc; + + rc = eth_dev->od->ops->init_tx(eth_dev); + if (rc) { + ipa_eth_dev_err(eth_dev, "Failed to init offload for tx"); + return rc; + } + + rc = eth_dev->od->ops->init_rx(eth_dev); + if (rc) { + ipa_eth_dev_err(eth_dev, "Failed to init offload for rx"); + (void) eth_dev->od->ops->deinit_tx(eth_dev); + return rc; + } + + return 0; +} + +int ipa_eth_offload_deinit(struct ipa_eth_device *eth_dev) +{ + int rc_rx, rc_tx; + + rc_rx = eth_dev->od->ops->deinit_rx(eth_dev); + if (rc_rx) + ipa_eth_dev_err(eth_dev, "Failed to deinit offload for rx"); + + rc_tx = eth_dev->od->ops->deinit_tx(eth_dev); + if (rc_tx) + ipa_eth_dev_err(eth_dev, "Failed to deinit offload for tx"); + + return rc_rx || rc_tx; +} + +int ipa_eth_offload_start(struct ipa_eth_device *eth_dev) +{ + int rc; + + rc = eth_dev->od->ops->start_tx(eth_dev); + if (rc) { + ipa_eth_dev_err(eth_dev, "Failed to start offload for tx"); + return rc; + } + + rc = eth_dev->od->ops->start_rx(eth_dev); + if (rc) { + ipa_eth_dev_err(eth_dev, "Failed to start offload for rx"); + (void) eth_dev->od->ops->stop_tx(eth_dev); + return rc; + } + + return 0; +} + +int ipa_eth_offload_stop(struct ipa_eth_device *eth_dev) +{ + int rc_rx, rc_tx; + + rc_rx = eth_dev->od->ops->stop_rx(eth_dev); + if (rc_rx) + ipa_eth_dev_err(eth_dev, "Failed to stop offload for rx"); + + rc_tx = eth_dev->od->ops->stop_tx(eth_dev); + if (rc_tx) + ipa_eth_dev_err(eth_dev, "Failed to stop offload for tx"); + + return rc_rx || rc_tx; +} + +static int __try_pair_device(struct ipa_eth_device *eth_dev, + struct ipa_eth_offload_driver *od) +{ + int rc = od->ops->pair(eth_dev); + + if (rc) { + ipa_eth_dev_dbg(eth_dev, + "Offload driver %s passed up paring with %s", + od->name, eth_dev->nd->name); + return rc; + } + + ipa_eth_dev_log(eth_dev, + "Offload driver %s is paired with device from %s", + od->name, eth_dev->nd->name); + + eth_dev->od = od; + + ipa_eth_dev_log(eth_dev, + "Offload driver %s successfully paired with device from %s", + od->name, eth_dev->nd->name); + + ipa_eth_device_refresh_sched(eth_dev); + + return 0; +} + +static int try_pair_device(struct ipa_eth_device *eth_dev, + struct ipa_eth_offload_driver *od) +{ + int rc; + + mutex_lock(&od->mutex); + + if (od->bus != eth_dev->dev->bus) { + rc = -EOPNOTSUPP; + ipa_eth_dev_dbg(eth_dev, + "Offload driver %s is not a bus match for %s", + od->name, eth_dev->nd->name); + } else { + rc = __try_pair_device(eth_dev, od); + } + + mutex_unlock(&od->mutex); + + return rc; +} + +int ipa_eth_offload_pair_device(struct ipa_eth_device *eth_dev) +{ + int rc = -ENODEV; + + mutex_lock(&ipa_eth_offload_drivers_lock); + + /* Check if device is already paired while holding the offload_drivers + * mutex to ensure this API is re-entrant safe. + */ + if (!eth_dev->od) { + struct ipa_eth_offload_driver *od; + + list_for_each_entry(od, &ipa_eth_offload_drivers, driver_list) { + if (!try_pair_device(eth_dev, od)) { + rc = 0; + break; + } + } + } + + mutex_unlock(&ipa_eth_offload_drivers_lock); + + return rc; +} + +void ipa_eth_offload_unpair_device(struct ipa_eth_device *eth_dev) +{ + mutex_lock(&ipa_eth_offload_drivers_lock); + + /* Check if device is already unpaired while holding the offload_drivers + * mutex to ensure this API is re-entrant safe. + */ + if (eth_dev->od) { + struct ipa_eth_offload_driver *od = eth_dev->od; + + mutex_lock(&od->mutex); + od->ops->unpair(eth_dev); + eth_dev->od = NULL; + mutex_unlock(&od->mutex); + } + + mutex_unlock(&ipa_eth_offload_drivers_lock); +} + +static bool ipa_eth_offload_check_ops(struct ipa_eth_offload_ops *ops) +{ + /* Following call backs are optional: + * - get_stats() + * - clear_stats() + * - save_regs() + * - prepare_reset() + * - complete_reset() + */ + return + ops && + ops->pair && + ops->unpair && + ops->init_tx && + ops->start_tx && + ops->stop_tx && + ops->deinit_tx && + ops->init_rx && + ops->start_rx && + ops->stop_rx && + ops->deinit_rx; +} + +static bool ipa_eth_offload_check_driver(struct ipa_eth_offload_driver *od) +{ + return + od && + od->bus && + od->name && + ipa_eth_offload_check_ops(od->ops); +} + +int ipa_eth_offload_register_driver(struct ipa_eth_offload_driver *od) +{ + if (!ipa_eth_offload_check_driver(od)) { + ipa_eth_err("Offload driver validation failed"); + return -EINVAL; + } + + mutex_init(&od->mutex); + + mutex_lock(&ipa_eth_offload_drivers_lock); + list_add(&od->driver_list, &ipa_eth_offload_drivers); + mutex_unlock(&ipa_eth_offload_drivers_lock); + + (void) ipa_eth_debugfs_add_offload_driver(od); + + return 0; +} + +void ipa_eth_offload_unregister_driver(struct ipa_eth_offload_driver *od) +{ + ipa_eth_debugfs_remove_offload_driver(od); + + mutex_lock(&ipa_eth_offload_drivers_lock); + list_del(&od->driver_list); + mutex_unlock(&ipa_eth_offload_drivers_lock); +} + +int ipa_eth_offload_save_regs(struct ipa_eth_device *eth_dev) +{ + struct ipa_eth_offload_driver *od = eth_dev->od; + + if (od && od->ops->save_regs) + return eth_dev->od->ops->save_regs(eth_dev, NULL, NULL); + + return 0; +} + +int ipa_eth_offload_prepare_reset(struct ipa_eth_device *eth_dev, void *data) +{ + struct ipa_eth_offload_driver *od = eth_dev->od; + + if (od && od->ops->prepare_reset) + return eth_dev->od->ops->prepare_reset(eth_dev, data); + + return 0; +} + +int ipa_eth_offload_complete_reset(struct ipa_eth_device *eth_dev, void *data) +{ + struct ipa_eth_offload_driver *od = eth_dev->od; + + if (od && od->ops->complete_reset) + return eth_dev->od->ops->complete_reset(eth_dev, data); + + return 0; +} diff --git a/drivers/platform/msm/ethernet/ipa_eth_pci.c b/drivers/platform/msm/ethernet/ipa_eth_pci.c new file mode 100644 index 000000000000..6a2a477b4e4c --- /dev/null +++ b/drivers/platform/msm/ethernet/ipa_eth_pci.c @@ -0,0 +1,612 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2019-2020, The Linux Foundation. All rights reserved + */ + +#include +#include + +#include +#include + +#include "ipa_eth_i.h" + +struct ipa_eth_pci_driver { + struct ipa_eth_net_driver *nd; + + int (*probe_real)(struct pci_dev *dev, const struct pci_device_id *id); + void (*remove_real)(struct pci_dev *dev); + + const struct dev_pm_ops *pm_ops_real; +}; + +struct ipa_eth_pci_device { + struct ipa_eth_device *eth_dev; + struct ipa_eth_pci_driver *eth_pdrv; + struct msm_pcie_register_event pcie_event; +}; + +#define eth_dev_to_pdev(e) \ + ((struct ipa_eth_pci_device *)e->bus_priv) + +#define eth_dev_pm_ops(e) \ + (eth_dev_to_pdev(e)->eth_pdrv->pm_ops_real) + +static LIST_HEAD(ipa_eth_pci_drivers); +static DEFINE_MUTEX(ipa_eth_pci_drivers_mutex); + +static LIST_HEAD(ipa_eth_pci_devices); +static DEFINE_MUTEX(ipa_eth_pci_devices_mutex); + +static bool ipa_eth_pci_is_ready; + +static struct ipa_eth_pci_driver *__lookup_eth_pdrv( + struct pci_driver *pdrv) +{ + struct ipa_eth_net_driver *nd; + + list_for_each_entry(nd, &ipa_eth_pci_drivers, bus_driver_list) { + if (nd->driver == &pdrv->driver) + return nd->bus_priv; + } + + return NULL; +} + +static struct ipa_eth_pci_driver *lookup_eth_pdrv(struct pci_driver *pdrv) +{ + struct ipa_eth_pci_driver *eth_pdrv; + + mutex_lock(&ipa_eth_pci_drivers_mutex); + eth_pdrv = __lookup_eth_pdrv(pdrv); + mutex_unlock(&ipa_eth_pci_drivers_mutex); + + return eth_pdrv; +} + +static struct ipa_eth_device *__lookup_eth_dev(struct pci_dev *pdev) +{ + struct device *dev = &pdev->dev; + struct ipa_eth_device *eth_dev; + + list_for_each_entry(eth_dev, &ipa_eth_pci_devices, bus_device_list) { + if (eth_dev->dev == dev) + return eth_dev; + } + + return NULL; +} + +static struct ipa_eth_device *lookup_eth_dev(struct pci_dev *pdev) +{ + struct ipa_eth_device *eth_dev; + + mutex_lock(&ipa_eth_pci_devices_mutex); + eth_dev = __lookup_eth_dev(pdev); + mutex_unlock(&ipa_eth_pci_devices_mutex); + + return eth_dev; +} + +static struct ipa_eth_device *dev_to_eth_dev(struct device *dev) +{ + return lookup_eth_dev(to_pci_dev(dev)); +} + +static bool is_driver_used(struct ipa_eth_pci_driver *eth_pdrv) +{ + bool in_use = false; + struct ipa_eth_device *eth_dev; + + list_for_each_entry(eth_dev, &ipa_eth_pci_devices, bus_device_list) { + if (eth_dev_to_pdev(eth_dev)->eth_pdrv == eth_pdrv) { + in_use = true; + break; + } + } + + return in_use; +} + +static void ipa_eth_pcie_event_wakeup(struct pci_dev *pdev) +{ + struct ipa_eth_device *eth_dev; + + /* We are not expected to be re-entrant while processing wake up + * event. + */ + eth_dev = __lookup_eth_dev(pdev); + if (!eth_dev) { + ipa_eth_bug("Failed to lookup eth device"); + return; + } + + if (eth_dev->start_on_wakeup && !eth_dev->start) { + eth_dev->start = true; + ipa_eth_device_refresh_sched(eth_dev); + } +} + +static void ipa_eth_pcie_event_cb(struct msm_pcie_notify *notify) +{ + struct pci_dev *pdev = notify->user; + + ipa_eth_log("Received PCIe event %d", notify->event); + + switch (notify->event) { + case MSM_PCIE_EVENT_WAKEUP: + ipa_eth_pcie_event_wakeup(pdev); + break; + default: + break; + } +} + +static struct ipa_eth_device *ipa_eth_pci_alloc_device( + struct pci_dev *pdev, + struct ipa_eth_pci_driver *eth_pdrv) +{ + struct device *dev = &pdev->dev; + struct ipa_eth_device *eth_dev; + struct ipa_eth_pci_device *eth_pdev; + + eth_pdev = kzalloc(sizeof(*eth_pdev), GFP_KERNEL); + if (!eth_pdev) + return NULL; + + eth_dev = ipa_eth_alloc_device(dev, eth_pdrv->nd); + if (!eth_dev) { + ipa_eth_err("Failed to alloc eth device"); + kfree(eth_pdev); + return NULL; + } + + eth_pdev->eth_dev = eth_dev; + eth_pdev->eth_pdrv = eth_pdrv; + + eth_pdev->pcie_event.events = MSM_PCIE_EVENT_WAKEUP; + eth_pdev->pcie_event.user = pdev; + eth_pdev->pcie_event.mode = MSM_PCIE_TRIGGER_CALLBACK; + eth_pdev->pcie_event.callback = ipa_eth_pcie_event_cb; + + eth_dev->bus_priv = eth_pdev; + + return eth_dev; +} + +static void ipa_eth_pci_free_device(struct ipa_eth_device *eth_dev) +{ + struct device *dev = eth_dev->dev; + struct ipa_eth_pci_device *eth_pdev = eth_dev->bus_priv; + + if (eth_pdev) { + memset(eth_pdev, 0, sizeof(*eth_pdev)); + kfree(eth_pdev); + } + + ipa_eth_free_device(eth_dev); +} + +static int ipa_eth_pci_register_device( + struct pci_dev *pdev, + struct ipa_eth_pci_driver *eth_pdrv) +{ + int rc; + struct ipa_eth_device *eth_dev; + + eth_dev = ipa_eth_pci_alloc_device(pdev, eth_pdrv); + if (!eth_dev) { + rc = -ENOMEM; + ipa_eth_err("Failed to alloc eth pci device"); + goto err_alloc; + } + + rc = ipa_eth_register_device(eth_dev); + if (rc) { + ipa_eth_dev_err(eth_dev, "Failed to register eth device"); + goto err_register; + } + + mutex_lock(&ipa_eth_pci_devices_mutex); + list_add(ð_dev->bus_device_list, &ipa_eth_pci_devices); + mutex_unlock(&ipa_eth_pci_devices_mutex); + + rc = msm_pcie_register_event(ð_dev_to_pdev(eth_dev)->pcie_event); + if (rc) { + ipa_eth_dev_err(eth_dev, "Failed to register for PCIe event"); + goto err_pcie_register; + } + + return 0; + +err_pcie_register: + ipa_eth_unregister_device(eth_dev); +err_register: + ipa_eth_pci_free_device(eth_dev); +err_alloc: + return rc; +} + +static void ipa_eth_pci_unregister_device(struct ipa_eth_device *eth_dev) +{ + /* Deregister event handler first to prevent possible race with + * __lookup_eth_dev() while remove the device from devices list. + */ + msm_pcie_deregister_event(ð_dev_to_pdev(eth_dev)->pcie_event); + + mutex_lock(&ipa_eth_pci_devices_mutex); + list_del(ð_dev->bus_device_list); + mutex_unlock(&ipa_eth_pci_devices_mutex); + + ipa_eth_unregister_device(eth_dev); + ipa_eth_pci_free_device(eth_dev); +} + +static int ipa_eth_pci_probe_handler(struct pci_dev *pdev, + const struct pci_device_id *id) +{ + int rc = 0; + struct ipa_eth_pci_driver *eth_pdrv; + + ipa_eth_dbg("PCI probe called for %s driver with devfn %u", + pdev->driver->name, pdev->devfn); + + if (!ipa_eth_is_ready()) { + ipa_eth_log( + "Offload sub-system not initialized, deferring probe"); + return -EPROBE_DEFER; + } + + eth_pdrv = lookup_eth_pdrv(pdev->driver); + if (!eth_pdrv) { + ipa_eth_bug("Failed to lookup epci driver"); + return -EFAULT; + } + + rc = eth_pdrv->probe_real(pdev, id); + if (rc) { + ipa_eth_err("Failed real PCI probe of devfn=%u"); + goto err_probe; + } + + rc = ipa_eth_pci_register_device(pdev, eth_pdrv); + if (rc) { + ipa_eth_err("Failed to register PCI eth device"); + goto err_register; + } + + return 0; + +err_register: + eth_pdrv->remove_real(pdev); +err_probe: + return rc; +} + +static void ipa_eth_pci_remove_handler(struct pci_dev *pdev) +{ + struct ipa_eth_device *eth_dev = NULL; + struct ipa_eth_pci_driver *eth_pdrv = NULL; + + ipa_eth_dbg("PCI remove called for %s driver with devfn %u", + pdev->driver->name, pdev->devfn); + + eth_dev = lookup_eth_dev(pdev); + if (!eth_dev) { + ipa_eth_bug("Failed to lookup pci_dev -> eth_dev"); + return; + } + + eth_pdrv = eth_dev_to_pdev(eth_dev)->eth_pdrv; + + ipa_eth_pci_unregister_device(eth_dev); + + eth_pdrv->remove_real(pdev); +} + +static int ipa_eth_pci_suspend_handler(struct device *dev) +{ + int rc = 0; + struct ipa_eth_device *eth_dev; + + eth_dev = dev_to_eth_dev(dev); + if (!eth_dev) { + ipa_eth_bug("Failed to lookup pci_dev -> eth_dev"); + return -EFAULT; + } + + if (work_pending(ð_dev->refresh)) { + ipa_eth_dev_log(eth_dev, + "Refresh work is pending, aborting suspend"); + + /* Just abort suspend. Since the wq is freezable, the work item + * would get flushed before we get called again. + */ + return -EAGAIN; + } + + /* When offload is started, PCI power collapse is already disabled by + * the ipa_eth_pci_disable_pc() api. Nonetheless, we still need to do + * a dummy PCI config space save so that the PCIe framework will not by + * itself perform a config space save-restore. + */ + if (eth_dev->of_state == IPA_ETH_OF_ST_STARTED) { + ipa_eth_dev_dbg(eth_dev, + "Device suspend performing dummy config space save"); + rc = pci_save_state(to_pci_dev(dev)); + } else { + ipa_eth_dev_log(eth_dev, + "Device suspend delegated to net driver"); + rc = eth_dev_pm_ops(eth_dev)->suspend(dev); + } + + if (rc) + ipa_eth_dev_err(eth_dev, "Device suspend failed"); + else + ipa_eth_dev_dbg(eth_dev, "Device suspend complete"); + + return rc; +} + +static int ipa_eth_pci_suspend_late_handler(struct device *dev) +{ + struct ipa_eth_device *eth_dev; + + eth_dev = dev_to_eth_dev(dev); + if (!eth_dev) { + ipa_eth_bug("Failed to lookup pci_dev -> eth_dev"); + return -EFAULT; + } + + /* In rare case where we detect some interface activity between the + * time PM_SUSPEND_PREPARE event was processed and the device was + * actually frozen, abort the suspend operation. + */ + if (ipa_eth_net_check_active(eth_dev)) { + pr_info("%s: %s shows late stage activity, preventing suspend\n", + IPA_ETH_SUBSYS, eth_dev->net_dev->name); + + /* Have PM_SUSPEND_PREPARE give us one wakeup time quanta */ + ipa_eth_dev_assume_active_inc(eth_dev, 1); + + return -EAGAIN; + } + + return 0; +} + +static int ipa_eth_pci_resume_early_handler(struct device *dev) +{ + struct ipa_eth_device *eth_dev; + + eth_dev = dev_to_eth_dev(dev); + if (!eth_dev) { + ipa_eth_bug("Failed to lookup pci_dev -> eth_dev"); + return -EFAULT; + } + + /* We cannot check start_on_resume in the resume handler as it can get + * invoked also if .suspend_late() aborts due to interface activity. + */ + if (eth_dev->start_on_resume && !eth_dev->start) { + eth_dev->start = true; + ipa_eth_device_refresh_sched(eth_dev); + } + + return 0; +} + +static int ipa_eth_pci_resume_handler(struct device *dev) +{ + int rc = 0; + struct ipa_eth_device *eth_dev; + struct pci_dev *pci_dev = to_pci_dev(dev); + + eth_dev = lookup_eth_dev(pci_dev); + if (!eth_dev) { + ipa_eth_bug("Failed to lookup pci_dev -> eth_dev"); + return -EFAULT; + } + + /* During suspend, RC power collapse would not have happened if offload + * was started. Ignore resume callback since the device does not need + * to be re-initialized. + */ + if (eth_dev->of_state == IPA_ETH_OF_ST_STARTED) { + ipa_eth_dev_dbg(eth_dev, + "Device resume performing nop"); + rc = 0; + } else { + ipa_eth_dev_log(eth_dev, + "Device resume delegated to net driver"); + rc = eth_dev_pm_ops(eth_dev)->resume(dev); + + /* Give some time for device to settle after a resume */ + ipa_eth_dev_assume_active_ms(eth_dev, IPA_ETH_RESUME_SETTLE_MS); + } + + if (rc) + ipa_eth_dev_err(eth_dev, "Device resume failed"); + else + ipa_eth_dev_dbg(eth_dev, "Device resume complete"); + + return 0; +} + +/* MSM PCIe driver invokes only suspend and resume callbacks, other operations + * can be ignored unless we see a client requiring the feature. + */ +static const struct dev_pm_ops ipa_eth_pci_pm_ops = { + .suspend = ipa_eth_pci_suspend_handler, + .suspend_late = ipa_eth_pci_suspend_late_handler, + .resume_early = ipa_eth_pci_resume_early_handler, + .resume = ipa_eth_pci_resume_handler, +}; + +static bool ipa_eth_pci_check_driver(struct pci_driver *pdrv) +{ + return + pdrv->name && + pdrv->probe && + pdrv->remove && + pdrv->driver.pm && + pdrv->driver.pm->suspend && + pdrv->driver.pm->resume; +} + +static struct ipa_eth_pci_driver *ipa_eth_pci_setup_driver( + struct pci_driver *pdrv, + struct ipa_eth_net_driver *nd) +{ + struct ipa_eth_pci_driver *eth_pdrv; + + eth_pdrv = kzalloc(sizeof(*eth_pdrv), GFP_KERNEL); + if (!eth_pdrv) + return NULL; + + eth_pdrv->probe_real = pdrv->probe; + pdrv->probe = ipa_eth_pci_probe_handler; + + eth_pdrv->remove_real = pdrv->remove; + pdrv->remove = ipa_eth_pci_remove_handler; + + eth_pdrv->pm_ops_real = pdrv->driver.pm; + pdrv->driver.pm = &ipa_eth_pci_pm_ops; + + eth_pdrv->nd = nd; + nd->bus_priv = eth_pdrv; + + return eth_pdrv; +} + +static void ipa_eth_pci_reset_driver(struct ipa_eth_net_driver *nd) +{ + struct pci_driver *pdrv = to_pci_driver(nd->driver); + struct ipa_eth_pci_driver *eth_pdrv = nd->bus_priv; + + if (is_driver_used(eth_pdrv)) + ipa_eth_bug("Driver is still being used by a device"); + + nd->bus_priv = NULL; + + pdrv->probe = eth_pdrv->probe_real; + pdrv->remove = eth_pdrv->remove_real; + pdrv->driver.pm = eth_pdrv->pm_ops_real; + + memset(eth_pdrv, 0, sizeof(*eth_pdrv)); + kfree(eth_pdrv); +} + +static int ipa_eth_pci_register_driver(struct ipa_eth_net_driver *nd) +{ + struct pci_driver *pdrv; + struct ipa_eth_pci_driver *eth_pdrv; + + pdrv = to_pci_driver(nd->driver); + + if (!ipa_eth_pci_check_driver(pdrv)) { + ipa_eth_err("PCI driver validation failed for %s", nd->name); + return -EINVAL; + } + + eth_pdrv = ipa_eth_pci_setup_driver(pdrv, nd); + if (!eth_pdrv) + return -ENOMEM; + + mutex_lock(&ipa_eth_pci_drivers_mutex); + list_add(&nd->bus_driver_list, &ipa_eth_pci_drivers); + mutex_unlock(&ipa_eth_pci_drivers_mutex); + + return 0; +} + +static void ipa_eth_pci_unregister_driver(struct ipa_eth_net_driver *nd) +{ + + mutex_lock(&ipa_eth_pci_drivers_mutex); + list_del(&nd->bus_driver_list); + mutex_unlock(&ipa_eth_pci_drivers_mutex); + + ipa_eth_pci_reset_driver(nd); +} + +/** + * ipa_eth_pci_enable_pc() - Permit power collapse of the PCI root port + * @eth_dev: Device attached to the PCI bus + * + * This function instructs the MSM PCIe bus driver to permit power collapse + * of the root complex when Linux goes to suspend state. + * + * Return: 0 on success, non-zero otherwise + */ +static int ipa_eth_pci_enable_pc(struct ipa_eth_device *eth_dev) +{ + int rc; + struct pci_dev *pci_dev = to_pci_dev(eth_dev->dev); + + rc = msm_pcie_pm_control(MSM_PCIE_ENABLE_PC, + pci_dev->bus->number, pci_dev, NULL, MSM_PCIE_CONFIG_INVALID); + if (rc) { + ipa_eth_dev_err(eth_dev, + "Failed to enable MSM PCIe power collapse"); + } else { + ipa_eth_dev_log(eth_dev, + "Enabled MSM PCIe power collapse"); + } + + return rc; +} + +/** + * ipa_eth_pci_disable_pc() - Prevent power collapse of the PCI root port + * @eth_dev: Device attached to the PCI bus + * + * This function instructs the MSM PCIe bus driver to prevent power collapse + * of the root complex and connected EPs when Linux goes to suspend state. + * + * Return: 0 on success, non-zero otherwise + */ +static int ipa_eth_pci_disable_pc(struct ipa_eth_device *eth_dev) +{ + int rc; + struct pci_dev *pci_dev = to_pci_dev(eth_dev->dev); + + rc = msm_pcie_pm_control(MSM_PCIE_DISABLE_PC, + pci_dev->bus->number, pci_dev, NULL, MSM_PCIE_CONFIG_INVALID); + if (rc) { + ipa_eth_dev_err(eth_dev, + "Failed to disable MSM PCIe power collapse"); + } else { + ipa_eth_dev_log(eth_dev, + "Disabled MSM PCIe power collapse"); + } + + return rc; +} + +struct ipa_eth_bus ipa_eth_pci_bus = { + .bus = &pci_bus_type, + .register_driver = ipa_eth_pci_register_driver, + .unregister_driver = ipa_eth_pci_unregister_driver, + .enable_pc = ipa_eth_pci_enable_pc, + .disable_pc = ipa_eth_pci_disable_pc, +}; + +int ipa_eth_pci_modinit(void) +{ + ipa_eth_pci_is_ready = true; + + ipa_eth_log("Offload sub-system pci bus module init is complete"); + + return 0; +} + +void ipa_eth_pci_modexit(void) +{ + ipa_eth_log("De-initing offload sub-system pci bus module"); + + if (!ipa_eth_pci_is_ready) + return; + + ipa_eth_pci_is_ready = false; +} diff --git a/drivers/platform/msm/ethernet/ipa_eth_pm.c b/drivers/platform/msm/ethernet/ipa_eth_pm.c new file mode 100644 index 000000000000..f4517aebdd31 --- /dev/null +++ b/drivers/platform/msm/ethernet/ipa_eth_pm.c @@ -0,0 +1,131 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2019-2020, The Linux Foundation. All rights reserved + */ + +#include +#include + +#include "ipa_eth_i.h" + +#define IPA_ETH_MIN_BW_MBPS 1 +#define IPA_ETH_MAX_BW_MBPS 100000 + +static void ipa_eth_pm_callback(void *arg, enum ipa_pm_cb_event event) +{ + struct ipa_eth_device *eth_dev = (struct ipa_eth_device *)arg; + + ipa_eth_dev_log(eth_dev, "Received PM event 0x%x", event); +} + +/** + * ipa_eth_pm_register() - Register offload device with IPA PM + * @eth_dev: Offload device to register with PM + * + * Return: 0 on success, negative errno otherwise + */ +int ipa_eth_pm_register(struct ipa_eth_device *eth_dev) +{ + int rc = 0; + char name[IPA_PM_MAX_EX_CL]; + struct ipa_pm_register_params pm_params; + + if (!ipa_pm_is_used()) + return 0; + + if (eth_dev->pm_handle != IPA_PM_MAX_CLIENTS) + return -EEXIST; + + snprintf(name, sizeof(name), IPA_ETH_SUBSYS ":%s", + eth_dev->net_dev->name); + + memset(&pm_params, 0, sizeof(pm_params)); + pm_params.name = name; + pm_params.callback = ipa_eth_pm_callback; + pm_params.user_data = eth_dev; + pm_params.group = IPA_PM_GROUP_DEFAULT; + pm_params.skip_clk_vote = false; + + rc = ipa_pm_register(&pm_params, ð_dev->pm_handle); + if (rc) { + ipa_eth_dev_err(eth_dev, "Failed to register with IPA PM"); + + eth_dev->pm_handle = IPA_PM_MAX_CLIENTS; + } + + return rc; +} + +/** + * ipa_eth_pm_unregister() - Unregister offload device from IPA PM + * @eth_dev: Offload device to unregister + * + * Return: 0 on success, negative errno otherwise + */ +int ipa_eth_pm_unregister(struct ipa_eth_device *eth_dev) +{ + int rc = 0; + + if (!ipa_pm_is_used()) + return 0; + + rc = ipa_pm_deregister(eth_dev->pm_handle); + if (rc) + ipa_eth_dev_err(eth_dev, "Failed to deregister from IPA PM"); + else + eth_dev->pm_handle = IPA_PM_MAX_CLIENTS; + + return rc; +} + +int ipa_eth_pm_activate(struct ipa_eth_device *eth_dev) +{ + if (!ipa_pm_is_used()) + return 0; + + return ipa_pm_activate_sync(eth_dev->pm_handle); +} + +int ipa_eth_pm_deactivate(struct ipa_eth_device *eth_dev) +{ + if (!ipa_pm_is_used()) + return 0; + + return ipa_pm_deactivate_sync(eth_dev->pm_handle); +} + +static u32 __fetch_ethtool_link_speed(struct ipa_eth_device *eth_dev) +{ + int rc; + struct ethtool_link_ksettings link_ksettings; + + rtnl_lock(); + rc = __ethtool_get_link_ksettings(eth_dev->net_dev, &link_ksettings); + rtnl_unlock(); + + if (rc) { + ipa_eth_dev_err(eth_dev, + "Failed to obtain link settings via ethtool"); + return 0; + } + + return link_ksettings.base.speed; +} + +int ipa_eth_pm_vote_bw(struct ipa_eth_device *eth_dev) +{ + u32 link_speed; + int throughput; + + if (!ipa_pm_is_used()) + return 0; + + link_speed = __fetch_ethtool_link_speed(eth_dev); + throughput = clamp_val(link_speed, + IPA_ETH_MIN_BW_MBPS, IPA_ETH_MAX_BW_MBPS); + + ipa_eth_dev_log(eth_dev, "Link speed is %u, setting throughput as %d", + link_speed, throughput); + + return ipa_pm_set_throughput(eth_dev->pm_handle, throughput); +} diff --git a/drivers/platform/msm/ethernet/ipa_eth_trace.c b/drivers/platform/msm/ethernet/ipa_eth_trace.c new file mode 100644 index 000000000000..a1076e7306c3 --- /dev/null +++ b/drivers/platform/msm/ethernet/ipa_eth_trace.c @@ -0,0 +1,7 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2019-2020, The Linux Foundation. All rights reserved + */ + +#define CREATE_TRACE_POINTS +#include "ipa_eth_trace.h" diff --git a/drivers/platform/msm/ethernet/ipa_eth_trace.h b/drivers/platform/msm/ethernet/ipa_eth_trace.h new file mode 100644 index 000000000000..74c213b74207 --- /dev/null +++ b/drivers/platform/msm/ethernet/ipa_eth_trace.h @@ -0,0 +1,85 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (c) 2019-2020, The Linux Foundation. All rights reserved + */ + +#undef TRACE_SYSTEM +#define TRACE_SYSTEM ipa_eth + +#define TRACE_INCLUDE_PATH . +#define TRACE_INCLUDE_FILE ipa_eth_trace + +#if !defined(_IPA_ETH_TRACE_H) || defined(TRACE_HEADER_MULTI_READ) +#define _IPA_ETH_TRACE_H + +#include + +#include "ipa_eth_i.h" + +/* + * Tracepoint for tracking all packets received through IPA LAN exception. + */ +TRACE_EVENT(lan_rx_skb, + TP_PROTO(struct ipa_eth_channel *ch, struct sk_buff *skb), + TP_ARGS(ch, skb), + TP_STRUCT__entry( + __field(const char *, dev) + __field(const struct ethhdr *, eth_hdr) + __field(unsigned int, eth_proto) + __field(unsigned int, skb_len) + __field(u64, count_total) + __field(u64, count_drops) + __field(u64, count_loopback) + ), + TP_fast_assign( + __entry->dev = ch->eth_dev->net_dev->name; + __entry->eth_hdr = (struct ethhdr *)skb->data; + __entry->eth_proto = ntohs(__entry->eth_hdr->h_proto); + __entry->skb_len = skb->len; + __entry->count_total = ch->exception_total; + __entry->count_drops = ch->exception_drops; + __entry->count_loopback = ch->exception_loopback; + ), + TP_printk( + "dev=%s count_total=%llu count_drops=%llu count_loopback=%llu eth_proto=%04X skb_len=%u", + __entry->dev, __entry->count_total, __entry->count_drops, + __entry->count_loopback, __entry->eth_proto, __entry->skb_len + ) +); + +/* + * Tracepoint for ethernet link activity check by PM. + */ +TRACE_EVENT(net_check_active, + TP_PROTO( + struct ipa_eth_device *eth_dev, + struct rtnl_link_stats64 *old_stats, + struct rtnl_link_stats64 *new_stats, + unsigned long assume_active + ), + TP_ARGS(eth_dev, old_stats, new_stats, assume_active), + TP_STRUCT__entry( + __field(const char *, dev) + __field(u64, old_rx_packets) + __field(u64, new_rx_packets) + __field(u64, diff_rx_packets) + __field(unsigned long, assume_active) + ), + TP_fast_assign( + __entry->dev = eth_dev->net_dev->name; + __entry->old_rx_packets = old_stats->rx_packets; + __entry->new_rx_packets = new_stats->rx_packets; + __entry->diff_rx_packets = + __entry->new_rx_packets - __entry->old_rx_packets; + __entry->assume_active = assume_active; + ), + TP_printk("dev=%s assume_active=%lu rx_total=%llu rx_diff=+%llu", + __entry->dev, __entry->assume_active, + __entry->new_rx_packets, __entry->diff_rx_packets + ) +); + +#endif /* _IPA_ETH_TRACE_H */ + +/* This part must be outside protection */ +#include diff --git a/drivers/platform/msm/ethernet/ipa_eth_uc.c b/drivers/platform/msm/ethernet/ipa_eth_uc.c new file mode 100644 index 000000000000..3801dbbedb40 --- /dev/null +++ b/drivers/platform/msm/ethernet/ipa_eth_uc.c @@ -0,0 +1,224 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2019-2020, The Linux Foundation. All rights reserved + */ + +#include "../ipa_i.h" +#include "../ipa_uc_offload_i.h" + +#include "ipa_eth_i.h" + +#define IPA_ETH_UC_RESPONSE_SUCCESS \ + FEATURE_ENUM_VAL(IPA_HW_FEATURE_OFFLOAD, IPA_ETH_UC_RSP_SUCCESS) + +struct ipa_eth_uc_cmd_param { + u32 protocol; + u8 protocol_data[]; +} __packed; + +struct ipa_eth_uc_cmd_param_mem { + struct ipa_eth_uc_cmd_param *param; + dma_addr_t dma_addr; + size_t size; +}; + +struct ipa_eth_uc_cmd_ctx { + u8 cmd_op; + u32 protocol; + const void *protocol_data; + size_t data_size; + + struct ipa_eth_uc_cmd_param_mem pmem; +}; + +static int ipa_eth_uc_init_cmd_ctx(struct ipa_eth_uc_cmd_ctx *cmd_ctx, + enum ipa_eth_uc_op op, u32 protocol, + const void *prot_data, size_t datasz) +{ + struct ipa_eth_uc_cmd_param_mem *pmem = &cmd_ctx->pmem; + + if (op == IPA_ETH_UC_OP_NOP || op >= IPA_ETH_UC_OP_MAX) { + ipa_eth_err("Invalid uC op code"); + return -EINVAL; + } + + cmd_ctx->cmd_op = FEATURE_ENUM_VAL(IPA_HW_FEATURE_OFFLOAD, op); + cmd_ctx->protocol = protocol; + cmd_ctx->protocol_data = prot_data; + cmd_ctx->data_size = datasz; + + pmem->size = sizeof(*pmem->param) + datasz; + pmem->param = dma_alloc_coherent(ipa3_ctx->uc_pdev, pmem->size, + &pmem->dma_addr, GFP_KERNEL); + if (!pmem->param) + return -ENOMEM; + + pmem->param->protocol = protocol; + memcpy(pmem->param->protocol_data, prot_data, datasz); + + return 0; +} + +static void ipa_eth_uc_deinit_cmd_ctx(struct ipa_eth_uc_cmd_ctx *cmd_ctx) +{ + struct ipa_eth_uc_cmd_param_mem *pmem = &cmd_ctx->pmem; + + dma_free_coherent(ipa3_ctx->uc_pdev, pmem->size, + pmem->param, pmem->dma_addr); +} + +/** + * ipa_eth_uc_send_cmd() - Send an offload command to IPA uC + * @op: uC offload op code + * @protocol: uC offload protocol value + * @prot_data: uC offload command data, specific to the protocol + * @datasz: size of command data + * + * Return: 0 on success, negative errno otherwise + */ +int ipa_eth_uc_send_cmd(enum ipa_eth_uc_op op, u32 protocol, + const void *prot_data, size_t datasz) +{ + int rc = 0; + struct ipa_eth_uc_cmd_ctx cmd_ctx; + + rc = ipa_eth_uc_init_cmd_ctx(&cmd_ctx, op, protocol, prot_data, datasz); + if (rc) { + ipa_eth_err("Failed to init command context for op=%u", op); + return rc; + } + + ipa_eth_log("Sending uC command, op=%u, prot=%u, data size=%u", + cmd_ctx.cmd_op, cmd_ctx.protocol, cmd_ctx.data_size); + + rc = ipa3_uc_send_cmd((u32)cmd_ctx.pmem.dma_addr, cmd_ctx.cmd_op, + IPA_ETH_UC_RESPONSE_SUCCESS, false, 10*HZ); + + ipa_eth_uc_deinit_cmd_ctx(&cmd_ctx); + + return rc; +} +EXPORT_SYMBOL(ipa_eth_uc_send_cmd); + +/* Define IPA hardware constants not available from IPA header files */ + +#define IPA_HW_DIR_PRODUCER 0 +#define IPA_HW_DIR_CONSUMER 1 + +#define IPA_HW_CH_ID_INVALID 0xFF +#define IPA_HW_PROTOCOL_INVALID 0xFFFFFFFF + +static u32 find_uc_protocol(struct ipa_eth_device *eth_dev) +{ + int protocol; + struct ipa_eth_channel *ch; + + list_for_each_entry(ch, ð_dev->rx_channels, channel_list) { + protocol = ipa_get_prot_id(ch->ipa_client); + if (protocol >= 0) + return lower_32_bits(protocol); + } + + list_for_each_entry(ch, ð_dev->tx_channels, channel_list) { + protocol = ipa_get_prot_id(ch->ipa_client); + if (protocol >= 0) + return lower_32_bits(protocol); + } + + return IPA_HW_PROTOCOL_INVALID; +} + +static int find_client_channel(enum ipa_client_type client) +{ + const struct ipa_gsi_ep_config *gsi_ep_cfg; + + gsi_ep_cfg = ipa3_get_gsi_ep_info(client); + if (!gsi_ep_cfg) + return -EFAULT; + + return gsi_ep_cfg->ipa_gsi_chan_num; +} + +int ipa_eth_uc_stats_init(struct ipa_eth_device *eth_dev) +{ + return 0; +} + +int ipa_eth_uc_stats_deinit(struct ipa_eth_device *eth_dev) +{ + u32 protocol = find_uc_protocol(eth_dev); + + if (protocol == IPA_HW_PROTOCOL_INVALID) + return -EFAULT; + + return ipa_uc_debug_stats_dealloc(protocol); +} + +static void __fill_stats_info( + struct ipa_eth_channel *ch, + struct IpaOffloadStatschannel_info *ch_info, + bool start) +{ + ch_info->dir = IPA_ETH_CH_IS_RX(ch) ? + IPA_HW_DIR_CONSUMER : IPA_HW_DIR_PRODUCER; + + if (start) { + int gsi_ch = find_client_channel(ch->ipa_client); + + if (gsi_ch < 0) { + ipa_eth_dev_err(ch->eth_dev, + "Failed to determine GSI channel for client %d", + ch->ipa_client); + gsi_ch = IPA_HW_CH_ID_INVALID; + } + + ch_info->ch_id = (u8) gsi_ch; + } else { + ch_info->ch_id = IPA_HW_CH_ID_INVALID; + } +} + +static int ipa_eth_uc_stats_control(struct ipa_eth_device *eth_dev, bool start) +{ + int stats_idx = 0; + struct ipa_eth_channel *ch; + u32 protocol = find_uc_protocol(eth_dev); + struct IpaHwOffloadStatsAllocCmdData_t stats_info; + + if (protocol == IPA_HW_PROTOCOL_INVALID) { + ipa_eth_dev_err(eth_dev, "Failed find to uC protocol"); + return -EFAULT; + } + + memset(&stats_info, 0, sizeof(stats_info)); + + stats_info.protocol = protocol; + + list_for_each_entry(ch, ð_dev->rx_channels, channel_list) { + if (stats_idx == IPA_MAX_CH_STATS_SUPPORTED) + break; + + __fill_stats_info(ch, + &stats_info.ch_id_info[stats_idx++], start); + } + + list_for_each_entry(ch, ð_dev->tx_channels, channel_list) { + if (stats_idx == IPA_MAX_CH_STATS_SUPPORTED) + break; + + __fill_stats_info(ch, + &stats_info.ch_id_info[stats_idx++], start); + } + + return ipa_uc_debug_stats_alloc(stats_info); +} + +int ipa_eth_uc_stats_start(struct ipa_eth_device *eth_dev) +{ + return ipa_eth_uc_stats_control(eth_dev, true); +} + +int ipa_eth_uc_stats_stop(struct ipa_eth_device *eth_dev) +{ + return ipa_eth_uc_stats_control(eth_dev, false); +} diff --git a/drivers/platform/msm/ethernet/ipa_eth_utils.c b/drivers/platform/msm/ethernet/ipa_eth_utils.c new file mode 100644 index 000000000000..70f9ecf2d5d5 --- /dev/null +++ b/drivers/platform/msm/ethernet/ipa_eth_utils.c @@ -0,0 +1,154 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2019-2020, The Linux Foundation. All rights reserved + */ + +#include +#include + +#include "ipa_eth_i.h" + +#ifdef CONFIG_IPA_ETH_DEBUG +bool ipa_eth_ipc_logdbg = true; +#else +bool ipa_eth_ipc_logdbg; +#endif + +static const char * const ipa_eth_device_events[IPA_ETH_DEV_EVENT_COUNT] = { + [IPA_ETH_DEV_RESET_PREPARE] = "RESET_PREPARE", + [IPA_ETH_DEV_RESET_COMPLETE] = "RESET_COMPLETE", + [IPA_ETH_DEV_ADD_MACSEC_IF] = "ADD_MACSEC_IF", + [IPA_ETH_DEV_DEL_MACSEC_IF] = "DEL_MACSEC_IF", +}; + +const char *ipa_eth_device_event_name(enum ipa_eth_device_event event) +{ + const char *name = ""; + + if (event < IPA_ETH_DEV_EVENT_COUNT && ipa_eth_device_events[event]) + name = ipa_eth_device_events[event]; + + return name; +} + +static const char * const + ipa_eth_net_device_events[IPA_ETH_NET_DEVICE_MAX_EVENTS] = { + [NETDEV_REGISTER] = "REGISTER", + [NETDEV_UNREGISTER] = "UNREGISTER", + [NETDEV_CHANGENAME] = "CHANGE_NAME", + [NETDEV_PRE_UP] = "PRE_UP", + [NETDEV_UP] = "UP", + [NETDEV_GOING_DOWN] = "GOING_DOWN", + [NETDEV_DOWN] = "DOWN", + [NETDEV_CHANGE] = "CHANGE", + [NETDEV_CHANGELOWERSTATE] = "CHANGE_LOWER_STATE", + [NETDEV_PRECHANGEUPPER] = "PRE_CHANGE_UPPER", + [NETDEV_CHANGEUPPER] = "CHANGE_UPPER", + [NETDEV_JOIN] = "JOIN", +}; + +const char *ipa_eth_net_device_event_name(unsigned long event) +{ + const char *name = ""; + + if (event < IPA_ETH_NET_DEVICE_MAX_EVENTS && + ipa_eth_net_device_events[event]) + name = ipa_eth_net_device_events[event]; + + return name; +} + +static const char * const + ipa_eth_pm_notifier_events[IPA_ETH_PM_NOTIFIER_MAX_EVENTS] = { + [PM_HIBERNATION_PREPARE] = "HIBERNATION_PREPARE", + [PM_POST_HIBERNATION] = "POST_HIBERNATION", + [PM_SUSPEND_PREPARE] = "SUSPEND_PREPARE", + [PM_POST_SUSPEND] = "POST_SUSPEND", + [PM_RESTORE_PREPARE] = "RESTORE_PREPARE", + [PM_POST_RESTORE] = "POST_RESTORE", +}; + +const char *ipa_eth_pm_notifier_event_name(unsigned long event) +{ + const char *name = ""; + + if (event < IPA_ETH_PM_NOTIFIER_MAX_EVENTS && + ipa_eth_pm_notifier_events[event]) + name = ipa_eth_pm_notifier_events[event]; + + return name; +} + +static void __ipa_eth_free_msg(void *buff, u32 len, u32 type) {} + +static int ipa_eth_send_ecm_msg(struct net_device *net_dev, + enum ipa_ecm_event ecm_event) +{ + struct ipa_msg_meta msg_meta; + struct ipa_ecm_msg ecm_msg; + + if (!net_dev) + return -EFAULT; + + memset(&msg_meta, 0, sizeof(msg_meta)); + memset(&ecm_msg, 0, sizeof(ecm_msg)); + + ecm_msg.ifindex = net_dev->ifindex; + strlcpy(ecm_msg.name, net_dev->name, IPA_RESOURCE_NAME_MAX); + + msg_meta.msg_type = ecm_event; + msg_meta.msg_len = sizeof(struct ipa_ecm_msg); + + return ipa_send_msg(&msg_meta, &ecm_msg, __ipa_eth_free_msg); +} + +int ipa_eth_send_msg_connect(struct net_device *net_dev) +{ + ipa_eth_log("Sending ECM_CONNECT for %s", net_dev->name); + return ipa_eth_send_ecm_msg(net_dev, ECM_CONNECT); +} + +int ipa_eth_send_msg_disconnect(struct net_device *net_dev) +{ + ipa_eth_log("Sending ECM_DISCONNECT for %s", net_dev->name); + return ipa_eth_send_ecm_msg(net_dev, ECM_DISCONNECT); +} + +/* IPC Logging */ +module_param(ipa_eth_ipc_logdbg, bool, 0444); +MODULE_PARM_DESC(ipa_eth_ipc_logdbg, "Log debug IPC messages"); + +static void *ipa_eth_ipc_logbuf; + +void *ipa_eth_get_ipc_logbuf(void) +{ + return ipa_eth_ipc_logbuf; +} +EXPORT_SYMBOL(ipa_eth_get_ipc_logbuf); + +void *ipa_eth_get_ipc_logbuf_dbg(void) +{ + return ipa_eth_ipc_logdbg ? ipa_eth_ipc_logbuf : NULL; +} +EXPORT_SYMBOL(ipa_eth_get_ipc_logbuf_dbg); + +#define IPA_ETH_IPC_LOG_PAGES 128 + +int ipa_eth_ipc_log_init(void) +{ + if (ipa_eth_ipc_logbuf) + return 0; + + ipa_eth_ipc_logbuf = ipc_log_context_create( + IPA_ETH_IPC_LOG_PAGES, IPA_ETH_SUBSYS, 0); + + return ipa_eth_ipc_logbuf ? 0 : -EFAULT; +} + +void ipa_eth_ipc_log_cleanup(void) +{ + if (ipa_eth_ipc_logbuf) { + ipc_log_context_destroy(ipa_eth_ipc_logbuf); + ipa_eth_ipc_logbuf = NULL; + } +} diff --git a/include/linux/ipa_eth.h b/include/linux/ipa_eth.h index ce0775accde1..721e4f0c0621 100644 --- a/include/linux/ipa_eth.h +++ b/include/linux/ipa_eth.h @@ -6,10 +6,1108 @@ #ifndef _IPA_ETH_H_ #define _IPA_ETH_H_ +#include +#include +#include +#include +#include +#include + +#include +#include + #include #include #include +/** + * API Version Changes + * --------------------------------------------------------------------------- + * 1 - Initial version + * 2 - API separation for use by network and offload drivers + * - New ipa_eth_net_*() APIs offer safer interface for offload + * drivers to call into ipa_eth_net_ops + * - ipa_eth_net_ops.request_channel() to accept additional + * memory allocation params, including custom memory allocator + * defined via struct ipa_eth_dma_allocator interface + * - probe() and remove() offload bus ops are replaced by pair() + * and unpair() callbacks respectively + * 3 - Added .save_regs() callback for network and offload drivers + * 4 - Added ipa_eth_device_notify() interface for client drivers + * to notify of various device events. + * 5 - Removed ipa_eth_{gsi,uc}_iommu_*{} APIs that were used for + * mapping memory to GSI and IPA uC IOMMU CBs. + * 6 - Added ipa_eth_ep_deinit() + * 7 - ipa_eth_net_ops.receive_skb() now accepts in_napi parameter + * 8 - Added IPA rx/tx intf properties to ipa_eth_device + * 9 - Support for skipping IPA API call from offload sub-system + */ + +#define IPA_ETH_API_VER 9 + +/** + * enum ipa_eth_dev_features - Features supported by an ethernet device or + * driver that may be requested via offload APIs + * @IPA_ETH_DEV_F_L2_CSUM_BIT: Ethernet L2 checksum offload + * @IPA_ETH_DEV_F_L3_CSUM_BIT: Ethernet L2 checksum offload + * @IPA_ETH_DEV_F_TCP_CSUM_BIT: TCP checksum offload + * @IPA_ETH_DEV_F_UDP_CSUM_BIT: UDP checksum offload + * @IPA_ETH_DEV_F_LSO_BIT: Large Send Offload / TCP Segmentation Offload + * @IPA_ETH_DEV_F_LRO_BIT: Large Receive Offload / Receive Side Coalescing + * @IPA_ETH_DEV_F_VLAN_BIT: VLAN Offload + * @IPA_ETH_DEV_F_MODC_BIT: Counter based event moderation + * @IPA_ETH_DEV_F_MODT_BIT: Timer based event moderation + * @IPA_ETH_DEV_F_IPA_API: Driver supports direct IPA API calls + * + * Ethernet hardware features represented as bit numbers below are used in + * IPA_ETH_DEV_F_* feature flags that are to be used in offload APIs. + */ +enum ipa_eth_dev_features { + IPA_ETH_DEV_F_L2_CSUM_BIT, + IPA_ETH_DEV_F_L3_CSUM_BIT, + IPA_ETH_DEV_F_TCP_CSUM_BIT, + IPA_ETH_DEV_F_UDP_CSUM_BIT, + IPA_ETH_DEV_F_LSO_BIT, + IPA_ETH_DEV_F_LRO_BIT, + IPA_ETH_DEV_F_VLAN_BIT, + IPA_ETH_DEV_F_MODC_BIT, + IPA_ETH_DEV_F_MODT_BIT, + IPA_ETH_DEV_F_IPA_API_BIT, +}; + +#define ipa_eth_dev_f(f) BIT(IPA_ETH_DEV_F_##f##_BIT) + +#define IPA_ETH_DEV_F_L2_CSUM ipa_eth_dev_f(L2_CSUM) +#define IPA_ETH_DEV_F_L3_CSUM ipa_eth_dev_f(L3_CSUM) +#define IPA_ETH_DEV_F_TCP_CSUM ipa_eth_dev_f(TCP_CSUM) +#define IPA_ETH_DEV_F_UDP_CSUM ipa_eth_dev_f(UDP_CSUM) +#define IPA_ETH_DEV_F_LSO ipa_eth_dev_f(LSO) +#define IPA_ETH_DEV_F_LRO ipa_eth_dev_f(LRO) +#define IPA_ETH_DEV_F_VLAN ipa_eth_dev_f(VLAN) +#define IPA_ETH_DEV_F_MODC ipa_eth_dev_f(MODC) +#define IPA_ETH_DEV_F_MODT ipa_eth_dev_f(MODT) +#define IPA_ETH_DEV_F_IPA_API ipa_eth_dev_f(IPA_API) + +/** + * enum ipa_eth_dev_events - Events supported by an ethernet device that may be + * requested via offload APIs + * @IPA_ETH_DEV_EV_RX_INT_BIT: Rx interrupt + * @IPA_ETH_DEV_EV_TX_INT_BIT: Tx interrupt + * @IPA_ETH_DEV_EV_RX_PTR_BIT: Rx (head) pointer write-back + * @IPA_ETH_DEV_EV_TX_PTR_BIT: Tx (head) pointer write-back + */ +enum ipa_eth_dev_events { + IPA_ETH_DEV_EV_RX_INT_BIT, + IPA_ETH_DEV_EV_TX_INT_BIT, + IPA_ETH_DEV_EV_RX_PTR_BIT, + IPA_ETH_DEV_EV_TX_PTR_BIT +}; + +#define ipa_eth_dev_ev(ev) BIT(IPA_ETH_DEV_EV_##ev##_BIT) + +#define IPA_ETH_DEV_EV_RX_INT ipa_eth_dev_ev(RX_INT) +#define IPA_ETH_DEV_EV_TX_INT ipa_eth_dev_ev(TX_INT) +#define IPA_ETH_DEV_EV_RX_PTR ipa_eth_dev_ev(RX_PTR) +#define IPA_ETH_DEV_EV_TX_PTR ipa_eth_dev_ev(TX_PTR) + +/** + * enum ipa_eth_channel_dir - Direction of a ring / channel + * @IPA_ETH_CH_DIR_RX: Traffic flowing from device to IPA + * @IPA_ETH_CH_DIR_TX: Traffic flowing from IPA to device + */ +enum ipa_eth_channel_dir { + IPA_ETH_CH_DIR_RX, + IPA_ETH_CH_DIR_TX, +}; + +#define IPA_ETH_DIR_RX IPA_ETH_CH_DIR_RX +#define IPA_ETH_DIR_TX IPA_ETH_CH_DIR_TX + +/** + * enum ipa_eth_offload_state - Offload state of an ethernet device + * @IPA_ETH_OF_ST_DEINITED: No offload path resources are allocated + * @IPA_ETH_OF_ST_INITED: Offload path resources are allocated, but not started + * @IPA_ETH_OF_ST_STARTED: Offload path is started and ready to handle traffic + * @IPA_ETH_OF_ST_ERROR: One or more offload path components are in error state + * @IPA_ETH_OF_ST_RECOVERY: Offload path is attempting to recover from error + */ +enum ipa_eth_offload_state { + IPA_ETH_OF_ST_DEINITED = 0, + IPA_ETH_OF_ST_INITED, + IPA_ETH_OF_ST_STARTED, + IPA_ETH_OF_ST_ERROR, + IPA_ETH_OF_ST_RECOVERY, + IPA_ETH_OF_ST_MAX, +}; + +/** + * enum ipa_eth_offload_state - States of the network interface + * @IPA_ETH_IF_ST_UP: Network interface is up in software/Linux + * @IPA_ETH_IF_ST_LOWER_UP: Network interface PHY link is up / cable connected + */ +enum ipa_eth_interface_states { + IPA_ETH_IF_ST_UP, + IPA_ETH_IF_ST_LOWER_UP, + IPA_ETH_IF_ST_MAX, +}; + +struct ipa_eth_device; +struct ipa_eth_net_driver; + +/** + * struct ipa_eth_resource - Memory based resource info + * @size: Size of the memory accessible + * @vaddr: Kernel mapped address of the resource + * @daddr: DMA address of the resource + * @paddr: Physical address of the resource + */ +struct ipa_eth_resource { + size_t size; + + void *vaddr; + dma_addr_t daddr; + phys_addr_t paddr; +}; + +/** + * ipa_eth_mem_it_t() - Callback used by the ipa_eth_dma_allocator.walk() as it + * iterates through each contiguous chunk of memory + * @eth_dev: Device to which the memory belong + * @cmem: Contigous mapping. If cmem->paddr is NULL, it could be determined from + * ipa_eth_dma_allocator.paddr(cmem->vaddr) + * @arg: Private argument passed to ipa_eth_dma_allocator.walk() that can hold + * necessary context required by the iterator + * + * See &struct ipa_eth_dma_allocator for more details. + */ +typedef int (*ipa_eth_mem_it_t)(struct ipa_eth_device *eth_dev, + const struct ipa_eth_resource *cmem, void *arg); + +/** + * struct ipa_eth_dma_allocator - Custom DMA memory allocator interface + * @name: Name of the allocator + */ +struct ipa_eth_dma_allocator { + const char *name; + + /** + * .paddr() - Convert a virtual address previously allocated by this + * ipa_eth_dma_allocator, to physical address + * @eth_dev: Device which owns the memory + * @vaddr: Kernel mapped address of the resource + * + * Return: Physical address of @vaddr + */ + phys_addr_t (*paddr)(struct ipa_eth_device *eth_dev, + const void *vaddr); + + /** + * .alloc() - Allocates DMA memory for a given device + * @eth_dev: Device which will own the memory. Note that @eth_dev->dev + * should be a valid struct device pointer. + * @size: Minimum number of bytes to allocate + * @gfp: Kernel GFP_* flags to use, if the allocator supports it + * @mem: Memory info if the allocation was successful. @mem->paddr may + * be NUll or not valid, use ipa_eth_dma_allocator.paddr() to get + * the correct physical address for a page of virtual address. + * + * This API callback is typically called by the network driver to + * allocate memory for descriptor rings, buffers, etc. Any memory + * allocated by this callback should be traversable by the .remap() + * API callback implementation. + * + * Return: 0 on success, non-zero otherwise + */ + int (*alloc)(struct ipa_eth_device *eth_dev, size_t size, gfp_t gfp, + struct ipa_eth_resource *mem); + + /** + * .free() - Free a memory previously allocated using .alloc() API + * @eth_dev: Device which owns the memory + * @mem: Memory info + * + * This API callback is typically called by the network driver to free + * memory resources associated with descriptor rings, buffers, etc. once + * their associated hardware queues are stopped. + */ + void (*free)(struct ipa_eth_device *eth_dev, + struct ipa_eth_resource *mem); + + /** + * .walk() - Iterates over memory previously allocated by .alloc() + * @eth_dev: Device which owns the memory + * @mem: Allocated memory that need to be iterated over + * @it: Iterator that is invoked for each chunk (typically PAGE_SIZE) + * of memory + * @arg: Private argument passed to @it for each invocation + * + * This API is expected to iterate over pieces of an allocated memory + * for typical purposes remapping to additional peripherals, and later + * unmapping from them. .walk() is responsible to invoke the iterator + * @it for each mappable region within an allocated space, which is + * typically page sized. + * + * Iteration is expected to stop either when the page walk completes or + * when any of the @it() invocations return failure. Iterator can add + * additional checks for memory alignments, addressability, etc. that + * can also fail. In any case, the API is expected to return the number + * of bytes (<= mem->size) from the given memory that was successfully + * iterated. Although the API may re-align memory regions while invoking + * the iterator, it should still return only the number of bytes of the + * original memory that was successfully iterated. + * + * .walk() API could be used for any purpose of iteration, not limited + * to memory mapping and unmapping. Implementation of the API should be + * flexible for generalized use-cases. Caller of this API should always + * check for the return value against @mem->size to make sure if the + * intended operation was successful, and explicitly revert any partial + * operation. Ex, + * + * mapped_bytes = alctr->walk(dev, mem_region, iommu_mapper, map_ctx); + * if (mapped_bytes != mem_region->size) { + * struct ipa_eth_resource unmap_region = *mem_region; + * unmap_region.size = mapped_bytes; + * alctr->walk(dev, unmap_region, smmu_map, ctx); + * return -ENOMEM; + * } + * + * Return: the number of bytes in @mem that were successfully iterated + */ + size_t (*walk)(struct ipa_eth_device *eth_dev, + const struct ipa_eth_resource *mem, + ipa_eth_mem_it_t it, void *arg); +}; + +/** + * enum ipa_eth_hw_type - Types of hardware used in an offload path + * @IPA_ETH_HW_UC: IPA uC + * @IPA_ETH_HW_GSI: GSI + * @IPA_ETH_HW_IPA: IPA hardware/endpoint + */ +enum ipa_eth_hw_type { + IPA_ETH_HW_UC, + IPA_ETH_HW_GSI, + IPA_ETH_HW_IPA, + IPA_ETH_HW_MAX, +}; + +/** + * struct ipa_eth_hw_map_param - Params for mapping memory to IPA hardware + * @map: If true, perform mapping of memory to the given hardware + * @sym: If true, performs symmetric mapping where IO virtual address (IOVA) + * used is the same as the one used on original device. + * @read: Memory should be readable by hardware + * @write: Memory should be writable by hardware + */ +struct ipa_eth_hw_map_param { + bool map; + bool sym; + bool read; + bool write; +}; + +/** + * struct ipa_eth_desc_params - Params for allocating descriptor memory + * @size: Size of each descriptor. This field is usually filled in by the + * network driver. + * @count: Number of descriptors to be allocated + * @allocator: DMA allocator to be used for IO memory allocation and mapping + * @hw_map_params: Info on memory mapping requirements to IPA CBs + */ +struct ipa_eth_desc_params { + size_t size; + size_t count; + struct ipa_eth_dma_allocator *allocator; + struct ipa_eth_hw_map_param hw_map_params[IPA_ETH_HW_MAX]; +}; + +/** + * struct ipa_eth_buff_params - Params for allocating buffer memory + * @size: Size of data buffer associated with a descriptor + * @count: Number of buffers. This ield is usually filled in by the network + * driver and is typically the same value as descriptor count. + * @allocator: DMA allocator to be used for IO memory allocation and mapping + * @maps: Info on memory mapping requirements to IPA CBs + */ +struct ipa_eth_buff_params { + size_t size; + size_t count; + struct ipa_eth_dma_allocator *allocator; + struct ipa_eth_hw_map_param hw_map_params[IPA_ETH_HW_MAX]; +}; + +/** + * struct ipa_eth_channel_mem_params - Params for various channel memory + * @desc: Descriptor memory parameters + * @buff: Buffer memory parameters + */ +struct ipa_eth_channel_mem_params { + struct ipa_eth_desc_params desc; + struct ipa_eth_buff_params buff; +}; + +/** + * struct ipa_eth_channel_mem - Represents a piece of memory used by the + * network device channel for descriptrors, buffers, etc. + * @mem_list_entry: list entry in either of ipa_eth_channel.desc_mem or + * ipa_eth_channel.buff_mem + * @mem: Memory mapping info as used by the network device/driver + * @cb_mem: Memory mapping info as used by each of IPA context banks. When a + * mapping is present, cb_mem[cb_type].size would be non-zero. + */ +struct ipa_eth_channel_mem { + struct list_head mem_list_entry; + + struct ipa_eth_resource mem; + +/* private: for internal use by offload sub-system */ + struct ipa_eth_resource *cb_mem; +}; + +/** + * struct ipa_eth_channel - Represents a network device channel + * @channel_list: list entry in either of ipa_eth_device.rx_channels or + * ipa_eth_device.tx_channels + * @nd_priv: Private field for use by network driver + * @events: Events supported by the channel + * @features: Features enabled in the channel + * @direction: Channel direction + * @mem_params: Channel memory params filled in collectively by Offload driver, + * Network driver and Offload sub-system + * @queue: Network device queue/ring number + * @desc_mem: Descriptor ring memory list + * @buff_mem: Data buffer memory list + * @od_priv: Private field for use by offload driver + * @eth_dev: Associated ipa_eth_device + * @ipa_client: IPA client type enum to be used for the channel + * @ipa_ep_num: IPA endpoint number configured for the client type + * @process_skb: Callback to be called for processing IPA exception + * packets received from the IPA endpoint + * @ipa_priv: Private field to be used by offload subsystem + * @exception_total: Total number of exception packets received + * @exception_drops: Total number of dropped exception packets + * @exception_loopback: Total number of exception packets looped back + * into IPA + */ +struct ipa_eth_channel { + struct list_head channel_list; + + /* fields managed by network driver */ + void *nd_priv; + + unsigned long events; + unsigned long features; + enum ipa_eth_channel_dir direction; + struct ipa_eth_channel_mem_params mem_params; + + int queue; + struct list_head desc_mem; + struct list_head buff_mem; + + /* fields managed by offload driver */ + void *od_priv; + struct ipa_eth_device *eth_dev; + + enum ipa_client_type ipa_client; + int ipa_ep_num; + + int (*process_skb)(struct ipa_eth_channel *ch, struct sk_buff *skb); + + /* fields managed by offload subsystem */ + void *ipa_priv; + + u64 exception_total; + u64 exception_drops; + u64 exception_loopback; +}; + +#define IPA_ETH_CH_IS_RX(ch) ((ch)->direction == IPA_ETH_CH_DIR_RX) +#define IPA_ETH_CH_IS_TX(ch) ((ch)->direction == IPA_ETH_CH_DIR_TX) + +/** + * struct ipa_eth_device - Represents an ethernet device + * @device_list: Entry in the global offload device list + * @bus_device_list: Entry in the per-bus offload device list + * @net_dev: Netdev registered by the network driver + * @nd_priv: Private field for use by network driver + * @od_priv: Private field for use by offload driver + * @rx_channels: Rx channels allocated for the offload path + * @tx_channels: Tx channels allocated for the offload path + * @ipa_rx_intf: IPA properties Rx interface + * @ipa_tx_intf: IPA properties Tx interface + * @of_state: Offload state of the device + * @dev: Pointer to struct device + * @nd: IPA offload net driver associated with the device + * @od: IPA offload driver that is managing the device + * @netdevice_nb: Notifier block for receiving netdev events from the + * network device (to monitor link state changes) + * @init: Allowed to initialize offload path for the device + * @start: Allowed to start offload data path for the device + * @start_on_wakeup: Allow start upon wake up by device + * @start_on_resume: Allow start upon driver resume + * @start_on_timeout: Timeout in milliseconds after which @start is enabled + * @start_timer: Timer associated with @start_on_timer + * @flags: Device flags + * @if_state: Interface state - one or more bit numbers IPA_ETH_IF_ST_* + * @pm_handle: IPA PM client handle for the device + * @phdr_v4_handle: Partial header handle for IPv4 + * @phdr_v6_handle: Partial header handle for IPv6 + * @bus_priv: Private field for use by offload subsystem bus layer + * @ipa_priv: Private field for use by offload subsystem + * @debugfs: Debugfs root for the device + * @skip_ipa: Skip IPA API calls from the sub-system + * @refresh: Work struct used to perform device refresh + */ +struct ipa_eth_device { + struct list_head device_list; + struct list_head bus_device_list; + + /* fields managed by the network driver */ + struct net_device *net_dev; + void *nd_priv; + + /* fields managed by offload driver */ + void *od_priv; + + /* fields managed by offload subsystem */ + struct list_head rx_channels; + struct list_head tx_channels; + + struct ipa_rx_intf ipa_rx_intf; + struct ipa_tx_intf ipa_tx_intf; + + enum ipa_eth_offload_state of_state; + + struct device *dev; + struct ipa_eth_net_driver *nd; + struct ipa_eth_offload_driver *od; + + struct notifier_block netdevice_nb; + + bool init; + bool start; + + bool start_on_wakeup; + bool start_on_resume; + u32 start_on_timeout; + struct timer_list start_timer; + + unsigned long flags; + unsigned long if_state; + + u32 pm_handle; + u32 phdr_v4_handle; + u32 phdr_v6_handle; + + void *bus_priv; + void *ipa_priv; + struct dentry *debugfs; + + bool skip_ipa; + + struct work_struct refresh; +}; + +/** + * enum ipa_eth_device_event - Events related to device state + * @IPA_ETH_DEV_RESET_PREPARE: Device is entering reset and is requesting + * offload path to stop using the device + * @IPA_ETH_DEV_RESET_COMPLETE: Device has completed resetting and is + * requesting offload path to resume its operations + * @IPA_ETH_DEV_ADD_MACSEC_IF: A MACSec interface is coming up + * @IPA_ETH_DEV_DEL_MACSEC_IF: A MACSec interface is going down + */ +enum ipa_eth_device_event { + IPA_ETH_DEV_RESET_PREPARE, + IPA_ETH_DEV_RESET_COMPLETE, + IPA_ETH_DEV_ADD_MACSEC_IF, + IPA_ETH_DEV_DEL_MACSEC_IF, + IPA_ETH_DEV_EVENT_COUNT, +}; + +int ipa_eth_device_notify(struct ipa_eth_device *eth_dev, + enum ipa_eth_device_event event, void *data); + +#ifdef IPA_ETH_NET_DRIVER + +/** + * struct ipa_eth_net_ops - Network device operations required for IPA offload + */ +struct ipa_eth_net_ops { + /** + * .open_device() - Initialize the network device if needed and fill in + * @eth_dev->net_dev field + * @eth_dev: Device to initialize + * + * Some network perform complete device initialization only in driver + * .ndo_open(). Offload sub-system may try to use the network hardware + * for offload path even before .ndo_open() is called. .open_device() is + * expected to perform all device initialization that may be required + * to make the hardware functional for offload path, irrespective of + * whether .ndo_open() gets called. + * + * Once the device is initialized, .open_device() should initialize the + * @eth_dev->net_dev field with the driver struct net_device pointer + * before returning. + * + * Return: 0 on success, negative errno otherwise + */ + int (*open_device)(struct ipa_eth_device *eth_dev); + + /** + * .close_device() Deinitialize the device if required + * @eth_dev: Device to deinitialize + * + * This is called when the offload data path is deinitialized and the + * offload subsystem do not see any offload drivers registered to + * handle the device anymore. Typically the device is maintained in + * the initialized state until both Linux and IPA data paths are + * deinitialized. + * + * Return: 0 on success, negative errno otherwise + */ + void (*close_device)(struct ipa_eth_device *eth_dev); + + /** + * .request_channel() - Request a channel/ring for IPA offload data + * path to use + * @eth_dev: Device from which to allocate channel + * @dir: Requested channel direction + * @events: Device events requested for the channel. Value is zero or + * more IPA_ETH_DEV_EV_* flags. + * @features: Device featured requested for the channel. Value is zero + * or more IPA_ETH_DEV_F_* feature flags. + * @mem_params: Channel memory parameters. Values to be passed in is + * specific to the network driver. This info is typically + * passed on to ipa_eth_net_alloc_channel() which will + * memcpy() the contents to mem_params inside the + * ipa_eth_channel that is returned back. + * + * Arguments @dir, @features and @events are used to inform the network + * driver about the capabilities of the offload subsystem/driver. The + * API implementation may choose to allocate a channel with only a + * subset of capablities enabled. Caller of this API need to check the + * corresponding values in returned the channel and proceed only if the + * allocated capability set is acceptable for data path operation. + * + * The allocated channel is expected to be in disabled state with no + * events allocated or enabled. It is recommended to use the offload + * sub-system API ipa_eth_net_alloc_channel() for allocating the + * ipa_eth_channel object. + * + * Return: Channel object pointer, or NULL if the channel allocation + * failed + */ + struct ipa_eth_channel * (*request_channel)( + struct ipa_eth_device *eth_dev, enum ipa_eth_channel_dir dir, + unsigned long events, unsigned long features, + const struct ipa_eth_channel_mem_params *mem_params); + + /** + * .release_channel() - Free a channel/ring previously allocated using + * .request_channel() + * @ch: Channel to be freed + */ + void (*release_channel)(struct ipa_eth_channel *ch); + + /** + * .enable_channel() - Enable a channel, allowing data to flow + * + * @ch: Channel to be enabled + * + * Return: 0 on success, negative errno otherwise + */ + int (*enable_channel)(struct ipa_eth_channel *ch); + + /** + * .disable_channel() - Disable a channel, stopping the data flow + * + * @ch: Channel to be disabled + * + * Return: 0 on success, negative errno otherwise + */ + int (*disable_channel)(struct ipa_eth_channel *ch); + + /** + * .request_event() - Allocate an event for a channel + * + * @ch: Channel for which event need to be allocated + * @event: Event to be allocated + * @addr: Address to which the event need to be reported + * @data: Data value to be associated with the event + * + * The allocated event is expected to be unmoderated. + * + * Return: 0 on success, negative errno otherwise + */ + int (*request_event)(struct ipa_eth_channel *ch, unsigned long event, + phys_addr_t addr, u64 data); + + /** + * .release_event() - Deallocate a channel event + * + * @ch: Channel for which event need to be deallocated + * @event: Event to be deallocated + * + * Return: 0 on success, negative errno otherwise + */ + void (*release_event)(struct ipa_eth_channel *ch, unsigned long event); + + /** + * .enable_event() - Enable a channel event + * + * @ch: Channel for which event need to be enabled + * @event: Event to be enabled + * + * This API is called when IPA or GIC is ready to receive events from + * the network device. + * + * Return: 0 on success, negative errno otherwise + */ + int (*enable_event)(struct ipa_eth_channel *ch, unsigned long event); + + /** + * .disable_event() - Disable a channel event + * + * @ch: Channel for which event need to be disabled + * @event: Event to be disabled + * + * Once this API is called, events may no more be reported to IPA/GIC + * although they may still be queued in the device for later delivery. + * + * Return: 0 on success, negative errno otherwise + */ + int (*disable_event)(struct ipa_eth_channel *ch, unsigned long event); + + /** + * .moderate_event() - Moderate a channel event + * + * @ch: Channel for which event need to be disabled + * @event: Event to be disabled + * @min_count: Min threshold for counter based moderation + * @max_count: Max threshold for counter based moderation + * @min_usecs: Min microseconds for timer based moderation + * @max_usecs: Max microseconds for timer based moderation + * + * This API enables event moderation when supported by the device. A + * value of 0 in either of the @max_* arguments would disable moderation + * of that specific type. If both types of moderation are enabled, the + * event is triggered with either of them expires. + * + * It is expected from the device/driver to make sure there is always a + * default min_X value for each moderation type such that there would + * be no data stalls or queue overflow when a moderation target is not + * reached. + * + * Return: 0 on success, negative errno otherwise + */ + int (*moderate_event)(struct ipa_eth_channel *ch, unsigned long event, + u64 min_count, u64 max_count, + u64 min_usecs, u64 max_usecs); + + /** + * .receive_skb() - Receive an skb from IPA and push it to Linux network + * stack + * @eth_dev: Device to which the skb need to belong + * @skb: Skb to be provided to Linux network stack + * @in_napi: IPA LAN Rx is executing in NAPI poll + * + * When a network packet received by the IPA connected device queue can + * not be routed within IPA, it will be sent to Linux as an exception + * skb. Offload subsystem receives such packets and forwards to this API + * to be provided to Linux network stack to perform the necessary packet + * routing/filtering in the software path. + * + * Network packets received by this API is expected to emerge from the + * device's Linux network interface as it would have, had the packet + * arrived directly to the Linux connected queues. + * + * Return: 0 on success, negative errno otherwise. On error, skb is NOT + * expected to have been freed. + */ + int (*receive_skb)(struct ipa_eth_device *eth_dev, + struct sk_buff *skb, bool in_napi); + + /** + * .transmit_skb() - Transmit an skb given IPA + * @eth_dev: Device through which the packet need to be transmitted + * @skb: Skb to be transmitted + * + * Return: 0 on success, negative errno otherwise. On error, skb is NOT + * expected to have been freed. + */ + int (*transmit_skb)(struct ipa_eth_device *eth_dev, + struct sk_buff *skb); + + /** + * .save_regs() - Save registers for debugging + * @eth_dev: Offloaded device + * @regs: if not NULL, write saved data address to the given pointer + * @size: if not NULL, write the size of saved data to the given pointer + * + * Return: 0 on success, errno otherwise. + */ + int (*save_regs)(struct ipa_eth_device *eth_dev, + void **regs, size_t *size); +}; + +/** + * struct ipa_eth_net_driver - Network driver to be registered with IPA offload + * subsystem + * @driver_list: Entry in the offload sub-system network driver list + * @bus_driver_list: Entry in the bus specific driver list + * @ops: Network device operations + * @bus_priv: Bus private data + * @name: Name of the network driver + * @bus: Pointer to the bus object. Must use &pci_bus_type for PCI and + * &platform_bus_type for platform drivers. This property is used by the + * offload subsystem to deduce the network driver's original pci_driver + * or plaform_driver object from the @driver argument during driver + * registration. + * Beyond registration, the bus type is also used to deduce a pci_dev or + * platform_device object from a `struct device` pointer. + * @driver: Pointer to the device_driver object embedded within a PCI/plaform + * driver object used by the network driver + * @events: Events supported by the network device + * @features: Capabilities of the network device + * @debugfs: Debugfs directory for the device + */ +struct ipa_eth_net_driver { + struct list_head driver_list; + struct list_head bus_driver_list; + + struct ipa_eth_net_ops *ops; + void *bus_priv; + + const char *name; + struct bus_type *bus; + struct device_driver *driver; + + unsigned long events; + unsigned long features; + + struct dentry *debugfs; +}; + +int ipa_eth_register_net_driver(struct ipa_eth_net_driver *nd); +void ipa_eth_unregister_net_driver(struct ipa_eth_net_driver *nd); + +struct ipa_eth_channel *ipa_eth_net_alloc_channel( + struct ipa_eth_device *eth_dev, enum ipa_eth_channel_dir dir, + unsigned long events, unsigned long features, + const struct ipa_eth_channel_mem_params *mem_params); +void ipa_eth_net_free_channel(struct ipa_eth_channel *channel); + +#endif /* IPA_ETH_NET_DRIVER */ + +#ifdef IPA_ETH_OFFLOAD_DRIVER + +/** + * struct ipa_eth_offload_link_stats - Stats for each link within an + * offload data path + * @valid: Stats are not available for the link + * @events: Number of events reported to the link + * @frames: Number of frames received by the link + * @packets: Number of packets received by the link + * @octets: Number of bytes received by the link + */ +struct ipa_eth_offload_link_stats { + bool valid; + u64 events; + u64 frames; + u64 packets; + u64 octets; +}; + +struct ipa_eth_offload_ch_stats { + struct ipa_eth_offload_link_stats ndev; + struct ipa_eth_offload_link_stats host; + struct ipa_eth_offload_link_stats uc; + struct ipa_eth_offload_link_stats gsi; + struct ipa_eth_offload_link_stats ipa; +}; + +struct ipa_eth_offload_stats { + struct ipa_eth_offload_ch_stats rx; + struct ipa_eth_offload_ch_stats tx; +}; + +/** + * struct ipa_eth_offload_ops - Offload operations provided by an offload driver + */ +struct ipa_eth_offload_ops { + /** + * .pair() - Pair a new device, if compatible, with the offload driver + * @eth_dev: Device to pair with the offload driver + * + * This API is called by offload sub-system in order to pair an ethernet + * device with the offload driver. Typically this API is called soon + * after the device probe is completed and registered with the offload + * sub-system. When the API is called, offload driver is expected to + * check if the device is compatible with the driver and that the driver + * has enough offload resources for offloading the device to IPA. + * + * The API implementation can expect the eth_dev->dev to have been + * already initialized by the offload subsystem, from which a bus + * specific device pointer (pci_dev, platform_device, etc.) can be + * derived. The API is expected to perform at least the following: + * + * 1. Ensure the device is compatible with the offload driver. For + * plaform drivers, the .compatible DT property could be used while + * PCI IDs could be used for matching with a PCI device. + * 2. If multiple network devices of the same type can be managed by the + * offload driver, it may attempt to pair @eth_dev with an available + * resource set for a single instance. + * + * Return: 0 on success, negative errno otherwise + */ + int (*pair)(struct ipa_eth_device *eth_dev); + + /** + * .unpair() - Unpair a device from the offload driver + * @eth_dev: Device to unpair + * + * Unpairing the device from offload driver should make sure that all + * resources allocated for the device are freed and the data path is + * stopped and deinitialized, and device is readied for removal. The + * implementation should be able to handle plug-n-play devices by not + * assuming the device to be connected anymore to the bus/system when + * this API is called. + */ + void (*unpair)(struct ipa_eth_device *eth_dev); + + /** + * .init_tx() - Initialize offload path in Tx direction + * + * Return: 0 on success, negative errno otherwise + */ + int (*init_tx)(struct ipa_eth_device *eth_dev); + + /** + * .start_tx() - Start offload path in Tx direction + * + * Return: 0 on success, negative errno otherwise + */ + int (*start_tx)(struct ipa_eth_device *eth_dev); + + /** + * .stop_tx() - Stop offload path in Tx direction + * + * Return: 0 on success, negative errno otherwise + */ + int (*stop_tx)(struct ipa_eth_device *eth_dev); + + /** + * .deinit_tx() - Deinitialize offload path in Tx direction + * + * Return: 0 on success, negative errno otherwise + */ + int (*deinit_tx)(struct ipa_eth_device *eth_dev); + + /** + * .init_rx() - Initialize offload path in Rx direction + * + * Return: 0 on success, negative errno otherwise + */ + int (*init_rx)(struct ipa_eth_device *eth_dev); + + /** + * .start_rx() - Start offload path in Rx direction + * + * Return: 0 on success, negative errno otherwise + */ + int (*start_rx)(struct ipa_eth_device *eth_dev); + + /** + * .stop_rx() - Stop offload path in Rx direction + * + * Return: 0 on success, negative errno otherwise + */ + int (*stop_rx)(struct ipa_eth_device *eth_dev); + + /** + * .deinit_rx() - Deinitialize offload path in Rx direction + * + * Return: 0 on success, negative errno otherwise + */ + int (*deinit_rx)(struct ipa_eth_device *eth_dev); + + /** + * .get_stats() - Get offload statistics + * + * Return: 0 on success, negative errno otherwise + */ + int (*get_stats)(struct ipa_eth_device *eth_dev, + struct ipa_eth_offload_stats *stats); + + /** + * .clear_stats() - Clear offload statistics + * + * Return: 0 on success, negative errno otherwise + */ + int (*clear_stats)(struct ipa_eth_device *eth_dev); + + /** + * .save_regs() - Save registers for debugging + * @eth_dev: Offloaded device + * @regs: if not NULL, write saved data address to the given pointer + * @size: if not NULL, write the size of saved data to the given pointer + * + * Return: 0 on success, errno otherwise. + */ + int (*save_regs)(struct ipa_eth_device *eth_dev, + void **regs, size_t *size); + + /** + * .prepare_reset() - Prepare offload path for netdev reset + * @eth_dev: Offloaded device + * @data: Private data the network driver has provided + * + * Return: 0 on success, errno otherwise. + */ + int (*prepare_reset)(struct ipa_eth_device *eth_dev, void *data); + + /** + * .complete_reset() - Netdev reset completed, offload path can resume + * @eth_dev: Offloaded device + * @data: Private data the network driver has provided + * + * Return: 0 on success, errno otherwise. + */ + int (*complete_reset)(struct ipa_eth_device *eth_dev, void *data); + +}; + +/** + * struct ipa_eth_offload_driver - Offload driver to be registered with the + * offload sub-system + * @driver_list: Entry in the global offload driver list + * @mutex: Mutex to protect offload driver struct + * @name: Name of the offload driver + * @bus: Supported network device bus type + * @ops: Offload operations + * @bus_ops: Bus level callbacks and operations for the offload driver + * @debugfs: Debugfs directory for the offload driver + */ +struct ipa_eth_offload_driver { + struct list_head driver_list; + struct mutex mutex; + + const char *name; + struct bus_type *bus; + + struct ipa_eth_offload_ops *ops; + + struct dentry *debugfs; +}; + +int ipa_eth_register_offload_driver(struct ipa_eth_offload_driver *od); +void ipa_eth_unregister_offload_driver(struct ipa_eth_offload_driver *od); + +struct ipa_eth_channel *ipa_eth_net_request_channel( + struct ipa_eth_device *eth_dev, enum ipa_client_type ipa_client, + unsigned long events, unsigned long features, + const struct ipa_eth_channel_mem_params *mem_params); +void ipa_eth_net_release_channel(struct ipa_eth_channel *ch); +int ipa_eth_net_enable_channel(struct ipa_eth_channel *ch); +int ipa_eth_net_disable_channel(struct ipa_eth_channel *ch); + +int ipa_eth_net_request_event(struct ipa_eth_channel *ch, unsigned long event, + phys_addr_t addr, u64 data); +void ipa_eth_net_release_event(struct ipa_eth_channel *ch, unsigned long event); +int ipa_eth_net_enable_event(struct ipa_eth_channel *ch, unsigned long event); +int ipa_eth_net_disable_event(struct ipa_eth_channel *ch, unsigned long event); +int ipa_eth_net_moderate_event(struct ipa_eth_channel *ch, unsigned long event, + u64 min_count, u64 max_count, + u64 min_usecs, u64 max_usecs); + +int ipa_eth_net_receive_skb(struct ipa_eth_device *eth_dev, + struct sk_buff *skb); +int ipa_eth_net_transmit_skb(struct ipa_eth_device *eth_dev, + struct sk_buff *skb); + +struct ipa_eth_resource *ipa_eth_net_ch_to_cb_mem( + struct ipa_eth_channel *ch, + struct ipa_eth_channel_mem *ch_mem, + enum ipa_eth_hw_type hw_type); + +int ipa_eth_ep_init(struct ipa_eth_channel *ch); +int ipa_eth_ep_deinit(struct ipa_eth_channel *ch); +int ipa_eth_ep_start(struct ipa_eth_channel *ch); +int ipa_eth_ep_stop(struct ipa_eth_channel *ch); + +int ipa_eth_gsi_alloc(struct ipa_eth_channel *ch, + struct gsi_evt_ring_props *gsi_ev_props, + union gsi_evt_scratch *gsi_ev_scratch, + phys_addr_t *gsi_ev_db, + struct gsi_chan_props *gsi_ch_props, + union gsi_channel_scratch *gsi_ch_scratch, + phys_addr_t *gsi_ch_db); +int ipa_eth_gsi_dealloc(struct ipa_eth_channel *ch); +int ipa_eth_gsi_ring_evtring(struct ipa_eth_channel *ch, u64 value); +int ipa_eth_gsi_ring_channel(struct ipa_eth_channel *ch, u64 value); +int ipa_eth_gsi_start(struct ipa_eth_channel *ch); +int ipa_eth_gsi_stop(struct ipa_eth_channel *ch); +#endif /* IPA_ETH_OFFLOAD_DRIVER */ + +/* IPA uC interface for ethernet devices */ + +enum ipa_eth_uc_op { + IPA_ETH_UC_OP_NOP = 0, + IPA_ETH_UC_OP_CH_SETUP = 1, + IPA_ETH_UC_OP_CH_TEARDOWN = 2, + IPA_ETH_UC_OP_PER_INIT = 3, + IPA_ETH_UC_OP_PER_DEINIT = 4, + IPA_ETH_UC_OP_MAX, +}; + +enum ipa_eth_uc_resp { + IPA_ETH_UC_RSP_SUCCESS = 0, + IPA_ETH_UC_RSP_MAX_TX_CHANNELS = 1, + IPA_ETH_UC_RSP_TX_RING_OVERRUN_POSSIBILITY = 2, + IPA_ETH_UC_RSP_TX_RING_SET_UP_FAILURE = 3, + IPA_ETH_UC_RSP_TX_RING_PARAMS_UNALIGNED = 4, + IPA_ETH_UC_RSP_UNKNOWN_TX_CHANNEL = 5, + IPA_ETH_UC_RSP_TX_INVALID_FSM_TRANSITION = 6, + IPA_ETH_UC_RSP_TX_FSM_TRANSITION_ERROR = 7, + IPA_ETH_UC_RSP_MAX_RX_CHANNELS = 8, + IPA_ETH_UC_RSP_RX_RING_PARAMS_UNALIGNED = 9, + IPA_ETH_UC_RSP_RX_RING_SET_UP_FAILURE = 10, + IPA_ETH_UC_RSP_UNKNOWN_RX_CHANNEL = 11, + IPA_ETH_UC_RSP_RX_INVALID_FSM_TRANSITION = 12, + IPA_ETH_UC_RSP_RX_FSM_TRANSITION_ERROR = 13, + IPA_ETH_UC_RSP_RX_RING_OVERRUN_POSSIBILITY = 14, +}; + +int ipa_eth_uc_send_cmd(enum ipa_eth_uc_op op, u32 protocol, + const void *prot_data, size_t datasz); + + + +/* IPC logging interface */ + +#define ipa_eth_ipc_do_log(ipcbuf, fmt, args...) \ + do { \ + void *__buf = (ipcbuf); \ + ipc_log_string(__buf, " %s:%d " fmt "\n", \ + __func__, __LINE__, ## args); \ + } while (0) + +#define ipa_eth_ipc_log(fmt, args...) \ + do { \ + void *ipa_eth_get_ipc_logbuf(void); \ + ipa_eth_ipc_do_log(ipa_eth_get_ipc_logbuf(), \ + fmt, ## args); \ + } while (0) + +#define ipa_eth_ipc_dbg(fmt, args...) \ + do { \ + void *ipa_eth_get_ipc_logbuf_dbg(void); \ + ipa_eth_ipc_do_log(ipa_eth_get_ipc_logbuf_dbg(), \ + fmt, ## args); \ + } while (0) + + /* New architecture prototypes */ typedef void (*ipa_eth_ready_cb)(void *user_data);