From f226883d3b3f3700ad11a63d9d3635fe27178e87 Mon Sep 17 00:00:00 2001 From: Git User Date: Thu, 28 Nov 2019 01:39:39 -0800 Subject: [PATCH 01/54] Initial empty repository From 08d4972b2ac9cb2894cfbebd9a23f5152575442c Mon Sep 17 00:00:00 2001 From: Subash Abhinov Kasiviswanathan Date: Mon, 9 Dec 2019 23:20:02 -0700 Subject: [PATCH 02/54] core: Rmnet initial commit Inital commit of rmnet_core net device driver in dlkm form in datarmnet. This requires rmnet to be disabled in the kernel and for it to be loaded before dependent modules. CRs-Fixed: 2558810 Change-Id: I742e85033fa0999bf9069d43ce73ab9a622a8388 Acked-by: Raul Martinez Signed-off-by: Subash Abhinov Kasiviswanathan --- core/Android.mk | 26 + core/Kbuild | 5 + core/Kconfig | 12 + core/Makefile | 13 + core/dfc.h | 254 +++++ core/dfc_defs.h | 96 ++ core/dfc_qmap.c | 513 ++++++++++ core/dfc_qmi.c | 1596 ++++++++++++++++++++++++++++++++ core/qmi_rmnet.c | 1032 +++++++++++++++++++++ core/qmi_rmnet.h | 146 +++ core/qmi_rmnet_i.h | 203 ++++ core/rmnet_config.c | 738 +++++++++++++++ core/rmnet_config.h | 179 ++++ core/rmnet_descriptor.c | 1087 ++++++++++++++++++++++ core/rmnet_descriptor.h | 147 +++ core/rmnet_handlers.c | 448 +++++++++ core/rmnet_handlers.h | 35 + core/rmnet_map.h | 289 ++++++ core/rmnet_map_command.c | 366 ++++++++ core/rmnet_map_data.c | 1245 +++++++++++++++++++++++++ core/rmnet_private.h | 42 + core/rmnet_qmi.h | 34 + core/rmnet_trace.h | 253 +++++ core/rmnet_vnd.c | 399 ++++++++ core/rmnet_vnd.h | 30 + core/wda.h | 77 ++ core/wda_qmi.c | 458 +++++++++ datarmnet_dlkm_vendor_board.mk | 8 + 28 files changed, 9731 insertions(+) create mode 100644 core/Android.mk create mode 100644 core/Kbuild create mode 100644 core/Kconfig create mode 100644 core/Makefile create mode 100644 core/dfc.h create mode 100644 core/dfc_defs.h create mode 100644 core/dfc_qmap.c create mode 100644 core/dfc_qmi.c create mode 100644 core/qmi_rmnet.c create mode 100644 core/qmi_rmnet.h create mode 100644 core/qmi_rmnet_i.h create mode 100644 core/rmnet_config.c create mode 100644 core/rmnet_config.h create mode 100644 core/rmnet_descriptor.c create mode 100644 core/rmnet_descriptor.h create mode 100644 core/rmnet_handlers.c create mode 100644 core/rmnet_handlers.h create mode 100644 core/rmnet_map.h create mode 100644 core/rmnet_map_command.c create mode 100644 core/rmnet_map_data.c create mode 100644 core/rmnet_private.h create mode 100644 core/rmnet_qmi.h create mode 100644 core/rmnet_trace.h create mode 100644 core/rmnet_vnd.c create mode 100644 core/rmnet_vnd.h create mode 100644 core/wda.h create mode 100644 core/wda_qmi.c create mode 100644 datarmnet_dlkm_vendor_board.mk diff --git a/core/Android.mk b/core/Android.mk new file mode 100644 index 000000000000..3c9aaa72ec57 --- /dev/null +++ b/core/Android.mk @@ -0,0 +1,26 @@ +ifneq ($(TARGET_PRODUCT),qssi) +RMNET_CORE_DLKM_PLATFORMS_LIST := lahaina + +ifeq ($(call is-board-platform-in-list, $(RMNET_CORE_DLKM_PLATFORMS_LIST)),true) +#Make file to create RMNET_CORE DLKM +LOCAL_PATH := $(call my-dir) +include $(CLEAR_VARS) + +LOCAL_CFLAGS := -Wno-macro-redefined -Wno-unused-function -Wall -Werror +LOCAL_CLANG :=true + +LOCAL_MODULE_PATH := $(KERNEL_MODULES_OUT) +LOCAL_MODULE := rmnet_core.ko + +LOCAL_SRC_FILES := rmnet_config.c rmnet_descriptor.c rmnet_handlers.c rmnet_map_command.c rmnet_map_data.c rmnet_vnd.c dfc_qmi.c qmi_rmnet.c wda_qmi.c + +RMNET_BLD_DIR := ../../vendor/qcom/opensource/datarmnet/core +DLKM_DIR := $(TOP)/device/qcom/common/dlkm + +KBUILD_OPTIONS := $(RMNET_BLD_DIR) + +$(warning $(DLKM_DIR)) +include $(DLKM_DIR)/AndroidKernelModule.mk + +endif #End of Check for target +endif #End of Check for qssi target diff --git a/core/Kbuild b/core/Kbuild new file mode 100644 index 000000000000..d63e0ab4f690 --- /dev/null +++ b/core/Kbuild @@ -0,0 +1,5 @@ +obj-m += rmnet_core.o +rmnet_core-y := rmnet_config.o rmnet_handlers.o rmnet_descriptor.o rmnet_map_command.o rmnet_map_data.o rmnet_vnd.o rmnet_descriptor.o\ + qmi_rmnet.o wda_qmi.o dfc_qmi.o + + diff --git a/core/Kconfig b/core/Kconfig new file mode 100644 index 000000000000..b6356c1ee2a4 --- /dev/null +++ b/core/Kconfig @@ -0,0 +1,12 @@ +# +# RMNET MAP driver +# + +menuconfig RMNET_CORE + default m + select GRO_CELLS + ---help--- + If you select this, you will enable the RMNET module which is used + for handling data in the multiplexing and aggregation protocol (MAP) + format in the embedded data path. RMNET devices can be attached to + any IP mode physical device. diff --git a/core/Makefile b/core/Makefile new file mode 100644 index 000000000000..ed85c84a4124 --- /dev/null +++ b/core/Makefile @@ -0,0 +1,13 @@ +KERNEL_SRC ?= /lib/modules/$(shell uname -r)/build + +KBUILD_OPTIONS := RMNET_CORE_ROOT=$(PWD) +KBUILD_OPTIONS += MODNAME?=rmnet_core + +all: + $(MAKE) -C $(KERNEL_SRC) M=$(shell pwd) modules $(KBUILD_OPTIONS) + +modules_install: + $(MAKE) INSTALL_MOD_STRIP=1 -C $(KERNEL_SRC) M=$(shell pwd) modules_install + +clean: + $(MAKE) -C $(KERNEL_SRC) M=$(PWD) clean diff --git a/core/dfc.h b/core/dfc.h new file mode 100644 index 000000000000..48a20935fb42 --- /dev/null +++ b/core/dfc.h @@ -0,0 +1,254 @@ +/* Copyright (c) 2018-2019, The Linux Foundation. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 and + * only version 2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ + +#undef TRACE_SYSTEM +#define TRACE_SYSTEM dfc +#undef TRACE_INCLUDE_PATH +#define TRACE_INCLUDE_PATH ../../../../vendor/qcom/opensource/datarmnet/core +#define TRACE_INCLUDE_FILE dfc + +#if !defined(_TRACE_DFC_H) || defined(TRACE_HEADER_MULTI_READ) +#define _TRACE_DFC_H + +#include + +TRACE_EVENT(dfc_qmi_tc, + + TP_PROTO(const char *name, u8 bearer_id, u32 flow_id, u32 grant, + int qlen, u32 tcm_handle, int enable), + + TP_ARGS(name, bearer_id, flow_id, grant, qlen, tcm_handle, enable), + + TP_STRUCT__entry( + __string(dev_name, name) + __field(u8, bid) + __field(u32, fid) + __field(u32, grant) + __field(int, qlen) + __field(u32, tcm_handle) + __field(int, enable) + ), + + TP_fast_assign( + __assign_str(dev_name, name); + __entry->bid = bearer_id; + __entry->fid = flow_id; + __entry->grant = grant; + __entry->qlen = qlen; + __entry->tcm_handle = tcm_handle; + __entry->enable = enable; + ), + + TP_printk("dev=%s bearer_id=%u grant=%u len=%d flow_id=%u q=%d %s", + __get_str(dev_name), + __entry->bid, __entry->grant, __entry->qlen, __entry->fid, + __entry->tcm_handle, + __entry->enable ? "enable" : "disable") +); + +TRACE_EVENT(dfc_flow_ind, + + TP_PROTO(int src, int idx, u8 mux_id, u8 bearer_id, u32 grant, + u16 seq_num, u8 ack_req, u32 ancillary), + + TP_ARGS(src, idx, mux_id, bearer_id, grant, seq_num, ack_req, + ancillary), + + TP_STRUCT__entry( + __field(int, src) + __field(int, idx) + __field(u8, mid) + __field(u8, bid) + __field(u32, grant) + __field(u16, seq) + __field(u8, ack_req) + __field(u32, ancillary) + ), + + TP_fast_assign( + __entry->src = src; + __entry->idx = idx; + __entry->mid = mux_id; + __entry->bid = bearer_id; + __entry->grant = grant; + __entry->seq = seq_num; + __entry->ack_req = ack_req; + __entry->ancillary = ancillary; + ), + + TP_printk("src=%d [%d]: mid=%u bid=%u grant=%u seq=%u ack=%u anc=%u", + __entry->src, __entry->idx, __entry->mid, __entry->bid, + __entry->grant, __entry->seq, __entry->ack_req, + __entry->ancillary) +); + +TRACE_EVENT(dfc_flow_check, + + TP_PROTO(const char *name, u8 bearer_id, unsigned int len, u32 grant), + + TP_ARGS(name, bearer_id, len, grant), + + TP_STRUCT__entry( + __string(dev_name, name) + __field(u8, bearer_id) + __field(unsigned int, len) + __field(u32, grant) + ), + + TP_fast_assign( + __assign_str(dev_name, name) + __entry->bearer_id = bearer_id; + __entry->len = len; + __entry->grant = grant; + ), + + TP_printk("dev=%s bearer_id=%u skb_len=%u current_grant=%u", + __get_str(dev_name), + __entry->bearer_id, __entry->len, __entry->grant) +); + +TRACE_EVENT(dfc_flow_info, + + TP_PROTO(const char *name, u8 bearer_id, u32 flow_id, int ip_type, + u32 handle, int add), + + TP_ARGS(name, bearer_id, flow_id, ip_type, handle, add), + + TP_STRUCT__entry( + __string(dev_name, name) + __field(u8, bid) + __field(u32, fid) + __field(int, ip) + __field(u32, handle) + __field(int, action) + ), + + TP_fast_assign( + __assign_str(dev_name, name) + __entry->bid = bearer_id; + __entry->fid = flow_id; + __entry->ip = ip_type; + __entry->handle = handle; + __entry->action = add; + ), + + TP_printk("%s: dev=%s bearer_id=%u flow_id=%u ip_type=%d q=%d", + __entry->action ? "add flow" : "delete flow", + __get_str(dev_name), + __entry->bid, __entry->fid, __entry->ip, __entry->handle) +); + +TRACE_EVENT(dfc_client_state_up, + + TP_PROTO(int idx, u32 instance, u32 ep_type, u32 iface), + + TP_ARGS(idx, instance, ep_type, iface), + + TP_STRUCT__entry( + __field(int, idx) + __field(u32, instance) + __field(u32, ep_type) + __field(u32, iface) + ), + + TP_fast_assign( + __entry->idx = idx; + __entry->instance = instance; + __entry->ep_type = ep_type; + __entry->iface = iface; + ), + + TP_printk("Client[%d]: Connection established with DFC Service " + "instance=%u ep_type=%u iface_id=%u", + __entry->idx, __entry->instance, + __entry->ep_type, __entry->iface) +); + +TRACE_EVENT(dfc_client_state_down, + + TP_PROTO(int idx, int from_cb), + + TP_ARGS(idx, from_cb), + + TP_STRUCT__entry( + __field(int, idx) + __field(int, from_cb) + ), + + TP_fast_assign( + __entry->idx = idx; + __entry->from_cb = from_cb; + ), + + TP_printk("Client[%d]: Connection with DFC service lost. " + "Exit by callback %d", + __entry->idx, __entry->from_cb) +); + +TRACE_EVENT(dfc_qmap_cmd, + + TP_PROTO(u8 mux_id, u8 bearer_id, u16 seq_num, u8 type, u32 tran), + + TP_ARGS(mux_id, bearer_id, seq_num, type, tran), + + TP_STRUCT__entry( + __field(u8, mid) + __field(u8, bid) + __field(u16, seq) + __field(u8, type) + __field(u32, tran) + ), + + TP_fast_assign( + __entry->mid = mux_id; + __entry->bid = bearer_id; + __entry->seq = seq_num; + __entry->type = type; + __entry->tran = tran; + ), + + TP_printk("mux_id=%u bearer_id=%u seq_num=%u type=%u tran=%u", + __entry->mid, __entry->bid, __entry->seq, + __entry->type, __entry->tran) +); + +TRACE_EVENT(dfc_tx_link_status_ind, + + TP_PROTO(int src, int idx, u8 status, u8 mux_id, u8 bearer_id), + + TP_ARGS(src, idx, status, mux_id, bearer_id), + + TP_STRUCT__entry( + __field(int, src) + __field(int, idx) + __field(u8, status) + __field(u8, mid) + __field(u8, bid) + ), + + TP_fast_assign( + __entry->src = src; + __entry->idx = idx; + __entry->status = status; + __entry->mid = mux_id; + __entry->bid = bearer_id; + ), + + TP_printk("src=%d [%d]: status=%u mux_id=%u bearer_id=%u", + __entry->src, __entry->idx, __entry->status, + __entry->mid, __entry->bid) +); + +#endif /* _TRACE_DFC_H */ + +/* This part must be outside protection */ +#include diff --git a/core/dfc_defs.h b/core/dfc_defs.h new file mode 100644 index 000000000000..7553707d3ab6 --- /dev/null +++ b/core/dfc_defs.h @@ -0,0 +1,96 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (c) 2019, The Linux Foundation. All rights reserved. + */ + +#ifndef _DFC_DEFS_H +#define _DFC_DEFS_H + +#include +#include "qmi_rmnet_i.h" + +#define DFC_ACK_TYPE_DISABLE 1 +#define DFC_ACK_TYPE_THRESHOLD 2 + +#define DFC_MASK_TCP_BIDIR 0x1 +#define DFC_MASK_RAT_SWITCH 0x2 +#define DFC_IS_TCP_BIDIR(r) (bool)((r) & DFC_MASK_TCP_BIDIR) +#define DFC_IS_RAT_SWITCH(r) (bool)((r) & DFC_MASK_RAT_SWITCH) + +#define DFC_MAX_QOS_ID_V01 2 + +struct dfc_qmi_data { + void *rmnet_port; + struct workqueue_struct *dfc_wq; + struct work_struct svc_arrive; + struct qmi_handle handle; + struct sockaddr_qrtr ssctl; + struct svc_info svc; + struct work_struct qmi_ind_work; + struct list_head qmi_ind_q; + spinlock_t qmi_ind_lock; + int index; + int restart_state; +}; + +enum dfc_ip_type_enum_v01 { + DFC_IP_TYPE_ENUM_MIN_ENUM_VAL_V01 = -2147483647, + DFC_IPV4_TYPE_V01 = 0x4, + DFC_IPV6_TYPE_V01 = 0x6, + DFC_IP_TYPE_ENUM_MAX_ENUM_VAL_V01 = 2147483647 +}; + +struct dfc_qos_id_type_v01 { + u32 qos_id; + enum dfc_ip_type_enum_v01 ip_type; +}; + +struct dfc_flow_status_info_type_v01 { + u8 subs_id; + u8 mux_id; + u8 bearer_id; + u32 num_bytes; + u16 seq_num; + u8 qos_ids_len; + struct dfc_qos_id_type_v01 qos_ids[DFC_MAX_QOS_ID_V01]; +}; + +struct dfc_ancillary_info_type_v01 { + u8 subs_id; + u8 mux_id; + u8 bearer_id; + u32 reserved; +}; + +struct dfc_flow_status_ind_msg_v01 { + u8 flow_status_valid; + u8 flow_status_len; + struct dfc_flow_status_info_type_v01 flow_status[DFC_MAX_BEARERS_V01]; + u8 eod_ack_reqd_valid; + u8 eod_ack_reqd; + u8 ancillary_info_valid; + u8 ancillary_info_len; + struct dfc_ancillary_info_type_v01 ancillary_info[DFC_MAX_BEARERS_V01]; +}; + +struct dfc_bearer_info_type_v01 { + u8 subs_id; + u8 mux_id; + u8 bearer_id; + enum dfc_ip_type_enum_v01 ip_type; +}; + +struct dfc_tx_link_status_ind_msg_v01 { + u8 tx_status; + u8 bearer_info_valid; + u8 bearer_info_len; + struct dfc_bearer_info_type_v01 bearer_info[DFC_MAX_BEARERS_V01]; +}; + +void dfc_do_burst_flow_control(struct dfc_qmi_data *dfc, + struct dfc_flow_status_ind_msg_v01 *ind); + +void dfc_handle_tx_link_status_ind(struct dfc_qmi_data *dfc, + struct dfc_tx_link_status_ind_msg_v01 *ind); + +#endif /* _DFC_DEFS_H */ diff --git a/core/dfc_qmap.c b/core/dfc_qmap.c new file mode 100644 index 000000000000..a4b2095b9ec5 --- /dev/null +++ b/core/dfc_qmap.c @@ -0,0 +1,513 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2019, The Linux Foundation. All rights reserved. + */ + +#include +#include +#include +#include +#include +#include "dfc_defs.h" + +#define QMAP_DFC_VER 1 + +#define QMAP_CMD_DONE -1 + +#define QMAP_CMD_REQUEST 0 +#define QMAP_CMD_ACK 1 +#define QMAP_CMD_UNSUPPORTED 2 +#define QMAP_CMD_INVALID 3 + +#define QMAP_DFC_CONFIG 10 +#define QMAP_DFC_IND 11 +#define QMAP_DFC_QUERY 12 +#define QMAP_DFC_END_MARKER 13 + +struct qmap_hdr { + u8 cd_pad; + u8 mux_id; + __be16 pkt_len; +} __aligned(1); + +#define QMAP_HDR_LEN sizeof(struct qmap_hdr) + +struct qmap_cmd_hdr { + u8 pad_len:6; + u8 reserved_bit:1; + u8 cd_bit:1; + u8 mux_id; + __be16 pkt_len; + u8 cmd_name; + u8 cmd_type:2; + u8 reserved:6; + u16 reserved2; + __be32 tx_id; +} __aligned(1); + +struct qmap_dfc_config { + struct qmap_cmd_hdr hdr; + u8 cmd_ver; + u8 cmd_id; + u8 reserved; + u8 tx_info:1; + u8 reserved2:7; + __be32 ep_type; + __be32 iface_id; + u32 reserved3; +} __aligned(1); + +struct qmap_dfc_ind { + struct qmap_cmd_hdr hdr; + u8 cmd_ver; + u8 reserved; + __be16 seq_num; + u8 reserved2; + u8 tx_info_valid:1; + u8 tx_info:1; + u8 reserved3:6; + u8 bearer_id; + u8 tcp_bidir:1; + u8 bearer_status:3; + u8 reserved4:4; + __be32 grant; + u32 reserved5; + u32 reserved6; +} __aligned(1); + +struct qmap_dfc_query { + struct qmap_cmd_hdr hdr; + u8 cmd_ver; + u8 reserved; + u8 bearer_id; + u8 reserved2; + u32 reserved3; +} __aligned(1); + +struct qmap_dfc_query_resp { + struct qmap_cmd_hdr hdr; + u8 cmd_ver; + u8 bearer_id; + u8 tcp_bidir:1; + u8 reserved:7; + u8 invalid:1; + u8 reserved2:7; + __be32 grant; + u32 reserved3; + u32 reserved4; +} __aligned(1); + +struct qmap_dfc_end_marker_req { + struct qmap_cmd_hdr hdr; + u8 cmd_ver; + u8 reserved; + u8 bearer_id; + u8 reserved2; + u16 reserved3; + __be16 seq_num; + u32 reserved4; +} __aligned(1); + +struct qmap_dfc_end_marker_cnf { + struct qmap_cmd_hdr hdr; + u8 cmd_ver; + u8 reserved; + u8 bearer_id; + u8 reserved2; + u16 reserved3; + __be16 seq_num; + u32 reserved4; +} __aligned(1); + +static struct dfc_flow_status_ind_msg_v01 qmap_flow_ind; +static struct dfc_tx_link_status_ind_msg_v01 qmap_tx_ind; +static struct dfc_qmi_data __rcu *qmap_dfc_data; +static atomic_t qmap_txid; +static void *rmnet_ctl_handle; + +static void dfc_qmap_send_end_marker_cnf(struct qos_info *qos, + u8 bearer_id, u16 seq, u32 tx_id); + +static void dfc_qmap_send_cmd(struct sk_buff *skb) +{ + trace_dfc_qmap(skb->data, skb->len, false); + + if (rmnet_ctl_send_client(rmnet_ctl_handle, skb)) { + pr_err("Failed to send to rmnet ctl\n"); + kfree_skb(skb); + } +} + +static void dfc_qmap_send_inband_ack(struct dfc_qmi_data *dfc, + struct sk_buff *skb) +{ + struct qmap_cmd_hdr *cmd; + + cmd = (struct qmap_cmd_hdr *)skb->data; + + skb->protocol = htons(ETH_P_MAP); + skb->dev = rmnet_get_real_dev(dfc->rmnet_port); + + trace_dfc_qmap(skb->data, skb->len, false); + dev_queue_xmit(skb); +} + +static int dfc_qmap_handle_ind(struct dfc_qmi_data *dfc, + struct sk_buff *skb) +{ + struct qmap_dfc_ind *cmd; + + if (skb->len < sizeof(struct qmap_dfc_ind)) + return QMAP_CMD_INVALID; + + cmd = (struct qmap_dfc_ind *)skb->data; + + if (cmd->tx_info_valid) { + memset(&qmap_tx_ind, 0, sizeof(qmap_tx_ind)); + qmap_tx_ind.tx_status = cmd->tx_info; + qmap_tx_ind.bearer_info_valid = 1; + qmap_tx_ind.bearer_info_len = 1; + qmap_tx_ind.bearer_info[0].mux_id = cmd->hdr.mux_id; + qmap_tx_ind.bearer_info[0].bearer_id = cmd->bearer_id; + + dfc_handle_tx_link_status_ind(dfc, &qmap_tx_ind); + + /* Ignore grant since it is always 0 */ + goto done; + } + + memset(&qmap_flow_ind, 0, sizeof(qmap_flow_ind)); + qmap_flow_ind.flow_status_valid = 1; + qmap_flow_ind.flow_status_len = 1; + qmap_flow_ind.flow_status[0].mux_id = cmd->hdr.mux_id; + qmap_flow_ind.flow_status[0].bearer_id = cmd->bearer_id; + qmap_flow_ind.flow_status[0].num_bytes = ntohl(cmd->grant); + qmap_flow_ind.flow_status[0].seq_num = ntohs(cmd->seq_num); + + if (cmd->tcp_bidir) { + qmap_flow_ind.ancillary_info_valid = 1; + qmap_flow_ind.ancillary_info_len = 1; + qmap_flow_ind.ancillary_info[0].mux_id = cmd->hdr.mux_id; + qmap_flow_ind.ancillary_info[0].bearer_id = cmd->bearer_id; + qmap_flow_ind.ancillary_info[0].reserved = DFC_MASK_TCP_BIDIR; + } + + dfc_do_burst_flow_control(dfc, &qmap_flow_ind); + +done: + return QMAP_CMD_ACK; +} + +static int dfc_qmap_handle_query_resp(struct dfc_qmi_data *dfc, + struct sk_buff *skb) +{ + struct qmap_dfc_query_resp *cmd; + + if (skb->len < sizeof(struct qmap_dfc_query_resp)) + return QMAP_CMD_DONE; + + cmd = (struct qmap_dfc_query_resp *)skb->data; + + if (cmd->invalid) + return QMAP_CMD_DONE; + + memset(&qmap_flow_ind, 0, sizeof(qmap_flow_ind)); + qmap_flow_ind.flow_status_valid = 1; + qmap_flow_ind.flow_status_len = 1; + + qmap_flow_ind.flow_status[0].mux_id = cmd->hdr.mux_id; + qmap_flow_ind.flow_status[0].bearer_id = cmd->bearer_id; + qmap_flow_ind.flow_status[0].num_bytes = ntohl(cmd->grant); + qmap_flow_ind.flow_status[0].seq_num = 0xFFFF; + + if (cmd->tcp_bidir) { + qmap_flow_ind.ancillary_info_valid = 1; + qmap_flow_ind.ancillary_info_len = 1; + qmap_flow_ind.ancillary_info[0].mux_id = cmd->hdr.mux_id; + qmap_flow_ind.ancillary_info[0].bearer_id = cmd->bearer_id; + qmap_flow_ind.ancillary_info[0].reserved = DFC_MASK_TCP_BIDIR; + } + + dfc_do_burst_flow_control(dfc, &qmap_flow_ind); + + return QMAP_CMD_DONE; +} + +static void dfc_qmap_set_end_marker(struct dfc_qmi_data *dfc, u8 mux_id, + u8 bearer_id, u16 seq_num, u32 tx_id) +{ + struct net_device *dev; + struct qos_info *qos; + struct rmnet_bearer_map *bearer; + + dev = rmnet_get_rmnet_dev(dfc->rmnet_port, mux_id); + if (!dev) + return; + + qos = (struct qos_info *)rmnet_get_qos_pt(dev); + if (!qos) + return; + + spin_lock_bh(&qos->qos_lock); + + bearer = qmi_rmnet_get_bearer_map(qos, bearer_id); + + if (bearer && bearer->last_seq == seq_num && bearer->grant_size) { + bearer->ack_req = 1; + bearer->ack_txid = tx_id; + } else { + dfc_qmap_send_end_marker_cnf(qos, bearer_id, seq_num, tx_id); + } + + spin_unlock_bh(&qos->qos_lock); +} + +static int dfc_qmap_handle_end_marker_req(struct dfc_qmi_data *dfc, + struct sk_buff *skb) +{ + struct qmap_dfc_end_marker_req *cmd; + + if (skb->len < sizeof(struct qmap_dfc_end_marker_req)) + return QMAP_CMD_INVALID; + + cmd = (struct qmap_dfc_end_marker_req *)skb->data; + + dfc_qmap_set_end_marker(dfc, cmd->hdr.mux_id, cmd->bearer_id, + ntohs(cmd->seq_num), ntohl(cmd->hdr.tx_id)); + + return QMAP_CMD_DONE; +} + +static void dfc_qmap_cmd_handler(struct sk_buff *skb) +{ + struct qmap_cmd_hdr *cmd; + struct dfc_qmi_data *dfc; + int rc = QMAP_CMD_DONE; + + if (!skb) + return; + + trace_dfc_qmap(skb->data, skb->len, true); + + if (skb->len < sizeof(struct qmap_cmd_hdr)) + goto free_skb; + + cmd = (struct qmap_cmd_hdr *)skb->data; + if (!cmd->cd_bit || skb->len != ntohs(cmd->pkt_len) + QMAP_HDR_LEN) + goto free_skb; + + if (cmd->cmd_name == QMAP_DFC_QUERY) { + if (cmd->cmd_type != QMAP_CMD_ACK) + goto free_skb; + } else if (cmd->cmd_type != QMAP_CMD_REQUEST) { + goto free_skb; + } + + rcu_read_lock(); + + dfc = rcu_dereference(qmap_dfc_data); + if (!dfc || READ_ONCE(dfc->restart_state)) { + rcu_read_unlock(); + goto free_skb; + } + + switch (cmd->cmd_name) { + case QMAP_DFC_IND: + rc = dfc_qmap_handle_ind(dfc, skb); + qmi_rmnet_set_dl_msg_active(dfc->rmnet_port); + break; + + case QMAP_DFC_QUERY: + rc = dfc_qmap_handle_query_resp(dfc, skb); + break; + + case QMAP_DFC_END_MARKER: + rc = dfc_qmap_handle_end_marker_req(dfc, skb); + break; + + default: + rc = QMAP_CMD_UNSUPPORTED; + } + + /* Send ack */ + if (rc != QMAP_CMD_DONE) { + cmd->cmd_type = rc; + if (cmd->cmd_name == QMAP_DFC_IND) + dfc_qmap_send_inband_ack(dfc, skb); + else + dfc_qmap_send_cmd(skb); + + rcu_read_unlock(); + return; + } + + rcu_read_unlock(); + +free_skb: + kfree_skb(skb); +} + +static void dfc_qmap_send_config(struct dfc_qmi_data *data) +{ + struct sk_buff *skb; + struct qmap_dfc_config *dfc_config; + unsigned int len = sizeof(struct qmap_dfc_config); + + skb = alloc_skb(len, GFP_ATOMIC); + if (!skb) + return; + + skb->protocol = htons(ETH_P_MAP); + dfc_config = (struct qmap_dfc_config *)skb_put(skb, len); + memset(dfc_config, 0, len); + + dfc_config->hdr.cd_bit = 1; + dfc_config->hdr.mux_id = 0; + dfc_config->hdr.pkt_len = htons(len - QMAP_HDR_LEN); + dfc_config->hdr.cmd_name = QMAP_DFC_CONFIG; + dfc_config->hdr.cmd_type = QMAP_CMD_REQUEST; + dfc_config->hdr.tx_id = htonl(atomic_inc_return(&qmap_txid)); + + dfc_config->cmd_ver = QMAP_DFC_VER; + dfc_config->cmd_id = QMAP_DFC_IND; + dfc_config->tx_info = 1; + dfc_config->ep_type = htonl(data->svc.ep_type); + dfc_config->iface_id = htonl(data->svc.iface_id); + + dfc_qmap_send_cmd(skb); +} + +static void dfc_qmap_send_query(u8 mux_id, u8 bearer_id) +{ + struct sk_buff *skb; + struct qmap_dfc_query *dfc_query; + unsigned int len = sizeof(struct qmap_dfc_query); + + skb = alloc_skb(len, GFP_ATOMIC); + if (!skb) + return; + + skb->protocol = htons(ETH_P_MAP); + dfc_query = (struct qmap_dfc_query *)skb_put(skb, len); + memset(dfc_query, 0, len); + + dfc_query->hdr.cd_bit = 1; + dfc_query->hdr.mux_id = mux_id; + dfc_query->hdr.pkt_len = htons(len - QMAP_HDR_LEN); + dfc_query->hdr.cmd_name = QMAP_DFC_QUERY; + dfc_query->hdr.cmd_type = QMAP_CMD_REQUEST; + dfc_query->hdr.tx_id = htonl(atomic_inc_return(&qmap_txid)); + + dfc_query->cmd_ver = QMAP_DFC_VER; + dfc_query->bearer_id = bearer_id; + + dfc_qmap_send_cmd(skb); +} + +static void dfc_qmap_send_end_marker_cnf(struct qos_info *qos, + u8 bearer_id, u16 seq, u32 tx_id) +{ + struct sk_buff *skb; + struct qmap_dfc_end_marker_cnf *em_cnf; + unsigned int len = sizeof(struct qmap_dfc_end_marker_cnf); + + skb = alloc_skb(len, GFP_ATOMIC); + if (!skb) + return; + + em_cnf = (struct qmap_dfc_end_marker_cnf *)skb_put(skb, len); + memset(em_cnf, 0, len); + + em_cnf->hdr.cd_bit = 1; + em_cnf->hdr.mux_id = qos->mux_id; + em_cnf->hdr.pkt_len = htons(len - QMAP_HDR_LEN); + em_cnf->hdr.cmd_name = QMAP_DFC_END_MARKER; + em_cnf->hdr.cmd_type = QMAP_CMD_ACK; + em_cnf->hdr.tx_id = htonl(tx_id); + + em_cnf->cmd_ver = QMAP_DFC_VER; + em_cnf->bearer_id = bearer_id; + em_cnf->seq_num = htons(seq); + + skb->protocol = htons(ETH_P_MAP); + skb->dev = qos->real_dev; + + /* This cmd needs to be sent in-band */ + trace_dfc_qmap(skb->data, skb->len, false); + rmnet_map_tx_qmap_cmd(skb); +} + +void dfc_qmap_send_ack(struct qos_info *qos, u8 bearer_id, u16 seq, u8 type) +{ + struct rmnet_bearer_map *bearer; + + if (type == DFC_ACK_TYPE_DISABLE) { + bearer = qmi_rmnet_get_bearer_map(qos, bearer_id); + if (bearer) + dfc_qmap_send_end_marker_cnf(qos, bearer_id, + seq, bearer->ack_txid); + } else if (type == DFC_ACK_TYPE_THRESHOLD) { + dfc_qmap_send_query(qos->mux_id, bearer_id); + } +} + +static struct rmnet_ctl_client_hooks cb = { + .ctl_dl_client_hook = dfc_qmap_cmd_handler, +}; + +int dfc_qmap_client_init(void *port, int index, struct svc_info *psvc, + struct qmi_info *qmi) +{ + struct dfc_qmi_data *data; + + if (!port || !qmi) + return -EINVAL; + + data = kzalloc(sizeof(struct dfc_qmi_data), GFP_KERNEL); + if (!data) + return -ENOMEM; + + data->rmnet_port = port; + data->index = index; + memcpy(&data->svc, psvc, sizeof(data->svc)); + + qmi->dfc_clients[index] = (void *)data; + rcu_assign_pointer(qmap_dfc_data, data); + + atomic_set(&qmap_txid, 0); + + rmnet_ctl_handle = rmnet_ctl_register_client(&cb); + if (!rmnet_ctl_handle) + pr_err("Failed to register with rmnet ctl\n"); + + trace_dfc_client_state_up(data->index, data->svc.instance, + data->svc.ep_type, data->svc.iface_id); + + pr_info("DFC QMAP init\n"); + + dfc_qmap_send_config(data); + + return 0; +} + +void dfc_qmap_client_exit(void *dfc_data) +{ + struct dfc_qmi_data *data = (struct dfc_qmi_data *)dfc_data; + + if (!data) { + pr_err("%s() data is null\n", __func__); + return; + } + + trace_dfc_client_state_down(data->index, 0); + + rmnet_ctl_unregister_client(rmnet_ctl_handle); + + WRITE_ONCE(data->restart_state, 1); + RCU_INIT_POINTER(qmap_dfc_data, NULL); + synchronize_rcu(); + + kfree(data); + + pr_info("DFC QMAP exit\n"); +} diff --git a/core/dfc_qmi.c b/core/dfc_qmi.c new file mode 100644 index 000000000000..7d3e46b849fd --- /dev/null +++ b/core/dfc_qmi.c @@ -0,0 +1,1596 @@ +/* + * Copyright (c) 2018-2019, The Linux Foundation. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 and + * only version 2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ + +#include +#include +#include +#include "rmnet_qmi.h" +#include "qmi_rmnet.h" + +#include "qmi_rmnet_i.h" +#include "dfc.h" + +#define DFC_MASK_TCP_BIDIR 0x1 +#define DFC_MASK_RAT_SWITCH 0x2 +#define DFC_IS_TCP_BIDIR(r) (bool)((r) & DFC_MASK_TCP_BIDIR) +#define DFC_IS_RAT_SWITCH(r) (bool)((r) & DFC_MASK_RAT_SWITCH) + +#define DFC_IS_ANCILLARY(type) ((type) != AF_INET && (type) != AF_INET6) + +#define DFC_MAX_QOS_ID_V01 2 + +#define DFC_ACK_TYPE_DISABLE 1 +#define DFC_ACK_TYPE_THRESHOLD 2 + +struct dfc_qmap_header { + u8 pad_len:6; + u8 reserved_bit:1; + u8 cd_bit:1; + u8 mux_id; + __be16 pkt_len; +} __aligned(1); + +struct dfc_ack_cmd { + struct dfc_qmap_header header; + u8 command_name; + u8 cmd_type:2; + u8 reserved:6; + u16 reserved2; + u32 transaction_id; + u8 ver:2; + u8 reserved3:6; + u8 type:2; + u8 reserved4:6; + u16 dfc_seq; + u8 reserved5[3]; + u8 bearer_id; +} __aligned(1); + +struct dfc_qmi_data { + void *rmnet_port; + struct workqueue_struct *dfc_wq; + struct work_struct svc_arrive; + struct qmi_handle handle; + struct sockaddr_qrtr ssctl; + struct svc_info svc; + struct work_struct qmi_ind_work; + struct list_head qmi_ind_q; + spinlock_t qmi_ind_lock; + int index; + int restart_state; +}; + +static void dfc_svc_init(struct work_struct *work); + +/* **************************************************** */ +#define DFC_SERVICE_ID_V01 0x4E +#define DFC_SERVICE_VERS_V01 0x01 +#define DFC_TIMEOUT_JF msecs_to_jiffies(1000) + +#define QMI_DFC_BIND_CLIENT_REQ_V01 0x0020 +#define QMI_DFC_BIND_CLIENT_RESP_V01 0x0020 +#define QMI_DFC_BIND_CLIENT_REQ_V01_MAX_MSG_LEN 11 +#define QMI_DFC_BIND_CLIENT_RESP_V01_MAX_MSG_LEN 7 + +#define QMI_DFC_INDICATION_REGISTER_REQ_V01 0x0001 +#define QMI_DFC_INDICATION_REGISTER_RESP_V01 0x0001 +#define QMI_DFC_INDICATION_REGISTER_REQ_V01_MAX_MSG_LEN 8 +#define QMI_DFC_INDICATION_REGISTER_RESP_V01_MAX_MSG_LEN 7 + +#define QMI_DFC_FLOW_STATUS_IND_V01 0x0022 +#define QMI_DFC_TX_LINK_STATUS_IND_V01 0x0024 + +#define QMI_DFC_GET_FLOW_STATUS_REQ_V01 0x0023 +#define QMI_DFC_GET_FLOW_STATUS_RESP_V01 0x0023 +#define QMI_DFC_GET_FLOW_STATUS_REQ_V01_MAX_MSG_LEN 20 +#define QMI_DFC_GET_FLOW_STATUS_RESP_V01_MAX_MSG_LEN 543 + +struct dfc_bind_client_req_msg_v01 { + u8 ep_id_valid; + struct data_ep_id_type_v01 ep_id; +}; + +struct dfc_bind_client_resp_msg_v01 { + struct qmi_response_type_v01 resp; +}; + +struct dfc_indication_register_req_msg_v01 { + u8 report_flow_status_valid; + u8 report_flow_status; + u8 report_tx_link_status_valid; + u8 report_tx_link_status; +}; + +struct dfc_indication_register_resp_msg_v01 { + struct qmi_response_type_v01 resp; +}; + +enum dfc_ip_type_enum_v01 { + DFC_IP_TYPE_ENUM_MIN_ENUM_VAL_V01 = -2147483647, + DFC_IPV4_TYPE_V01 = 0x4, + DFC_IPV6_TYPE_V01 = 0x6, + DFC_IP_TYPE_ENUM_MAX_ENUM_VAL_V01 = 2147483647 +}; + +struct dfc_qos_id_type_v01 { + u32 qos_id; + enum dfc_ip_type_enum_v01 ip_type; +}; + +struct dfc_flow_status_info_type_v01 { + u8 subs_id; + u8 mux_id; + u8 bearer_id; + u32 num_bytes; + u16 seq_num; + u8 qos_ids_len; + struct dfc_qos_id_type_v01 qos_ids[DFC_MAX_QOS_ID_V01]; +}; + +static struct qmi_elem_info dfc_qos_id_type_v01_ei[] = { + { + .data_type = QMI_UNSIGNED_4_BYTE, + .elem_len = 1, + .elem_size = sizeof(u32), + .array_type = NO_ARRAY, + .tlv_type = QMI_COMMON_TLV_TYPE, + .offset = offsetof(struct dfc_qos_id_type_v01, + qos_id), + .ei_array = NULL, + }, + { + .data_type = QMI_SIGNED_4_BYTE_ENUM, + .elem_len = 1, + .elem_size = sizeof(enum dfc_ip_type_enum_v01), + .array_type = NO_ARRAY, + .tlv_type = QMI_COMMON_TLV_TYPE, + .offset = offsetof(struct dfc_qos_id_type_v01, + ip_type), + .ei_array = NULL, + }, + { + .data_type = QMI_EOTI, + .array_type = NO_ARRAY, + .tlv_type = QMI_COMMON_TLV_TYPE, + }, +}; + +static struct qmi_elem_info dfc_flow_status_info_type_v01_ei[] = { + { + .data_type = QMI_UNSIGNED_1_BYTE, + .elem_len = 1, + .elem_size = sizeof(u8), + .array_type = NO_ARRAY, + .tlv_type = QMI_COMMON_TLV_TYPE, + .offset = offsetof(struct + dfc_flow_status_info_type_v01, + subs_id), + .ei_array = NULL, + }, + { + .data_type = QMI_UNSIGNED_1_BYTE, + .elem_len = 1, + .elem_size = sizeof(u8), + .array_type = NO_ARRAY, + .tlv_type = QMI_COMMON_TLV_TYPE, + .offset = offsetof(struct + dfc_flow_status_info_type_v01, + mux_id), + .ei_array = NULL, + }, + { + .data_type = QMI_UNSIGNED_1_BYTE, + .elem_len = 1, + .elem_size = sizeof(u8), + .array_type = NO_ARRAY, + .tlv_type = QMI_COMMON_TLV_TYPE, + .offset = offsetof(struct + dfc_flow_status_info_type_v01, + bearer_id), + .ei_array = NULL, + }, + { + .data_type = QMI_UNSIGNED_4_BYTE, + .elem_len = 1, + .elem_size = sizeof(u32), + .array_type = NO_ARRAY, + .tlv_type = QMI_COMMON_TLV_TYPE, + .offset = offsetof(struct + dfc_flow_status_info_type_v01, + num_bytes), + .ei_array = NULL, + }, + { + .data_type = QMI_UNSIGNED_2_BYTE, + .elem_len = 1, + .elem_size = sizeof(u16), + .array_type = NO_ARRAY, + .tlv_type = QMI_COMMON_TLV_TYPE, + .offset = offsetof(struct + dfc_flow_status_info_type_v01, + seq_num), + .ei_array = NULL, + }, + { + .data_type = QMI_DATA_LEN, + .elem_len = 1, + .elem_size = sizeof(u8), + .array_type = NO_ARRAY, + .tlv_type = QMI_COMMON_TLV_TYPE, + .offset = offsetof(struct + dfc_flow_status_info_type_v01, + qos_ids_len), + .ei_array = NULL, + }, + { + .data_type = QMI_STRUCT, + .elem_len = DFC_MAX_QOS_ID_V01, + .elem_size = sizeof(struct dfc_qos_id_type_v01), + .array_type = VAR_LEN_ARRAY, + .tlv_type = 0x10, + .offset = offsetof(struct + dfc_flow_status_info_type_v01, + qos_ids), + .ei_array = dfc_qos_id_type_v01_ei, + }, + { + .data_type = QMI_EOTI, + .array_type = NO_ARRAY, + .tlv_type = QMI_COMMON_TLV_TYPE, + }, +}; + +struct dfc_ancillary_info_type_v01 { + u8 subs_id; + u8 mux_id; + u8 bearer_id; + u32 reserved; +}; + +static struct qmi_elem_info dfc_ancillary_info_type_v01_ei[] = { + { + .data_type = QMI_UNSIGNED_1_BYTE, + .elem_len = 1, + .elem_size = sizeof(u8), + .array_type = NO_ARRAY, + .tlv_type = QMI_COMMON_TLV_TYPE, + .offset = offsetof(struct + dfc_ancillary_info_type_v01, + subs_id), + .ei_array = NULL, + }, + { + .data_type = QMI_UNSIGNED_1_BYTE, + .elem_len = 1, + .elem_size = sizeof(u8), + .array_type = NO_ARRAY, + .tlv_type = QMI_COMMON_TLV_TYPE, + .offset = offsetof(struct + dfc_ancillary_info_type_v01, + mux_id), + .ei_array = NULL, + }, + { + .data_type = QMI_UNSIGNED_1_BYTE, + .elem_len = 1, + .elem_size = sizeof(u8), + .array_type = NO_ARRAY, + .tlv_type = QMI_COMMON_TLV_TYPE, + .offset = offsetof(struct + dfc_ancillary_info_type_v01, + bearer_id), + .ei_array = NULL, + }, + { + .data_type = QMI_UNSIGNED_4_BYTE, + .elem_len = 1, + .elem_size = sizeof(u32), + .array_type = NO_ARRAY, + .tlv_type = QMI_COMMON_TLV_TYPE, + .offset = offsetof(struct + dfc_ancillary_info_type_v01, + reserved), + .ei_array = NULL, + }, + { + .data_type = QMI_EOTI, + .array_type = NO_ARRAY, + .tlv_type = QMI_COMMON_TLV_TYPE, + }, +}; + +struct dfc_flow_status_ind_msg_v01 { + u8 flow_status_valid; + u8 flow_status_len; + struct dfc_flow_status_info_type_v01 flow_status[DFC_MAX_BEARERS_V01]; + u8 eod_ack_reqd_valid; + u8 eod_ack_reqd; + u8 ancillary_info_valid; + u8 ancillary_info_len; + struct dfc_ancillary_info_type_v01 ancillary_info[DFC_MAX_BEARERS_V01]; +}; + +struct dfc_bearer_info_type_v01 { + u8 subs_id; + u8 mux_id; + u8 bearer_id; + enum dfc_ip_type_enum_v01 ip_type; +}; + +struct dfc_tx_link_status_ind_msg_v01 { + u8 tx_status; + u8 bearer_info_valid; + u8 bearer_info_len; + struct dfc_bearer_info_type_v01 bearer_info[DFC_MAX_BEARERS_V01]; +}; + +struct dfc_get_flow_status_req_msg_v01 { + u8 bearer_id_list_valid; + u8 bearer_id_list_len; + u8 bearer_id_list[DFC_MAX_BEARERS_V01]; +}; + +struct dfc_get_flow_status_resp_msg_v01 { + struct qmi_response_type_v01 resp; + u8 flow_status_valid; + u8 flow_status_len; + struct dfc_flow_status_info_type_v01 flow_status[DFC_MAX_BEARERS_V01]; +}; + +struct dfc_svc_ind { + struct list_head list; + u16 msg_id; + union { + struct dfc_flow_status_ind_msg_v01 dfc_info; + struct dfc_tx_link_status_ind_msg_v01 tx_status; + } d; +}; + +static struct qmi_elem_info dfc_bind_client_req_msg_v01_ei[] = { + { + .data_type = QMI_OPT_FLAG, + .elem_len = 1, + .elem_size = sizeof(u8), + .array_type = NO_ARRAY, + .tlv_type = 0x10, + .offset = offsetof(struct dfc_bind_client_req_msg_v01, + ep_id_valid), + .ei_array = NULL, + }, + { + .data_type = QMI_STRUCT, + .elem_len = 1, + .elem_size = sizeof(struct data_ep_id_type_v01), + .array_type = NO_ARRAY, + .tlv_type = 0x10, + .offset = offsetof(struct dfc_bind_client_req_msg_v01, + ep_id), + .ei_array = data_ep_id_type_v01_ei, + }, + { + .data_type = QMI_EOTI, + .array_type = NO_ARRAY, + .tlv_type = QMI_COMMON_TLV_TYPE, + }, +}; + +static struct qmi_elem_info dfc_bind_client_resp_msg_v01_ei[] = { + { + .data_type = QMI_STRUCT, + .elem_len = 1, + .elem_size = sizeof(struct qmi_response_type_v01), + .array_type = NO_ARRAY, + .tlv_type = 0x02, + .offset = offsetof(struct dfc_bind_client_resp_msg_v01, + resp), + .ei_array = qmi_response_type_v01_ei, + }, + { + .data_type = QMI_EOTI, + .array_type = NO_ARRAY, + .tlv_type = QMI_COMMON_TLV_TYPE, + }, +}; + +static struct qmi_elem_info dfc_indication_register_req_msg_v01_ei[] = { + { + .data_type = QMI_OPT_FLAG, + .elem_len = 1, + .elem_size = sizeof(u8), + .array_type = NO_ARRAY, + .tlv_type = 0x10, + .offset = offsetof(struct + dfc_indication_register_req_msg_v01, + report_flow_status_valid), + .ei_array = NULL, + }, + { + .data_type = QMI_UNSIGNED_1_BYTE, + .elem_len = 1, + .elem_size = sizeof(u8), + .array_type = NO_ARRAY, + .tlv_type = 0x10, + .offset = offsetof(struct + dfc_indication_register_req_msg_v01, + report_flow_status), + .ei_array = NULL, + }, + { + .data_type = QMI_OPT_FLAG, + .elem_len = 1, + .elem_size = sizeof(u8), + .array_type = NO_ARRAY, + .tlv_type = 0x11, + .offset = offsetof(struct + dfc_indication_register_req_msg_v01, + report_tx_link_status_valid), + .ei_array = NULL, + }, + { + .data_type = QMI_UNSIGNED_1_BYTE, + .elem_len = 1, + .elem_size = sizeof(u8), + .array_type = NO_ARRAY, + .tlv_type = 0x11, + .offset = offsetof(struct + dfc_indication_register_req_msg_v01, + report_tx_link_status), + .ei_array = NULL, + }, + { + .data_type = QMI_EOTI, + .array_type = NO_ARRAY, + .tlv_type = QMI_COMMON_TLV_TYPE, + }, +}; + +static struct qmi_elem_info dfc_indication_register_resp_msg_v01_ei[] = { + { + .data_type = QMI_STRUCT, + .elem_len = 1, + .elem_size = sizeof(struct qmi_response_type_v01), + .array_type = NO_ARRAY, + .tlv_type = 0x02, + .offset = offsetof(struct + dfc_indication_register_resp_msg_v01, + resp), + .ei_array = qmi_response_type_v01_ei, + }, + { + .data_type = QMI_EOTI, + .array_type = NO_ARRAY, + .tlv_type = QMI_COMMON_TLV_TYPE, + }, +}; + +static struct qmi_elem_info dfc_flow_status_ind_v01_ei[] = { + { + .data_type = QMI_OPT_FLAG, + .elem_len = 1, + .elem_size = sizeof(u8), + .array_type = NO_ARRAY, + .tlv_type = 0x10, + .offset = offsetof(struct + dfc_flow_status_ind_msg_v01, + flow_status_valid), + .ei_array = NULL, + }, + { + .data_type = QMI_DATA_LEN, + .elem_len = 1, + .elem_size = sizeof(u8), + .array_type = NO_ARRAY, + .tlv_type = 0x10, + .offset = offsetof(struct + dfc_flow_status_ind_msg_v01, + flow_status_len), + .ei_array = NULL, + }, + { + .data_type = QMI_STRUCT, + .elem_len = DFC_MAX_BEARERS_V01, + .elem_size = sizeof(struct + dfc_flow_status_info_type_v01), + .array_type = VAR_LEN_ARRAY, + .tlv_type = 0x10, + .offset = offsetof(struct + dfc_flow_status_ind_msg_v01, + flow_status), + .ei_array = dfc_flow_status_info_type_v01_ei, + }, + { + .data_type = QMI_OPT_FLAG, + .elem_len = 1, + .elem_size = sizeof(u8), + .array_type = NO_ARRAY, + .tlv_type = 0x11, + .offset = offsetof(struct + dfc_flow_status_ind_msg_v01, + eod_ack_reqd_valid), + .ei_array = NULL, + }, + { + .data_type = QMI_UNSIGNED_1_BYTE, + .elem_len = 1, + .elem_size = sizeof(u8), + .array_type = NO_ARRAY, + .tlv_type = 0x11, + .offset = offsetof(struct + dfc_flow_status_ind_msg_v01, + eod_ack_reqd), + .ei_array = NULL, + }, + { + .data_type = QMI_OPT_FLAG, + .elem_len = 1, + .elem_size = sizeof(u8), + .array_type = NO_ARRAY, + .tlv_type = 0x12, + .offset = offsetof(struct + dfc_flow_status_ind_msg_v01, + ancillary_info_valid), + .ei_array = NULL, + }, + { + .data_type = QMI_DATA_LEN, + .elem_len = 1, + .elem_size = sizeof(u8), + .array_type = NO_ARRAY, + .tlv_type = 0x12, + .offset = offsetof(struct + dfc_flow_status_ind_msg_v01, + ancillary_info_len), + .ei_array = NULL, + }, + { + .data_type = QMI_STRUCT, + .elem_len = DFC_MAX_BEARERS_V01, + .elem_size = sizeof(struct + dfc_ancillary_info_type_v01), + .array_type = VAR_LEN_ARRAY, + .tlv_type = 0x12, + .offset = offsetof(struct + dfc_flow_status_ind_msg_v01, + ancillary_info), + .ei_array = dfc_ancillary_info_type_v01_ei, + }, + { + .data_type = QMI_EOTI, + .array_type = NO_ARRAY, + .tlv_type = QMI_COMMON_TLV_TYPE, + }, +}; + +static struct qmi_elem_info dfc_get_flow_status_req_msg_v01_ei[] = { + { + .data_type = QMI_OPT_FLAG, + .elem_len = 1, + .elem_size = sizeof(u8), + .array_type = NO_ARRAY, + .tlv_type = 0x10, + .offset = offsetof(struct + dfc_get_flow_status_req_msg_v01, + bearer_id_list_valid), + .ei_array = NULL, + }, + { + .data_type = QMI_DATA_LEN, + .elem_len = 1, + .elem_size = sizeof(u8), + .array_type = NO_ARRAY, + .tlv_type = 0x10, + .offset = offsetof(struct + dfc_get_flow_status_req_msg_v01, + bearer_id_list_len), + .ei_array = NULL, + }, + { + .data_type = QMI_UNSIGNED_1_BYTE, + .elem_len = DFC_MAX_BEARERS_V01, + .elem_size = sizeof(u8), + .array_type = VAR_LEN_ARRAY, + .tlv_type = 0x10, + .offset = offsetof(struct + dfc_get_flow_status_req_msg_v01, + bearer_id_list), + .ei_array = NULL, + }, + { + .data_type = QMI_EOTI, + .array_type = NO_ARRAY, + .tlv_type = QMI_COMMON_TLV_TYPE, + }, +}; + +static struct qmi_elem_info dfc_get_flow_status_resp_msg_v01_ei[] = { + { + .data_type = QMI_STRUCT, + .elem_len = 1, + .elem_size = sizeof(struct qmi_response_type_v01), + .array_type = NO_ARRAY, + .tlv_type = 0x02, + .offset = offsetof(struct + dfc_get_flow_status_resp_msg_v01, + resp), + .ei_array = qmi_response_type_v01_ei, + }, + { + .data_type = QMI_OPT_FLAG, + .elem_len = 1, + .elem_size = sizeof(u8), + .array_type = NO_ARRAY, + .tlv_type = 0x10, + .offset = offsetof(struct + dfc_get_flow_status_resp_msg_v01, + flow_status_valid), + .ei_array = NULL, + }, + { + .data_type = QMI_DATA_LEN, + .elem_len = 1, + .elem_size = sizeof(u8), + .array_type = NO_ARRAY, + .tlv_type = 0x10, + .offset = offsetof(struct + dfc_get_flow_status_resp_msg_v01, + flow_status_len), + .ei_array = NULL, + }, + { + .data_type = QMI_STRUCT, + .elem_len = DFC_MAX_BEARERS_V01, + .elem_size = sizeof(struct + dfc_flow_status_info_type_v01), + .array_type = VAR_LEN_ARRAY, + .tlv_type = 0x10, + .offset = offsetof(struct + dfc_get_flow_status_resp_msg_v01, + flow_status), + .ei_array = dfc_flow_status_info_type_v01_ei, + }, + { + .data_type = QMI_EOTI, + .array_type = NO_ARRAY, + .tlv_type = QMI_COMMON_TLV_TYPE, + }, +}; + +static struct qmi_elem_info dfc_bearer_info_type_v01_ei[] = { + { + .data_type = QMI_UNSIGNED_1_BYTE, + .elem_len = 1, + .elem_size = sizeof(u8), + .array_type = NO_ARRAY, + .tlv_type = QMI_COMMON_TLV_TYPE, + .offset = offsetof(struct + dfc_bearer_info_type_v01, + subs_id), + .ei_array = NULL, + }, + { + .data_type = QMI_UNSIGNED_1_BYTE, + .elem_len = 1, + .elem_size = sizeof(u8), + .array_type = NO_ARRAY, + .tlv_type = QMI_COMMON_TLV_TYPE, + .offset = offsetof(struct + dfc_bearer_info_type_v01, + mux_id), + .ei_array = NULL, + }, + { + .data_type = QMI_UNSIGNED_1_BYTE, + .elem_len = 1, + .elem_size = sizeof(u8), + .array_type = NO_ARRAY, + .tlv_type = QMI_COMMON_TLV_TYPE, + .offset = offsetof(struct + dfc_bearer_info_type_v01, + bearer_id), + .ei_array = NULL, + }, + { + .data_type = QMI_SIGNED_4_BYTE_ENUM, + .elem_len = 1, + .elem_size = sizeof(enum dfc_ip_type_enum_v01), + .array_type = NO_ARRAY, + .tlv_type = QMI_COMMON_TLV_TYPE, + .offset = offsetof(struct + dfc_bearer_info_type_v01, + ip_type), + .ei_array = NULL, + }, + { + .data_type = QMI_EOTI, + .array_type = NO_ARRAY, + .tlv_type = QMI_COMMON_TLV_TYPE, + }, +}; + +static struct qmi_elem_info dfc_tx_link_status_ind_v01_ei[] = { + { + .data_type = QMI_UNSIGNED_1_BYTE, + .elem_len = 1, + .elem_size = sizeof(u8), + .array_type = NO_ARRAY, + .tlv_type = 0x01, + .offset = offsetof(struct + dfc_tx_link_status_ind_msg_v01, + tx_status), + .ei_array = NULL, + }, + { + .data_type = QMI_OPT_FLAG, + .elem_len = 1, + .elem_size = sizeof(u8), + .array_type = NO_ARRAY, + .tlv_type = 0x10, + .offset = offsetof(struct + dfc_tx_link_status_ind_msg_v01, + bearer_info_valid), + .ei_array = NULL, + }, + { + .data_type = QMI_DATA_LEN, + .elem_len = 1, + .elem_size = sizeof(u8), + .array_type = NO_ARRAY, + .tlv_type = 0x10, + .offset = offsetof(struct + dfc_tx_link_status_ind_msg_v01, + bearer_info_len), + .ei_array = NULL, + }, + { + .data_type = QMI_STRUCT, + .elem_len = DFC_MAX_BEARERS_V01, + .elem_size = sizeof(struct + dfc_bearer_info_type_v01), + .array_type = VAR_LEN_ARRAY, + .tlv_type = 0x10, + .offset = offsetof(struct + dfc_tx_link_status_ind_msg_v01, + bearer_info), + .ei_array = dfc_bearer_info_type_v01_ei, + }, + { + .data_type = QMI_EOTI, + .array_type = NO_ARRAY, + .tlv_type = QMI_COMMON_TLV_TYPE, + }, +}; + +static int +dfc_bind_client_req(struct qmi_handle *dfc_handle, + struct sockaddr_qrtr *ssctl, struct svc_info *svc) +{ + struct dfc_bind_client_resp_msg_v01 *resp; + struct dfc_bind_client_req_msg_v01 *req; + struct qmi_txn txn; + int ret; + + req = kzalloc(sizeof(*req), GFP_ATOMIC); + if (!req) + return -ENOMEM; + + resp = kzalloc(sizeof(*resp), GFP_ATOMIC); + if (!resp) { + kfree(req); + return -ENOMEM; + } + + ret = qmi_txn_init(dfc_handle, &txn, + dfc_bind_client_resp_msg_v01_ei, resp); + if (ret < 0) { + pr_err("%s() Failed init for response, err: %d\n", + __func__, ret); + goto out; + } + + req->ep_id_valid = 1; + req->ep_id.ep_type = svc->ep_type; + req->ep_id.iface_id = svc->iface_id; + ret = qmi_send_request(dfc_handle, ssctl, &txn, + QMI_DFC_BIND_CLIENT_REQ_V01, + QMI_DFC_BIND_CLIENT_REQ_V01_MAX_MSG_LEN, + dfc_bind_client_req_msg_v01_ei, req); + if (ret < 0) { + qmi_txn_cancel(&txn); + pr_err("%s() Failed sending request, err: %d\n", + __func__, ret); + goto out; + } + + ret = qmi_txn_wait(&txn, DFC_TIMEOUT_JF); + if (ret < 0) { + pr_err("%s() Response waiting failed, err: %d\n", + __func__, ret); + } else if (resp->resp.result != QMI_RESULT_SUCCESS_V01) { + pr_err("%s() Request rejected, result: %d, err: %d\n", + __func__, resp->resp.result, resp->resp.error); + ret = -resp->resp.result; + } + +out: + kfree(resp); + kfree(req); + return ret; +} + +static int +dfc_indication_register_req(struct qmi_handle *dfc_handle, + struct sockaddr_qrtr *ssctl, u8 reg) +{ + struct dfc_indication_register_resp_msg_v01 *resp; + struct dfc_indication_register_req_msg_v01 *req; + struct qmi_txn txn; + int ret; + + req = kzalloc(sizeof(*req), GFP_ATOMIC); + if (!req) + return -ENOMEM; + + resp = kzalloc(sizeof(*resp), GFP_ATOMIC); + if (!resp) { + kfree(req); + return -ENOMEM; + } + + ret = qmi_txn_init(dfc_handle, &txn, + dfc_indication_register_resp_msg_v01_ei, resp); + if (ret < 0) { + pr_err("%s() Failed init for response, err: %d\n", + __func__, ret); + goto out; + } + + req->report_flow_status_valid = 1; + req->report_flow_status = reg; + req->report_tx_link_status_valid = 1; + req->report_tx_link_status = reg; + + ret = qmi_send_request(dfc_handle, ssctl, &txn, + QMI_DFC_INDICATION_REGISTER_REQ_V01, + QMI_DFC_INDICATION_REGISTER_REQ_V01_MAX_MSG_LEN, + dfc_indication_register_req_msg_v01_ei, req); + if (ret < 0) { + qmi_txn_cancel(&txn); + pr_err("%s() Failed sending request, err: %d\n", + __func__, ret); + goto out; + } + + ret = qmi_txn_wait(&txn, DFC_TIMEOUT_JF); + if (ret < 0) { + pr_err("%s() Response waiting failed, err: %d\n", + __func__, ret); + } else if (resp->resp.result != QMI_RESULT_SUCCESS_V01) { + pr_err("%s() Request rejected, result: %d, err: %d\n", + __func__, resp->resp.result, resp->resp.error); + ret = -resp->resp.result; + } + +out: + kfree(resp); + kfree(req); + return ret; +} + +static int +dfc_get_flow_status_req(struct qmi_handle *dfc_handle, + struct sockaddr_qrtr *ssctl, + struct dfc_get_flow_status_resp_msg_v01 *resp) +{ + struct dfc_get_flow_status_req_msg_v01 *req; + struct qmi_txn *txn; + int ret; + + req = kzalloc(sizeof(*req), GFP_ATOMIC); + if (!req) + return -ENOMEM; + + txn = kzalloc(sizeof(*txn), GFP_ATOMIC); + if (!txn) { + kfree(req); + return -ENOMEM; + } + + ret = qmi_txn_init(dfc_handle, txn, + dfc_get_flow_status_resp_msg_v01_ei, resp); + if (ret < 0) { + pr_err("%s() Failed init for response, err: %d\n", + __func__, ret); + goto out; + } + + ret = qmi_send_request(dfc_handle, ssctl, txn, + QMI_DFC_GET_FLOW_STATUS_REQ_V01, + QMI_DFC_GET_FLOW_STATUS_REQ_V01_MAX_MSG_LEN, + dfc_get_flow_status_req_msg_v01_ei, req); + if (ret < 0) { + qmi_txn_cancel(txn); + pr_err("%s() Failed sending request, err: %d\n", + __func__, ret); + goto out; + } + + ret = qmi_txn_wait(txn, DFC_TIMEOUT_JF); + if (ret < 0) { + pr_err("%s() Response waiting failed, err: %d\n", + __func__, ret); + } else if (resp->resp.result != QMI_RESULT_SUCCESS_V01) { + pr_err("%s() Request rejected, result: %d, err: %d\n", + __func__, resp->resp.result, resp->resp.error); + ret = -resp->resp.result; + } + +out: + kfree(txn); + kfree(req); + return ret; +} + +static int dfc_init_service(struct dfc_qmi_data *data) +{ + int rc; + + rc = dfc_bind_client_req(&data->handle, &data->ssctl, &data->svc); + if (rc < 0) + return rc; + + return dfc_indication_register_req(&data->handle, &data->ssctl, 1); +} + +static void +dfc_send_ack(struct net_device *dev, u8 bearer_id, u16 seq, u8 mux_id, u8 type) +{ + struct qos_info *qos = rmnet_get_qos_pt(dev); + struct sk_buff *skb; + struct dfc_ack_cmd *msg; + int data_size = sizeof(struct dfc_ack_cmd); + int header_size = sizeof(struct dfc_qmap_header); + + if (!qos) + return; + + skb = alloc_skb(data_size, GFP_ATOMIC); + if (!skb) + return; + + msg = (struct dfc_ack_cmd *)skb_put(skb, data_size); + memset(msg, 0, data_size); + + msg->header.cd_bit = 1; + msg->header.mux_id = mux_id; + msg->header.pkt_len = htons(data_size - header_size); + + msg->bearer_id = bearer_id; + msg->command_name = 4; + msg->cmd_type = 0; + msg->dfc_seq = htons(seq); + msg->type = type; + msg->ver = 2; + msg->transaction_id = htonl(qos->tran_num); + + skb->dev = qos->real_dev; + skb->protocol = htons(ETH_P_MAP); + + trace_dfc_qmap_cmd(mux_id, bearer_id, seq, type, qos->tran_num); + qos->tran_num++; + + rmnet_map_tx_qmap_cmd(skb); +} + +int dfc_bearer_flow_ctl(struct net_device *dev, + struct rmnet_bearer_map *bearer, + struct qos_info *qos) +{ + struct rmnet_flow_map *itm; + int rc = 0, qlen; + int enable; + + enable = bearer->grant_size ? 1 : 0; + + list_for_each_entry(itm, &qos->flow_head, list) { + if (itm->bearer_id == bearer->bearer_id) { + /* + * Do not flow disable ancillary q if ancillary is true + */ + if (bearer->tcp_bidir && enable == 0 && + DFC_IS_ANCILLARY(itm->ip_type)) + continue; + + qlen = qmi_rmnet_flow_control(dev, itm->tcm_handle, + enable); + trace_dfc_qmi_tc(dev->name, itm->bearer_id, + itm->flow_id, bearer->grant_size, + qlen, itm->tcm_handle, enable); + rc++; + } + } + + if (enable == 0 && bearer->ack_req) + dfc_send_ack(dev, bearer->bearer_id, + bearer->seq, qos->mux_id, + DFC_ACK_TYPE_DISABLE); + + return rc; +} + +static int dfc_all_bearer_flow_ctl(struct net_device *dev, + struct qos_info *qos, u8 ack_req, u32 ancillary, + struct dfc_flow_status_info_type_v01 *fc_info) +{ + struct rmnet_bearer_map *bearer_itm; + struct rmnet_flow_map *flow_itm; + int rc = 0, qlen; + bool enable; + + enable = fc_info->num_bytes > 0 ? 1 : 0; + + list_for_each_entry(bearer_itm, &qos->bearer_head, list) { + bearer_itm->grant_size = fc_info->num_bytes; + bearer_itm->grant_thresh = + qmi_rmnet_grant_per(bearer_itm->grant_size); + bearer_itm->seq = fc_info->seq_num; + bearer_itm->ack_req = ack_req; + bearer_itm->tcp_bidir = DFC_IS_TCP_BIDIR(ancillary); + bearer_itm->last_grant = fc_info->num_bytes; + bearer_itm->last_seq = fc_info->seq_num; + } + + list_for_each_entry(flow_itm, &qos->flow_head, list) { + qlen = qmi_rmnet_flow_control(dev, flow_itm->tcm_handle, + enable); + trace_dfc_qmi_tc(dev->name, flow_itm->bearer_id, + flow_itm->flow_id, fc_info->num_bytes, + qlen, flow_itm->tcm_handle, enable); + rc++; + } + + if (enable == 0 && ack_req) + dfc_send_ack(dev, fc_info->bearer_id, + fc_info->seq_num, fc_info->mux_id, + DFC_ACK_TYPE_DISABLE); + + return rc; +} + +static int dfc_update_fc_map(struct net_device *dev, struct qos_info *qos, + u8 ack_req, u32 ancillary, + struct dfc_flow_status_info_type_v01 *fc_info) +{ + struct rmnet_bearer_map *itm = NULL; + int rc = 0; + bool action = false; + + itm = qmi_rmnet_get_bearer_map(qos, fc_info->bearer_id); + if (itm) { + /* The RAT switch flag indicates the start and end of + * the switch. Ignore indications in between. + */ + if (DFC_IS_RAT_SWITCH(ancillary)) + itm->rat_switch = !fc_info->num_bytes; + else + if (itm->rat_switch) + return 0; + + /* If TX is OFF but we received grant, ignore it */ + if (itm->tx_off && fc_info->num_bytes > 0) + return 0; + + if ((itm->grant_size == 0 && fc_info->num_bytes > 0) || + (itm->grant_size > 0 && fc_info->num_bytes == 0)) + action = true; + + itm->grant_size = fc_info->num_bytes; + itm->grant_thresh = qmi_rmnet_grant_per(itm->grant_size); + itm->seq = fc_info->seq_num; + itm->ack_req = ack_req; + itm->tcp_bidir = DFC_IS_TCP_BIDIR(ancillary); + itm->last_grant = fc_info->num_bytes; + itm->last_seq = fc_info->seq_num; + + if (action) + rc = dfc_bearer_flow_ctl(dev, itm, qos); + } else { + pr_debug("grant %u before flow activate", fc_info->num_bytes); + qos->default_grant = fc_info->num_bytes; + } + return rc; +} + +static void dfc_do_burst_flow_control(struct dfc_qmi_data *dfc, + struct dfc_svc_ind *svc_ind) +{ + struct dfc_flow_status_ind_msg_v01 *ind = &svc_ind->d.dfc_info; + struct net_device *dev; + struct qos_info *qos; + struct dfc_flow_status_info_type_v01 *flow_status; + struct dfc_ancillary_info_type_v01 *ai; + u8 ack_req = ind->eod_ack_reqd_valid ? ind->eod_ack_reqd : 0; + u32 ancillary; + int i, j; + + rcu_read_lock(); + + for (i = 0; i < ind->flow_status_len; i++) { + flow_status = &ind->flow_status[i]; + + ancillary = 0; + if (ind->ancillary_info_valid) { + for (j = 0; j < ind->ancillary_info_len; j++) { + ai = &ind->ancillary_info[j]; + if (ai->mux_id == flow_status->mux_id && + ai->bearer_id == flow_status->bearer_id) { + ancillary = ai->reserved; + break; + } + } + } + + trace_dfc_flow_ind(dfc->index, + i, flow_status->mux_id, + flow_status->bearer_id, + flow_status->num_bytes, + flow_status->seq_num, + ack_req, + ancillary); + + dev = rmnet_get_rmnet_dev(dfc->rmnet_port, + flow_status->mux_id); + if (!dev) + goto clean_out; + + qos = (struct qos_info *)rmnet_get_qos_pt(dev); + if (!qos) + continue; + + spin_lock_bh(&qos->qos_lock); + + if (qmi_rmnet_ignore_grant(dfc->rmnet_port)) { + spin_unlock_bh(&qos->qos_lock); + continue; + } + + if (unlikely(flow_status->bearer_id == 0xFF)) + dfc_all_bearer_flow_ctl( + dev, qos, ack_req, ancillary, flow_status); + else + dfc_update_fc_map( + dev, qos, ack_req, ancillary, flow_status); + + spin_unlock_bh(&qos->qos_lock); + } + +clean_out: + rcu_read_unlock(); +} + +static void dfc_update_tx_link_status(struct net_device *dev, + struct qos_info *qos, u8 tx_status, + struct dfc_bearer_info_type_v01 *binfo) +{ + struct rmnet_bearer_map *itm = NULL; + + itm = qmi_rmnet_get_bearer_map(qos, binfo->bearer_id); + if (!itm) + return; + + if (itm->grant_size && !tx_status) { + itm->grant_size = 0; + itm->tcp_bidir = false; + dfc_bearer_flow_ctl(dev, itm, qos); + } else if (itm->grant_size == 0 && tx_status && !itm->rat_switch) { + itm->grant_size = DEFAULT_GRANT; + itm->grant_thresh = DEFAULT_GRANT; + itm->seq = 0; + itm->ack_req = 0; + dfc_bearer_flow_ctl(dev, itm, qos); + } + + itm->tx_off = !tx_status; +} + +static void dfc_handle_tx_link_status_ind(struct dfc_qmi_data *dfc, + struct dfc_svc_ind *svc_ind) +{ + struct dfc_tx_link_status_ind_msg_v01 *ind = &svc_ind->d.tx_status; + struct net_device *dev; + struct qos_info *qos; + struct dfc_bearer_info_type_v01 *bearer_info; + int i; + + rcu_read_lock(); + + for (i = 0; i < ind->bearer_info_len; i++) { + bearer_info = &ind->bearer_info[i]; + + trace_dfc_tx_link_status_ind(dfc->index, i, + ind->tx_status, + bearer_info->mux_id, + bearer_info->bearer_id); + + dev = rmnet_get_rmnet_dev(dfc->rmnet_port, + bearer_info->mux_id); + if (!dev) + goto clean_out; + + qos = (struct qos_info *)rmnet_get_qos_pt(dev); + if (!qos) + continue; + + spin_lock_bh(&qos->qos_lock); + + dfc_update_tx_link_status( + dev, qos, ind->tx_status, bearer_info); + + spin_unlock_bh(&qos->qos_lock); + } + +clean_out: + rcu_read_unlock(); +} + +static void dfc_qmi_ind_work(struct work_struct *work) +{ + struct dfc_qmi_data *dfc = container_of(work, struct dfc_qmi_data, + qmi_ind_work); + struct dfc_svc_ind *svc_ind; + unsigned long flags; + + if (!dfc) + return; + + local_bh_disable(); + + do { + spin_lock_irqsave(&dfc->qmi_ind_lock, flags); + svc_ind = list_first_entry_or_null(&dfc->qmi_ind_q, + struct dfc_svc_ind, list); + if (svc_ind) + list_del(&svc_ind->list); + spin_unlock_irqrestore(&dfc->qmi_ind_lock, flags); + + if (!svc_ind) + break; + + if (!dfc->restart_state) { + if (svc_ind->msg_id == QMI_DFC_FLOW_STATUS_IND_V01) + dfc_do_burst_flow_control(dfc, svc_ind); + else if (svc_ind->msg_id == + QMI_DFC_TX_LINK_STATUS_IND_V01) + dfc_handle_tx_link_status_ind(dfc, svc_ind); + } + kfree(svc_ind); + } while (1); + + local_bh_enable(); + + qmi_rmnet_set_dl_msg_active(dfc->rmnet_port); +} + +static void dfc_clnt_ind_cb(struct qmi_handle *qmi, struct sockaddr_qrtr *sq, + struct qmi_txn *txn, const void *data) +{ + struct dfc_qmi_data *dfc = container_of(qmi, struct dfc_qmi_data, + handle); + struct dfc_flow_status_ind_msg_v01 *ind_msg; + struct dfc_svc_ind *svc_ind; + unsigned long flags; + + if (qmi != &dfc->handle) + return; + + ind_msg = (struct dfc_flow_status_ind_msg_v01 *)data; + if (ind_msg->flow_status_valid) { + if (ind_msg->flow_status_len > DFC_MAX_BEARERS_V01) { + pr_err("%s() Invalid fc info len: %d\n", + __func__, ind_msg->flow_status_len); + return; + } + + svc_ind = kzalloc(sizeof(struct dfc_svc_ind), GFP_ATOMIC); + if (!svc_ind) + return; + + svc_ind->msg_id = QMI_DFC_FLOW_STATUS_IND_V01; + memcpy(&svc_ind->d.dfc_info, ind_msg, sizeof(*ind_msg)); + + spin_lock_irqsave(&dfc->qmi_ind_lock, flags); + list_add_tail(&svc_ind->list, &dfc->qmi_ind_q); + spin_unlock_irqrestore(&dfc->qmi_ind_lock, flags); + + queue_work(dfc->dfc_wq, &dfc->qmi_ind_work); + } +} + +static void dfc_tx_link_status_ind_cb(struct qmi_handle *qmi, + struct sockaddr_qrtr *sq, + struct qmi_txn *txn, const void *data) +{ + struct dfc_qmi_data *dfc = container_of(qmi, struct dfc_qmi_data, + handle); + struct dfc_tx_link_status_ind_msg_v01 *ind_msg; + struct dfc_svc_ind *svc_ind; + unsigned long flags; + + if (qmi != &dfc->handle) + return; + + ind_msg = (struct dfc_tx_link_status_ind_msg_v01 *)data; + if (ind_msg->bearer_info_valid) { + if (ind_msg->bearer_info_len > DFC_MAX_BEARERS_V01) { + pr_err("%s() Invalid bearer info len: %d\n", + __func__, ind_msg->bearer_info_len); + return; + } + + svc_ind = kzalloc(sizeof(struct dfc_svc_ind), GFP_ATOMIC); + if (!svc_ind) + return; + + svc_ind->msg_id = QMI_DFC_TX_LINK_STATUS_IND_V01; + memcpy(&svc_ind->d.tx_status, ind_msg, sizeof(*ind_msg)); + + spin_lock_irqsave(&dfc->qmi_ind_lock, flags); + list_add_tail(&svc_ind->list, &dfc->qmi_ind_q); + spin_unlock_irqrestore(&dfc->qmi_ind_lock, flags); + + queue_work(dfc->dfc_wq, &dfc->qmi_ind_work); + } +} + +static void dfc_svc_init(struct work_struct *work) +{ + int rc = 0; + struct dfc_qmi_data *data = container_of(work, struct dfc_qmi_data, + svc_arrive); + struct qmi_info *qmi; + + if (data->restart_state == 1) + return; + + rc = dfc_init_service(data); + if (rc < 0) { + pr_err("%s Failed to init service, err[%d]\n", __func__, rc); + return; + } + + if (data->restart_state == 1) + return; + while (!rtnl_trylock()) { + if (!data->restart_state) + cond_resched(); + else + return; + } + qmi = (struct qmi_info *)rmnet_get_qmi_pt(data->rmnet_port); + if (!qmi) { + rtnl_unlock(); + return; + } + + qmi->dfc_pending[data->index] = NULL; + qmi->dfc_clients[data->index] = (void *)data; + trace_dfc_client_state_up(data->index, + data->svc.instance, + data->svc.ep_type, + data->svc.iface_id); + + rtnl_unlock(); + + pr_info("Connection established with the DFC Service\n"); +} + +static int dfc_svc_arrive(struct qmi_handle *qmi, struct qmi_service *svc) +{ + struct dfc_qmi_data *data = container_of(qmi, struct dfc_qmi_data, + handle); + + data->ssctl.sq_family = AF_QIPCRTR; + data->ssctl.sq_node = svc->node; + data->ssctl.sq_port = svc->port; + + queue_work(data->dfc_wq, &data->svc_arrive); + + return 0; +} + +static void dfc_svc_exit(struct qmi_handle *qmi, struct qmi_service *svc) +{ + struct dfc_qmi_data *data = container_of(qmi, struct dfc_qmi_data, + handle); + + if (!data) + pr_debug("%s() data is null\n", __func__); +} + +static struct qmi_ops server_ops = { + .new_server = dfc_svc_arrive, + .del_server = dfc_svc_exit, +}; + +static struct qmi_msg_handler qmi_indication_handler[] = { + { + .type = QMI_INDICATION, + .msg_id = QMI_DFC_FLOW_STATUS_IND_V01, + .ei = dfc_flow_status_ind_v01_ei, + .decoded_size = sizeof(struct dfc_flow_status_ind_msg_v01), + .fn = dfc_clnt_ind_cb, + }, + { + .type = QMI_INDICATION, + .msg_id = QMI_DFC_TX_LINK_STATUS_IND_V01, + .ei = dfc_tx_link_status_ind_v01_ei, + .decoded_size = sizeof(struct dfc_tx_link_status_ind_msg_v01), + .fn = dfc_tx_link_status_ind_cb, + }, + {}, +}; + +int dfc_qmi_client_init(void *port, int index, struct svc_info *psvc, + struct qmi_info *qmi) +{ + struct dfc_qmi_data *data; + int rc = -ENOMEM; + + if (!port || !qmi) + return -EINVAL; + + data = kzalloc(sizeof(struct dfc_qmi_data), GFP_KERNEL); + if (!data) + return -ENOMEM; + + data->rmnet_port = port; + data->index = index; + data->restart_state = 0; + memcpy(&data->svc, psvc, sizeof(data->svc)); + + INIT_WORK(&data->qmi_ind_work, dfc_qmi_ind_work); + INIT_LIST_HEAD(&data->qmi_ind_q); + spin_lock_init(&data->qmi_ind_lock); + + data->dfc_wq = create_singlethread_workqueue("dfc_wq"); + if (!data->dfc_wq) { + pr_err("%s Could not create workqueue\n", __func__); + goto err0; + } + + INIT_WORK(&data->svc_arrive, dfc_svc_init); + rc = qmi_handle_init(&data->handle, + QMI_DFC_GET_FLOW_STATUS_RESP_V01_MAX_MSG_LEN, + &server_ops, qmi_indication_handler); + if (rc < 0) { + pr_err("%s: failed qmi_handle_init - rc[%d]\n", __func__, rc); + goto err1; + } + + rc = qmi_add_lookup(&data->handle, DFC_SERVICE_ID_V01, + DFC_SERVICE_VERS_V01, + psvc->instance); + if (rc < 0) { + pr_err("%s: failed qmi_add_lookup - rc[%d]\n", __func__, rc); + goto err2; + } + + qmi->dfc_pending[index] = (void *)data; + + return 0; + +err2: + qmi_handle_release(&data->handle); +err1: + destroy_workqueue(data->dfc_wq); +err0: + kfree(data); + return rc; +} + +void dfc_qmi_client_exit(void *dfc_data) +{ + struct dfc_qmi_data *data = (struct dfc_qmi_data *)dfc_data; + + if (!data) { + pr_err("%s() data is null\n", __func__); + return; + } + + data->restart_state = 1; + trace_dfc_client_state_down(data->index, 0); + qmi_handle_release(&data->handle); + + drain_workqueue(data->dfc_wq); + destroy_workqueue(data->dfc_wq); + kfree(data); +} + +void dfc_qmi_burst_check(struct net_device *dev, struct qos_info *qos, + int ip_type, u32 mark, unsigned int len) +{ + struct rmnet_bearer_map *bearer; + struct rmnet_flow_map *itm; + u32 start_grant; + + spin_lock_bh(&qos->qos_lock); + + itm = qmi_rmnet_get_flow_map(qos, mark, ip_type); + if (unlikely(!itm)) + goto out; + + bearer = qmi_rmnet_get_bearer_map(qos, itm->bearer_id); + if (unlikely(!bearer)) + goto out; + + trace_dfc_flow_check(dev->name, bearer->bearer_id, + len, bearer->grant_size); + + if (!bearer->grant_size) + goto out; + + start_grant = bearer->grant_size; + if (len >= bearer->grant_size) + bearer->grant_size = 0; + else + bearer->grant_size -= len; + + if (start_grant > bearer->grant_thresh && + bearer->grant_size <= bearer->grant_thresh) { + dfc_send_ack(dev, bearer->bearer_id, + bearer->seq, qos->mux_id, + DFC_ACK_TYPE_THRESHOLD); + } + + if (!bearer->grant_size) + dfc_bearer_flow_ctl(dev, bearer, qos); + +out: + spin_unlock_bh(&qos->qos_lock); +} + +void dfc_qmi_query_flow(void *dfc_data) +{ + struct dfc_qmi_data *data = (struct dfc_qmi_data *)dfc_data; + struct dfc_get_flow_status_resp_msg_v01 *resp; + struct dfc_svc_ind *svc_ind; + int rc; + + resp = kzalloc(sizeof(*resp), GFP_ATOMIC); + if (!resp) + return; + + svc_ind = kzalloc(sizeof(*svc_ind), GFP_ATOMIC); + if (!svc_ind) { + kfree(resp); + return; + } + + if (!data) + goto done; + + rc = dfc_get_flow_status_req(&data->handle, &data->ssctl, resp); + + if (rc < 0 || !resp->flow_status_valid || resp->flow_status_len < 1 || + resp->flow_status_len > DFC_MAX_BEARERS_V01) + goto done; + + svc_ind->d.dfc_info.flow_status_valid = resp->flow_status_valid; + svc_ind->d.dfc_info.flow_status_len = resp->flow_status_len; + memcpy(&svc_ind->d.dfc_info.flow_status, resp->flow_status, + sizeof(resp->flow_status[0]) * resp->flow_status_len); + dfc_do_burst_flow_control(data, svc_ind); + +done: + kfree(svc_ind); + kfree(resp); +} diff --git a/core/qmi_rmnet.c b/core/qmi_rmnet.c new file mode 100644 index 000000000000..9de6769011c8 --- /dev/null +++ b/core/qmi_rmnet.c @@ -0,0 +1,1032 @@ +/* + * Copyright (c) 2018-2019, The Linux Foundation. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 and + * only version 2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ +#include + +#include "qmi_rmnet_i.h" +#include "qmi_rmnet.h" +#include "rmnet_qmi.h" +#include +#include +#include +#define CREATE_TRACE_POINTS +#include "dfc.h" +#undef CREATE_TRACE_POINTS +#include +#include +#include +#include + + +#define NLMSG_FLOW_ACTIVATE 1 +#define NLMSG_FLOW_DEACTIVATE 2 +#define NLMSG_CLIENT_SETUP 4 +#define NLMSG_CLIENT_DELETE 5 + +#define FLAG_DFC_MASK 0x000F +#define FLAG_POWERSAVE_MASK 0x0010 +#define DFC_MODE_MULTIQ 2 + +unsigned int rmnet_wq_frequency __read_mostly = 1000; +module_param(rmnet_wq_frequency, uint, 0644); +MODULE_PARM_DESC(rmnet_wq_frequency, "Frequency of PS check in ms"); + +#define PS_WORK_ACTIVE_BIT 0 +#define PS_INTERVAL (((!rmnet_wq_frequency) ? \ + 1 : rmnet_wq_frequency/10) * (HZ/100)) +#define NO_DELAY (0x0000 * HZ) + +#ifdef CONFIG_QTI_QMI_DFC +static unsigned int qmi_rmnet_scale_factor = 5; +#endif + +struct qmi_elem_info data_ep_id_type_v01_ei[] = { + { + .data_type = QMI_SIGNED_4_BYTE_ENUM, + .elem_len = 1, + .elem_size = sizeof(enum data_ep_type_enum_v01), + .array_type = NO_ARRAY, + .tlv_type = QMI_COMMON_TLV_TYPE, + .offset = offsetof(struct data_ep_id_type_v01, + ep_type), + .ei_array = NULL, + }, + { + .data_type = QMI_UNSIGNED_4_BYTE, + .elem_len = 1, + .elem_size = sizeof(u32), + .array_type = NO_ARRAY, + .tlv_type = QMI_COMMON_TLV_TYPE, + .offset = offsetof(struct data_ep_id_type_v01, + iface_id), + .ei_array = NULL, + }, + { + .data_type = QMI_EOTI, + .elem_len = 0, + .elem_size = 0, + .array_type = NO_ARRAY, + .tlv_type = QMI_COMMON_TLV_TYPE, + .offset = 0, + .ei_array = NULL, + }, +}; +EXPORT_SYMBOL(data_ep_id_type_v01_ei); + +void *qmi_rmnet_has_dfc_client(struct qmi_info *qmi) +{ + int i; + + if (!qmi || ((qmi->flag & FLAG_DFC_MASK) != DFC_MODE_MULTIQ)) + return NULL; + + for (i = 0; i < MAX_CLIENT_NUM; i++) { + if (qmi->dfc_clients[i]) + return qmi->dfc_clients[i]; + } + + return NULL; +} + +static inline int +qmi_rmnet_has_client(struct qmi_info *qmi) +{ + if (qmi->wda_client) + return 1; + + return qmi_rmnet_has_dfc_client(qmi) ? 1 : 0; +} + +static int +qmi_rmnet_has_pending(struct qmi_info *qmi) +{ + int i; + + if (qmi->wda_pending) + return 1; + + for (i = 0; i < MAX_CLIENT_NUM; i++) { + if (qmi->dfc_pending[i]) + return 1; + } + + return 0; +} + +#ifdef CONFIG_QTI_QMI_DFC +static void +qmi_rmnet_clean_flow_list(struct qmi_info *qmi, struct net_device *dev, + struct qos_info *qos) +{ + struct rmnet_bearer_map *bearer, *br_tmp; + struct rmnet_flow_map *itm, *fl_tmp; + + ASSERT_RTNL(); + + list_for_each_entry_safe(itm, fl_tmp, &qos->flow_head, list) { + list_del(&itm->list); + kfree(itm); + } + + list_for_each_entry_safe(bearer, br_tmp, &qos->bearer_head, list) { + list_del(&bearer->list); + kfree(bearer); + } +} + +struct rmnet_flow_map * +qmi_rmnet_get_flow_map(struct qos_info *qos, u32 flow_id, int ip_type) +{ + struct rmnet_flow_map *itm; + + if (!qos) + return NULL; + + list_for_each_entry(itm, &qos->flow_head, list) { + if ((itm->flow_id == flow_id) && (itm->ip_type == ip_type)) + return itm; + } + return NULL; +} + +struct rmnet_bearer_map * +qmi_rmnet_get_bearer_map(struct qos_info *qos, uint8_t bearer_id) +{ + struct rmnet_bearer_map *itm; + + if (!qos) + return NULL; + + list_for_each_entry(itm, &qos->bearer_head, list) { + if (itm->bearer_id == bearer_id) + return itm; + } + return NULL; +} + +static void qmi_rmnet_update_flow_map(struct rmnet_flow_map *itm, + struct rmnet_flow_map *new_map) +{ + itm->bearer_id = new_map->bearer_id; + itm->flow_id = new_map->flow_id; + itm->ip_type = new_map->ip_type; + itm->tcm_handle = new_map->tcm_handle; +} + +int qmi_rmnet_flow_control(struct net_device *dev, u32 tcm_handle, int enable) +{ + struct netdev_queue *q; + + if (unlikely(tcm_handle >= dev->num_tx_queues)) + return 0; + + q = netdev_get_tx_queue(dev, tcm_handle); + if (unlikely(!q)) + return 0; + + if (enable) + netif_tx_wake_queue(q); + else + netif_tx_stop_queue(q); + + return 0; +} + +static void qmi_rmnet_reset_txq(struct net_device *dev, unsigned int txq) +{ + struct Qdisc *qdisc; + + if (unlikely(txq >= dev->num_tx_queues)) + return; + + qdisc = rtnl_dereference(netdev_get_tx_queue(dev, txq)->qdisc); + if (qdisc) { + spin_lock_bh(qdisc_lock(qdisc)); + qdisc_reset(qdisc); + spin_unlock_bh(qdisc_lock(qdisc)); + } +} + +static int qmi_rmnet_add_flow(struct net_device *dev, struct tcmsg *tcm, + struct qmi_info *qmi) +{ + struct qos_info *qos_info = (struct qos_info *)rmnet_get_qos_pt(dev); + struct rmnet_flow_map new_map, *itm; + struct rmnet_bearer_map *bearer; + + if (!qos_info) + return -EINVAL; + + ASSERT_RTNL(); + + /* flow activate + * tcm->tcm__pad1 - bearer_id, tcm->tcm_parent - flow_id, + * tcm->tcm_ifindex - ip_type, tcm->tcm_handle - tcm_handle + */ + + new_map.bearer_id = tcm->tcm__pad1; + new_map.flow_id = tcm->tcm_parent; + new_map.ip_type = tcm->tcm_ifindex; + new_map.tcm_handle = tcm->tcm_handle; + trace_dfc_flow_info(dev->name, new_map.bearer_id, new_map.flow_id, + new_map.ip_type, new_map.tcm_handle, 1); + + spin_lock_bh(&qos_info->qos_lock); + + itm = qmi_rmnet_get_flow_map(qos_info, new_map.flow_id, + new_map.ip_type); + if (itm) { + qmi_rmnet_update_flow_map(itm, &new_map); + } else { + itm = kzalloc(sizeof(*itm), GFP_ATOMIC); + if (!itm) { + spin_unlock_bh(&qos_info->qos_lock); + return -ENOMEM; + } + + qmi_rmnet_update_flow_map(itm, &new_map); + list_add(&itm->list, &qos_info->flow_head); + + bearer = qmi_rmnet_get_bearer_map(qos_info, new_map.bearer_id); + if (bearer) { + bearer->flow_ref++; + } else { + bearer = kzalloc(sizeof(*bearer), GFP_ATOMIC); + if (!bearer) { + spin_unlock_bh(&qos_info->qos_lock); + return -ENOMEM; + } + + bearer->bearer_id = new_map.bearer_id; + bearer->flow_ref = 1; + bearer->grant_size = qos_info->default_grant; + bearer->grant_thresh = + qmi_rmnet_grant_per(bearer->grant_size); + qos_info->default_grant = DEFAULT_GRANT; + list_add(&bearer->list, &qos_info->bearer_head); + } + + qmi_rmnet_flow_control(dev, itm->tcm_handle, + bearer->grant_size > 0 ? 1 : 0); + + trace_dfc_qmi_tc(dev->name, itm->bearer_id, itm->flow_id, + bearer->grant_size, 0, itm->tcm_handle, + bearer->grant_size > 0 ? 1 : 0); + } + + spin_unlock_bh(&qos_info->qos_lock); + + return 0; +} + +static int +qmi_rmnet_del_flow(struct net_device *dev, struct tcmsg *tcm, + struct qmi_info *qmi) +{ + struct qos_info *qos_info = (struct qos_info *)rmnet_get_qos_pt(dev); + struct rmnet_flow_map new_map, *itm; + struct rmnet_bearer_map *bearer; + + if (!qos_info) + return -EINVAL; + + ASSERT_RTNL(); + + /* flow deactivate + * tcm->tcm__pad1 - bearer_id, tcm->tcm_parent - flow_id, + * tcm->tcm_ifindex - ip_type + */ + + spin_lock_bh(&qos_info->qos_lock); + + new_map.bearer_id = tcm->tcm__pad1; + new_map.flow_id = tcm->tcm_parent; + new_map.ip_type = tcm->tcm_ifindex; + itm = qmi_rmnet_get_flow_map(qos_info, new_map.flow_id, + new_map.ip_type); + if (itm) { + trace_dfc_flow_info(dev->name, new_map.bearer_id, + new_map.flow_id, new_map.ip_type, + itm->tcm_handle, 0); + list_del(&itm->list); + + /*clear bearer map*/ + bearer = qmi_rmnet_get_bearer_map(qos_info, new_map.bearer_id); + if (bearer && --bearer->flow_ref == 0) { + list_del(&bearer->list); + kfree(bearer); + + /* Purge pending packets for dedicated flow */ + if (itm->flow_id) + qmi_rmnet_reset_txq(dev, itm->tcm_handle); + } + + /* Enable flow to allow new flow setup */ + qmi_rmnet_flow_control(dev, itm->tcm_handle, 1); + trace_dfc_qmi_tc(dev->name, itm->bearer_id, itm->flow_id, + 0, 0, itm->tcm_handle, 1); + + kfree(itm); + } + + if (list_empty(&qos_info->flow_head)) { + netif_tx_wake_all_queues(dev); + trace_dfc_qmi_tc(dev->name, 0xFF, 0, DEFAULT_GRANT, 0, 0, 1); + } + + spin_unlock_bh(&qos_info->qos_lock); + + return 0; +} + +static void qmi_rmnet_query_flows(struct qmi_info *qmi) +{ + int i; + + for (i = 0; i < MAX_CLIENT_NUM; i++) { + if (qmi->dfc_clients[i]) + dfc_qmi_query_flow(qmi->dfc_clients[i]); + } +} + +static int qmi_rmnet_set_scale_factor(const char *val, + const struct kernel_param *kp) +{ + int ret; + unsigned int num = 0; + + ret = kstrtouint(val, 10, &num); + if (ret != 0 || num == 0) + return -EINVAL; + + return param_set_uint(val, kp); +} + +static const struct kernel_param_ops qmi_rmnet_scale_ops = { + .set = qmi_rmnet_set_scale_factor, + .get = param_get_uint, +}; + +module_param_cb(qmi_rmnet_scale_factor, &qmi_rmnet_scale_ops, + &qmi_rmnet_scale_factor, 0664); +#else +static inline void +qmi_rmnet_update_flow_link(struct qmi_info *qmi, struct net_device *dev, + struct rmnet_flow_map *itm, int add_flow) +{ +} + +static inline void qmi_rmnet_clean_flow_list(struct qos_info *qos) +{ +} + +static inline void +qmi_rmnet_update_flow_map(struct rmnet_flow_map *itm, + struct rmnet_flow_map *new_map) +{ +} + +static inline int +qmi_rmnet_add_flow(struct net_device *dev, struct tcmsg *tcm, + struct qmi_info *qmi) +{ + return -EINVAL; +} + +static inline int +qmi_rmnet_del_flow(struct net_device *dev, struct tcmsg *tcm, + struct qmi_info *qmi) +{ + return -EINVAL; +} + +static inline void qmi_rmnet_query_flows(struct qmi_info *qmi) +{ +} +#endif + +static int +qmi_rmnet_setup_client(void *port, struct qmi_info *qmi, struct tcmsg *tcm) +{ + int idx, rc, err = 0; + struct svc_info svc; + + ASSERT_RTNL(); + + /* client setup + * tcm->tcm_handle - instance, tcm->tcm_info - ep_type, + * tcm->tcm_parent - iface_id, tcm->tcm_ifindex - flags + */ + idx = (tcm->tcm_handle == 0) ? 0 : 1; + + if (!qmi) { + qmi = kzalloc(sizeof(struct qmi_info), GFP_ATOMIC); + if (!qmi) + return -ENOMEM; + + rmnet_init_qmi_pt(port, qmi); + } + + qmi->flag = tcm->tcm_ifindex; + svc.instance = tcm->tcm_handle; + svc.ep_type = tcm->tcm_info; + svc.iface_id = tcm->tcm_parent; + + if (((tcm->tcm_ifindex & FLAG_DFC_MASK) == DFC_MODE_MULTIQ) && + !qmi->dfc_clients[idx] && !qmi->dfc_pending[idx]) { + rc = dfc_qmi_client_init(port, idx, &svc, qmi); + if (rc < 0) + err = rc; + } + + if ((tcm->tcm_ifindex & FLAG_POWERSAVE_MASK) && + (idx == 0) && !qmi->wda_client && !qmi->wda_pending) { + rc = wda_qmi_client_init(port, &svc, qmi); + if (rc < 0) + err = rc; + } + + return err; +} + +static int +__qmi_rmnet_delete_client(void *port, struct qmi_info *qmi, int idx) +{ + void *data = NULL; + + ASSERT_RTNL(); + + if (qmi->dfc_clients[idx]) + data = qmi->dfc_clients[idx]; + else if (qmi->dfc_pending[idx]) + data = qmi->dfc_pending[idx]; + + if (data) { + dfc_qmi_client_exit(data); + qmi->dfc_clients[idx] = NULL; + qmi->dfc_pending[idx] = NULL; + } + + if (!qmi_rmnet_has_client(qmi) && !qmi_rmnet_has_pending(qmi)) { + rmnet_reset_qmi_pt(port); + kfree(qmi); + return 0; + } + + return 1; +} + +static void +qmi_rmnet_delete_client(void *port, struct qmi_info *qmi, struct tcmsg *tcm) +{ + int idx; + void *data = NULL; + + /* client delete: tcm->tcm_handle - instance*/ + idx = (tcm->tcm_handle == 0) ? 0 : 1; + + ASSERT_RTNL(); + if (qmi->wda_client) + data = qmi->wda_client; + else if (qmi->wda_pending) + data = qmi->wda_pending; + + if ((idx == 0) && data) { + wda_qmi_client_exit(data); + qmi->wda_client = NULL; + qmi->wda_pending = NULL; + } else { + qmi_rmnet_flush_ps_wq(); + } + + __qmi_rmnet_delete_client(port, qmi, idx); +} + +void qmi_rmnet_change_link(struct net_device *dev, void *port, void *tcm_pt) +{ + struct qmi_info *qmi = (struct qmi_info *)rmnet_get_qmi_pt(port); + struct tcmsg *tcm = (struct tcmsg *)tcm_pt; + + switch (tcm->tcm_family) { + case NLMSG_FLOW_ACTIVATE: + if (!qmi || ((qmi->flag & FLAG_DFC_MASK) != DFC_MODE_MULTIQ) || + !qmi_rmnet_has_dfc_client(qmi)) + return; + + qmi_rmnet_add_flow(dev, tcm, qmi); + break; + case NLMSG_FLOW_DEACTIVATE: + if (!qmi || ((qmi->flag & FLAG_DFC_MASK) != DFC_MODE_MULTIQ)) + return; + + qmi_rmnet_del_flow(dev, tcm, qmi); + break; + case NLMSG_CLIENT_SETUP: + if (((tcm->tcm_ifindex & FLAG_DFC_MASK) != DFC_MODE_MULTIQ) && + !(tcm->tcm_ifindex & FLAG_POWERSAVE_MASK)) + return; + + if (qmi_rmnet_setup_client(port, qmi, tcm) < 0) { + /* retrieve qmi again as it could have been changed */ + qmi = (struct qmi_info *)rmnet_get_qmi_pt(port); + if (qmi && + !qmi_rmnet_has_client(qmi) && + !qmi_rmnet_has_pending(qmi)) { + rmnet_reset_qmi_pt(port); + kfree(qmi); + } + } else if (tcm->tcm_ifindex & FLAG_POWERSAVE_MASK) { + qmi_rmnet_work_init(port); + rmnet_set_powersave_format(port); + } + break; + case NLMSG_CLIENT_DELETE: + if (!qmi) + return; + if (tcm->tcm_handle == 0) { /* instance 0 */ + rmnet_clear_powersave_format(port); + qmi_rmnet_work_exit(port); + } + qmi_rmnet_delete_client(port, qmi, tcm); + break; + default: + pr_debug("%s(): No handler\n", __func__); + break; + } +} +EXPORT_SYMBOL(qmi_rmnet_change_link); + +void qmi_rmnet_qmi_exit(void *qmi_pt, void *port) +{ + struct qmi_info *qmi = (struct qmi_info *)qmi_pt; + int i; + void *data = NULL; + + if (!qmi) + return; + + ASSERT_RTNL(); + + qmi_rmnet_work_exit(port); + + if (qmi->wda_client) + data = qmi->wda_client; + else if (qmi->wda_pending) + data = qmi->wda_pending; + + if (data) { + wda_qmi_client_exit(data); + qmi->wda_client = NULL; + qmi->wda_pending = NULL; + } + + for (i = 0; i < MAX_CLIENT_NUM; i++) { + if (!__qmi_rmnet_delete_client(port, qmi, i)) + return; + } +} +EXPORT_SYMBOL(qmi_rmnet_qmi_exit); + +void qmi_rmnet_enable_all_flows(struct net_device *dev) +{ + struct qos_info *qos; + struct rmnet_bearer_map *bearer; + bool do_wake; + + qos = (struct qos_info *)rmnet_get_qos_pt(dev); + if (!qos) + return; + + spin_lock_bh(&qos->qos_lock); + + list_for_each_entry(bearer, &qos->bearer_head, list) { + if (bearer->tx_off) + continue; + do_wake = !bearer->grant_size; + bearer->grant_size = DEFAULT_GRANT; + bearer->grant_thresh = DEFAULT_GRANT; + bearer->seq = 0; + bearer->ack_req = 0; + bearer->tcp_bidir = false; + bearer->rat_switch = false; + + if (do_wake) + dfc_bearer_flow_ctl(dev, bearer, qos); + } + + spin_unlock_bh(&qos->qos_lock); +} +EXPORT_SYMBOL(qmi_rmnet_enable_all_flows); + +bool qmi_rmnet_all_flows_enabled(struct net_device *dev) +{ + struct qos_info *qos; + struct rmnet_bearer_map *bearer; + bool ret = true; + + qos = (struct qos_info *)rmnet_get_qos_pt(dev); + if (!qos) + return true; + + spin_lock_bh(&qos->qos_lock); + + list_for_each_entry(bearer, &qos->bearer_head, list) { + if (!bearer->grant_size) { + ret = false; + break; + } + } + + spin_unlock_bh(&qos->qos_lock); + + return ret; +} +EXPORT_SYMBOL(qmi_rmnet_all_flows_enabled); + +#ifdef CONFIG_QTI_QMI_DFC +void qmi_rmnet_burst_fc_check(struct net_device *dev, + int ip_type, u32 mark, unsigned int len) +{ + struct qos_info *qos = rmnet_get_qos_pt(dev); + + if (!qos) + return; + + dfc_qmi_burst_check(dev, qos, ip_type, mark, len); +} +EXPORT_SYMBOL(qmi_rmnet_burst_fc_check); + +int qmi_rmnet_get_queue(struct net_device *dev, struct sk_buff *skb) +{ + struct qos_info *qos = rmnet_get_qos_pt(dev); + int txq = 0, ip_type = AF_INET; + unsigned int len = skb->len; + struct rmnet_flow_map *itm; + u32 mark = skb->mark; + + if (!qos) + return 0; + + switch (skb->protocol) { + /* TCPv4 ACKs */ + case htons(ETH_P_IP): + ip_type = AF_INET; + if ((!mark) && + (ip_hdr(skb)->protocol == IPPROTO_TCP) && + (len == 40 || len == 52) && + (ip_hdr(skb)->ihl == 5) && + ((tcp_flag_word(tcp_hdr(skb)) & 0xFF00) == TCP_FLAG_ACK)) + return 1; + break; + + /* TCPv6 ACKs */ + case htons(ETH_P_IPV6): + ip_type = AF_INET6; + if ((!mark) && + (ipv6_hdr(skb)->nexthdr == IPPROTO_TCP) && + (len == 60 || len == 72) && + ((tcp_flag_word(tcp_hdr(skb)) & 0xFF00) == TCP_FLAG_ACK)) + return 1; + /* Fall through */ + } + + /* Default flows */ + if (!mark) + return 0; + + /* Dedicated flows */ + spin_lock_bh(&qos->qos_lock); + + itm = qmi_rmnet_get_flow_map(qos, mark, ip_type); + if (unlikely(!itm)) + goto done; + + txq = itm->tcm_handle; + +done: + spin_unlock_bh(&qos->qos_lock); + return txq; +} +EXPORT_SYMBOL(qmi_rmnet_get_queue); + +inline unsigned int qmi_rmnet_grant_per(unsigned int grant) +{ + return grant / qmi_rmnet_scale_factor; +} +EXPORT_SYMBOL(qmi_rmnet_grant_per); + +void *qmi_rmnet_qos_init(struct net_device *real_dev, u8 mux_id) +{ + struct qos_info *qos; + + qos = kmalloc(sizeof(*qos), GFP_KERNEL); + if (!qos) + return NULL; + + qos->mux_id = mux_id; + qos->real_dev = real_dev; + qos->default_grant = DEFAULT_GRANT; + qos->tran_num = 0; + INIT_LIST_HEAD(&qos->flow_head); + INIT_LIST_HEAD(&qos->bearer_head); + spin_lock_init(&qos->qos_lock); + + return qos; +} +EXPORT_SYMBOL(qmi_rmnet_qos_init); + +void qmi_rmnet_qos_exit(struct net_device *dev, void *qos) +{ + void *port = rmnet_get_rmnet_port(dev); + struct qmi_info *qmi = rmnet_get_qmi_pt(port); + struct qos_info *qos_info = (struct qos_info *)qos; + + if (!qmi || !qos) + return; + + qmi_rmnet_clean_flow_list(qmi, dev, qos_info); + kfree(qos); +} +EXPORT_SYMBOL(qmi_rmnet_qos_exit); +#endif + +#ifdef CONFIG_QTI_QMI_POWER_COLLAPSE +static struct workqueue_struct *rmnet_ps_wq; +static struct rmnet_powersave_work *rmnet_work; +static LIST_HEAD(ps_list); + +struct rmnet_powersave_work { + struct delayed_work work; + void *port; + u64 old_rx_pkts; + u64 old_tx_pkts; +}; + +void qmi_rmnet_ps_on_notify(void *port) +{ + struct qmi_rmnet_ps_ind *tmp; + + list_for_each_entry(tmp, &ps_list, list) + tmp->ps_on_handler(port); +} +EXPORT_SYMBOL(qmi_rmnet_ps_on_notify); + +void qmi_rmnet_ps_off_notify(void *port) +{ + struct qmi_rmnet_ps_ind *tmp; + + list_for_each_entry(tmp, &ps_list, list) + tmp->ps_off_handler(port); +} +EXPORT_SYMBOL(qmi_rmnet_ps_off_notify); + +int qmi_rmnet_ps_ind_register(void *port, + struct qmi_rmnet_ps_ind *ps_ind) +{ + + if (!port || !ps_ind || !ps_ind->ps_on_handler || + !ps_ind->ps_off_handler) + return -EINVAL; + + list_add_rcu(&ps_ind->list, &ps_list); + + return 0; +} +EXPORT_SYMBOL(qmi_rmnet_ps_ind_register); + +int qmi_rmnet_ps_ind_deregister(void *port, + struct qmi_rmnet_ps_ind *ps_ind) +{ + struct qmi_rmnet_ps_ind *tmp; + + if (!port || !ps_ind) + return -EINVAL; + + list_for_each_entry(tmp, &ps_list, list) { + if (tmp == ps_ind) { + list_del_rcu(&ps_ind->list); + goto done; + } + } + +done: + return 0; +} +EXPORT_SYMBOL(qmi_rmnet_ps_ind_deregister); + +int qmi_rmnet_set_powersave_mode(void *port, uint8_t enable) +{ + int rc = -EINVAL; + struct qmi_info *qmi = (struct qmi_info *)rmnet_get_qmi_pt(port); + + if (!qmi || !qmi->wda_client) + return rc; + + rc = wda_set_powersave_mode(qmi->wda_client, enable); + if (rc < 0) { + pr_err("%s() failed set powersave mode[%u], err=%d\n", + __func__, enable, rc); + return rc; + } + + return 0; +} +EXPORT_SYMBOL(qmi_rmnet_set_powersave_mode); + +static void qmi_rmnet_work_restart(void *port) +{ + if (!rmnet_ps_wq || !rmnet_work) + return; + queue_delayed_work(rmnet_ps_wq, &rmnet_work->work, NO_DELAY); +} + +static void qmi_rmnet_check_stats(struct work_struct *work) +{ + struct rmnet_powersave_work *real_work; + struct qmi_info *qmi; + u64 rxd, txd; + u64 rx, tx; + bool dl_msg_active; + + real_work = container_of(to_delayed_work(work), + struct rmnet_powersave_work, work); + + if (unlikely(!real_work || !real_work->port)) + return; + + qmi = (struct qmi_info *)rmnet_get_qmi_pt(real_work->port); + if (unlikely(!qmi)) + return; + + if (qmi->ps_enabled) { + + /* Ready to accept grant */ + qmi->ps_ignore_grant = false; + + /* Register to get QMI DFC and DL marker */ + if (qmi_rmnet_set_powersave_mode(real_work->port, 0) < 0) { + /* If this failed need to retry quickly */ + queue_delayed_work(rmnet_ps_wq, + &real_work->work, HZ / 50); + return; + + } + qmi->ps_enabled = false; + + /* Do a query when coming out of powersave */ + qmi_rmnet_query_flows(qmi); + + if (rmnet_get_powersave_notif(real_work->port)) + qmi_rmnet_ps_off_notify(real_work->port); + + goto end; + } + + rmnet_get_packets(real_work->port, &rx, &tx); + rxd = rx - real_work->old_rx_pkts; + txd = tx - real_work->old_tx_pkts; + real_work->old_rx_pkts = rx; + real_work->old_tx_pkts = tx; + + dl_msg_active = qmi->dl_msg_active; + qmi->dl_msg_active = false; + + if (!rxd && !txd) { + /* If no DL msg received and there is a flow disabled, + * (likely in RLF), no need to enter powersave + */ + if (!dl_msg_active && + !rmnet_all_flows_enabled(real_work->port)) + goto end; + + /* Deregister to suppress QMI DFC and DL marker */ + if (qmi_rmnet_set_powersave_mode(real_work->port, 1) < 0) { + queue_delayed_work(rmnet_ps_wq, + &real_work->work, PS_INTERVAL); + return; + } + qmi->ps_enabled = true; + + /* Ignore grant after going into powersave */ + qmi->ps_ignore_grant = true; + + /* Clear the bit before enabling flow so pending packets + * can trigger the work again + */ + clear_bit(PS_WORK_ACTIVE_BIT, &qmi->ps_work_active); + rmnet_enable_all_flows(real_work->port); + + if (rmnet_get_powersave_notif(real_work->port)) + qmi_rmnet_ps_on_notify(real_work->port); + + return; + } +end: + queue_delayed_work(rmnet_ps_wq, &real_work->work, PS_INTERVAL); +} + +static void qmi_rmnet_work_set_active(void *port, int status) +{ + struct qmi_info *qmi; + + qmi = (struct qmi_info *)rmnet_get_qmi_pt(port); + if (unlikely(!qmi)) + return; + + if (status) + set_bit(PS_WORK_ACTIVE_BIT, &qmi->ps_work_active); + else + clear_bit(PS_WORK_ACTIVE_BIT, &qmi->ps_work_active); +} + +void qmi_rmnet_work_init(void *port) +{ + if (rmnet_ps_wq) + return; + + rmnet_ps_wq = alloc_workqueue("rmnet_powersave_work", + WQ_MEM_RECLAIM | WQ_CPU_INTENSIVE, 1); + + rmnet_work = kmalloc(sizeof(*rmnet_work), GFP_ATOMIC); + if (!rmnet_work) { + destroy_workqueue(rmnet_ps_wq); + rmnet_ps_wq = NULL; + return; + } + INIT_DEFERRABLE_WORK(&rmnet_work->work, qmi_rmnet_check_stats); + rmnet_work->port = port; + rmnet_get_packets(rmnet_work->port, &rmnet_work->old_rx_pkts, + &rmnet_work->old_tx_pkts); + + qmi_rmnet_work_set_active(rmnet_work->port, 1); + queue_delayed_work(rmnet_ps_wq, &rmnet_work->work, PS_INTERVAL); +} +EXPORT_SYMBOL(qmi_rmnet_work_init); + +void qmi_rmnet_work_maybe_restart(void *port) +{ + struct qmi_info *qmi; + + qmi = (struct qmi_info *)rmnet_get_qmi_pt(port); + if (unlikely(!qmi)) + return; + + if (!test_and_set_bit(PS_WORK_ACTIVE_BIT, &qmi->ps_work_active)) + qmi_rmnet_work_restart(port); +} +EXPORT_SYMBOL(qmi_rmnet_work_maybe_restart); + +void qmi_rmnet_work_exit(void *port) +{ + if (!rmnet_ps_wq || !rmnet_work) + return; + cancel_delayed_work_sync(&rmnet_work->work); + destroy_workqueue(rmnet_ps_wq); + qmi_rmnet_work_set_active(port, 0); + rmnet_ps_wq = NULL; + kfree(rmnet_work); + rmnet_work = NULL; +} +EXPORT_SYMBOL(qmi_rmnet_work_exit); + +void qmi_rmnet_set_dl_msg_active(void *port) +{ + struct qmi_info *qmi; + + qmi = (struct qmi_info *)rmnet_get_qmi_pt(port); + if (unlikely(!qmi)) + return; + + qmi->dl_msg_active = true; +} +EXPORT_SYMBOL(qmi_rmnet_set_dl_msg_active); + +void qmi_rmnet_flush_ps_wq(void) +{ + if (rmnet_ps_wq) + flush_workqueue(rmnet_ps_wq); +} + +bool qmi_rmnet_ignore_grant(void *port) +{ + struct qmi_info *qmi; + + qmi = (struct qmi_info *)rmnet_get_qmi_pt(port); + if (unlikely(!qmi)) + return false; + + return qmi->ps_ignore_grant; +} +EXPORT_SYMBOL(qmi_rmnet_ignore_grant); + +#endif +MODULE_LICENSE("GPL v2"); diff --git a/core/qmi_rmnet.h b/core/qmi_rmnet.h new file mode 100644 index 000000000000..6fb284e2b937 --- /dev/null +++ b/core/qmi_rmnet.h @@ -0,0 +1,146 @@ +/* + * Copyright (c) 2018-2019, The Linux Foundation. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 and + * only version 2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ + +#ifndef _QMI_RMNET_H +#define _QMI_RMNET_H + +#include +#include +#define CONFIG_QTI_QMI_RMNET 1 +#define CONFIG_QTI_QMI_DFC 1 +#define CONFIG_QTI_QMI_POWER_COLLAPSE 1 + +struct qmi_rmnet_ps_ind { + void (*ps_on_handler)(void *); + void (*ps_off_handler)(void *); + struct list_head list; +}; + + +#ifdef CONFIG_QTI_QMI_RMNET +void qmi_rmnet_qmi_exit(void *qmi_pt, void *port); +void qmi_rmnet_change_link(struct net_device *dev, void *port, void *tcm_pt); +void qmi_rmnet_enable_all_flows(struct net_device *dev); +bool qmi_rmnet_all_flows_enabled(struct net_device *dev); +#else +static inline void qmi_rmnet_qmi_exit(void *qmi_pt, void *port) +{ +} + +static inline void +qmi_rmnet_change_link(struct net_device *dev, void *port, void *tcm_pt) +{ +} + +static inline void +qmi_rmnet_enable_all_flows(struct net_device *dev) +{ +} + +static inline bool +qmi_rmnet_all_flows_enabled(struct net_device *dev) +{ + return true; +} +#endif + +#ifdef CONFIG_QTI_QMI_DFC +void *qmi_rmnet_qos_init(struct net_device *real_dev, u8 mux_id); +void qmi_rmnet_qos_exit(struct net_device *dev, void *qos); +void qmi_rmnet_burst_fc_check(struct net_device *dev, + int ip_type, u32 mark, unsigned int len); +int qmi_rmnet_get_queue(struct net_device *dev, struct sk_buff *skb); +#else +static inline void * +qmi_rmnet_qos_init(struct net_device *real_dev, u8 mux_id) +{ + return NULL; +} + +static inline void qmi_rmnet_qos_exit(struct net_device *dev, void *qos) +{ +} + +static inline void +qmi_rmnet_burst_fc_check(struct net_device *dev, + int ip_type, u32 mark, unsigned int len) +{ +} + +static inline int qmi_rmnet_get_queue(struct net_device *dev, + struct sk_buff *skb) +{ + return 0; +} +#endif + +#ifdef CONFIG_QTI_QMI_POWER_COLLAPSE +int qmi_rmnet_set_powersave_mode(void *port, uint8_t enable); +void qmi_rmnet_work_init(void *port); +void qmi_rmnet_work_exit(void *port); +void qmi_rmnet_work_maybe_restart(void *port); +void qmi_rmnet_set_dl_msg_active(void *port); +bool qmi_rmnet_ignore_grant(void *port); + +int qmi_rmnet_ps_ind_register(void *port, + struct qmi_rmnet_ps_ind *ps_ind); +int qmi_rmnet_ps_ind_deregister(void *port, + struct qmi_rmnet_ps_ind *ps_ind); +void qmi_rmnet_ps_off_notify(void *port); +void qmi_rmnet_ps_on_notify(void *port); + +#else +static inline int qmi_rmnet_set_powersave_mode(void *port, uint8_t enable) +{ + return 0; +} +static inline void qmi_rmnet_work_init(void *port) +{ +} +static inline void qmi_rmnet_work_exit(void *port) +{ +} +static inline void qmi_rmnet_work_maybe_restart(void *port) +{ + +} +static inline void qmi_rmnet_set_dl_msg_active(void *port) +{ +} +static inline bool qmi_rmnet_ignore_grant(void *port) +{ + return false; +} + +static inline int qmi_rmnet_ps_ind_register(void *port, + struct qmi_rmnet_ps_ind *ps_ind) +{ + return 0; +} +static inline int qmi_rmnet_ps_ind_deregister(void *port, + struct qmi_rmnet_ps_ind *ps_ind) +{ + return 0; +} + +static inline void qmi_rmnet_ps_off_notify(void *port) +{ + +} + +static inline void qmi_rmnet_ps_on_notify(void *port) +{ + +} +#endif +#endif /*_QMI_RMNET_H*/ diff --git a/core/qmi_rmnet_i.h b/core/qmi_rmnet_i.h new file mode 100644 index 000000000000..27a34df98a36 --- /dev/null +++ b/core/qmi_rmnet_i.h @@ -0,0 +1,203 @@ +/* + * Copyright (c) 2018-2019, The Linux Foundation. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 and + * only version 2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ + +#ifndef _RMNET_QMI_I_H +#define _RMNET_QMI_I_H + +#include +#include + +#define IP_VER_4 4 +#define IP_VER_6 6 + +#define MAX_CLIENT_NUM 2 +#define MAX_FLOW_NUM 32 +#define DEFAULT_GRANT 1 +#define DFC_MAX_BEARERS_V01 16 + +#define CONFIG_QTI_QMI_RMNET 1 +#define CONFIG_QTI_QMI_DFC 1 +#define CONFIG_QTI_QMI_POWER_COLLAPSE 1 + +struct rmnet_flow_map { + struct list_head list; + u8 bearer_id; + u32 flow_id; + int ip_type; + u32 tcm_handle; +}; + +struct rmnet_bearer_map { + struct list_head list; + u8 bearer_id; + int flow_ref; + u32 grant_size; + u32 grant_thresh; + u16 seq; + u8 ack_req; + u32 last_grant; + u16 last_seq; + bool tcp_bidir; + bool rat_switch; + bool tx_off; +}; + +struct svc_info { + u32 instance; + u32 ep_type; + u32 iface_id; +}; + +struct qos_info { + u8 mux_id; + struct net_device *real_dev; + struct list_head flow_head; + struct list_head bearer_head; + u32 default_grant; + u32 tran_num; + spinlock_t qos_lock; +}; + +struct flow_info { + struct net_device *dev; + struct rmnet_flow_map *itm; +}; + +struct qmi_info { + int flag; + void *wda_client; + void *wda_pending; + void *dfc_clients[MAX_CLIENT_NUM]; + void *dfc_pending[MAX_CLIENT_NUM]; + unsigned long ps_work_active; + bool ps_enabled; + bool dl_msg_active; + bool ps_ignore_grant; +}; + +enum data_ep_type_enum_v01 { + DATA_EP_TYPE_ENUM_MIN_ENUM_VAL_V01 = INT_MIN, + DATA_EP_TYPE_RESERVED_V01 = 0x00, + DATA_EP_TYPE_HSIC_V01 = 0x01, + DATA_EP_TYPE_HSUSB_V01 = 0x02, + DATA_EP_TYPE_PCIE_V01 = 0x03, + DATA_EP_TYPE_EMBEDDED_V01 = 0x04, + DATA_EP_TYPE_ENUM_MAX_ENUM_VAL_V01 = INT_MAX +}; + +struct data_ep_id_type_v01 { + + enum data_ep_type_enum_v01 ep_type; + u32 iface_id; +}; + +extern struct qmi_elem_info data_ep_id_type_v01_ei[]; + +void *qmi_rmnet_has_dfc_client(struct qmi_info *qmi); + +#ifdef CONFIG_QTI_QMI_DFC +struct rmnet_flow_map * +qmi_rmnet_get_flow_map(struct qos_info *qos_info, + u32 flow_id, int ip_type); + +struct rmnet_bearer_map * +qmi_rmnet_get_bearer_map(struct qos_info *qos_info, u8 bearer_id); + +unsigned int qmi_rmnet_grant_per(unsigned int grant); + +int dfc_qmi_client_init(void *port, int index, struct svc_info *psvc, + struct qmi_info *qmi); + +void dfc_qmi_client_exit(void *dfc_data); + +void dfc_qmi_burst_check(struct net_device *dev, struct qos_info *qos, + int ip_type, u32 mark, unsigned int len); + +int qmi_rmnet_flow_control(struct net_device *dev, u32 tcm_handle, int enable); + +void dfc_qmi_query_flow(void *dfc_data); + +int dfc_bearer_flow_ctl(struct net_device *dev, + struct rmnet_bearer_map *bearer, + struct qos_info *qos); +#else +static inline struct rmnet_flow_map * +qmi_rmnet_get_flow_map(struct qos_info *qos_info, + uint32_t flow_id, int ip_type) +{ + return NULL; +} + +static inline struct rmnet_bearer_map * +qmi_rmnet_get_bearer_map(struct qos_info *qos_info, u8 bearer_id) +{ + return NULL; +} + +static inline int +dfc_qmi_client_init(void *port, int index, struct svc_info *psvc, + struct qmi_info *qmi) +{ + return -EINVAL; +} + +static inline void dfc_qmi_client_exit(void *dfc_data) +{ +} + +static inline void +dfc_qmi_burst_check(struct net_device *dev, struct qos_info *qos, + int ip_type, u32 mark, unsigned int len) +{ +} + +static inline void +dfc_qmi_query_flow(void *dfc_data) +{ +} + +static inline int +dfc_bearer_flow_ctl(struct net_device *dev, + struct rmnet_bearer_map *bearer, + struct qos_info *qos) +{ + return 0; +} +#endif + +#ifdef CONFIG_QTI_QMI_POWER_COLLAPSE +int +wda_qmi_client_init(void *port, struct svc_info *psvc, struct qmi_info *qmi); +void wda_qmi_client_exit(void *wda_data); +int wda_set_powersave_mode(void *wda_data, u8 enable); +void qmi_rmnet_flush_ps_wq(void); +#else +static inline int +wda_qmi_client_init(void *port, struct svc_info *psvc, struct qmi_info *qmi) +{ + return -EINVAL; +} + +static inline void wda_qmi_client_exit(void *wda_data) +{ +} + +static inline int wda_set_powersave_mode(void *wda_data, u8 enable) +{ + return -EINVAL; +} +static inline void qmi_rmnet_flush_ps_wq(void) +{ +} +#endif +#endif /*_RMNET_QMI_I_H*/ diff --git a/core/rmnet_config.c b/core/rmnet_config.c new file mode 100644 index 000000000000..907acd305cd2 --- /dev/null +++ b/core/rmnet_config.c @@ -0,0 +1,738 @@ +/* Copyright (c) 2013-2019, The Linux Foundation. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 and + * only version 2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * RMNET configuration engine + * + */ + +#include +#include +#include +#include +#include "rmnet_config.h" +#include "rmnet_handlers.h" +#include "rmnet_vnd.h" +#include "rmnet_private.h" +#include "rmnet_map.h" +#include "rmnet_descriptor.h" +#include "rmnet_qmi.h" +#include "qmi_rmnet.h" +#define CONFIG_QTI_QMI_RMNET 1 +#define CONFIG_QTI_QMI_DFC 1 +#define CONFIG_QTI_QMI_POWER_COLLAPSE 1 + +/* Locking scheme - + * The shared resource which needs to be protected is realdev->rx_handler_data. + * For the writer path, this is using rtnl_lock(). The writer paths are + * rmnet_newlink(), rmnet_dellink() and rmnet_force_unassociate_device(). These + * paths are already called with rtnl_lock() acquired in. There is also an + * ASSERT_RTNL() to ensure that we are calling with rtnl acquired. For + * dereference here, we will need to use rtnl_dereference(). Dev list writing + * needs to happen with rtnl_lock() acquired for netdev_master_upper_dev_link(). + * For the reader path, the real_dev->rx_handler_data is called in the TX / RX + * path. We only need rcu_read_lock() for these scenarios. In these cases, + * the rcu_read_lock() is held in __dev_queue_xmit() and + * netif_receive_skb_internal(), so readers need to use rcu_dereference_rtnl() + * to get the relevant information. For dev list reading, we again acquire + * rcu_read_lock() in rmnet_dellink() for netdev_master_upper_dev_get_rcu(). + * We also use unregister_netdevice_many() to free all rmnet devices in + * rmnet_force_unassociate_device() so we dont lose the rtnl_lock() and free in + * same context. + */ + +/* Local Definitions and Declarations */ + +enum { + IFLA_RMNET_DFC_QOS = __IFLA_RMNET_MAX, + IFLA_RMNET_UL_AGG_PARAMS, + __IFLA_RMNET_EXT_MAX, +}; + +static const struct nla_policy rmnet_policy[__IFLA_RMNET_EXT_MAX] = { + [IFLA_RMNET_MUX_ID] = { + .type = NLA_U16 + }, + [IFLA_RMNET_FLAGS] = { + .len = sizeof(struct ifla_rmnet_flags) + }, + [IFLA_RMNET_DFC_QOS] = { + .len = sizeof(struct tcmsg) + }, + [IFLA_RMNET_UL_AGG_PARAMS] = { + .len = sizeof(struct rmnet_egress_agg_params) + }, +}; + +int rmnet_is_real_dev_registered(const struct net_device *real_dev) +{ + return rcu_access_pointer(real_dev->rx_handler) == rmnet_rx_handler; +} +EXPORT_SYMBOL(rmnet_is_real_dev_registered); + +/* Needs rtnl lock */ +static struct rmnet_port* +rmnet_get_port_rtnl(const struct net_device *real_dev) +{ + return rtnl_dereference(real_dev->rx_handler_data); +} + +static int rmnet_unregister_real_device(struct net_device *real_dev, + struct rmnet_port *port) +{ + if (port->nr_rmnet_devs) + return -EINVAL; + + rmnet_map_cmd_exit(port); + rmnet_map_tx_aggregate_exit(port); + + rmnet_descriptor_deinit(port); + + kfree(port); + + netdev_rx_handler_unregister(real_dev); + + /* release reference on real_dev */ + dev_put(real_dev); + + netdev_dbg(real_dev, "Removed from rmnet\n"); + return 0; +} + +static int rmnet_register_real_device(struct net_device *real_dev) +{ + struct rmnet_port *port; + int rc, entry; + + ASSERT_RTNL(); + + if (rmnet_is_real_dev_registered(real_dev)) + return 0; + + port = kzalloc(sizeof(*port), GFP_ATOMIC); + if (!port) + return -ENOMEM; + + port->dev = real_dev; + rc = netdev_rx_handler_register(real_dev, rmnet_rx_handler, port); + if (rc) { + kfree(port); + return -EBUSY; + } + /* hold on to real dev for MAP data */ + dev_hold(real_dev); + + for (entry = 0; entry < RMNET_MAX_LOGICAL_EP; entry++) + INIT_HLIST_HEAD(&port->muxed_ep[entry]); + + rc = rmnet_descriptor_init(port); + if (rc) { + rmnet_descriptor_deinit(port); + return rc; + } + + rmnet_map_tx_aggregate_init(port); + rmnet_map_cmd_init(port); + + netdev_dbg(real_dev, "registered with rmnet\n"); + return 0; +} + +static void rmnet_unregister_bridge(struct net_device *dev, + struct rmnet_port *port) +{ + struct rmnet_port *bridge_port; + struct net_device *bridge_dev; + + if (port->rmnet_mode != RMNET_EPMODE_BRIDGE) + return; + + /* bridge slave handling */ + if (!port->nr_rmnet_devs) { + bridge_dev = port->bridge_ep; + + bridge_port = rmnet_get_port_rtnl(bridge_dev); + bridge_port->bridge_ep = NULL; + bridge_port->rmnet_mode = RMNET_EPMODE_VND; + } else { + bridge_dev = port->bridge_ep; + + bridge_port = rmnet_get_port_rtnl(bridge_dev); + rmnet_unregister_real_device(bridge_dev, bridge_port); + } +} + +static int rmnet_newlink(struct net *src_net, struct net_device *dev, + struct nlattr *tb[], struct nlattr *data[], + struct netlink_ext_ack *extack) +{ + u32 data_format = RMNET_FLAGS_INGRESS_DEAGGREGATION; + struct net_device *real_dev; + int mode = RMNET_EPMODE_VND; + struct rmnet_endpoint *ep; + struct rmnet_port *port; + int err = 0; + u16 mux_id; + + real_dev = __dev_get_by_index(src_net, nla_get_u32(tb[IFLA_LINK])); + if (!real_dev || !dev) + return -ENODEV; + + if (!data[IFLA_RMNET_MUX_ID]) + return -EINVAL; + + ep = kzalloc(sizeof(*ep), GFP_ATOMIC); + if (!ep) + return -ENOMEM; + + mux_id = nla_get_u16(data[IFLA_RMNET_MUX_ID]); + + err = rmnet_register_real_device(real_dev); + if (err) + goto err0; + + port = rmnet_get_port_rtnl(real_dev); + err = rmnet_vnd_newlink(mux_id, dev, port, real_dev, ep); + if (err) + goto err1; + + port->rmnet_mode = mode; + + hlist_add_head_rcu(&ep->hlnode, &port->muxed_ep[mux_id]); + + if (data[IFLA_RMNET_FLAGS]) { + struct ifla_rmnet_flags *flags; + + flags = nla_data(data[IFLA_RMNET_FLAGS]); + data_format = flags->flags & flags->mask; + } + + netdev_dbg(dev, "data format [0x%08X]\n", data_format); + port->data_format = data_format; + + if (data[IFLA_RMNET_UL_AGG_PARAMS]) { + void *agg_params; + unsigned long irq_flags; + + agg_params = nla_data(data[IFLA_RMNET_UL_AGG_PARAMS]); + spin_lock_irqsave(&port->agg_lock, irq_flags); + memcpy(&port->egress_agg_params, agg_params, + sizeof(port->egress_agg_params)); + spin_unlock_irqrestore(&port->agg_lock, irq_flags); + } + + return 0; + +err1: + rmnet_unregister_real_device(real_dev, port); +err0: + kfree(ep); + return err; +} + +static void rmnet_dellink(struct net_device *dev, struct list_head *head) +{ + struct rmnet_priv *priv = netdev_priv(dev); + struct net_device *real_dev; + struct rmnet_endpoint *ep; + struct rmnet_port *port; + u8 mux_id; + + real_dev = priv->real_dev; + + if (!real_dev || !rmnet_is_real_dev_registered(real_dev)) + return; + + port = rmnet_get_port_rtnl(real_dev); + + mux_id = rmnet_vnd_get_mux(dev); + + ep = rmnet_get_endpoint(port, mux_id); + if (ep) { + hlist_del_init_rcu(&ep->hlnode); + rmnet_unregister_bridge(dev, port); + rmnet_vnd_dellink(mux_id, port, ep); + synchronize_rcu(); + kfree(ep); + } + + if (!port->nr_rmnet_devs) + qmi_rmnet_qmi_exit(port->qmi_info, port); + + rmnet_unregister_real_device(real_dev, port); + + unregister_netdevice_queue(dev, head); +} + +static void rmnet_force_unassociate_device(struct net_device *dev) +{ + struct net_device *real_dev = dev; + struct hlist_node *tmp_ep; + struct rmnet_endpoint *ep; + struct rmnet_port *port; + unsigned long bkt_ep; + LIST_HEAD(list); + + if (!rmnet_is_real_dev_registered(real_dev)) + return; + + ASSERT_RTNL(); + + port = rmnet_get_port_rtnl(dev); + qmi_rmnet_qmi_exit(port->qmi_info, port); + + rmnet_unregister_bridge(dev, port); + + hash_for_each_safe(port->muxed_ep, bkt_ep, tmp_ep, ep, hlnode) { + unregister_netdevice_queue(ep->egress_dev, &list); + rmnet_vnd_dellink(ep->mux_id, port, ep); + + hlist_del_init_rcu(&ep->hlnode); + synchronize_rcu(); + kfree(ep); + } + + unregister_netdevice_many(&list); + + rmnet_unregister_real_device(real_dev, port); +} + +static int rmnet_config_notify_cb(struct notifier_block *nb, + unsigned long event, void *data) +{ + struct net_device *dev = netdev_notifier_info_to_dev(data); + + if (!dev) + return NOTIFY_DONE; + + switch (event) { + case NETDEV_UNREGISTER: + netdev_dbg(dev, "Kernel unregister\n"); + rmnet_force_unassociate_device(dev); + break; + + default: + break; + } + + return NOTIFY_DONE; +} + +static struct notifier_block rmnet_dev_notifier __read_mostly = { + .notifier_call = rmnet_config_notify_cb, +}; + +static int rmnet_rtnl_validate(struct nlattr *tb[], struct nlattr *data[], + struct netlink_ext_ack *extack) +{ + struct rmnet_egress_agg_params *agg_params; + u16 mux_id; + + if (!data) { + return -EINVAL; + } else { + if (data[IFLA_RMNET_MUX_ID]) { + mux_id = nla_get_u16(data[IFLA_RMNET_MUX_ID]); + if (mux_id > (RMNET_MAX_LOGICAL_EP - 1)) + return -ERANGE; + } + + if (data[IFLA_RMNET_UL_AGG_PARAMS]) { + agg_params = nla_data(data[IFLA_RMNET_UL_AGG_PARAMS]); + if (agg_params->agg_time < 3000000) + return -EINVAL; + } + } + + return 0; +} + +static int rmnet_changelink(struct net_device *dev, struct nlattr *tb[], + struct nlattr *data[], + struct netlink_ext_ack *extack) +{ + struct rmnet_priv *priv = netdev_priv(dev); + struct net_device *real_dev; + struct rmnet_endpoint *ep; + struct rmnet_port *port; + u16 mux_id; + + real_dev = __dev_get_by_index(dev_net(dev), + nla_get_u32(tb[IFLA_LINK])); + + if (!real_dev || !dev || !rmnet_is_real_dev_registered(real_dev)) + return -ENODEV; + + port = rmnet_get_port_rtnl(real_dev); + + if (data[IFLA_RMNET_MUX_ID]) { + mux_id = nla_get_u16(data[IFLA_RMNET_MUX_ID]); + ep = rmnet_get_endpoint(port, priv->mux_id); + if (!ep) + return -ENODEV; + + hlist_del_init_rcu(&ep->hlnode); + hlist_add_head_rcu(&ep->hlnode, &port->muxed_ep[mux_id]); + + ep->mux_id = mux_id; + priv->mux_id = mux_id; + } + + if (data[IFLA_RMNET_FLAGS]) { + struct ifla_rmnet_flags *flags; + + flags = nla_data(data[IFLA_RMNET_FLAGS]); + port->data_format = flags->flags & flags->mask; + } + + if (data[IFLA_RMNET_DFC_QOS]) { + struct tcmsg *tcm; + + tcm = nla_data(data[IFLA_RMNET_DFC_QOS]); + qmi_rmnet_change_link(dev, port, tcm); + } + + if (data[IFLA_RMNET_UL_AGG_PARAMS]) { + void *agg_params; + unsigned long irq_flags; + + agg_params = nla_data(data[IFLA_RMNET_UL_AGG_PARAMS]); + spin_lock_irqsave(&port->agg_lock, irq_flags); + memcpy(&port->egress_agg_params, agg_params, + sizeof(port->egress_agg_params)); + spin_unlock_irqrestore(&port->agg_lock, irq_flags); + } + + return 0; +} + +static size_t rmnet_get_size(const struct net_device *dev) +{ + return + /* IFLA_RMNET_MUX_ID */ + nla_total_size(2) + + /* IFLA_RMNET_FLAGS */ + nla_total_size(sizeof(struct ifla_rmnet_flags)) + + /* IFLA_RMNET_DFC_QOS */ + nla_total_size(sizeof(struct tcmsg)) + + /* IFLA_RMNET_UL_AGG_PARAMS */ + nla_total_size(sizeof(struct rmnet_egress_agg_params)); +} + +static int rmnet_fill_info(struct sk_buff *skb, const struct net_device *dev) +{ + struct rmnet_priv *priv = netdev_priv(dev); + struct net_device *real_dev; + struct ifla_rmnet_flags f; + struct rmnet_port *port = NULL; + + real_dev = priv->real_dev; + + if (nla_put_u16(skb, IFLA_RMNET_MUX_ID, priv->mux_id)) + goto nla_put_failure; + + if (rmnet_is_real_dev_registered(real_dev)) { + port = rmnet_get_port_rtnl(real_dev); + f.flags = port->data_format; + } else { + f.flags = 0; + } + + f.mask = ~0; + + if (nla_put(skb, IFLA_RMNET_FLAGS, sizeof(f), &f)) + goto nla_put_failure; + + if (port) { + if (nla_put(skb, IFLA_RMNET_UL_AGG_PARAMS, + sizeof(port->egress_agg_params), + &port->egress_agg_params)) + goto nla_put_failure; + } + + return 0; + +nla_put_failure: + return -EMSGSIZE; +} + +struct rtnl_link_ops rmnet_link_ops __read_mostly = { + .kind = "rmnet", + .maxtype = __IFLA_RMNET_EXT_MAX, + .priv_size = sizeof(struct rmnet_priv), + .setup = rmnet_vnd_setup, + .validate = rmnet_rtnl_validate, + .newlink = rmnet_newlink, + .dellink = rmnet_dellink, + .get_size = rmnet_get_size, + .changelink = rmnet_changelink, + .policy = rmnet_policy, + .fill_info = rmnet_fill_info, +}; + +/* Needs either rcu_read_lock() or rtnl lock */ +struct rmnet_port *rmnet_get_port(struct net_device *real_dev) +{ + if (rmnet_is_real_dev_registered(real_dev)) + return rcu_dereference_rtnl(real_dev->rx_handler_data); + else + return NULL; +} +EXPORT_SYMBOL(rmnet_get_port); + +struct rmnet_endpoint *rmnet_get_endpoint(struct rmnet_port *port, u8 mux_id) +{ + struct rmnet_endpoint *ep; + + hlist_for_each_entry_rcu(ep, &port->muxed_ep[mux_id], hlnode) { + if (ep->mux_id == mux_id) + return ep; + } + + return NULL; +} +EXPORT_SYMBOL(rmnet_get_endpoint); + +int rmnet_add_bridge(struct net_device *rmnet_dev, + struct net_device *slave_dev, + struct netlink_ext_ack *extack) +{ + struct rmnet_priv *priv = netdev_priv(rmnet_dev); + struct net_device *real_dev = priv->real_dev; + struct rmnet_port *port, *slave_port; + int err; + + port = rmnet_get_port(real_dev); + + /* If there is more than one rmnet dev attached, its probably being + * used for muxing. Skip the briding in that case + */ + if (port->nr_rmnet_devs > 1) + return -EINVAL; + + if (rmnet_is_real_dev_registered(slave_dev)) + return -EBUSY; + + err = rmnet_register_real_device(slave_dev); + if (err) + return -EBUSY; + + slave_port = rmnet_get_port(slave_dev); + slave_port->rmnet_mode = RMNET_EPMODE_BRIDGE; + slave_port->bridge_ep = real_dev; + + port->rmnet_mode = RMNET_EPMODE_BRIDGE; + port->bridge_ep = slave_dev; + + netdev_dbg(slave_dev, "registered with rmnet as slave\n"); + return 0; +} + +int rmnet_del_bridge(struct net_device *rmnet_dev, + struct net_device *slave_dev) +{ + struct rmnet_priv *priv = netdev_priv(rmnet_dev); + struct net_device *real_dev = priv->real_dev; + struct rmnet_port *port, *slave_port; + + port = rmnet_get_port(real_dev); + port->rmnet_mode = RMNET_EPMODE_VND; + port->bridge_ep = NULL; + + slave_port = rmnet_get_port(slave_dev); + rmnet_unregister_real_device(slave_dev, slave_port); + + netdev_dbg(slave_dev, "removed from rmnet as slave\n"); + return 0; +} + +void *rmnet_get_qmi_pt(void *port) +{ + if (port) + return ((struct rmnet_port *)port)->qmi_info; + + return NULL; +} +EXPORT_SYMBOL(rmnet_get_qmi_pt); + +void *rmnet_get_qos_pt(struct net_device *dev) +{ + if (dev) + return rcu_dereference( + ((struct rmnet_priv *)netdev_priv(dev))->qos_info); + + return NULL; +} +EXPORT_SYMBOL(rmnet_get_qos_pt); + +void *rmnet_get_rmnet_port(struct net_device *dev) +{ + struct rmnet_priv *priv; + + if (dev) { + priv = netdev_priv(dev); + return (void *)rmnet_get_port(priv->real_dev); + } + + return NULL; +} +EXPORT_SYMBOL(rmnet_get_rmnet_port); + +struct net_device *rmnet_get_rmnet_dev(void *port, u8 mux_id) +{ + struct rmnet_endpoint *ep; + + if (port) { + ep = rmnet_get_endpoint((struct rmnet_port *)port, mux_id); + if (ep) + return ep->egress_dev; + } + + return NULL; +} +EXPORT_SYMBOL(rmnet_get_rmnet_dev); + +void rmnet_reset_qmi_pt(void *port) +{ + if (port) + ((struct rmnet_port *)port)->qmi_info = NULL; +} +EXPORT_SYMBOL(rmnet_reset_qmi_pt); + +void rmnet_init_qmi_pt(void *port, void *qmi) +{ + if (port) + ((struct rmnet_port *)port)->qmi_info = qmi; +} +EXPORT_SYMBOL(rmnet_init_qmi_pt); + +void rmnet_get_packets(void *port, u64 *rx, u64 *tx) +{ + struct rmnet_priv *priv; + struct rmnet_pcpu_stats *ps; + unsigned int cpu, start; + + struct rmnet_endpoint *ep; + unsigned long bkt; + + if (!port || !tx || !rx) + return; + + *tx = 0; + *rx = 0; + rcu_read_lock(); + hash_for_each(((struct rmnet_port *)port)->muxed_ep, bkt, ep, hlnode) { + priv = netdev_priv(ep->egress_dev); + for_each_possible_cpu(cpu) { + ps = per_cpu_ptr(priv->pcpu_stats, cpu); + do { + start = u64_stats_fetch_begin_irq(&ps->syncp); + *tx += ps->stats.tx_pkts; + *rx += ps->stats.rx_pkts; + } while (u64_stats_fetch_retry_irq(&ps->syncp, start)); + } + } + rcu_read_unlock(); +} +EXPORT_SYMBOL(rmnet_get_packets); + +void rmnet_set_powersave_format(void *port) +{ + if (!port) + return; + ((struct rmnet_port *)port)->data_format |= RMNET_INGRESS_FORMAT_PS; +} +EXPORT_SYMBOL(rmnet_set_powersave_format); + +void rmnet_clear_powersave_format(void *port) +{ + if (!port) + return; + ((struct rmnet_port *)port)->data_format &= ~RMNET_INGRESS_FORMAT_PS; +} +EXPORT_SYMBOL(rmnet_clear_powersave_format); + +void rmnet_enable_all_flows(void *port) +{ + struct rmnet_endpoint *ep; + unsigned long bkt; + + if (unlikely(!port)) + return; + + rcu_read_lock(); + hash_for_each_rcu(((struct rmnet_port *)port)->muxed_ep, + bkt, ep, hlnode) { + qmi_rmnet_enable_all_flows(ep->egress_dev); + } + rcu_read_unlock(); +} +EXPORT_SYMBOL(rmnet_enable_all_flows); + +bool rmnet_all_flows_enabled(void *port) +{ + struct rmnet_endpoint *ep; + unsigned long bkt; + bool ret = true; + + if (unlikely(!port)) + return true; + + rcu_read_lock(); + hash_for_each_rcu(((struct rmnet_port *)port)->muxed_ep, + bkt, ep, hlnode) { + if (!qmi_rmnet_all_flows_enabled(ep->egress_dev)) { + ret = false; + goto out; + } + } +out: + rcu_read_unlock(); + + return ret; +} +EXPORT_SYMBOL(rmnet_all_flows_enabled); + +int rmnet_get_powersave_notif(void *port) +{ + if (!port) + return 0; + return ((struct rmnet_port *)port)->data_format & RMNET_FORMAT_PS_NOTIF; +} +EXPORT_SYMBOL(rmnet_get_powersave_notif); + +/* Startup/Shutdown */ + +static int __init rmnet_init(void) +{ + int rc; + + WARN_ON(1); + rc = register_netdevice_notifier(&rmnet_dev_notifier); + if (rc != 0) + return rc; + + rc = rtnl_link_register(&rmnet_link_ops); + if (rc != 0) { + unregister_netdevice_notifier(&rmnet_dev_notifier); + return rc; + } + return rc; +} + +static void __exit rmnet_exit(void) +{ + unregister_netdevice_notifier(&rmnet_dev_notifier); + rtnl_link_unregister(&rmnet_link_ops); +} + +module_init(rmnet_init) +module_exit(rmnet_exit) +MODULE_LICENSE("GPL v2"); diff --git a/core/rmnet_config.h b/core/rmnet_config.h new file mode 100644 index 000000000000..f2f20a913468 --- /dev/null +++ b/core/rmnet_config.h @@ -0,0 +1,179 @@ +/* Copyright (c) 2013-2014, 2016-2019 The Linux Foundation. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 and + * only version 2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * RMNET Data configuration engine + * + */ + +#include +#include + +#ifndef _RMNET_CONFIG_H_ +#define _RMNET_CONFIG_H_ + +#define RMNET_MAX_LOGICAL_EP 255 +#define RMNET_MAX_VEID 4 + +struct rmnet_endpoint { + u8 mux_id; + struct net_device *egress_dev; + struct hlist_node hlnode; +}; + +struct rmnet_port_priv_stats { + u64 dl_hdr_last_qmap_vers; + u64 dl_hdr_last_ep_id; + u64 dl_hdr_last_trans_id; + u64 dl_hdr_last_seq; + u64 dl_hdr_last_bytes; + u64 dl_hdr_last_pkts; + u64 dl_hdr_last_flows; + u64 dl_hdr_count; + u64 dl_hdr_total_bytes; + u64 dl_hdr_total_pkts; + u64 dl_trl_last_seq; + u64 dl_trl_count; +}; + +struct rmnet_egress_agg_params { + u16 agg_size; + u16 agg_count; + u32 agg_time; +}; + +/* One instance of this structure is instantiated for each real_dev associated + * with rmnet. + */ +struct rmnet_port { + struct net_device *dev; + u32 data_format; + u8 nr_rmnet_devs; + u8 rmnet_mode; + struct hlist_head muxed_ep[RMNET_MAX_LOGICAL_EP]; + struct net_device *bridge_ep; + void *rmnet_perf; + + struct rmnet_egress_agg_params egress_agg_params; + + /* Protect aggregation related elements */ + spinlock_t agg_lock; + + struct sk_buff *agg_skb; + int agg_state; + u8 agg_count; + struct timespec64 agg_time; + struct timespec64 agg_last; + struct hrtimer hrtimer; + struct work_struct agg_wq; + + void *qmi_info; + + /* dl marker elements */ + struct list_head dl_list; + struct rmnet_port_priv_stats stats; + + /* Descriptor pool */ + spinlock_t desc_pool_lock; + struct rmnet_frag_descriptor_pool *frag_desc_pool; +}; + +extern struct rtnl_link_ops rmnet_link_ops; + +struct rmnet_vnd_stats { + u64 rx_pkts; + u64 rx_bytes; + u64 tx_pkts; + u64 tx_bytes; + u32 tx_drops; +}; + +struct rmnet_pcpu_stats { + struct rmnet_vnd_stats stats; + struct u64_stats_sync syncp; +}; + +struct rmnet_coal_close_stats { + u64 non_coal; + u64 ip_miss; + u64 trans_miss; + u64 hw_nl; + u64 hw_pkt; + u64 hw_byte; + u64 hw_time; + u64 hw_evict; + u64 coal; +}; + +struct rmnet_coal_stats { + u64 coal_rx; + u64 coal_pkts; + u64 coal_hdr_nlo_err; + u64 coal_hdr_pkt_err; + u64 coal_csum_err; + u64 coal_reconstruct; + u64 coal_ip_invalid; + u64 coal_trans_invalid; + struct rmnet_coal_close_stats close; + u64 coal_veid[RMNET_MAX_VEID]; +}; + +struct rmnet_priv_stats { + u64 csum_ok; + u64 csum_valid_unset; + u64 csum_validation_failed; + u64 csum_err_bad_buffer; + u64 csum_err_invalid_ip_version; + u64 csum_err_invalid_transport; + u64 csum_fragmented_pkt; + u64 csum_skipped; + u64 csum_sw; + u64 csum_hw; + struct rmnet_coal_stats coal; +}; + +struct rmnet_priv { + u8 mux_id; + struct net_device *real_dev; + struct rmnet_pcpu_stats __percpu *pcpu_stats; + struct gro_cells gro_cells; + struct rmnet_priv_stats stats; + void __rcu *qos_info; +}; + +enum rmnet_dl_marker_prio { + RMNET_PERF, + RMNET_SHS, +}; + +enum rmnet_trace_func { + RMNET_MODULE, + NW_STACK_MODULE, +}; + +enum rmnet_trace_evt { + RMNET_DLVR_SKB, + RMNET_RCV_FROM_PND, + RMNET_TX_UL_PKT, + NW_STACK_DEV_Q_XMIT, + NW_STACK_NAPI_GRO_FLUSH, + NW_STACK_RX, + NW_STACK_TX, +}; + +int rmnet_is_real_dev_registered(const struct net_device *real_dev); +struct rmnet_port *rmnet_get_port(struct net_device *real_dev); +struct rmnet_endpoint *rmnet_get_endpoint(struct rmnet_port *port, u8 mux_id); +int rmnet_add_bridge(struct net_device *rmnet_dev, + struct net_device *slave_dev, + struct netlink_ext_ack *extack); +int rmnet_del_bridge(struct net_device *rmnet_dev, + struct net_device *slave_dev); +#endif /* _RMNET_CONFIG_H_ */ diff --git a/core/rmnet_descriptor.c b/core/rmnet_descriptor.c new file mode 100644 index 000000000000..ddc9f12fa689 --- /dev/null +++ b/core/rmnet_descriptor.c @@ -0,0 +1,1087 @@ +/* Copyright (c) 2013-2019, The Linux Foundation. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 and + * only version 2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * RMNET Packet Descriptor Framework + * + */ + +#include +#include +#include +#include +#include "rmnet_config.h" +#include "rmnet_descriptor.h" +#include "rmnet_handlers.h" +#include "rmnet_private.h" +#include "rmnet_vnd.h" +#include "rmnet_qmi.h" +#include "qmi_rmnet.h" + +#define RMNET_FRAG_DESCRIPTOR_POOL_SIZE 64 +#define RMNET_DL_IND_HDR_SIZE (sizeof(struct rmnet_map_dl_ind_hdr) + \ + sizeof(struct rmnet_map_header) + \ + sizeof(struct rmnet_map_control_command_header)) +#define RMNET_DL_IND_TRL_SIZE (sizeof(struct rmnet_map_dl_ind_trl) + \ + sizeof(struct rmnet_map_header) + \ + sizeof(struct rmnet_map_control_command_header)) + +typedef void (*rmnet_perf_desc_hook_t)(struct rmnet_frag_descriptor *frag_desc, + struct rmnet_port *port); +typedef void (*rmnet_perf_chain_hook_t)(void); + +struct rmnet_frag_descriptor * +rmnet_get_frag_descriptor(struct rmnet_port *port) +{ + struct rmnet_frag_descriptor_pool *pool = port->frag_desc_pool; + struct rmnet_frag_descriptor *frag_desc; + + spin_lock(&port->desc_pool_lock); + if (!list_empty(&pool->free_list)) { + frag_desc = list_first_entry(&pool->free_list, + struct rmnet_frag_descriptor, + list); + list_del_init(&frag_desc->list); + } else { + frag_desc = kzalloc(sizeof(*frag_desc), GFP_ATOMIC); + if (!frag_desc) + goto out; + + INIT_LIST_HEAD(&frag_desc->list); + INIT_LIST_HEAD(&frag_desc->sub_frags); + pool->pool_size++; + } + +out: + spin_unlock(&port->desc_pool_lock); + return frag_desc; +} +EXPORT_SYMBOL(rmnet_get_frag_descriptor); + +void rmnet_recycle_frag_descriptor(struct rmnet_frag_descriptor *frag_desc, + struct rmnet_port *port) +{ + struct rmnet_frag_descriptor_pool *pool = port->frag_desc_pool; + + list_del(&frag_desc->list); + put_page(skb_frag_page(&frag_desc->frag)); + + memset(frag_desc, 0, sizeof(*frag_desc)); + INIT_LIST_HEAD(&frag_desc->list); + INIT_LIST_HEAD(&frag_desc->sub_frags); + spin_lock(&port->desc_pool_lock); + list_add_tail(&frag_desc->list, &pool->free_list); + spin_unlock(&port->desc_pool_lock); +} +EXPORT_SYMBOL(rmnet_recycle_frag_descriptor); + +void rmnet_descriptor_add_frag(struct rmnet_port *port, struct list_head *list, + struct page *p, u32 page_offset, u32 len) +{ + struct rmnet_frag_descriptor *frag_desc; + + frag_desc = rmnet_get_frag_descriptor(port); + if (!frag_desc) + return; + + rmnet_frag_fill(frag_desc, p, page_offset, len); + list_add_tail(&frag_desc->list, list); +} +EXPORT_SYMBOL(rmnet_descriptor_add_frag); + +int rmnet_frag_ipv6_skip_exthdr(struct rmnet_frag_descriptor *frag_desc, + int start, u8 *nexthdrp, __be16 *fragp) +{ + u8 nexthdr = *nexthdrp; + + *fragp = 0; + + while (ipv6_ext_hdr(nexthdr)) { + struct ipv6_opt_hdr *hp; + int hdrlen; + + if (nexthdr == NEXTHDR_NONE) + return -EINVAL; + + hp = rmnet_frag_data_ptr(frag_desc) + start; + + if (nexthdr == NEXTHDR_FRAGMENT) { + __be16 *fp; + + fp = rmnet_frag_data_ptr(frag_desc) + start + + offsetof(struct frag_hdr, frag_off); + *fragp = *fp; + if (ntohs(*fragp) & ~0x7) + break; + hdrlen = 8; + } else if (nexthdr == NEXTHDR_AUTH) { + hdrlen = (hp->hdrlen + 2) << 2; + } else { + hdrlen = ipv6_optlen(hp); + } + + nexthdr = hp->nexthdr; + start += hdrlen; + } + + *nexthdrp = nexthdr; + return start; +} +EXPORT_SYMBOL(rmnet_frag_ipv6_skip_exthdr); + +static u8 rmnet_frag_do_flow_control(struct rmnet_map_header *qmap, + struct rmnet_port *port, + int enable) +{ + struct rmnet_map_control_command *cmd; + struct rmnet_endpoint *ep; + struct net_device *vnd; + u16 ip_family; + u16 fc_seq; + u32 qos_id; + u8 mux_id; + int r; + + mux_id = qmap->mux_id; + cmd = (struct rmnet_map_control_command *) + ((char *)qmap + sizeof(*qmap)); + + if (mux_id >= RMNET_MAX_LOGICAL_EP) + return RX_HANDLER_CONSUMED; + + ep = rmnet_get_endpoint(port, mux_id); + if (!ep) + return RX_HANDLER_CONSUMED; + + vnd = ep->egress_dev; + + ip_family = cmd->flow_control.ip_family; + fc_seq = ntohs(cmd->flow_control.flow_control_seq_num); + qos_id = ntohl(cmd->flow_control.qos_id); + + /* Ignore the ip family and pass the sequence number for both v4 and v6 + * sequence. User space does not support creating dedicated flows for + * the 2 protocols + */ + r = rmnet_vnd_do_flow_control(vnd, enable); + if (r) + return RMNET_MAP_COMMAND_UNSUPPORTED; + else + return RMNET_MAP_COMMAND_ACK; +} + +static void rmnet_frag_send_ack(struct rmnet_map_header *qmap, + unsigned char type, + struct rmnet_port *port) +{ + struct rmnet_map_control_command *cmd; + struct net_device *dev = port->dev; + struct sk_buff *skb; + u16 alloc_len = ntohs(qmap->pkt_len) + sizeof(*qmap); + + skb = alloc_skb(alloc_len, GFP_ATOMIC); + if (!skb) + return; + + skb->protocol = htons(ETH_P_MAP); + skb->dev = dev; + + cmd = rmnet_map_get_cmd_start(skb); + cmd->cmd_type = type & 0x03; + + netif_tx_lock(dev); + dev->netdev_ops->ndo_start_xmit(skb, dev); + netif_tx_unlock(dev); +} + +static void +rmnet_frag_process_flow_start(struct rmnet_map_control_command_header *cmd, + struct rmnet_port *port, + u16 cmd_len) +{ + struct rmnet_map_dl_ind_hdr *dlhdr; + u32 data_format; + bool is_dl_mark_v2; + + if (cmd_len + sizeof(struct rmnet_map_header) < RMNET_DL_IND_HDR_SIZE) + return; + + data_format = port->data_format; + is_dl_mark_v2 = data_format & RMNET_INGRESS_FORMAT_DL_MARKER_V2; + dlhdr = (struct rmnet_map_dl_ind_hdr *)((char *)cmd + sizeof(*cmd)); + + port->stats.dl_hdr_last_ep_id = cmd->source_id; + port->stats.dl_hdr_last_qmap_vers = cmd->reserved; + port->stats.dl_hdr_last_trans_id = cmd->transaction_id; + port->stats.dl_hdr_last_seq = dlhdr->le.seq; + port->stats.dl_hdr_last_bytes = dlhdr->le.bytes; + port->stats.dl_hdr_last_pkts = dlhdr->le.pkts; + port->stats.dl_hdr_last_flows = dlhdr->le.flows; + port->stats.dl_hdr_total_bytes += port->stats.dl_hdr_last_bytes; + port->stats.dl_hdr_total_pkts += port->stats.dl_hdr_last_pkts; + port->stats.dl_hdr_count++; + + /* If a target is taking frag path, we can assume DL marker v2 is in + * play + */ + if (is_dl_mark_v2) + rmnet_map_dl_hdr_notify_v2(port, dlhdr, cmd); + else + rmnet_map_dl_hdr_notify(port, dlhdr); +} + +static void +rmnet_frag_process_flow_end(struct rmnet_map_control_command_header *cmd, + struct rmnet_port *port, u16 cmd_len) +{ + struct rmnet_map_dl_ind_trl *dltrl; + u32 data_format; + bool is_dl_mark_v2; + + + if (cmd_len + sizeof(struct rmnet_map_header) < RMNET_DL_IND_TRL_SIZE) + return; + + data_format = port->data_format; + is_dl_mark_v2 = data_format & RMNET_INGRESS_FORMAT_DL_MARKER_V2; + dltrl = (struct rmnet_map_dl_ind_trl *)((char *)cmd + sizeof(*cmd)); + + port->stats.dl_trl_last_seq = dltrl->seq_le; + port->stats.dl_trl_count++; + + /* If a target is taking frag path, we can assume DL marker v2 is in + * play + */ + if (is_dl_mark_v2) + rmnet_map_dl_trl_notify_v2(port, dltrl, cmd); + else + rmnet_map_dl_trl_notify(port, dltrl); +} + +/* Process MAP command frame and send N/ACK message as appropriate. Message cmd + * name is decoded here and appropriate handler is called. + */ +void rmnet_frag_command(struct rmnet_map_header *qmap, struct rmnet_port *port) +{ + struct rmnet_map_control_command *cmd; + unsigned char command_name; + unsigned char rc = 0; + + cmd = (struct rmnet_map_control_command *) + ((char *)qmap + sizeof(*qmap)); + command_name = cmd->command_name; + + switch (command_name) { + case RMNET_MAP_COMMAND_FLOW_ENABLE: + rc = rmnet_frag_do_flow_control(qmap, port, 1); + break; + + case RMNET_MAP_COMMAND_FLOW_DISABLE: + rc = rmnet_frag_do_flow_control(qmap, port, 0); + break; + + default: + rc = RMNET_MAP_COMMAND_UNSUPPORTED; + break; + } + if (rc == RMNET_MAP_COMMAND_ACK) + rmnet_frag_send_ack(qmap, rc, port); +} + +int rmnet_frag_flow_command(struct rmnet_map_header *qmap, + struct rmnet_port *port, u16 pkt_len) +{ + struct rmnet_map_control_command_header *cmd; + unsigned char command_name; + + cmd = (struct rmnet_map_control_command_header *) + ((char *)qmap + sizeof(*qmap)); + command_name = cmd->command_name; + + switch (command_name) { + case RMNET_MAP_COMMAND_FLOW_START: + rmnet_frag_process_flow_start(cmd, port, pkt_len); + break; + + case RMNET_MAP_COMMAND_FLOW_END: + rmnet_frag_process_flow_end(cmd, port, pkt_len); + break; + + default: + return 1; + } + + return 0; +} +EXPORT_SYMBOL(rmnet_frag_flow_command); + +void rmnet_frag_deaggregate(skb_frag_t *frag, struct rmnet_port *port, + struct list_head *list) +{ + struct rmnet_map_header *maph; + u8 *data = skb_frag_address(frag); + u32 offset = 0; + u32 packet_len; + + while (offset < skb_frag_size(frag)) { + maph = (struct rmnet_map_header *)data; + packet_len = ntohs(maph->pkt_len); + + /* Some hardware can send us empty frames. Catch them */ + if (packet_len == 0) + return; + + packet_len += sizeof(*maph); + + if (port->data_format & RMNET_FLAGS_INGRESS_MAP_CKSUMV4) { + packet_len += sizeof(struct rmnet_map_dl_csum_trailer); + } else if (port->data_format & + (RMNET_FLAGS_INGRESS_MAP_CKSUMV5 | + RMNET_FLAGS_INGRESS_COALESCE) && !maph->cd_bit) { + u32 hsize = 0; + u8 type; + + type = ((struct rmnet_map_v5_coal_header *) + (data + sizeof(*maph)))->header_type; + switch (type) { + case RMNET_MAP_HEADER_TYPE_COALESCING: + hsize = sizeof(struct rmnet_map_v5_coal_header); + break; + case RMNET_MAP_HEADER_TYPE_CSUM_OFFLOAD: + hsize = sizeof(struct rmnet_map_v5_csum_header); + break; + } + + packet_len += hsize; + } + + if ((int)skb_frag_size(frag) - (int)packet_len < 0) + return; + + rmnet_descriptor_add_frag(port, list, skb_frag_page(frag), + frag->bv_offset + offset, + packet_len); + + offset += packet_len; + data += packet_len; + } +} + +/* Fill in GSO metadata to allow the SKB to be segmented by the NW stack + * if needed (i.e. forwarding, UDP GRO) + */ +static void rmnet_frag_gso_stamp(struct sk_buff *skb, + struct rmnet_frag_descriptor *frag_desc) +{ + struct skb_shared_info *shinfo = skb_shinfo(skb); + struct iphdr *iph = (struct iphdr *)skb->data; + __sum16 pseudo; + u16 pkt_len = skb->len - frag_desc->ip_len; + bool ipv4 = frag_desc->ip_proto == 4; + + if (ipv4) { + iph->tot_len = htons(skb->len); + iph->check = 0; + iph->check = ip_fast_csum(iph, iph->ihl); + pseudo = ~csum_tcpudp_magic(iph->saddr, iph->daddr, + pkt_len, frag_desc->trans_proto, + 0); + } else { + struct ipv6hdr *ip6h = (struct ipv6hdr *)iph; + + /* Payload length includes any extension headers */ + ip6h->payload_len = htons(skb->len - sizeof(*ip6h)); + pseudo = ~csum_ipv6_magic(&ip6h->saddr, &ip6h->daddr, + pkt_len, frag_desc->trans_proto, 0); + } + + if (frag_desc->trans_proto == IPPROTO_TCP) { + struct tcphdr *tp = (struct tcphdr *) + ((u8 *)iph + frag_desc->ip_len); + + tp->check = pseudo; + shinfo->gso_type = (ipv4) ? SKB_GSO_TCPV4 : SKB_GSO_TCPV6; + skb->csum_offset = offsetof(struct tcphdr, check); + } else { + struct udphdr *up = (struct udphdr *) + ((u8 *)iph + frag_desc->ip_len); + + up->len = htons(pkt_len); + up->check = pseudo; + shinfo->gso_type = SKB_GSO_UDP_L4; + skb->csum_offset = offsetof(struct udphdr, check); + } + + skb->ip_summed = CHECKSUM_PARTIAL; + skb->csum_start = (u8 *)iph + frag_desc->ip_len - skb->head; + shinfo->gso_size = frag_desc->gso_size; + shinfo->gso_segs = frag_desc->gso_segs; +} + +/* Allocate and populate an skb to contain the packet represented by the + * frag descriptor. + */ +static struct sk_buff *rmnet_alloc_skb(struct rmnet_frag_descriptor *frag_desc, + struct rmnet_port *port) +{ + struct sk_buff *head_skb, *current_skb, *skb; + struct skb_shared_info *shinfo; + struct rmnet_frag_descriptor *sub_frag, *tmp; + + /* Use the exact sizes if we know them (i.e. RSB/RSC, rmnet_perf) */ + if (frag_desc->hdrs_valid) { + u16 hdr_len = frag_desc->ip_len + frag_desc->trans_len; + + head_skb = alloc_skb(hdr_len + RMNET_MAP_DEAGGR_HEADROOM, + GFP_ATOMIC); + if (!head_skb) + return NULL; + + skb_reserve(head_skb, RMNET_MAP_DEAGGR_HEADROOM); + skb_put_data(head_skb, frag_desc->hdr_ptr, hdr_len); + skb_reset_network_header(head_skb); + if (frag_desc->trans_len) + skb_set_transport_header(head_skb, frag_desc->ip_len); + + /* Packets that have no data portion don't need any frags */ + if (hdr_len == skb_frag_size(&frag_desc->frag)) + goto skip_frags; + + /* If the headers we added are the start of the page, + * we don't want to add them twice + */ + if (frag_desc->hdr_ptr == rmnet_frag_data_ptr(frag_desc)) + rmnet_frag_pull(frag_desc, port, hdr_len); + } else { + /* Allocate enough space to avoid penalties in the stack + * from __pskb_pull_tail() + */ + head_skb = alloc_skb(256 + RMNET_MAP_DEAGGR_HEADROOM, + GFP_ATOMIC); + if (!head_skb) + return NULL; + + skb_reserve(head_skb, RMNET_MAP_DEAGGR_HEADROOM); + } + + /* Add main fragment */ + get_page(skb_frag_page(&frag_desc->frag)); + skb_add_rx_frag(head_skb, 0, skb_frag_page(&frag_desc->frag), + frag_desc->frag.bv_offset, + skb_frag_size(&frag_desc->frag), + skb_frag_size(&frag_desc->frag)); + + shinfo = skb_shinfo(head_skb); + current_skb = head_skb; + + /* Add in any frags from rmnet_perf */ + list_for_each_entry_safe(sub_frag, tmp, &frag_desc->sub_frags, list) { + skb_frag_t *frag; + u32 frag_size; + + frag = &sub_frag->frag; + frag_size = skb_frag_size(frag); + +add_frag: + if (shinfo->nr_frags < MAX_SKB_FRAGS) { + get_page(skb_frag_page(frag)); + skb_add_rx_frag(current_skb, shinfo->nr_frags, + skb_frag_page(frag), frag->bv_offset, + frag_size, frag_size); + if (current_skb != head_skb) { + head_skb->len += frag_size; + head_skb->data_len += frag_size; + } + } else { + /* Alloc a new skb and try again */ + skb = alloc_skb(0, GFP_ATOMIC); + if (!skb) + break; + + if (current_skb == head_skb) + shinfo->frag_list = skb; + else + current_skb->next = skb; + + current_skb = skb; + shinfo = skb_shinfo(current_skb); + goto add_frag; + } + + rmnet_recycle_frag_descriptor(sub_frag, port); + } + +skip_frags: + head_skb->dev = frag_desc->dev; + rmnet_set_skb_proto(head_skb); + + /* Handle any header metadata that needs to be updated after RSB/RSC + * segmentation + */ + if (frag_desc->ip_id_set) { + struct iphdr *iph; + + iph = (struct iphdr *)rmnet_map_data_ptr(head_skb); + csum_replace2(&iph->check, iph->id, frag_desc->ip_id); + iph->id = frag_desc->ip_id; + } + + if (frag_desc->tcp_seq_set) { + struct tcphdr *th; + + th = (struct tcphdr *) + (rmnet_map_data_ptr(head_skb) + frag_desc->ip_len); + th->seq = frag_desc->tcp_seq; + } + + /* Handle csum offloading */ + if (frag_desc->csum_valid) + head_skb->ip_summed = CHECKSUM_UNNECESSARY; + + /* Handle any rmnet_perf metadata */ + if (frag_desc->hash) { + head_skb->hash = frag_desc->hash; + head_skb->sw_hash = 1; + } + + if (frag_desc->flush_shs) + head_skb->cb[0] = 1; + + /* Handle coalesced packets */ + if (frag_desc->gso_segs > 1) + rmnet_frag_gso_stamp(head_skb, frag_desc); + + return head_skb; +} + +/* Deliver the packets contained within a frag descriptor */ +void rmnet_frag_deliver(struct rmnet_frag_descriptor *frag_desc, + struct rmnet_port *port) +{ + struct sk_buff *skb; + + skb = rmnet_alloc_skb(frag_desc, port); + if (skb) + rmnet_deliver_skb(skb, port); + rmnet_recycle_frag_descriptor(frag_desc, port); +} +EXPORT_SYMBOL(rmnet_frag_deliver); + +static void __rmnet_frag_segment_data(struct rmnet_frag_descriptor *coal_desc, + struct rmnet_port *port, + struct list_head *list, + u8 pkt_id) +{ + struct rmnet_priv *priv = netdev_priv(coal_desc->dev); + struct rmnet_frag_descriptor *new_frag; + u8 *hdr_start = rmnet_frag_data_ptr(coal_desc); + u32 offset; + + new_frag = rmnet_get_frag_descriptor(port); + if (!new_frag) + return; + + /* Account for header lengths to access the data start */ + offset = coal_desc->frag.bv_offset + coal_desc->ip_len + + coal_desc->trans_len + coal_desc->data_offset; + + /* Header information and most metadata is the same as the original */ + memcpy(new_frag, coal_desc, sizeof(*coal_desc)); + INIT_LIST_HEAD(&new_frag->list); + INIT_LIST_HEAD(&new_frag->sub_frags); + rmnet_frag_fill(new_frag, skb_frag_page(&coal_desc->frag), offset, + coal_desc->gso_size * coal_desc->gso_segs); + + if (coal_desc->trans_proto == IPPROTO_TCP) { + struct tcphdr *th; + + th = (struct tcphdr *)(hdr_start + coal_desc->ip_len); + new_frag->tcp_seq_set = 1; + new_frag->tcp_seq = htonl(ntohl(th->seq) + + coal_desc->data_offset); + } + + if (coal_desc->ip_proto == 4) { + struct iphdr *iph; + + iph = (struct iphdr *)hdr_start; + new_frag->ip_id_set = 1; + new_frag->ip_id = htons(ntohs(iph->id) + coal_desc->pkt_id); + } + + new_frag->hdr_ptr = hdr_start; + new_frag->csum_valid = true; + priv->stats.coal.coal_reconstruct++; + + /* Update meta information to move past the data we just segmented */ + coal_desc->data_offset += coal_desc->gso_size * coal_desc->gso_segs; + coal_desc->pkt_id = pkt_id + 1; + coal_desc->gso_segs = 0; + + list_add_tail(&new_frag->list, list); +} + +/* Converts the coalesced frame into a list of descriptors. + * NLOs containing csum erros will not be included. + */ +static void +rmnet_frag_segment_coal_data(struct rmnet_frag_descriptor *coal_desc, + u64 nlo_err_mask, struct rmnet_port *port, + struct list_head *list) +{ + struct iphdr *iph; + struct rmnet_priv *priv = netdev_priv(coal_desc->dev); + struct rmnet_map_v5_coal_header *coal_hdr; + u16 pkt_len; + u8 pkt, total_pkt = 0; + u8 nlo; + bool gro = coal_desc->dev->features & NETIF_F_GRO_HW; + + /* Pull off the headers we no longer need */ + rmnet_frag_pull(coal_desc, port, sizeof(struct rmnet_map_header)); + coal_hdr = (struct rmnet_map_v5_coal_header *) + rmnet_frag_data_ptr(coal_desc); + rmnet_frag_pull(coal_desc, port, sizeof(*coal_hdr)); + + iph = (struct iphdr *)rmnet_frag_data_ptr(coal_desc); + + if (iph->version == 4) { + coal_desc->ip_proto = 4; + coal_desc->ip_len = iph->ihl * 4; + coal_desc->trans_proto = iph->protocol; + + /* Don't allow coalescing of any packets with IP options */ + if (iph->ihl != 5) + gro = false; + } else if (iph->version == 6) { + struct ipv6hdr *ip6h = (struct ipv6hdr *)iph; + __be16 frag_off; + u8 protocol = ip6h->nexthdr; + + coal_desc->ip_proto = 6; + coal_desc->ip_len = rmnet_frag_ipv6_skip_exthdr(coal_desc, + sizeof(*ip6h), + &protocol, + &frag_off); + coal_desc->trans_proto = protocol; + + /* If we run into a problem, or this has a fragment header + * (which should technically not be possible, if the HW + * works as intended...), bail. + */ + if (coal_desc->ip_len < 0 || frag_off) { + priv->stats.coal.coal_ip_invalid++; + return; + } else if (coal_desc->ip_len > sizeof(*ip6h)) { + /* Don't allow coalescing of any packets with IPv6 + * extension headers. + */ + gro = false; + } + } else { + priv->stats.coal.coal_ip_invalid++; + return; + } + + if (coal_desc->trans_proto == IPPROTO_TCP) { + struct tcphdr *th; + + th = (struct tcphdr *)((u8 *)iph + coal_desc->ip_len); + coal_desc->trans_len = th->doff * 4; + } else if (coal_desc->trans_proto == IPPROTO_UDP) { + coal_desc->trans_len = sizeof(struct udphdr); + } else { + priv->stats.coal.coal_trans_invalid++; + return; + } + + coal_desc->hdrs_valid = 1; + for (nlo = 0; nlo < coal_hdr->num_nlos; nlo++) { + pkt_len = ntohs(coal_hdr->nl_pairs[nlo].pkt_len); + pkt_len -= coal_desc->ip_len + coal_desc->trans_len; + coal_desc->gso_size = pkt_len; + for (pkt = 0; pkt < coal_hdr->nl_pairs[nlo].num_packets; + pkt++, total_pkt++) { + nlo_err_mask <<= 1; + if (nlo_err_mask & (1ULL << 63)) { + priv->stats.coal.coal_csum_err++; + + /* Segment out the good data */ + if (gro && coal_desc->gso_segs) + __rmnet_frag_segment_data(coal_desc, + port, + list, + total_pkt); + + /* skip over bad packet */ + coal_desc->data_offset += pkt_len; + coal_desc->pkt_id = total_pkt + 1; + } else { + coal_desc->gso_segs++; + + /* Segment the packet if we aren't sending the + * larger packet up the stack. + */ + if (!gro) + __rmnet_frag_segment_data(coal_desc, + port, + list, + total_pkt); + } + } + + /* If we're switching NLOs, we need to send out everything from + * the previous one, if we haven't done so. NLOs only switch + * when the packet length changes. + */ + if (gro && coal_desc->gso_segs) { + /* Fast forward the (hopefully) common case. + * Frames with only one NLO (i.e. one packet length) and + * no checksum errors don't need to be segmented here. + * We can just pass off the original skb. + */ + if (coal_desc->gso_size * coal_desc->gso_segs == + skb_frag_size(&coal_desc->frag) - + coal_desc->ip_len - coal_desc->trans_len) { + coal_desc->hdr_ptr = + rmnet_frag_data_ptr(coal_desc); + coal_desc->csum_valid = true; + list_add_tail(&coal_desc->list, list); + return; + } + + __rmnet_frag_segment_data(coal_desc, port, list, + total_pkt); + } + } +} + +/* Record reason for coalescing pipe closure */ +static void rmnet_frag_data_log_close_stats(struct rmnet_priv *priv, u8 type, + u8 code) +{ + struct rmnet_coal_close_stats *stats = &priv->stats.coal.close; + + switch (type) { + case RMNET_MAP_COAL_CLOSE_NON_COAL: + stats->non_coal++; + break; + case RMNET_MAP_COAL_CLOSE_IP_MISS: + stats->ip_miss++; + break; + case RMNET_MAP_COAL_CLOSE_TRANS_MISS: + stats->trans_miss++; + break; + case RMNET_MAP_COAL_CLOSE_HW: + switch (code) { + case RMNET_MAP_COAL_CLOSE_HW_NL: + stats->hw_nl++; + break; + case RMNET_MAP_COAL_CLOSE_HW_PKT: + stats->hw_pkt++; + break; + case RMNET_MAP_COAL_CLOSE_HW_BYTE: + stats->hw_byte++; + break; + case RMNET_MAP_COAL_CLOSE_HW_TIME: + stats->hw_time++; + break; + case RMNET_MAP_COAL_CLOSE_HW_EVICT: + stats->hw_evict++; + break; + default: + break; + } + break; + case RMNET_MAP_COAL_CLOSE_COAL: + stats->coal++; + break; + default: + break; + } +} + +/* Check if the coalesced header has any incorrect values, in which case, the + * entire coalesced frame must be dropped. Then check if there are any + * checksum issues + */ +static int +rmnet_frag_data_check_coal_header(struct rmnet_frag_descriptor *frag_desc, + u64 *nlo_err_mask) +{ + struct rmnet_map_v5_coal_header *coal_hdr; + unsigned char *data = rmnet_frag_data_ptr(frag_desc); + struct rmnet_priv *priv = netdev_priv(frag_desc->dev); + u64 mask = 0; + int i; + u8 veid, pkts = 0; + + coal_hdr = (struct rmnet_map_v5_coal_header *) + (data + sizeof(struct rmnet_map_header)); + veid = coal_hdr->virtual_channel_id; + + if (coal_hdr->num_nlos == 0 || + coal_hdr->num_nlos > RMNET_MAP_V5_MAX_NLOS) { + priv->stats.coal.coal_hdr_nlo_err++; + return -EINVAL; + } + + for (i = 0; i < RMNET_MAP_V5_MAX_NLOS; i++) { + /* If there is a checksum issue, we need to split + * up the skb. Rebuild the full csum error field + */ + u8 err = coal_hdr->nl_pairs[i].csum_error_bitmap; + u8 pkt = coal_hdr->nl_pairs[i].num_packets; + + mask |= ((u64)err) << (7 - i) * 8; + + /* Track total packets in frame */ + pkts += pkt; + if (pkts > RMNET_MAP_V5_MAX_PACKETS) { + priv->stats.coal.coal_hdr_pkt_err++; + return -EINVAL; + } + } + + /* Track number of packets we get inside of coalesced frames */ + priv->stats.coal.coal_pkts += pkts; + + /* Update ethtool stats */ + rmnet_frag_data_log_close_stats(priv, + coal_hdr->close_type, + coal_hdr->close_value); + if (veid < RMNET_MAX_VEID) + priv->stats.coal.coal_veid[veid]++; + + *nlo_err_mask = mask; + + return 0; +} + +/* Process a QMAPv5 packet header */ +int rmnet_frag_process_next_hdr_packet(struct rmnet_frag_descriptor *frag_desc, + struct rmnet_port *port, + struct list_head *list, + u16 len) +{ + struct rmnet_priv *priv = netdev_priv(frag_desc->dev); + u64 nlo_err_mask; + int rc = 0; + + switch (rmnet_frag_get_next_hdr_type(frag_desc)) { + case RMNET_MAP_HEADER_TYPE_COALESCING: + priv->stats.coal.coal_rx++; + rc = rmnet_frag_data_check_coal_header(frag_desc, + &nlo_err_mask); + if (rc) + return rc; + + rmnet_frag_segment_coal_data(frag_desc, nlo_err_mask, port, + list); + if (list_first_entry(list, struct rmnet_frag_descriptor, + list) != frag_desc) + rmnet_recycle_frag_descriptor(frag_desc, port); + break; + case RMNET_MAP_HEADER_TYPE_CSUM_OFFLOAD: + if (rmnet_frag_get_csum_valid(frag_desc)) { + priv->stats.csum_ok++; + frag_desc->csum_valid = true; + } else { + priv->stats.csum_valid_unset++; + } + + rmnet_frag_pull(frag_desc, port, + sizeof(struct rmnet_map_header) + + sizeof(struct rmnet_map_v5_csum_header)); + frag_desc->hdr_ptr = rmnet_frag_data_ptr(frag_desc); + + /* Remove padding only for csum offload packets. + * Coalesced packets should never have padding. + */ + rmnet_frag_trim(frag_desc, port, len); + list_del_init(&frag_desc->list); + list_add_tail(&frag_desc->list, list); + break; + default: + rc = -EINVAL; + break; + } + + return rc; +} + +/* Perf hook handler */ +rmnet_perf_desc_hook_t rmnet_perf_desc_entry __rcu __read_mostly; +EXPORT_SYMBOL(rmnet_perf_desc_entry); + +static void +__rmnet_frag_ingress_handler(struct rmnet_frag_descriptor *frag_desc, + struct rmnet_port *port) +{ + rmnet_perf_desc_hook_t rmnet_perf_ingress; + struct rmnet_map_header *qmap; + struct rmnet_endpoint *ep; + struct rmnet_frag_descriptor *frag, *tmp; + LIST_HEAD(segs); + u16 len, pad; + u8 mux_id; + + qmap = (struct rmnet_map_header *)skb_frag_address(&frag_desc->frag); + mux_id = qmap->mux_id; + pad = qmap->pad_len; + len = ntohs(qmap->pkt_len) - pad; + + if (qmap->cd_bit) { + qmi_rmnet_set_dl_msg_active(port); + if (port->data_format & RMNET_INGRESS_FORMAT_DL_MARKER) { + rmnet_frag_flow_command(qmap, port, len); + goto recycle; + } + + if (port->data_format & RMNET_FLAGS_INGRESS_MAP_COMMANDS) + rmnet_frag_command(qmap, port); + + goto recycle; + } + + if (mux_id >= RMNET_MAX_LOGICAL_EP) + goto recycle; + + ep = rmnet_get_endpoint(port, mux_id); + if (!ep) + goto recycle; + + frag_desc->dev = ep->egress_dev; + + /* Handle QMAPv5 packet */ + if (qmap->next_hdr && + (port->data_format & (RMNET_FLAGS_INGRESS_COALESCE | + RMNET_FLAGS_INGRESS_MAP_CKSUMV5))) { + if (rmnet_frag_process_next_hdr_packet(frag_desc, port, &segs, + len)) + goto recycle; + } else { + /* We only have the main QMAP header to worry about */ + rmnet_frag_pull(frag_desc, port, sizeof(*qmap)); + frag_desc->hdr_ptr = rmnet_frag_data_ptr(frag_desc); + + rmnet_frag_trim(frag_desc, port, len); + list_add_tail(&frag_desc->list, &segs); + } + + if (port->data_format & RMNET_INGRESS_FORMAT_PS) + qmi_rmnet_work_maybe_restart(port); + + rcu_read_lock(); + rmnet_perf_ingress = rcu_dereference(rmnet_perf_desc_entry); + if (rmnet_perf_ingress) { + list_for_each_entry_safe(frag, tmp, &segs, list) { + list_del_init(&frag->list); + rmnet_perf_ingress(frag, port); + } + rcu_read_unlock(); + return; + } + rcu_read_unlock(); + + list_for_each_entry_safe(frag, tmp, &segs, list) { + list_del_init(&frag->list); + rmnet_frag_deliver(frag, port); + } + return; + +recycle: + rmnet_recycle_frag_descriptor(frag_desc, port); +} + +/* Notify perf at the end of SKB chain */ +rmnet_perf_chain_hook_t rmnet_perf_chain_end __rcu __read_mostly; +EXPORT_SYMBOL(rmnet_perf_chain_end); + +void rmnet_frag_ingress_handler(struct sk_buff *skb, + struct rmnet_port *port) +{ + rmnet_perf_chain_hook_t rmnet_perf_opt_chain_end; + LIST_HEAD(desc_list); + + /* Deaggregation and freeing of HW originating + * buffers is done within here + */ + while (skb) { + struct sk_buff *skb_frag; + + rmnet_frag_deaggregate(skb_shinfo(skb)->frags, port, + &desc_list); + if (!list_empty(&desc_list)) { + struct rmnet_frag_descriptor *frag_desc, *tmp; + + list_for_each_entry_safe(frag_desc, tmp, &desc_list, + list) { + list_del_init(&frag_desc->list); + __rmnet_frag_ingress_handler(frag_desc, port); + } + } + + skb_frag = skb_shinfo(skb)->frag_list; + skb_shinfo(skb)->frag_list = NULL; + consume_skb(skb); + skb = skb_frag; + } + + rcu_read_lock(); + rmnet_perf_opt_chain_end = rcu_dereference(rmnet_perf_chain_end); + if (rmnet_perf_opt_chain_end) + rmnet_perf_opt_chain_end(); + rcu_read_unlock(); +} + +void rmnet_descriptor_deinit(struct rmnet_port *port) +{ + struct rmnet_frag_descriptor_pool *pool; + struct rmnet_frag_descriptor *frag_desc, *tmp; + + pool = port->frag_desc_pool; + + list_for_each_entry_safe(frag_desc, tmp, &pool->free_list, list) { + kfree(frag_desc); + pool->pool_size--; + } + + kfree(pool); +} + +int rmnet_descriptor_init(struct rmnet_port *port) +{ + struct rmnet_frag_descriptor_pool *pool; + int i; + + spin_lock_init(&port->desc_pool_lock); + pool = kzalloc(sizeof(*pool), GFP_ATOMIC); + if (!pool) + return -ENOMEM; + + INIT_LIST_HEAD(&pool->free_list); + port->frag_desc_pool = pool; + + for (i = 0; i < RMNET_FRAG_DESCRIPTOR_POOL_SIZE; i++) { + struct rmnet_frag_descriptor *frag_desc; + + frag_desc = kzalloc(sizeof(*frag_desc), GFP_ATOMIC); + if (!frag_desc) + return -ENOMEM; + + INIT_LIST_HEAD(&frag_desc->list); + INIT_LIST_HEAD(&frag_desc->sub_frags); + list_add_tail(&frag_desc->list, &pool->free_list); + pool->pool_size++; + } + + return 0; +} diff --git a/core/rmnet_descriptor.h b/core/rmnet_descriptor.h new file mode 100644 index 000000000000..996a6caa1dd8 --- /dev/null +++ b/core/rmnet_descriptor.h @@ -0,0 +1,147 @@ +/* Copyright (c) 2013-2019, The Linux Foundation. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 and + * only version 2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * RMNET Packet Descriptor Framework + * + */ + +#ifndef _RMNET_DESCRIPTOR_H_ +#define _RMNET_DESCRIPTOR_H_ + +#include +#include +#include +#include "rmnet_config.h" +#include "rmnet_map.h" + +struct rmnet_frag_descriptor_pool { + struct list_head free_list; + u32 pool_size; +}; + +struct rmnet_frag_descriptor { + struct list_head list; + struct list_head sub_frags; + skb_frag_t frag; + u8 *hdr_ptr; + struct net_device *dev; + u32 hash; + __be32 tcp_seq; + __be16 ip_id; + u16 data_offset; + u16 gso_size; + u16 gso_segs; + u16 ip_len; + u16 trans_len; + u8 ip_proto; + u8 trans_proto; + u8 pkt_id; + u8 csum_valid:1, + hdrs_valid:1, + ip_id_set:1, + tcp_seq_set:1, + flush_shs:1, + reserved:3; +}; + +/* Descriptor management */ +struct rmnet_frag_descriptor * +rmnet_get_frag_descriptor(struct rmnet_port *port); +void rmnet_recycle_frag_descriptor(struct rmnet_frag_descriptor *frag_desc, + struct rmnet_port *port); +void rmnet_descriptor_add_frag(struct rmnet_port *port, struct list_head *list, + struct page *p, u32 page_offset, u32 len); +int rmnet_frag_ipv6_skip_exthdr(struct rmnet_frag_descriptor *frag_desc, + int start, u8 *nexthdrp, __be16 *fragp); + +/* QMAP command packets */ +void rmnet_frag_command(struct rmnet_map_header *qmap, struct rmnet_port *port); +int rmnet_frag_flow_command(struct rmnet_map_header *qmap, + struct rmnet_port *port, u16 pkt_len); + +/* Ingress data handlers */ +void rmnet_frag_deaggregate(skb_frag_t *frag, struct rmnet_port *port, + struct list_head *list); +void rmnet_frag_deliver(struct rmnet_frag_descriptor *frag_desc, + struct rmnet_port *port); +int rmnet_frag_process_next_hdr_packet(struct rmnet_frag_descriptor *frag_desc, + struct rmnet_port *port, + struct list_head *list, + u16 len); +void rmnet_frag_ingress_handler(struct sk_buff *skb, + struct rmnet_port *port); + +int rmnet_descriptor_init(struct rmnet_port *port); +void rmnet_descriptor_deinit(struct rmnet_port *port); + +static inline void *rmnet_frag_data_ptr(struct rmnet_frag_descriptor *frag_desc) +{ + return skb_frag_address(&frag_desc->frag); +} + +static inline void *rmnet_frag_pull(struct rmnet_frag_descriptor *frag_desc, + struct rmnet_port *port, + unsigned int size) +{ + if (size >= skb_frag_size(&frag_desc->frag)) { + rmnet_recycle_frag_descriptor(frag_desc, port); + return NULL; + } + + frag_desc->frag.bv_offset += size; + skb_frag_size_sub(&frag_desc->frag, size); + + return rmnet_frag_data_ptr(frag_desc); +} + +static inline void *rmnet_frag_trim(struct rmnet_frag_descriptor *frag_desc, + struct rmnet_port *port, + unsigned int size) +{ + if (!size) { + rmnet_recycle_frag_descriptor(frag_desc, port); + return NULL; + } + + if (size < skb_frag_size(&frag_desc->frag)) + skb_frag_size_set(&frag_desc->frag, size); + + return rmnet_frag_data_ptr(frag_desc); +} + +static inline void rmnet_frag_fill(struct rmnet_frag_descriptor *frag_desc, + struct page *p, u32 page_offset, u32 len) +{ + get_page(p); + __skb_frag_set_page(&frag_desc->frag, p); + skb_frag_size_set(&frag_desc->frag, len); + frag_desc->frag.bv_offset = page_offset; +} + +static inline u8 +rmnet_frag_get_next_hdr_type(struct rmnet_frag_descriptor *frag_desc) +{ + unsigned char *data = rmnet_frag_data_ptr(frag_desc); + + data += sizeof(struct rmnet_map_header); + return ((struct rmnet_map_v5_coal_header *)data)->header_type; +} + +static inline bool +rmnet_frag_get_csum_valid(struct rmnet_frag_descriptor *frag_desc) +{ + unsigned char *data = rmnet_frag_data_ptr(frag_desc); + + data += sizeof(struct rmnet_map_header); + return ((struct rmnet_map_v5_csum_header *)data)->csum_valid_required; +} + +#endif /* _RMNET_DESCRIPTOR_H_ */ diff --git a/core/rmnet_handlers.c b/core/rmnet_handlers.c new file mode 100644 index 000000000000..a4e5d36488b9 --- /dev/null +++ b/core/rmnet_handlers.c @@ -0,0 +1,448 @@ +/* Copyright (c) 2013-2019, The Linux Foundation. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 and + * only version 2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * RMNET Data ingress/egress handler + * + */ + +#include +#include +#include +#include +#include +#include +#include +#include "rmnet_private.h" +#include "rmnet_config.h" +#include "rmnet_vnd.h" +#include "rmnet_map.h" +#include "rmnet_handlers.h" +#include "rmnet_descriptor.h" + +#include "rmnet_qmi.h" +#include "qmi_rmnet.h" + +#define RMNET_IP_VERSION_4 0x40 +#define RMNET_IP_VERSION_6 0x60 +#define CREATE_TRACE_POINTS +#include "rmnet_trace.h" + +EXPORT_TRACEPOINT_SYMBOL(rmnet_shs_low); +EXPORT_TRACEPOINT_SYMBOL(rmnet_shs_high); +EXPORT_TRACEPOINT_SYMBOL(rmnet_shs_err); +EXPORT_TRACEPOINT_SYMBOL(rmnet_shs_wq_low); +EXPORT_TRACEPOINT_SYMBOL(rmnet_shs_wq_high); +EXPORT_TRACEPOINT_SYMBOL(rmnet_shs_wq_err); +EXPORT_TRACEPOINT_SYMBOL(rmnet_perf_low); +EXPORT_TRACEPOINT_SYMBOL(rmnet_perf_high); +EXPORT_TRACEPOINT_SYMBOL(rmnet_perf_err); +EXPORT_TRACEPOINT_SYMBOL(rmnet_low); +EXPORT_TRACEPOINT_SYMBOL(rmnet_high); +EXPORT_TRACEPOINT_SYMBOL(rmnet_err); +EXPORT_TRACEPOINT_SYMBOL(rmnet_freq_update); +EXPORT_TRACEPOINT_SYMBOL(rmnet_freq_reset); +EXPORT_TRACEPOINT_SYMBOL(rmnet_freq_boost); + +/* Helper Functions */ + +void rmnet_set_skb_proto(struct sk_buff *skb) +{ + switch (rmnet_map_data_ptr(skb)[0] & 0xF0) { + case RMNET_IP_VERSION_4: + skb->protocol = htons(ETH_P_IP); + break; + case RMNET_IP_VERSION_6: + skb->protocol = htons(ETH_P_IPV6); + break; + default: + skb->protocol = htons(ETH_P_MAP); + break; + } +} +EXPORT_SYMBOL(rmnet_set_skb_proto); + +/* Shs hook handler */ +int (*rmnet_shs_skb_entry)(struct sk_buff *skb, + struct rmnet_port *port) __rcu __read_mostly; +EXPORT_SYMBOL(rmnet_shs_skb_entry); + +/* Shs hook handler for work queue*/ +int (*rmnet_shs_skb_entry_wq)(struct sk_buff *skb, + struct rmnet_port *port) __rcu __read_mostly; +EXPORT_SYMBOL(rmnet_shs_skb_entry_wq); + +/* Generic handler */ + +void +rmnet_deliver_skb(struct sk_buff *skb, struct rmnet_port *port) +{ + int (*rmnet_shs_stamp)(struct sk_buff *skb, struct rmnet_port *port); + + trace_rmnet_low(RMNET_MODULE, RMNET_DLVR_SKB, 0xDEF, 0xDEF, + 0xDEF, 0xDEF, (void *)skb, NULL); + skb_reset_transport_header(skb); + skb_reset_network_header(skb); + rmnet_vnd_rx_fixup(skb->dev, skb->len); + + skb->pkt_type = PACKET_HOST; + skb_set_mac_header(skb, 0); + + rcu_read_lock(); + rmnet_shs_stamp = rcu_dereference(rmnet_shs_skb_entry); + if (rmnet_shs_stamp) { + rmnet_shs_stamp(skb, port); + rcu_read_unlock(); + return; + } + rcu_read_unlock(); + + netif_receive_skb(skb); +} +EXPORT_SYMBOL(rmnet_deliver_skb); + +/* Important to note, port cannot be used here if it has gone stale */ +void +rmnet_deliver_skb_wq(struct sk_buff *skb, struct rmnet_port *port, + enum rmnet_packet_context ctx) +{ + int (*rmnet_shs_stamp)(struct sk_buff *skb, struct rmnet_port *port); + struct rmnet_priv *priv = netdev_priv(skb->dev); + + trace_rmnet_low(RMNET_MODULE, RMNET_DLVR_SKB, 0xDEF, 0xDEF, + 0xDEF, 0xDEF, (void *)skb, NULL); + skb_reset_transport_header(skb); + skb_reset_network_header(skb); + rmnet_vnd_rx_fixup(skb->dev, skb->len); + + skb->pkt_type = PACKET_HOST; + skb_set_mac_header(skb, 0); + + /* packets coming from work queue context due to packet flush timer + * must go through the special workqueue path in SHS driver + */ + rcu_read_lock(); + rmnet_shs_stamp = (!ctx) ? rcu_dereference(rmnet_shs_skb_entry) : + rcu_dereference(rmnet_shs_skb_entry_wq); + if (rmnet_shs_stamp) { + rmnet_shs_stamp(skb, port); + rcu_read_unlock(); + return; + } + rcu_read_unlock(); + + if (ctx == RMNET_NET_RX_CTX) + netif_receive_skb(skb); + else + gro_cells_receive(&priv->gro_cells, skb); +} +EXPORT_SYMBOL(rmnet_deliver_skb_wq); + +/* Deliver a list of skbs after undoing coalescing */ +static void rmnet_deliver_skb_list(struct sk_buff_head *head, + struct rmnet_port *port) +{ + struct sk_buff *skb; + + while ((skb = __skb_dequeue(head))) { + rmnet_set_skb_proto(skb); + rmnet_deliver_skb(skb, port); + } +} + +/* MAP handler */ + +static void +__rmnet_map_ingress_handler(struct sk_buff *skb, + struct rmnet_port *port) +{ + struct rmnet_map_header *qmap; + struct rmnet_endpoint *ep; + struct sk_buff_head list; + u16 len, pad; + u8 mux_id; + + /* We don't need the spinlock since only we touch this */ + __skb_queue_head_init(&list); + + qmap = (struct rmnet_map_header *)rmnet_map_data_ptr(skb); + if (qmap->cd_bit) { + qmi_rmnet_set_dl_msg_active(port); + if (port->data_format & RMNET_INGRESS_FORMAT_DL_MARKER) { + if (!rmnet_map_flow_command(skb, port, false)) + return; + } + + if (port->data_format & RMNET_FLAGS_INGRESS_MAP_COMMANDS) + return rmnet_map_command(skb, port); + + goto free_skb; + } + + mux_id = qmap->mux_id; + pad = qmap->pad_len; + len = ntohs(qmap->pkt_len) - pad; + + if (mux_id >= RMNET_MAX_LOGICAL_EP) + goto free_skb; + + ep = rmnet_get_endpoint(port, mux_id); + if (!ep) + goto free_skb; + + skb->dev = ep->egress_dev; + + /* Handle QMAPv5 packet */ + if (qmap->next_hdr && + (port->data_format & (RMNET_FLAGS_INGRESS_COALESCE | + RMNET_FLAGS_INGRESS_MAP_CKSUMV5))) { + if (rmnet_map_process_next_hdr_packet(skb, &list, len)) + goto free_skb; + } else { + /* We only have the main QMAP header to worry about */ + pskb_pull(skb, sizeof(*qmap)); + + rmnet_set_skb_proto(skb); + + if (port->data_format & RMNET_FLAGS_INGRESS_MAP_CKSUMV4) { + if (!rmnet_map_checksum_downlink_packet(skb, len + pad)) + skb->ip_summed = CHECKSUM_UNNECESSARY; + } + + pskb_trim(skb, len); + + /* Push the single packet onto the list */ + __skb_queue_tail(&list, skb); + } + + if (port->data_format & RMNET_INGRESS_FORMAT_PS) + qmi_rmnet_work_maybe_restart(port); + + rmnet_deliver_skb_list(&list, port); + return; + +free_skb: + kfree_skb(skb); +} + +int (*rmnet_perf_deag_entry)(struct sk_buff *skb, + struct rmnet_port *port) __rcu __read_mostly; +EXPORT_SYMBOL(rmnet_perf_deag_entry); + +static void +rmnet_map_ingress_handler(struct sk_buff *skb, + struct rmnet_port *port) +{ + struct sk_buff *skbn; + int (*rmnet_perf_core_deaggregate)(struct sk_buff *skb, + struct rmnet_port *port); + + if (skb->dev->type == ARPHRD_ETHER) { + if (pskb_expand_head(skb, ETH_HLEN, 0, GFP_KERNEL)) { + kfree_skb(skb); + return; + } + + skb_push(skb, ETH_HLEN); + } + + if (port->data_format & (RMNET_FLAGS_INGRESS_COALESCE | + RMNET_FLAGS_INGRESS_MAP_CKSUMV5)) { + if (skb_is_nonlinear(skb)) { + rmnet_frag_ingress_handler(skb, port); + return; + } + } + + /* No aggregation. Pass the frame on as is */ + if (!(port->data_format & RMNET_FLAGS_INGRESS_DEAGGREGATION)) { + __rmnet_map_ingress_handler(skb, port); + return; + } + + /* Pass off handling to rmnet_perf module, if present */ + rcu_read_lock(); + rmnet_perf_core_deaggregate = rcu_dereference(rmnet_perf_deag_entry); + if (rmnet_perf_core_deaggregate) { + rmnet_perf_core_deaggregate(skb, port); + rcu_read_unlock(); + return; + } + rcu_read_unlock(); + + /* Deaggregation and freeing of HW originating + * buffers is done within here + */ + while (skb) { + struct sk_buff *skb_frag = skb_shinfo(skb)->frag_list; + + skb_shinfo(skb)->frag_list = NULL; + while ((skbn = rmnet_map_deaggregate(skb, port)) != NULL) { + __rmnet_map_ingress_handler(skbn, port); + + if (skbn == skb) + goto next_skb; + } + + consume_skb(skb); +next_skb: + skb = skb_frag; + } +} + +static int rmnet_map_egress_handler(struct sk_buff *skb, + struct rmnet_port *port, u8 mux_id, + struct net_device *orig_dev) +{ + int required_headroom, additional_header_len, csum_type; + struct rmnet_map_header *map_header; + + additional_header_len = 0; + required_headroom = sizeof(struct rmnet_map_header); + csum_type = 0; + + if (port->data_format & RMNET_FLAGS_EGRESS_MAP_CKSUMV4) { + additional_header_len = sizeof(struct rmnet_map_ul_csum_header); + csum_type = RMNET_FLAGS_EGRESS_MAP_CKSUMV4; + } else if (port->data_format & RMNET_FLAGS_EGRESS_MAP_CKSUMV5) { + additional_header_len = sizeof(struct rmnet_map_v5_csum_header); + csum_type = RMNET_FLAGS_EGRESS_MAP_CKSUMV5; + } + + required_headroom += additional_header_len; + + if (skb_headroom(skb) < required_headroom) { + if (pskb_expand_head(skb, required_headroom, 0, GFP_ATOMIC)) + return -ENOMEM; + } + + if (csum_type) + rmnet_map_checksum_uplink_packet(skb, orig_dev, csum_type); + + map_header = rmnet_map_add_map_header(skb, additional_header_len, 0, + port); + if (!map_header) + return -ENOMEM; + + map_header->mux_id = mux_id; + + if (port->data_format & RMNET_EGRESS_FORMAT_AGGREGATION) { + int non_linear_skb; + + if (rmnet_map_tx_agg_skip(skb, required_headroom)) + goto done; + + non_linear_skb = (orig_dev->features & NETIF_F_GSO) && + skb_is_nonlinear(skb); + + if (non_linear_skb) { + if (unlikely(__skb_linearize(skb))) + goto done; + } + + rmnet_map_tx_aggregate(skb, port); + return -EINPROGRESS; + } + +done: + skb->protocol = htons(ETH_P_MAP); + return 0; +} + +static void +rmnet_bridge_handler(struct sk_buff *skb, struct net_device *bridge_dev) +{ + if (bridge_dev) { + skb->dev = bridge_dev; + dev_queue_xmit(skb); + } +} + +/* Ingress / Egress Entry Points */ + +/* Processes packet as per ingress data format for receiving device. Logical + * endpoint is determined from packet inspection. Packet is then sent to the + * egress device listed in the logical endpoint configuration. + */ +rx_handler_result_t rmnet_rx_handler(struct sk_buff **pskb) +{ + struct sk_buff *skb = *pskb; + struct rmnet_port *port; + struct net_device *dev; + + if (!skb) + goto done; + + if (skb->pkt_type == PACKET_LOOPBACK) + return RX_HANDLER_PASS; + + trace_rmnet_low(RMNET_MODULE, RMNET_RCV_FROM_PND, 0xDEF, + 0xDEF, 0xDEF, 0xDEF, NULL, NULL); + dev = skb->dev; + port = rmnet_get_port(dev); + + switch (port->rmnet_mode) { + case RMNET_EPMODE_VND: + rmnet_map_ingress_handler(skb, port); + break; + case RMNET_EPMODE_BRIDGE: + rmnet_bridge_handler(skb, port->bridge_ep); + break; + } + +done: + return RX_HANDLER_CONSUMED; +} +EXPORT_SYMBOL(rmnet_rx_handler); + +/* Modifies packet as per logical endpoint configuration and egress data format + * for egress device configured in logical endpoint. Packet is then transmitted + * on the egress device. + */ +void rmnet_egress_handler(struct sk_buff *skb) +{ + struct net_device *orig_dev; + struct rmnet_port *port; + struct rmnet_priv *priv; + u8 mux_id; + int err; + u32 skb_len; + + trace_rmnet_low(RMNET_MODULE, RMNET_TX_UL_PKT, 0xDEF, 0xDEF, 0xDEF, + 0xDEF, (void *)skb, NULL); + sk_pacing_shift_update(skb->sk, 8); + + orig_dev = skb->dev; + priv = netdev_priv(orig_dev); + skb->dev = priv->real_dev; + mux_id = priv->mux_id; + + port = rmnet_get_port(skb->dev); + if (!port) + goto drop; + + skb_len = skb->len; + err = rmnet_map_egress_handler(skb, port, mux_id, orig_dev); + if (err == -ENOMEM) + goto drop; + else if (err == -EINPROGRESS) { + rmnet_vnd_tx_fixup(orig_dev, skb_len); + return; + } + + rmnet_vnd_tx_fixup(orig_dev, skb_len); + + dev_queue_xmit(skb); + return; + +drop: + this_cpu_inc(priv->pcpu_stats->stats.tx_drops); + kfree_skb(skb); +} diff --git a/core/rmnet_handlers.h b/core/rmnet_handlers.h new file mode 100644 index 000000000000..f821b764a909 --- /dev/null +++ b/core/rmnet_handlers.h @@ -0,0 +1,35 @@ +/* Copyright (c) 2013, 2016-2017, 2019, The Linux Foundation. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 and + * only version 2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * RMNET Data ingress/egress handler + * + */ + +#ifndef _RMNET_HANDLERS_H_ +#define _RMNET_HANDLERS_H_ + +#include "rmnet_config.h" + +enum rmnet_packet_context { + RMNET_NET_RX_CTX, + RMNET_WQ_CTX, +}; + +void rmnet_egress_handler(struct sk_buff *skb); +void rmnet_deliver_skb(struct sk_buff *skb, struct rmnet_port *port); +void rmnet_deliver_skb_wq(struct sk_buff *skb, struct rmnet_port *port, + enum rmnet_packet_context ctx); +void rmnet_set_skb_proto(struct sk_buff *skb); +rx_handler_result_t _rmnet_map_ingress_handler(struct sk_buff *skb, + struct rmnet_port *port); +rx_handler_result_t rmnet_rx_handler(struct sk_buff **pskb); + +#endif /* _RMNET_HANDLERS_H_ */ diff --git a/core/rmnet_map.h b/core/rmnet_map.h new file mode 100644 index 000000000000..07164d952384 --- /dev/null +++ b/core/rmnet_map.h @@ -0,0 +1,289 @@ +/* Copyright (c) 2013-2019, The Linux Foundation. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 and + * only version 2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ + +#ifndef _RMNET_MAP_H_ +#define _RMNET_MAP_H_ + +#include +#include "rmnet_config.h" + +struct rmnet_map_control_command { + u8 command_name; + u8 cmd_type:2; + u8 reserved:6; + u16 reserved2; + u32 transaction_id; + union { + struct { + u16 ip_family:2; + u16 reserved:14; + __be16 flow_control_seq_num; + __be32 qos_id; + } flow_control; + u8 data[0]; + }; +} __aligned(1); + +enum rmnet_map_commands { + RMNET_MAP_COMMAND_NONE, + RMNET_MAP_COMMAND_FLOW_DISABLE, + RMNET_MAP_COMMAND_FLOW_ENABLE, + RMNET_MAP_COMMAND_FLOW_START = 7, + RMNET_MAP_COMMAND_FLOW_END = 8, + /* These should always be the last 2 elements */ + RMNET_MAP_COMMAND_UNKNOWN, + RMNET_MAP_COMMAND_ENUM_LENGTH +}; + +enum rmnet_map_v5_header_type { + RMNET_MAP_HEADER_TYPE_UNKNOWN, + RMNET_MAP_HEADER_TYPE_COALESCING = 0x1, + RMNET_MAP_HEADER_TYPE_CSUM_OFFLOAD = 0x2, + RMNET_MAP_HEADER_TYPE_ENUM_LENGTH +}; + +enum rmnet_map_v5_close_type { + RMNET_MAP_COAL_CLOSE_NON_COAL, + RMNET_MAP_COAL_CLOSE_IP_MISS, + RMNET_MAP_COAL_CLOSE_TRANS_MISS, + RMNET_MAP_COAL_CLOSE_HW, + RMNET_MAP_COAL_CLOSE_COAL, +}; + +enum rmnet_map_v5_close_value { + RMNET_MAP_COAL_CLOSE_HW_NL, + RMNET_MAP_COAL_CLOSE_HW_PKT, + RMNET_MAP_COAL_CLOSE_HW_BYTE, + RMNET_MAP_COAL_CLOSE_HW_TIME, + RMNET_MAP_COAL_CLOSE_HW_EVICT, +}; + +/* Main QMAP header */ +struct rmnet_map_header { + u8 pad_len:6; + u8 next_hdr:1; + u8 cd_bit:1; + u8 mux_id; + __be16 pkt_len; +} __aligned(1); + +/* QMAP v5 headers */ +struct rmnet_map_v5_csum_header { + u8 next_hdr:1; + u8 header_type:7; + u8 hw_reserved:7; + u8 csum_valid_required:1; + __be16 reserved; +} __aligned(1); + +struct rmnet_map_v5_nl_pair { + __be16 pkt_len; + u8 csum_error_bitmap; + u8 num_packets; +} __aligned(1); + +/* NLO: Number-length object */ +#define RMNET_MAP_V5_MAX_NLOS (6) +#define RMNET_MAP_V5_MAX_PACKETS (48) + +struct rmnet_map_v5_coal_header { + u8 next_hdr:1; + u8 header_type:7; + u8 reserved1:4; + u8 num_nlos:3; + u8 csum_valid:1; + u8 close_type:4; + u8 close_value:4; + u8 reserved2:4; + u8 virtual_channel_id:4; + + struct rmnet_map_v5_nl_pair nl_pairs[RMNET_MAP_V5_MAX_NLOS]; +} __aligned(1); + +/* QMAP v4 headers */ +struct rmnet_map_dl_csum_trailer { + u8 reserved1; + u8 valid:1; + u8 reserved2:7; + u16 csum_start_offset; + u16 csum_length; + __be16 csum_value; +} __aligned(1); + +struct rmnet_map_ul_csum_header { + __be16 csum_start_offset; + u16 csum_insert_offset:14; + u16 udp_ip4_ind:1; + u16 csum_enabled:1; +} __aligned(1); + +struct rmnet_map_control_command_header { + u8 command_name; + u8 cmd_type:2; + u8 reserved:5; + u8 e:1; + u16 source_id:15; + u16 ext:1; + u32 transaction_id; +} __aligned(1); + +struct rmnet_map_flow_info_le { + __be32 mux_id; + __be32 flow_id; + __be32 bytes; + __be32 pkts; +} __aligned(1); + +struct rmnet_map_flow_info_be { + u32 mux_id; + u32 flow_id; + u32 bytes; + u32 pkts; +} __aligned(1); + +struct rmnet_map_dl_ind_hdr { + union { + struct { + u32 seq; + u32 bytes; + u32 pkts; + u32 flows; + struct rmnet_map_flow_info_le flow[0]; + } le __aligned(1); + struct { + __be32 seq; + __be32 bytes; + __be32 pkts; + __be32 flows; + struct rmnet_map_flow_info_be flow[0]; + } be __aligned(1); + } __aligned(1); +} __aligned(1); + +struct rmnet_map_dl_ind_trl { + union { + __be32 seq_be; + u32 seq_le; + } __aligned(1); +} __aligned(1); + +struct rmnet_map_dl_ind { + u8 priority; + union { + void (*dl_hdr_handler)(struct rmnet_map_dl_ind_hdr *); + void (*dl_hdr_handler_v2)(struct rmnet_map_dl_ind_hdr *, + struct + rmnet_map_control_command_header *); + } __aligned(1); + union { + void (*dl_trl_handler)(struct rmnet_map_dl_ind_trl *); + void (*dl_trl_handler_v2)(struct rmnet_map_dl_ind_trl *, + struct + rmnet_map_control_command_header *); + } __aligned(1); + struct list_head list; +}; + +#define RMNET_MAP_GET_MUX_ID(Y) (((struct rmnet_map_header *) \ + (Y)->data)->mux_id) +#define RMNET_MAP_GET_CD_BIT(Y) (((struct rmnet_map_header *) \ + (Y)->data)->cd_bit) +#define RMNET_MAP_GET_PAD(Y) (((struct rmnet_map_header *) \ + (Y)->data)->pad_len) +#define RMNET_MAP_GET_CMD_START(Y) ((struct rmnet_map_control_command *) \ + ((Y)->data + \ + sizeof(struct rmnet_map_header))) +#define RMNET_MAP_GET_LENGTH(Y) (ntohs(((struct rmnet_map_header *) \ + (Y)->data)->pkt_len)) + +#define RMNET_MAP_DEAGGR_SPACING 64 +#define RMNET_MAP_DEAGGR_HEADROOM (RMNET_MAP_DEAGGR_SPACING / 2) + +#define RMNET_MAP_COMMAND_REQUEST 0 +#define RMNET_MAP_COMMAND_ACK 1 +#define RMNET_MAP_COMMAND_UNSUPPORTED 2 +#define RMNET_MAP_COMMAND_INVALID 3 + +#define RMNET_MAP_NO_PAD_BYTES 0 +#define RMNET_MAP_ADD_PAD_BYTES 1 + +static inline unsigned char *rmnet_map_data_ptr(struct sk_buff *skb) +{ + /* Nonlinear packets we receive are entirely within frag 0 */ + if (skb_is_nonlinear(skb) && skb->len == skb->data_len) + return skb_frag_address(skb_shinfo(skb)->frags); + + return skb->data; +} + +static inline struct rmnet_map_control_command * +rmnet_map_get_cmd_start(struct sk_buff *skb) +{ + unsigned char *data = rmnet_map_data_ptr(skb); + + data += sizeof(struct rmnet_map_header); + return (struct rmnet_map_control_command *)data; +} + +static inline u8 rmnet_map_get_next_hdr_type(struct sk_buff *skb) +{ + unsigned char *data = rmnet_map_data_ptr(skb); + + data += sizeof(struct rmnet_map_header); + return ((struct rmnet_map_v5_coal_header *)data)->header_type; +} + +static inline bool rmnet_map_get_csum_valid(struct sk_buff *skb) +{ + unsigned char *data = rmnet_map_data_ptr(skb); + + data += sizeof(struct rmnet_map_header); + return ((struct rmnet_map_v5_csum_header *)data)->csum_valid_required; +} + +struct sk_buff *rmnet_map_deaggregate(struct sk_buff *skb, + struct rmnet_port *port); +struct rmnet_map_header *rmnet_map_add_map_header(struct sk_buff *skb, + int hdrlen, int pad, + struct rmnet_port *port); +void rmnet_map_command(struct sk_buff *skb, struct rmnet_port *port); +int rmnet_map_checksum_downlink_packet(struct sk_buff *skb, u16 len); +void rmnet_map_checksum_uplink_packet(struct sk_buff *skb, + struct net_device *orig_dev, + int csum_type); +int rmnet_map_process_next_hdr_packet(struct sk_buff *skb, + struct sk_buff_head *list, + u16 len); +int rmnet_map_tx_agg_skip(struct sk_buff *skb, int offset); +void rmnet_map_tx_aggregate(struct sk_buff *skb, struct rmnet_port *port); +void rmnet_map_tx_aggregate_init(struct rmnet_port *port); +void rmnet_map_tx_aggregate_exit(struct rmnet_port *port); +void rmnet_map_dl_hdr_notify(struct rmnet_port *port, + struct rmnet_map_dl_ind_hdr *dl_hdr); +void rmnet_map_dl_hdr_notify_v2(struct rmnet_port *port, + struct rmnet_map_dl_ind_hdr *dl_hdr, + struct rmnet_map_control_command_header *qcmd); +void rmnet_map_dl_trl_notify(struct rmnet_port *port, + struct rmnet_map_dl_ind_trl *dltrl); +void rmnet_map_dl_trl_notify_v2(struct rmnet_port *port, + struct rmnet_map_dl_ind_trl *dltrl, + struct rmnet_map_control_command_header *qcmd); +int rmnet_map_flow_command(struct sk_buff *skb, + struct rmnet_port *port, + bool rmnet_perf); +void rmnet_map_cmd_init(struct rmnet_port *port); +int rmnet_map_dl_ind_register(struct rmnet_port *port, + struct rmnet_map_dl_ind *dl_ind); +int rmnet_map_dl_ind_deregister(struct rmnet_port *port, + struct rmnet_map_dl_ind *dl_ind); +void rmnet_map_cmd_exit(struct rmnet_port *port); +#endif /* _RMNET_MAP_H_ */ diff --git a/core/rmnet_map_command.c b/core/rmnet_map_command.c new file mode 100644 index 000000000000..ce058d2efedd --- /dev/null +++ b/core/rmnet_map_command.c @@ -0,0 +1,366 @@ +/* Copyright (c) 2013-2019, The Linux Foundation. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 and + * only version 2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ + +#include +#include "rmnet_config.h" +#include "rmnet_map.h" +#include "rmnet_private.h" +#include "rmnet_vnd.h" + +#define RMNET_DL_IND_HDR_SIZE (sizeof(struct rmnet_map_dl_ind_hdr) + \ + sizeof(struct rmnet_map_header) + \ + sizeof(struct rmnet_map_control_command_header)) + +#define RMNET_MAP_CMD_SIZE (sizeof(struct rmnet_map_header) + \ + sizeof(struct rmnet_map_control_command_header)) + +#define RMNET_DL_IND_TRL_SIZE (sizeof(struct rmnet_map_dl_ind_trl) + \ + sizeof(struct rmnet_map_header) + \ + sizeof(struct rmnet_map_control_command_header)) + +static u8 rmnet_map_do_flow_control(struct sk_buff *skb, + struct rmnet_port *port, + int enable) +{ + struct rmnet_map_header *qmap; + struct rmnet_map_control_command *cmd; + struct rmnet_endpoint *ep; + struct net_device *vnd; + u16 ip_family; + u16 fc_seq; + u32 qos_id; + u8 mux_id; + int r; + + qmap = (struct rmnet_map_header *)rmnet_map_data_ptr(skb); + mux_id = qmap->mux_id; + cmd = rmnet_map_get_cmd_start(skb); + + if (mux_id >= RMNET_MAX_LOGICAL_EP) { + kfree_skb(skb); + return RX_HANDLER_CONSUMED; + } + + ep = rmnet_get_endpoint(port, mux_id); + if (!ep) { + kfree_skb(skb); + return RX_HANDLER_CONSUMED; + } + + vnd = ep->egress_dev; + + ip_family = cmd->flow_control.ip_family; + fc_seq = ntohs(cmd->flow_control.flow_control_seq_num); + qos_id = ntohl(cmd->flow_control.qos_id); + + /* Ignore the ip family and pass the sequence number for both v4 and v6 + * sequence. User space does not support creating dedicated flows for + * the 2 protocols + */ + r = rmnet_vnd_do_flow_control(vnd, enable); + if (r) { + kfree_skb(skb); + return RMNET_MAP_COMMAND_UNSUPPORTED; + } else { + return RMNET_MAP_COMMAND_ACK; + } +} + +static void rmnet_map_send_ack(struct sk_buff *skb, + unsigned char type, + struct rmnet_port *port) +{ + struct rmnet_map_control_command *cmd; + struct net_device *dev = skb->dev; + + if (port->data_format & RMNET_FLAGS_INGRESS_MAP_CKSUMV4) + pskb_trim(skb, + skb->len - sizeof(struct rmnet_map_dl_csum_trailer)); + + skb->protocol = htons(ETH_P_MAP); + + cmd = rmnet_map_get_cmd_start(skb); + cmd->cmd_type = type & 0x03; + + netif_tx_lock(dev); + dev->netdev_ops->ndo_start_xmit(skb, dev); + netif_tx_unlock(dev); +} + +void +rmnet_map_dl_hdr_notify_v2(struct rmnet_port *port, + struct rmnet_map_dl_ind_hdr *dlhdr, + struct rmnet_map_control_command_header *qcmd) +{ + struct rmnet_map_dl_ind *tmp; + + list_for_each_entry(tmp, &port->dl_list, list) + tmp->dl_hdr_handler_v2(dlhdr, qcmd); +} + +void +rmnet_map_dl_hdr_notify(struct rmnet_port *port, + struct rmnet_map_dl_ind_hdr *dlhdr) +{ + struct rmnet_map_dl_ind *tmp; + + list_for_each_entry(tmp, &port->dl_list, list) + tmp->dl_hdr_handler(dlhdr); +} + +void +rmnet_map_dl_trl_notify_v2(struct rmnet_port *port, + struct rmnet_map_dl_ind_trl *dltrl, + struct rmnet_map_control_command_header *qcmd) +{ + struct rmnet_map_dl_ind *tmp; + + list_for_each_entry(tmp, &port->dl_list, list) + tmp->dl_trl_handler_v2(dltrl, qcmd); +} + +void +rmnet_map_dl_trl_notify(struct rmnet_port *port, + struct rmnet_map_dl_ind_trl *dltrl) +{ + struct rmnet_map_dl_ind *tmp; + + list_for_each_entry(tmp, &port->dl_list, list) + tmp->dl_trl_handler(dltrl); +} + +static void rmnet_map_process_flow_start(struct sk_buff *skb, + struct rmnet_port *port, + bool rmnet_perf) +{ + struct rmnet_map_dl_ind_hdr *dlhdr; + struct rmnet_map_control_command_header *qcmd; + u32 data_format; + bool is_dl_mark_v2; + + if (skb->len < RMNET_DL_IND_HDR_SIZE) + return; + + data_format = port->data_format; + is_dl_mark_v2 = data_format & RMNET_INGRESS_FORMAT_DL_MARKER_V2; + if (is_dl_mark_v2) { + pskb_pull(skb, sizeof(struct rmnet_map_header)); + qcmd = (struct rmnet_map_control_command_header *) + rmnet_map_data_ptr(skb); + port->stats.dl_hdr_last_ep_id = qcmd->source_id; + port->stats.dl_hdr_last_qmap_vers = qcmd->reserved; + port->stats.dl_hdr_last_trans_id = qcmd->transaction_id; + pskb_pull(skb, sizeof(struct rmnet_map_control_command_header)); + } else { + pskb_pull(skb, RMNET_MAP_CMD_SIZE); + } + + dlhdr = (struct rmnet_map_dl_ind_hdr *)rmnet_map_data_ptr(skb); + + port->stats.dl_hdr_last_seq = dlhdr->le.seq; + port->stats.dl_hdr_last_bytes = dlhdr->le.bytes; + port->stats.dl_hdr_last_pkts = dlhdr->le.pkts; + port->stats.dl_hdr_last_flows = dlhdr->le.flows; + port->stats.dl_hdr_total_bytes += port->stats.dl_hdr_last_bytes; + port->stats.dl_hdr_total_pkts += port->stats.dl_hdr_last_pkts; + port->stats.dl_hdr_count++; + + if (is_dl_mark_v2) + rmnet_map_dl_hdr_notify_v2(port, dlhdr, qcmd); + else + rmnet_map_dl_hdr_notify(port, dlhdr); + + if (rmnet_perf) { + unsigned int pull_size; + + pull_size = sizeof(struct rmnet_map_dl_ind_hdr); + if (data_format & RMNET_FLAGS_INGRESS_MAP_CKSUMV4) + pull_size += sizeof(struct rmnet_map_dl_csum_trailer); + pskb_pull(skb, pull_size); + } +} + +static void rmnet_map_process_flow_end(struct sk_buff *skb, + struct rmnet_port *port, + bool rmnet_perf) +{ + struct rmnet_map_dl_ind_trl *dltrl; + struct rmnet_map_control_command_header *qcmd; + u32 data_format; + bool is_dl_mark_v2; + + if (skb->len < RMNET_DL_IND_TRL_SIZE) + return; + + data_format = port->data_format; + is_dl_mark_v2 = data_format & RMNET_INGRESS_FORMAT_DL_MARKER_V2; + if (is_dl_mark_v2) { + pskb_pull(skb, sizeof(struct rmnet_map_header)); + qcmd = (struct rmnet_map_control_command_header *) + rmnet_map_data_ptr(skb); + pskb_pull(skb, sizeof(struct rmnet_map_control_command_header)); + } else { + pskb_pull(skb, RMNET_MAP_CMD_SIZE); + } + + dltrl = (struct rmnet_map_dl_ind_trl *)rmnet_map_data_ptr(skb); + + port->stats.dl_trl_last_seq = dltrl->seq_le; + port->stats.dl_trl_count++; + + if (is_dl_mark_v2) + rmnet_map_dl_trl_notify_v2(port, dltrl, qcmd); + else + rmnet_map_dl_trl_notify(port, dltrl); + + if (rmnet_perf) { + unsigned int pull_size; + + pull_size = sizeof(struct rmnet_map_dl_ind_trl); + if (data_format & RMNET_FLAGS_INGRESS_MAP_CKSUMV4) + pull_size += sizeof(struct rmnet_map_dl_csum_trailer); + pskb_pull(skb, pull_size); + } +} + +/* Process MAP command frame and send N/ACK message as appropriate. Message cmd + * name is decoded here and appropriate handler is called. + */ +void rmnet_map_command(struct sk_buff *skb, struct rmnet_port *port) +{ + struct rmnet_map_control_command *cmd; + unsigned char command_name; + unsigned char rc = 0; + + cmd = rmnet_map_get_cmd_start(skb); + command_name = cmd->command_name; + + switch (command_name) { + case RMNET_MAP_COMMAND_FLOW_ENABLE: + rc = rmnet_map_do_flow_control(skb, port, 1); + break; + + case RMNET_MAP_COMMAND_FLOW_DISABLE: + rc = rmnet_map_do_flow_control(skb, port, 0); + break; + + default: + rc = RMNET_MAP_COMMAND_UNSUPPORTED; + kfree_skb(skb); + break; + } + if (rc == RMNET_MAP_COMMAND_ACK) + rmnet_map_send_ack(skb, rc, port); +} + +int rmnet_map_flow_command(struct sk_buff *skb, struct rmnet_port *port, + bool rmnet_perf) +{ + struct rmnet_map_control_command *cmd; + unsigned char command_name; + + cmd = rmnet_map_get_cmd_start(skb); + command_name = cmd->command_name; + + switch (command_name) { + case RMNET_MAP_COMMAND_FLOW_START: + rmnet_map_process_flow_start(skb, port, rmnet_perf); + break; + + case RMNET_MAP_COMMAND_FLOW_END: + rmnet_map_process_flow_end(skb, port, rmnet_perf); + break; + + default: + return 1; + } + + /* rmnet_perf module will handle the consuming */ + if (!rmnet_perf) + consume_skb(skb); + + return 0; +} +EXPORT_SYMBOL(rmnet_map_flow_command); + +void rmnet_map_cmd_exit(struct rmnet_port *port) +{ + struct rmnet_map_dl_ind *tmp, *idx; + + list_for_each_entry_safe(tmp, idx, &port->dl_list, list) + list_del_rcu(&tmp->list); +} + +void rmnet_map_cmd_init(struct rmnet_port *port) +{ + INIT_LIST_HEAD(&port->dl_list); +} + +int rmnet_map_dl_ind_register(struct rmnet_port *port, + struct rmnet_map_dl_ind *dl_ind) +{ + struct rmnet_map_dl_ind *dl_ind_iterator; + bool empty_ind_list = true; + + if (!port || !dl_ind || !dl_ind->dl_hdr_handler || + !dl_ind->dl_trl_handler) + return -EINVAL; + + list_for_each_entry_rcu(dl_ind_iterator, &port->dl_list, list) { + empty_ind_list = false; + if (dl_ind_iterator->priority < dl_ind->priority) { + if (dl_ind_iterator->list.next) { + if (dl_ind->priority + < list_entry_rcu(dl_ind_iterator->list.next, + typeof(*dl_ind_iterator), list)->priority) { + list_add_rcu(&dl_ind->list, + &dl_ind_iterator->list); + break; + } + } else { + list_add_rcu(&dl_ind->list, + &dl_ind_iterator->list); + break; + } + } else { + list_add_tail_rcu(&dl_ind->list, + &dl_ind_iterator->list); + break; + } + } + + if (empty_ind_list) + list_add_rcu(&dl_ind->list, &port->dl_list); + + return 0; +} +EXPORT_SYMBOL(rmnet_map_dl_ind_register); + +int rmnet_map_dl_ind_deregister(struct rmnet_port *port, + struct rmnet_map_dl_ind *dl_ind) +{ + struct rmnet_map_dl_ind *tmp; + + if (!port || !dl_ind) + return -EINVAL; + + list_for_each_entry(tmp, &port->dl_list, list) { + if (tmp == dl_ind) { + list_del_rcu(&dl_ind->list); + goto done; + } + } + +done: + return 0; +} +EXPORT_SYMBOL(rmnet_map_dl_ind_deregister); diff --git a/core/rmnet_map_data.c b/core/rmnet_map_data.c new file mode 100644 index 000000000000..505fcc33de2f --- /dev/null +++ b/core/rmnet_map_data.c @@ -0,0 +1,1245 @@ +/* Copyright (c) 2013-2019, The Linux Foundation. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 and + * only version 2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * RMNET Data MAP protocol + * + */ + +#include +#include +#include +#include +#include "rmnet_config.h" +#include "rmnet_map.h" +#include "rmnet_private.h" +#include "rmnet_handlers.h" + +#define RMNET_MAP_PKT_COPY_THRESHOLD 64 +#define RMNET_MAP_DEAGGR_SPACING 64 +#define RMNET_MAP_DEAGGR_HEADROOM (RMNET_MAP_DEAGGR_SPACING / 2) + +struct rmnet_map_coal_metadata { + void *ip_header; + void *trans_header; + u16 ip_len; + u16 trans_len; + u16 data_offset; + u16 data_len; + u8 ip_proto; + u8 trans_proto; + u8 pkt_id; + u8 pkt_count; +}; + +static __sum16 *rmnet_map_get_csum_field(unsigned char protocol, + const void *txporthdr) +{ + __sum16 *check = NULL; + + switch (protocol) { + case IPPROTO_TCP: + check = &(((struct tcphdr *)txporthdr)->check); + break; + + case IPPROTO_UDP: + check = &(((struct udphdr *)txporthdr)->check); + break; + + default: + check = NULL; + break; + } + + return check; +} + +static int +rmnet_map_ipv4_dl_csum_trailer(struct sk_buff *skb, + struct rmnet_map_dl_csum_trailer *csum_trailer, + struct rmnet_priv *priv) +{ + __sum16 *csum_field, csum_temp, pseudo_csum, hdr_csum, ip_payload_csum; + u16 csum_value, csum_value_final; + struct iphdr *ip4h; + void *txporthdr; + __be16 addend; + + ip4h = (struct iphdr *)rmnet_map_data_ptr(skb); + if ((ntohs(ip4h->frag_off) & IP_MF) || + ((ntohs(ip4h->frag_off) & IP_OFFSET) > 0)) { + priv->stats.csum_fragmented_pkt++; + return -EOPNOTSUPP; + } + + txporthdr = rmnet_map_data_ptr(skb) + ip4h->ihl * 4; + + csum_field = rmnet_map_get_csum_field(ip4h->protocol, txporthdr); + + if (!csum_field) { + priv->stats.csum_err_invalid_transport++; + return -EPROTONOSUPPORT; + } + + /* RFC 768 - Skip IPv4 UDP packets where sender checksum field is 0 */ + if (*csum_field == 0 && ip4h->protocol == IPPROTO_UDP) { + priv->stats.csum_skipped++; + return 0; + } + + csum_value = ~ntohs(csum_trailer->csum_value); + hdr_csum = ~ip_fast_csum(ip4h, (int)ip4h->ihl); + ip_payload_csum = csum16_sub((__force __sum16)csum_value, + (__force __be16)hdr_csum); + + pseudo_csum = ~csum_tcpudp_magic(ip4h->saddr, ip4h->daddr, + ntohs(ip4h->tot_len) - ip4h->ihl * 4, + ip4h->protocol, 0); + addend = (__force __be16)ntohs((__force __be16)pseudo_csum); + pseudo_csum = csum16_add(ip_payload_csum, addend); + + addend = (__force __be16)ntohs((__force __be16)*csum_field); + csum_temp = ~csum16_sub(pseudo_csum, addend); + csum_value_final = (__force u16)csum_temp; + + if (unlikely(csum_value_final == 0)) { + switch (ip4h->protocol) { + case IPPROTO_UDP: + /* RFC 768 - DL4 1's complement rule for UDP csum 0 */ + csum_value_final = ~csum_value_final; + break; + + case IPPROTO_TCP: + /* DL4 Non-RFC compliant TCP checksum found */ + if (*csum_field == (__force __sum16)0xFFFF) + csum_value_final = ~csum_value_final; + break; + } + } + + if (csum_value_final == ntohs((__force __be16)*csum_field)) { + priv->stats.csum_ok++; + return 0; + } else { + priv->stats.csum_validation_failed++; + return -EINVAL; + } +} + +#if IS_ENABLED(CONFIG_IPV6) +static int +rmnet_map_ipv6_dl_csum_trailer(struct sk_buff *skb, + struct rmnet_map_dl_csum_trailer *csum_trailer, + struct rmnet_priv *priv) +{ + __sum16 *csum_field, ip6_payload_csum, pseudo_csum, csum_temp; + u16 csum_value, csum_value_final; + __be16 ip6_hdr_csum, addend; + struct ipv6hdr *ip6h; + void *txporthdr, *data = rmnet_map_data_ptr(skb); + u32 length; + + ip6h = data; + + txporthdr = data + sizeof(struct ipv6hdr); + csum_field = rmnet_map_get_csum_field(ip6h->nexthdr, txporthdr); + + if (!csum_field) { + priv->stats.csum_err_invalid_transport++; + return -EPROTONOSUPPORT; + } + + csum_value = ~ntohs(csum_trailer->csum_value); + ip6_hdr_csum = (__force __be16) + ~ntohs((__force __be16)ip_compute_csum(ip6h, + (int)(txporthdr - data))); + ip6_payload_csum = csum16_sub((__force __sum16)csum_value, + ip6_hdr_csum); + + length = (ip6h->nexthdr == IPPROTO_UDP) ? + ntohs(((struct udphdr *)txporthdr)->len) : + ntohs(ip6h->payload_len); + pseudo_csum = ~(csum_ipv6_magic(&ip6h->saddr, &ip6h->daddr, + length, ip6h->nexthdr, 0)); + addend = (__force __be16)ntohs((__force __be16)pseudo_csum); + pseudo_csum = csum16_add(ip6_payload_csum, addend); + + addend = (__force __be16)ntohs((__force __be16)*csum_field); + csum_temp = ~csum16_sub(pseudo_csum, addend); + csum_value_final = (__force u16)csum_temp; + + if (unlikely(csum_value_final == 0)) { + switch (ip6h->nexthdr) { + case IPPROTO_UDP: + /* RFC 2460 section 8.1 + * DL6 One's complement rule for UDP checksum 0 + */ + csum_value_final = ~csum_value_final; + break; + + case IPPROTO_TCP: + /* DL6 Non-RFC compliant TCP checksum found */ + if (*csum_field == (__force __sum16)0xFFFF) + csum_value_final = ~csum_value_final; + break; + } + } + + if (csum_value_final == ntohs((__force __be16)*csum_field)) { + priv->stats.csum_ok++; + return 0; + } else { + priv->stats.csum_validation_failed++; + return -EINVAL; + } +} +#endif + +static void rmnet_map_complement_ipv4_txporthdr_csum_field(void *iphdr) +{ + struct iphdr *ip4h = (struct iphdr *)iphdr; + void *txphdr; + u16 *csum; + + txphdr = iphdr + ip4h->ihl * 4; + + if (ip4h->protocol == IPPROTO_TCP || ip4h->protocol == IPPROTO_UDP) { + csum = (u16 *)rmnet_map_get_csum_field(ip4h->protocol, txphdr); + *csum = ~(*csum); + } +} + +static void +rmnet_map_ipv4_ul_csum_header(void *iphdr, + struct rmnet_map_ul_csum_header *ul_header, + struct sk_buff *skb) +{ + struct iphdr *ip4h = (struct iphdr *)iphdr; + __be16 *hdr = (__be16 *)ul_header, offset; + + offset = htons((__force u16)(skb_transport_header(skb) - + (unsigned char *)iphdr)); + ul_header->csum_start_offset = offset; + ul_header->csum_insert_offset = skb->csum_offset; + ul_header->csum_enabled = 1; + if (ip4h->protocol == IPPROTO_UDP) + ul_header->udp_ip4_ind = 1; + else + ul_header->udp_ip4_ind = 0; + + /* Changing remaining fields to network order */ + hdr++; + *hdr = htons((__force u16)*hdr); + + skb->ip_summed = CHECKSUM_NONE; + + rmnet_map_complement_ipv4_txporthdr_csum_field(iphdr); +} + +#if IS_ENABLED(CONFIG_IPV6) +static void rmnet_map_complement_ipv6_txporthdr_csum_field(void *ip6hdr) +{ + struct ipv6hdr *ip6h = (struct ipv6hdr *)ip6hdr; + void *txphdr; + u16 *csum; + + txphdr = ip6hdr + sizeof(struct ipv6hdr); + + if (ip6h->nexthdr == IPPROTO_TCP || ip6h->nexthdr == IPPROTO_UDP) { + csum = (u16 *)rmnet_map_get_csum_field(ip6h->nexthdr, txphdr); + *csum = ~(*csum); + } +} + +static void +rmnet_map_ipv6_ul_csum_header(void *ip6hdr, + struct rmnet_map_ul_csum_header *ul_header, + struct sk_buff *skb) +{ + __be16 *hdr = (__be16 *)ul_header, offset; + + offset = htons((__force u16)(skb_transport_header(skb) - + (unsigned char *)ip6hdr)); + ul_header->csum_start_offset = offset; + ul_header->csum_insert_offset = skb->csum_offset; + ul_header->csum_enabled = 1; + ul_header->udp_ip4_ind = 0; + + /* Changing remaining fields to network order */ + hdr++; + *hdr = htons((__force u16)*hdr); + + skb->ip_summed = CHECKSUM_NONE; + + rmnet_map_complement_ipv6_txporthdr_csum_field(ip6hdr); +} +#endif + +/* Adds MAP header to front of skb->data + * Padding is calculated and set appropriately in MAP header. Mux ID is + * initialized to 0. + */ +struct rmnet_map_header *rmnet_map_add_map_header(struct sk_buff *skb, + int hdrlen, int pad, + struct rmnet_port *port) +{ + struct rmnet_map_header *map_header; + u32 padding, map_datalen; + u8 *padbytes; + + map_datalen = skb->len - hdrlen; + map_header = (struct rmnet_map_header *) + skb_push(skb, sizeof(struct rmnet_map_header)); + memset(map_header, 0, sizeof(struct rmnet_map_header)); + + /* Set next_hdr bit for csum offload packets */ + if (port->data_format & RMNET_FLAGS_EGRESS_MAP_CKSUMV5) + map_header->next_hdr = 1; + + if (pad == RMNET_MAP_NO_PAD_BYTES) { + map_header->pkt_len = htons(map_datalen); + return map_header; + } + + padding = ALIGN(map_datalen, 4) - map_datalen; + + if (padding == 0) + goto done; + + if (skb_tailroom(skb) < padding) + return NULL; + + padbytes = (u8 *)skb_put(skb, padding); + memset(padbytes, 0, padding); + +done: + map_header->pkt_len = htons(map_datalen + padding); + map_header->pad_len = padding & 0x3F; + + return map_header; +} + +/* Deaggregates a single packet + * A whole new buffer is allocated for each portion of an aggregated frame. + * Caller should keep calling deaggregate() on the source skb until 0 is + * returned, indicating that there are no more packets to deaggregate. Caller + * is responsible for freeing the original skb. + */ +struct sk_buff *rmnet_map_deaggregate(struct sk_buff *skb, + struct rmnet_port *port) +{ + struct rmnet_map_header *maph; + struct sk_buff *skbn; + unsigned char *data = rmnet_map_data_ptr(skb), *next_hdr = NULL; + u32 packet_len; + + if (skb->len == 0) + return NULL; + + maph = (struct rmnet_map_header *)data; + packet_len = ntohs(maph->pkt_len) + sizeof(struct rmnet_map_header); + + if (port->data_format & RMNET_FLAGS_INGRESS_MAP_CKSUMV4) + packet_len += sizeof(struct rmnet_map_dl_csum_trailer); + else if (port->data_format & RMNET_FLAGS_INGRESS_MAP_CKSUMV5) { + if (!maph->cd_bit) { + packet_len += sizeof(struct rmnet_map_v5_csum_header); + + /* Coalescing headers require MAPv5 */ + next_hdr = data + sizeof(*maph); + } + } + + if (((int)skb->len - (int)packet_len) < 0) + return NULL; + + /* Some hardware can send us empty frames. Catch them */ + if (ntohs(maph->pkt_len) == 0) + return NULL; + + if (next_hdr && + ((struct rmnet_map_v5_coal_header *)next_hdr)->header_type == + RMNET_MAP_HEADER_TYPE_COALESCING) + return skb; + + if (skb_is_nonlinear(skb)) { + skb_frag_t *frag0 = skb_shinfo(skb)->frags; + struct page *page = skb_frag_page(frag0); + + skbn = alloc_skb(RMNET_MAP_DEAGGR_HEADROOM, GFP_ATOMIC); + if (!skbn) + return NULL; + + skb_append_pagefrags(skbn, page, frag0->bv_offset, + packet_len); + skbn->data_len += packet_len; + skbn->len += packet_len; + } else { + skbn = alloc_skb(packet_len + RMNET_MAP_DEAGGR_SPACING, + GFP_ATOMIC); + if (!skbn) + return NULL; + + skb_reserve(skbn, RMNET_MAP_DEAGGR_HEADROOM); + skb_put(skbn, packet_len); + memcpy(skbn->data, data, packet_len); + } + + pskb_pull(skb, packet_len); + + return skbn; +} + +/* Validates packet checksums. Function takes a pointer to + * the beginning of a buffer which contains the IP payload + + * padding + checksum trailer. + * Only IPv4 and IPv6 are supported along with TCP & UDP. + * Fragmented or tunneled packets are not supported. + */ +int rmnet_map_checksum_downlink_packet(struct sk_buff *skb, u16 len) +{ + struct rmnet_priv *priv = netdev_priv(skb->dev); + struct rmnet_map_dl_csum_trailer *csum_trailer; + + if (unlikely(!(skb->dev->features & NETIF_F_RXCSUM))) { + priv->stats.csum_sw++; + return -EOPNOTSUPP; + } + + csum_trailer = (struct rmnet_map_dl_csum_trailer *) + (rmnet_map_data_ptr(skb) + len); + + if (!csum_trailer->valid) { + priv->stats.csum_valid_unset++; + return -EINVAL; + } + + if (skb->protocol == htons(ETH_P_IP)) { + return rmnet_map_ipv4_dl_csum_trailer(skb, csum_trailer, priv); + } else if (skb->protocol == htons(ETH_P_IPV6)) { +#if IS_ENABLED(CONFIG_IPV6) + return rmnet_map_ipv6_dl_csum_trailer(skb, csum_trailer, priv); +#else + priv->stats.csum_err_invalid_ip_version++; + return -EPROTONOSUPPORT; +#endif + } else { + priv->stats.csum_err_invalid_ip_version++; + return -EPROTONOSUPPORT; + } + + return 0; +} +EXPORT_SYMBOL(rmnet_map_checksum_downlink_packet); + +void rmnet_map_v4_checksum_uplink_packet(struct sk_buff *skb, + struct net_device *orig_dev) +{ + struct rmnet_priv *priv = netdev_priv(orig_dev); + struct rmnet_map_ul_csum_header *ul_header; + void *iphdr; + + ul_header = (struct rmnet_map_ul_csum_header *) + skb_push(skb, sizeof(struct rmnet_map_ul_csum_header)); + + if (unlikely(!(orig_dev->features & + (NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM)))) + goto sw_csum; + + if (skb->ip_summed == CHECKSUM_PARTIAL) { + iphdr = (char *)ul_header + + sizeof(struct rmnet_map_ul_csum_header); + + if (skb->protocol == htons(ETH_P_IP)) { + rmnet_map_ipv4_ul_csum_header(iphdr, ul_header, skb); + priv->stats.csum_hw++; + return; + } else if (skb->protocol == htons(ETH_P_IPV6)) { +#if IS_ENABLED(CONFIG_IPV6) + rmnet_map_ipv6_ul_csum_header(iphdr, ul_header, skb); + priv->stats.csum_hw++; + return; +#else + priv->stats.csum_err_invalid_ip_version++; + goto sw_csum; +#endif + } else { + priv->stats.csum_err_invalid_ip_version++; + } + } + +sw_csum: + ul_header->csum_start_offset = 0; + ul_header->csum_insert_offset = 0; + ul_header->csum_enabled = 0; + ul_header->udp_ip4_ind = 0; + + priv->stats.csum_sw++; +} + +void rmnet_map_v5_checksum_uplink_packet(struct sk_buff *skb, + struct net_device *orig_dev) +{ + struct rmnet_priv *priv = netdev_priv(orig_dev); + struct rmnet_map_v5_csum_header *ul_header; + + ul_header = (struct rmnet_map_v5_csum_header *) + skb_push(skb, sizeof(*ul_header)); + memset(ul_header, 0, sizeof(*ul_header)); + ul_header->header_type = RMNET_MAP_HEADER_TYPE_CSUM_OFFLOAD; + + if (skb->ip_summed == CHECKSUM_PARTIAL) { + void *iph = (char *)ul_header + sizeof(*ul_header); + void *trans; + __sum16 *check; + u8 proto; + + if (skb->protocol == htons(ETH_P_IP)) { + u16 ip_len = ((struct iphdr *)iph)->ihl * 4; + + proto = ((struct iphdr *)iph)->protocol; + trans = iph + ip_len; + } else if (skb->protocol == htons(ETH_P_IPV6)) { + u16 ip_len = sizeof(struct ipv6hdr); + + proto = ((struct ipv6hdr *)iph)->nexthdr; + trans = iph + ip_len; + } else { + priv->stats.csum_err_invalid_ip_version++; + goto sw_csum; + } + + check = rmnet_map_get_csum_field(proto, trans); + if (check) { + *check = 0; + skb->ip_summed = CHECKSUM_NONE; + /* Ask for checksum offloading */ + ul_header->csum_valid_required = 1; + priv->stats.csum_hw++; + return; + } + } + +sw_csum: + priv->stats.csum_sw++; +} + +/* Generates UL checksum meta info header for IPv4 and IPv6 over TCP and UDP + * packets that are supported for UL checksum offload. + */ +void rmnet_map_checksum_uplink_packet(struct sk_buff *skb, + struct net_device *orig_dev, + int csum_type) +{ + switch (csum_type) { + case RMNET_FLAGS_EGRESS_MAP_CKSUMV4: + rmnet_map_v4_checksum_uplink_packet(skb, orig_dev); + break; + case RMNET_FLAGS_EGRESS_MAP_CKSUMV5: + rmnet_map_v5_checksum_uplink_packet(skb, orig_dev); + break; + default: + break; + } +} + +static void rmnet_map_move_headers(struct sk_buff *skb) +{ + struct iphdr *iph; + u16 ip_len; + u16 trans_len = 0; + u8 proto; + + /* This only applies to non-linear SKBs */ + if (!skb_is_nonlinear(skb)) + return; + + iph = (struct iphdr *)rmnet_map_data_ptr(skb); + if (iph->version == 4) { + ip_len = iph->ihl * 4; + proto = iph->protocol; + if (iph->frag_off & htons(IP_OFFSET)) + /* No transport header information */ + goto pull; + } else if (iph->version == 6) { + struct ipv6hdr *ip6h = (struct ipv6hdr *)iph; + __be16 frag_off; + u8 nexthdr = ip6h->nexthdr; + + ip_len = ipv6_skip_exthdr(skb, sizeof(*ip6h), &nexthdr, + &frag_off); + if (ip_len < 0) + return; + + proto = nexthdr; + } else { + return; + } + + if (proto == IPPROTO_TCP) { + struct tcphdr *tp = (struct tcphdr *)((u8 *)iph + ip_len); + + trans_len = tp->doff * 4; + } else if (proto == IPPROTO_UDP) { + trans_len = sizeof(struct udphdr); + } else if (proto == NEXTHDR_FRAGMENT) { + /* Non-first fragments don't have the fragment length added by + * ipv6_skip_exthdr() and sho up as proto NEXTHDR_FRAGMENT, so + * we account for the length here. + */ + ip_len += sizeof(struct frag_hdr); + } + +pull: + __pskb_pull_tail(skb, ip_len + trans_len); + skb_reset_network_header(skb); + if (trans_len) + skb_set_transport_header(skb, ip_len); +} + +static void rmnet_map_nonlinear_copy(struct sk_buff *coal_skb, + struct rmnet_map_coal_metadata *coal_meta, + struct sk_buff *dest) +{ + unsigned char *data_start = rmnet_map_data_ptr(coal_skb) + + coal_meta->ip_len + coal_meta->trans_len; + u32 copy_len = coal_meta->data_len * coal_meta->pkt_count; + + if (skb_is_nonlinear(coal_skb)) { + skb_frag_t *frag0 = skb_shinfo(coal_skb)->frags; + struct page *page = skb_frag_page(frag0); + + skb_append_pagefrags(dest, page, + frag0->bv_offset + coal_meta->ip_len + + coal_meta->trans_len + + coal_meta->data_offset, + copy_len); + dest->data_len += copy_len; + dest->len += copy_len; + } else { + skb_put_data(dest, data_start + coal_meta->data_offset, + copy_len); + } +} + +/* Fill in GSO metadata to allow the SKB to be segmented by the NW stack + * if needed (i.e. forwarding, UDP GRO) + */ +static void rmnet_map_gso_stamp(struct sk_buff *skb, + struct rmnet_map_coal_metadata *coal_meta) +{ + struct skb_shared_info *shinfo = skb_shinfo(skb); + struct iphdr *iph = ip_hdr(skb); + __sum16 pseudo; + u16 pkt_len = skb->len - coal_meta->ip_len; + bool ipv4 = coal_meta->ip_proto == 4; + + if (ipv4) { + pseudo = ~csum_tcpudp_magic(iph->saddr, iph->daddr, + pkt_len, coal_meta->trans_proto, + 0); + } else { + struct ipv6hdr *ip6h = ipv6_hdr(skb); + + pseudo = ~csum_ipv6_magic(&ip6h->saddr, &ip6h->daddr, + pkt_len, coal_meta->trans_proto, 0); + } + + if (coal_meta->trans_proto == IPPROTO_TCP) { + struct tcphdr *tp = tcp_hdr(skb); + + tp->check = pseudo; + shinfo->gso_type = (ipv4) ? SKB_GSO_TCPV4 : SKB_GSO_TCPV6; + skb->csum_offset = offsetof(struct tcphdr, check); + } else { + struct udphdr *up = udp_hdr(skb); + + up->check = pseudo; + shinfo->gso_type = SKB_GSO_UDP_L4; + skb->csum_offset = offsetof(struct udphdr, check); + } + + skb->ip_summed = CHECKSUM_PARTIAL; + skb->csum_start = skb->data + coal_meta->ip_len - skb->head; + shinfo->gso_size = coal_meta->data_len; + shinfo->gso_segs = coal_meta->pkt_count; +} + +static void +__rmnet_map_segment_coal_skb(struct sk_buff *coal_skb, + struct rmnet_map_coal_metadata *coal_meta, + struct sk_buff_head *list, u8 pkt_id) +{ + struct sk_buff *skbn; + struct rmnet_priv *priv = netdev_priv(coal_skb->dev); + u32 alloc_len; + + /* We can avoid copying the data if the SKB we got from the lower-level + * drivers was nonlinear. + */ + if (skb_is_nonlinear(coal_skb)) + alloc_len = coal_meta->ip_len + coal_meta->trans_len; + else + alloc_len = coal_meta->ip_len + coal_meta->trans_len + + coal_meta->data_len; + + skbn = alloc_skb(alloc_len, GFP_ATOMIC); + if (!skbn) + return; + + skb_reserve(skbn, coal_meta->ip_len + coal_meta->trans_len); + rmnet_map_nonlinear_copy(coal_skb, coal_meta, skbn); + + /* Push transport header and update necessary fields */ + skb_push(skbn, coal_meta->trans_len); + memcpy(skbn->data, coal_meta->trans_header, coal_meta->trans_len); + skb_reset_transport_header(skbn); + if (coal_meta->trans_proto == IPPROTO_TCP) { + struct tcphdr *th = tcp_hdr(skbn); + + th->seq = htonl(ntohl(th->seq) + coal_meta->data_offset); + } else if (coal_meta->trans_proto == IPPROTO_UDP) { + udp_hdr(skbn)->len = htons(skbn->len); + } + + /* Push IP header and update necessary fields */ + skb_push(skbn, coal_meta->ip_len); + memcpy(skbn->data, coal_meta->ip_header, coal_meta->ip_len); + skb_reset_network_header(skbn); + if (coal_meta->ip_proto == 4) { + struct iphdr *iph = ip_hdr(skbn); + + iph->id = htons(ntohs(iph->id) + coal_meta->pkt_id); + iph->tot_len = htons(skbn->len); + iph->check = 0; + iph->check = ip_fast_csum(iph, iph->ihl); + } else { + /* Payload length includes any extension headers */ + ipv6_hdr(skbn)->payload_len = htons(skbn->len - + sizeof(struct ipv6hdr)); + } + + skbn->ip_summed = CHECKSUM_UNNECESSARY; + skbn->dev = coal_skb->dev; + priv->stats.coal.coal_reconstruct++; + + /* Stamp GSO information if necessary */ + if (coal_meta->pkt_count > 1) + rmnet_map_gso_stamp(skbn, coal_meta); + + __skb_queue_tail(list, skbn); + + /* Update meta information to move past the data we just segmented */ + coal_meta->data_offset += coal_meta->data_len * coal_meta->pkt_count; + coal_meta->pkt_id = pkt_id + 1; + coal_meta->pkt_count = 0; +} + +/* Converts the coalesced SKB into a list of SKBs. + * NLOs containing csum erros will not be included. + * The original coalesced SKB should be treated as invalid and + * must be freed by the caller + */ +static void rmnet_map_segment_coal_skb(struct sk_buff *coal_skb, + u64 nlo_err_mask, + struct sk_buff_head *list) +{ + struct iphdr *iph; + struct rmnet_priv *priv = netdev_priv(coal_skb->dev); + struct rmnet_map_v5_coal_header *coal_hdr; + struct rmnet_map_coal_metadata coal_meta; + u16 pkt_len; + u8 pkt, total_pkt = 0; + u8 nlo; + bool gro = coal_skb->dev->features & NETIF_F_GRO_HW; + + memset(&coal_meta, 0, sizeof(coal_meta)); + + /* Pull off the headers we no longer need */ + pskb_pull(coal_skb, sizeof(struct rmnet_map_header)); + coal_hdr = (struct rmnet_map_v5_coal_header *) + rmnet_map_data_ptr(coal_skb); + pskb_pull(coal_skb, sizeof(*coal_hdr)); + + iph = (struct iphdr *)rmnet_map_data_ptr(coal_skb); + + if (iph->version == 4) { + coal_meta.ip_proto = 4; + coal_meta.ip_len = iph->ihl * 4; + coal_meta.trans_proto = iph->protocol; + coal_meta.ip_header = iph; + + /* Don't allow coalescing of any packets with IP options */ + if (iph->ihl != 5) + gro = false; + } else if (iph->version == 6) { + struct ipv6hdr *ip6h = (struct ipv6hdr *)iph; + __be16 frag_off; + u8 protocol = ip6h->nexthdr; + + coal_meta.ip_proto = 6; + coal_meta.ip_len = ipv6_skip_exthdr(coal_skb, sizeof(*ip6h), + &protocol, &frag_off); + coal_meta.trans_proto = protocol; + coal_meta.ip_header = ip6h; + + /* If we run into a problem, or this has a fragment header + * (which should technically not be possible, if the HW + * works as intended...), bail. + */ + if (coal_meta.ip_len < 0 || frag_off) { + priv->stats.coal.coal_ip_invalid++; + return; + } else if (coal_meta.ip_len > sizeof(*ip6h)) { + /* Don't allow coalescing of any packets with IPv6 + * extension headers. + */ + gro = false; + } + } else { + priv->stats.coal.coal_ip_invalid++; + return; + } + + if (coal_meta.trans_proto == IPPROTO_TCP) { + struct tcphdr *th; + + th = (struct tcphdr *)((u8 *)iph + coal_meta.ip_len); + coal_meta.trans_len = th->doff * 4; + coal_meta.trans_header = th; + } else if (coal_meta.trans_proto == IPPROTO_UDP) { + struct udphdr *uh; + + uh = (struct udphdr *)((u8 *)iph + coal_meta.ip_len); + coal_meta.trans_len = sizeof(*uh); + coal_meta.trans_header = uh; + } else { + priv->stats.coal.coal_trans_invalid++; + return; + } + + for (nlo = 0; nlo < coal_hdr->num_nlos; nlo++) { + pkt_len = ntohs(coal_hdr->nl_pairs[nlo].pkt_len); + pkt_len -= coal_meta.ip_len + coal_meta.trans_len; + coal_meta.data_len = pkt_len; + for (pkt = 0; pkt < coal_hdr->nl_pairs[nlo].num_packets; + pkt++, total_pkt++) { + nlo_err_mask <<= 1; + if (nlo_err_mask & (1ULL << 63)) { + priv->stats.coal.coal_csum_err++; + + /* Segment out the good data */ + if (gro && coal_meta.pkt_count) { + __rmnet_map_segment_coal_skb(coal_skb, + &coal_meta, + list, + total_pkt); + } + + /* skip over bad packet */ + coal_meta.data_offset += pkt_len; + coal_meta.pkt_id = total_pkt + 1; + } else { + coal_meta.pkt_count++; + + /* Segment the packet if we aren't sending the + * larger packet up the stack. + */ + if (!gro) + __rmnet_map_segment_coal_skb(coal_skb, + &coal_meta, + list, + total_pkt); + } + } + + /* If we're switching NLOs, we need to send out everything from + * the previous one, if we haven't done so. NLOs only switch + * when the packet length changes. + */ + if (gro && coal_meta.pkt_count) { + /* Fast forward the (hopefully) common case. + * Frames with only one NLO (i.e. one packet length) and + * no checksum errors don't need to be segmented here. + * We can just pass off the original skb. + */ + if (pkt_len * coal_meta.pkt_count == + coal_skb->len - coal_meta.ip_len - + coal_meta.trans_len) { + rmnet_map_move_headers(coal_skb); + coal_skb->ip_summed = CHECKSUM_UNNECESSARY; + if (coal_meta.pkt_count > 1) + rmnet_map_gso_stamp(coal_skb, + &coal_meta); + __skb_queue_tail(list, coal_skb); + return; + } + + __rmnet_map_segment_coal_skb(coal_skb, &coal_meta, list, + total_pkt); + } + } +} + +/* Record reason for coalescing pipe closure */ +static void rmnet_map_data_log_close_stats(struct rmnet_priv *priv, u8 type, + u8 code) +{ + struct rmnet_coal_close_stats *stats = &priv->stats.coal.close; + + switch (type) { + case RMNET_MAP_COAL_CLOSE_NON_COAL: + stats->non_coal++; + break; + case RMNET_MAP_COAL_CLOSE_IP_MISS: + stats->ip_miss++; + break; + case RMNET_MAP_COAL_CLOSE_TRANS_MISS: + stats->trans_miss++; + break; + case RMNET_MAP_COAL_CLOSE_HW: + switch (code) { + case RMNET_MAP_COAL_CLOSE_HW_NL: + stats->hw_nl++; + break; + case RMNET_MAP_COAL_CLOSE_HW_PKT: + stats->hw_pkt++; + break; + case RMNET_MAP_COAL_CLOSE_HW_BYTE: + stats->hw_byte++; + break; + case RMNET_MAP_COAL_CLOSE_HW_TIME: + stats->hw_time++; + break; + case RMNET_MAP_COAL_CLOSE_HW_EVICT: + stats->hw_evict++; + break; + default: + break; + } + break; + case RMNET_MAP_COAL_CLOSE_COAL: + stats->coal++; + break; + default: + break; + } +} + +/* Check if the coalesced header has any incorrect values, in which case, the + * entire coalesced skb must be dropped. Then check if there are any + * checksum issues + */ +static int rmnet_map_data_check_coal_header(struct sk_buff *skb, + u64 *nlo_err_mask) +{ + struct rmnet_map_v5_coal_header *coal_hdr; + unsigned char *data = rmnet_map_data_ptr(skb); + struct rmnet_priv *priv = netdev_priv(skb->dev); + u64 mask = 0; + int i; + u8 veid, pkts = 0; + + coal_hdr = ((struct rmnet_map_v5_coal_header *) + (data + sizeof(struct rmnet_map_header))); + veid = coal_hdr->virtual_channel_id; + + if (coal_hdr->num_nlos == 0 || + coal_hdr->num_nlos > RMNET_MAP_V5_MAX_NLOS) { + priv->stats.coal.coal_hdr_nlo_err++; + return -EINVAL; + } + + for (i = 0; i < RMNET_MAP_V5_MAX_NLOS; i++) { + /* If there is a checksum issue, we need to split + * up the skb. Rebuild the full csum error field + */ + u8 err = coal_hdr->nl_pairs[i].csum_error_bitmap; + u8 pkt = coal_hdr->nl_pairs[i].num_packets; + + mask |= ((u64)err) << (7 - i) * 8; + + /* Track total packets in frame */ + pkts += pkt; + if (pkts > RMNET_MAP_V5_MAX_PACKETS) { + priv->stats.coal.coal_hdr_pkt_err++; + return -EINVAL; + } + } + + /* Track number of packets we get inside of coalesced frames */ + priv->stats.coal.coal_pkts += pkts; + + /* Update ethtool stats */ + rmnet_map_data_log_close_stats(priv, + coal_hdr->close_type, + coal_hdr->close_value); + if (veid < RMNET_MAX_VEID) + priv->stats.coal.coal_veid[veid]++; + + *nlo_err_mask = mask; + + return 0; +} + +/* Process a QMAPv5 packet header */ +int rmnet_map_process_next_hdr_packet(struct sk_buff *skb, + struct sk_buff_head *list, + u16 len) +{ + struct rmnet_priv *priv = netdev_priv(skb->dev); + u64 nlo_err_mask; + int rc = 0; + + switch (rmnet_map_get_next_hdr_type(skb)) { + case RMNET_MAP_HEADER_TYPE_COALESCING: + priv->stats.coal.coal_rx++; + rc = rmnet_map_data_check_coal_header(skb, &nlo_err_mask); + if (rc) + return rc; + + rmnet_map_segment_coal_skb(skb, nlo_err_mask, list); + if (skb_peek(list) != skb) + consume_skb(skb); + break; + case RMNET_MAP_HEADER_TYPE_CSUM_OFFLOAD: + if (rmnet_map_get_csum_valid(skb)) { + priv->stats.csum_ok++; + skb->ip_summed = CHECKSUM_UNNECESSARY; + } else { + priv->stats.csum_valid_unset++; + } + + /* Pull unnecessary headers and move the rest to the linear + * section of the skb. + */ + pskb_pull(skb, + (sizeof(struct rmnet_map_header) + + sizeof(struct rmnet_map_v5_csum_header))); + rmnet_map_move_headers(skb); + + /* Remove padding only for csum offload packets. + * Coalesced packets should never have padding. + */ + pskb_trim(skb, len); + __skb_queue_tail(list, skb); + break; + default: + rc = -EINVAL; + break; + } + + return rc; +} + +long rmnet_agg_time_limit __read_mostly = 1000000L; +long rmnet_agg_bypass_time __read_mostly = 10000000L; + +int rmnet_map_tx_agg_skip(struct sk_buff *skb, int offset) +{ + u8 *packet_start = skb->data + offset; + int is_icmp = 0; + + if (skb->protocol == htons(ETH_P_IP)) { + struct iphdr *ip4h = (struct iphdr *)(packet_start); + + if (ip4h->protocol == IPPROTO_ICMP) + is_icmp = 1; + } else if (skb->protocol == htons(ETH_P_IPV6)) { + struct ipv6hdr *ip6h = (struct ipv6hdr *)(packet_start); + + if (ip6h->nexthdr == IPPROTO_ICMPV6) { + is_icmp = 1; + } else if (ip6h->nexthdr == NEXTHDR_FRAGMENT) { + struct frag_hdr *frag; + + frag = (struct frag_hdr *)(packet_start + + sizeof(struct ipv6hdr)); + if (frag->nexthdr == IPPROTO_ICMPV6) + is_icmp = 1; + } + } + + return is_icmp; +} + +static void rmnet_map_flush_tx_packet_work(struct work_struct *work) +{ + struct sk_buff *skb = NULL; + struct rmnet_port *port; + unsigned long flags; + + port = container_of(work, struct rmnet_port, agg_wq); + + spin_lock_irqsave(&port->agg_lock, flags); + if (likely(port->agg_state == -EINPROGRESS)) { + /* Buffer may have already been shipped out */ + if (likely(port->agg_skb)) { + skb = port->agg_skb; + port->agg_skb = NULL; + port->agg_count = 0; + memset(&port->agg_time, 0, sizeof(struct timespec)); + } + port->agg_state = 0; + } + + spin_unlock_irqrestore(&port->agg_lock, flags); + if (skb) + dev_queue_xmit(skb); +} + +enum hrtimer_restart rmnet_map_flush_tx_packet_queue(struct hrtimer *t) +{ + struct rmnet_port *port; + + port = container_of(t, struct rmnet_port, hrtimer); + + schedule_work(&port->agg_wq); + return HRTIMER_NORESTART; +} + +void rmnet_map_tx_aggregate(struct sk_buff *skb, struct rmnet_port *port) +{ + struct timespec64 diff, last; + int size, agg_count = 0; + struct sk_buff *agg_skb; + unsigned long flags; + u8 *dest_buff; + +new_packet: + spin_lock_irqsave(&port->agg_lock, flags); + memcpy(&last, &port->agg_last, sizeof(struct timespec)); + ktime_get_real_ts64(&port->agg_last); + + if (!port->agg_skb) { + /* Check to see if we should agg first. If the traffic is very + * sparse, don't aggregate. We will need to tune this later + */ + diff = timespec64_sub(port->agg_last, last); + size = port->egress_agg_params.agg_size - skb->len; + + if (diff.tv_sec > 0 || diff.tv_nsec > rmnet_agg_bypass_time || + size <= 0) { + spin_unlock_irqrestore(&port->agg_lock, flags); + skb->protocol = htons(ETH_P_MAP); + dev_queue_xmit(skb); + return; + } + + port->agg_skb = skb_copy_expand(skb, 0, size, GFP_ATOMIC); + if (!port->agg_skb) { + port->agg_skb = 0; + port->agg_count = 0; + memset(&port->agg_time, 0, sizeof(struct timespec)); + spin_unlock_irqrestore(&port->agg_lock, flags); + skb->protocol = htons(ETH_P_MAP); + dev_queue_xmit(skb); + return; + } + port->agg_skb->protocol = htons(ETH_P_MAP); + port->agg_count = 1; + ktime_get_real_ts64(&port->agg_time); + dev_kfree_skb_any(skb); + goto schedule; + } + diff = timespec64_sub(port->agg_last, port->agg_time); + size = port->egress_agg_params.agg_size - port->agg_skb->len; + + if (skb->len > size || + port->agg_count >= port->egress_agg_params.agg_count || + diff.tv_sec > 0 || diff.tv_nsec > rmnet_agg_time_limit) { + agg_skb = port->agg_skb; + agg_count = port->agg_count; + port->agg_skb = 0; + port->agg_count = 0; + memset(&port->agg_time, 0, sizeof(struct timespec)); + port->agg_state = 0; + spin_unlock_irqrestore(&port->agg_lock, flags); + hrtimer_cancel(&port->hrtimer); + dev_queue_xmit(agg_skb); + goto new_packet; + } + + dest_buff = skb_put(port->agg_skb, skb->len); + memcpy(dest_buff, skb->data, skb->len); + port->agg_count++; + dev_kfree_skb_any(skb); + +schedule: + if (port->agg_state != -EINPROGRESS) { + port->agg_state = -EINPROGRESS; + hrtimer_start(&port->hrtimer, + ns_to_ktime(port->egress_agg_params.agg_time), + HRTIMER_MODE_REL); + } + spin_unlock_irqrestore(&port->agg_lock, flags); +} + +void rmnet_map_tx_aggregate_init(struct rmnet_port *port) +{ + hrtimer_init(&port->hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); + port->hrtimer.function = rmnet_map_flush_tx_packet_queue; + port->egress_agg_params.agg_size = 8192; + port->egress_agg_params.agg_count = 20; + port->egress_agg_params.agg_time = 3000000; + spin_lock_init(&port->agg_lock); + + INIT_WORK(&port->agg_wq, rmnet_map_flush_tx_packet_work); +} + +void rmnet_map_tx_aggregate_exit(struct rmnet_port *port) +{ + unsigned long flags; + + hrtimer_cancel(&port->hrtimer); + cancel_work_sync(&port->agg_wq); + + spin_lock_irqsave(&port->agg_lock, flags); + if (port->agg_state == -EINPROGRESS) { + if (port->agg_skb) { + kfree_skb(port->agg_skb); + port->agg_skb = NULL; + port->agg_count = 0; + memset(&port->agg_time, 0, sizeof(struct timespec)); + } + + port->agg_state = 0; + } + + spin_unlock_irqrestore(&port->agg_lock, flags); +} + +void rmnet_map_tx_qmap_cmd(struct sk_buff *qmap_skb) +{ + struct rmnet_port *port; + struct sk_buff *agg_skb; + unsigned long flags; + + port = rmnet_get_port(qmap_skb->dev); + + if (port->data_format & RMNET_EGRESS_FORMAT_AGGREGATION) { + spin_lock_irqsave(&port->agg_lock, flags); + if (port->agg_skb) { + agg_skb = port->agg_skb; + port->agg_skb = 0; + port->agg_count = 0; + memset(&port->agg_time, 0, sizeof(struct timespec)); + port->agg_state = 0; + spin_unlock_irqrestore(&port->agg_lock, flags); + hrtimer_cancel(&port->hrtimer); + dev_queue_xmit(agg_skb); + } else { + spin_unlock_irqrestore(&port->agg_lock, flags); + } + } + + dev_queue_xmit(qmap_skb); +} +EXPORT_SYMBOL(rmnet_map_tx_qmap_cmd); diff --git a/core/rmnet_private.h b/core/rmnet_private.h new file mode 100644 index 000000000000..420c65683650 --- /dev/null +++ b/core/rmnet_private.h @@ -0,0 +1,42 @@ +/* Copyright (c) 2013-2014, 2016-2019 The Linux Foundation. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 and + * only version 2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ + +#ifndef _RMNET_PRIVATE_H_ +#define _RMNET_PRIVATE_H_ + +#define RMNET_MAX_PACKET_SIZE 16384 +#define RMNET_DFLT_PACKET_SIZE 1500 +#define RMNET_NEEDED_HEADROOM 16 +#define RMNET_TX_QUEUE_LEN 1000 + +/* Constants */ +#define RMNET_EGRESS_FORMAT_AGGREGATION BIT(31) +#define RMNET_INGRESS_FORMAT_DL_MARKER_V1 BIT(30) +#define RMNET_INGRESS_FORMAT_DL_MARKER_V2 BIT(29) + +#define RMNET_FLAGS_INGRESS_COALESCE BIT(4) +#define RMNET_FLAGS_INGRESS_MAP_CKSUMV5 BIT(5) +#define RMNET_FLAGS_EGRESS_MAP_CKSUMV5 BIT(6) + +#define RMNET_INGRESS_FORMAT_DL_MARKER (RMNET_INGRESS_FORMAT_DL_MARKER_V1 |\ +RMNET_INGRESS_FORMAT_DL_MARKER_V2) + +/* Power save feature*/ +#define RMNET_INGRESS_FORMAT_PS BIT(27) +#define RMNET_FORMAT_PS_NOTIF BIT(26) + +/* Replace skb->dev to a virtual rmnet device and pass up the stack */ +#define RMNET_EPMODE_VND (1) +/* Pass the frame directly to another device with dev_queue_xmit() */ +#define RMNET_EPMODE_BRIDGE (2) + +#endif /* _RMNET_PRIVATE_H_ */ diff --git a/core/rmnet_qmi.h b/core/rmnet_qmi.h new file mode 100644 index 000000000000..06844879ec0c --- /dev/null +++ b/core/rmnet_qmi.h @@ -0,0 +1,34 @@ +/* + * Copyright (c) 2018-2019, The Linux Foundation. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 and + * only version 2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ + +#ifndef _RMNET_QMI_H +#define _RMNET_QMI_H + +#include +#include + +void rmnet_map_tx_qmap_cmd(struct sk_buff *qmap_skb); + +void *rmnet_get_qmi_pt(void *port); +void *rmnet_get_qos_pt(struct net_device *dev); +void *rmnet_get_rmnet_port(struct net_device *dev); +struct net_device *rmnet_get_rmnet_dev(void *port, u8 mux_id); +void rmnet_reset_qmi_pt(void *port); +void rmnet_init_qmi_pt(void *port, void *qmi); +void rmnet_enable_all_flows(void *port); +bool rmnet_all_flows_enabled(void *port); +void rmnet_set_powersave_format(void *port); +void rmnet_clear_powersave_format(void *port); +void rmnet_get_packets(void *port, u64 *rx, u64 *tx); +int rmnet_get_powersave_notif(void *port); +#endif /*_RMNET_QMI_H*/ diff --git a/core/rmnet_trace.h b/core/rmnet_trace.h new file mode 100644 index 000000000000..b1ae59b30c6a --- /dev/null +++ b/core/rmnet_trace.h @@ -0,0 +1,253 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* Copyright (c) 2018-2019, The Linux Foundation. All rights reserved. + */ + +#undef TRACE_SYSTEM +#define TRACE_SYSTEM rmnet +#undef TRACE_INCLUDE_PATH +#define TRACE_INCLUDE_PATH ../../../../vendor/qcom/opensource/datarmnet/core +#define TRACE_INCLUDE_FILE rmnet_trace + +#if !defined(_TRACE_RMNET_H) || defined(TRACE_HEADER_MULTI_READ) +#define _TRACE_RMNET_H + +#include +#include + +/*****************************************************************************/ +/* Trace events for rmnet module */ +/*****************************************************************************/ +TRACE_EVENT(rmnet_xmit_skb, + + TP_PROTO(struct sk_buff *skb), + + TP_ARGS(skb), + + TP_STRUCT__entry( + __string(dev_name, skb->dev->name) + __field(unsigned int, len) + ), + + TP_fast_assign( + __assign_str(dev_name, skb->dev->name); + __entry->len = skb->len; + ), + + TP_printk("dev_name=%s len=%u", __get_str(dev_name), __entry->len) +); + +DECLARE_EVENT_CLASS + (rmnet_mod_template, + + TP_PROTO(u8 func, u8 evt, u32 uint1, u32 uint2, + u64 ulong1, u64 ulong2, void *ptr1, void *ptr2), + + TP_ARGS(func, evt, uint1, uint2, ulong1, ulong2, ptr1, ptr2), + + TP_STRUCT__entry(__field(u8, func) + __field(u8, evt) + __field(u32, uint1) + __field(u32, uint2) + __field(u64, ulong1) + __field(u64, ulong2) + __field(void *, ptr1) + __field(void *, ptr2) + ), + + TP_fast_assign(__entry->func = func; + __entry->evt = evt; + __entry->uint1 = uint1; + __entry->uint2 = uint2; + __entry->ulong1 = ulong1; + __entry->ulong2 = ulong2; + __entry->ptr1 = ptr1; + __entry->ptr2 = ptr2; + ), + +TP_printk("fun:%u ev:%u u1:%u u2:%u ul1:%llu ul2:%llu p1:0x%pK p2:0x%pK", + __entry->func, __entry->evt, + __entry->uint1, __entry->uint2, + __entry->ulong1, __entry->ulong2, + __entry->ptr1, __entry->ptr2) +) + +DEFINE_EVENT + (rmnet_mod_template, rmnet_low, + + TP_PROTO(u8 func, u8 evt, u32 uint1, u32 uint2, + u64 ulong1, u64 ulong2, void *ptr1, void *ptr2), + + TP_ARGS(func, evt, uint1, uint2, ulong1, ulong2, ptr1, ptr2) + +); + +DEFINE_EVENT + (rmnet_mod_template, rmnet_high, + + TP_PROTO(u8 func, u8 evt, u32 uint1, u32 uint2, + u64 ulong1, u64 ulong2, void *ptr1, void *ptr2), + + TP_ARGS(func, evt, uint1, uint2, ulong1, ulong2, ptr1, ptr2) + +); + +DEFINE_EVENT + (rmnet_mod_template, rmnet_err, + + TP_PROTO(u8 func, u8 evt, u32 uint1, u32 uint2, + u64 ulong1, u64 ulong2, void *ptr1, void *ptr2), + + TP_ARGS(func, evt, uint1, uint2, ulong1, ulong2, ptr1, ptr2) + +); + +/*****************************************************************************/ +/* Trace events for rmnet_perf module */ +/*****************************************************************************/ +DEFINE_EVENT + (rmnet_mod_template, rmnet_perf_low, + + TP_PROTO(u8 func, u8 evt, u32 uint1, u32 uint2, + u64 ulong1, u64 ulong2, void *ptr1, void *ptr2), + + TP_ARGS(func, evt, uint1, uint2, ulong1, ulong2, ptr1, ptr2) + +); + +DEFINE_EVENT + (rmnet_mod_template, rmnet_perf_high, + + TP_PROTO(u8 func, u8 evt, u32 uint1, u32 uint2, + u64 ulong1, u64 ulong2, void *ptr1, void *ptr2), + + TP_ARGS(func, evt, uint1, uint2, ulong1, ulong2, ptr1, ptr2) + +); + +DEFINE_EVENT + (rmnet_mod_template, rmnet_perf_err, + + TP_PROTO(u8 func, u8 evt, u32 uint1, u32 uint2, + u64 ulong1, u64 ulong2, void *ptr1, void *ptr2), + + TP_ARGS(func, evt, uint1, uint2, ulong1, ulong2, ptr1, ptr2) + +); + +/*****************************************************************************/ +/* Trace events for rmnet_shs module */ +/*****************************************************************************/ +DEFINE_EVENT + (rmnet_mod_template, rmnet_shs_low, + + TP_PROTO(u8 func, u8 evt, u32 uint1, u32 uint2, + u64 ulong1, u64 ulong2, void *ptr1, void *ptr2), + + TP_ARGS(func, evt, uint1, uint2, ulong1, ulong2, ptr1, ptr2) +); + +DEFINE_EVENT + (rmnet_mod_template, rmnet_shs_high, + + TP_PROTO(u8 func, u8 evt, u32 uint1, u32 uint2, + u64 ulong1, u64 ulong2, void *ptr1, void *ptr2), + + TP_ARGS(func, evt, uint1, uint2, ulong1, ulong2, ptr1, ptr2) +); + +DEFINE_EVENT + (rmnet_mod_template, rmnet_shs_err, + + TP_PROTO(u8 func, u8 evt, u32 uint1, u32 uint2, + u64 ulong1, u64 ulong2, void *ptr1, void *ptr2), + + TP_ARGS(func, evt, uint1, uint2, ulong1, ulong2, ptr1, ptr2) +); + +DEFINE_EVENT + (rmnet_mod_template, rmnet_shs_wq_low, + + TP_PROTO(u8 func, u8 evt, u32 uint1, u32 uint2, + u64 ulong1, u64 ulong2, void *ptr1, void *ptr2), + + TP_ARGS(func, evt, uint1, uint2, ulong1, ulong2, ptr1, ptr2) +); + +DEFINE_EVENT + (rmnet_mod_template, rmnet_shs_wq_high, + + TP_PROTO(u8 func, u8 evt, u32 uint1, u32 uint2, + u64 ulong1, u64 ulong2, void *ptr1, void *ptr2), + + TP_ARGS(func, evt, uint1, uint2, ulong1, ulong2, ptr1, ptr2) +); + +DEFINE_EVENT + (rmnet_mod_template, rmnet_shs_wq_err, + + TP_PROTO(u8 func, u8 evt, u32 uint1, u32 uint2, + u64 ulong1, u64 ulong2, void *ptr1, void *ptr2), + + TP_ARGS(func, evt, uint1, uint2, ulong1, ulong2, ptr1, ptr2) +); + +DECLARE_EVENT_CLASS + (rmnet_freq_template, + + TP_PROTO(u8 core, u32 newfreq), + + TP_ARGS(core, newfreq), + + TP_STRUCT__entry(__field(u8, core) + __field(u32, newfreq) + ), + + TP_fast_assign(__entry->core = core; + __entry->newfreq = newfreq; + ), + +TP_printk("freq policy core:%u freq floor :%u", + __entry->core, __entry->newfreq) + +); + +DEFINE_EVENT + (rmnet_freq_template, rmnet_freq_boost, + + TP_PROTO(u8 core, u32 newfreq), + + TP_ARGS(core, newfreq) +); + +DEFINE_EVENT + (rmnet_freq_template, rmnet_freq_reset, + + TP_PROTO(u8 core, u32 newfreq), + + TP_ARGS(core, newfreq) +); + +TRACE_EVENT + (rmnet_freq_update, + + TP_PROTO(u8 core, u32 lowfreq, u32 highfreq), + + TP_ARGS(core, lowfreq, highfreq), + + TP_STRUCT__entry(__field(u8, core) + __field(u32, lowfreq) + __field(u32, highfreq) + ), + + TP_fast_assign(__entry->core = core; + __entry->lowfreq = lowfreq; + __entry->highfreq = highfreq; + + ), + +TP_printk("freq policy update core:%u policy freq floor :%u freq ceil :%u", + __entry->core, __entry->lowfreq, __entry->highfreq) +); +#endif /* _TRACE_RMNET_H */ + +#include diff --git a/core/rmnet_vnd.c b/core/rmnet_vnd.c new file mode 100644 index 000000000000..c340f327d589 --- /dev/null +++ b/core/rmnet_vnd.c @@ -0,0 +1,399 @@ +/* Copyright (c) 2013-2019, The Linux Foundation. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 and + * only version 2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * + * RMNET Data virtual network driver + * + */ + +#include +#include +#include +#include +#include "rmnet_config.h" +#include "rmnet_handlers.h" +#include "rmnet_private.h" +#include "rmnet_map.h" +#include "rmnet_vnd.h" + +#include "qmi_rmnet.h" +#include "rmnet_qmi.h" +#include "rmnet_trace.h" + +/* RX/TX Fixup */ + +void rmnet_vnd_rx_fixup(struct net_device *dev, u32 skb_len) +{ + struct rmnet_priv *priv = netdev_priv(dev); + struct rmnet_pcpu_stats *pcpu_ptr; + + pcpu_ptr = this_cpu_ptr(priv->pcpu_stats); + + u64_stats_update_begin(&pcpu_ptr->syncp); + pcpu_ptr->stats.rx_pkts++; + pcpu_ptr->stats.rx_bytes += skb_len; + u64_stats_update_end(&pcpu_ptr->syncp); +} + +void rmnet_vnd_tx_fixup(struct net_device *dev, u32 skb_len) +{ + struct rmnet_priv *priv = netdev_priv(dev); + struct rmnet_pcpu_stats *pcpu_ptr; + + pcpu_ptr = this_cpu_ptr(priv->pcpu_stats); + + u64_stats_update_begin(&pcpu_ptr->syncp); + pcpu_ptr->stats.tx_pkts++; + pcpu_ptr->stats.tx_bytes += skb_len; + u64_stats_update_end(&pcpu_ptr->syncp); +} + +/* Network Device Operations */ + +static netdev_tx_t rmnet_vnd_start_xmit(struct sk_buff *skb, + struct net_device *dev) +{ + struct rmnet_priv *priv; + int ip_type; + u32 mark; + unsigned int len; + + priv = netdev_priv(dev); + if (priv->real_dev) { + ip_type = (ip_hdr(skb)->version == 4) ? + AF_INET : AF_INET6; + mark = skb->mark; + len = skb->len; + trace_rmnet_xmit_skb(skb); + rmnet_egress_handler(skb); + qmi_rmnet_burst_fc_check(dev, ip_type, mark, len); + qmi_rmnet_work_maybe_restart(rmnet_get_rmnet_port(dev)); + } else { + this_cpu_inc(priv->pcpu_stats->stats.tx_drops); + kfree_skb(skb); + } + return NETDEV_TX_OK; +} + +static int rmnet_vnd_change_mtu(struct net_device *rmnet_dev, int new_mtu) +{ + if (new_mtu < 0 || new_mtu > RMNET_MAX_PACKET_SIZE) + return -EINVAL; + + rmnet_dev->mtu = new_mtu; + return 0; +} + +static int rmnet_vnd_get_iflink(const struct net_device *dev) +{ + struct rmnet_priv *priv = netdev_priv(dev); + + return priv->real_dev->ifindex; +} + +static int rmnet_vnd_init(struct net_device *dev) +{ + struct rmnet_priv *priv = netdev_priv(dev); + int err; + + priv->pcpu_stats = alloc_percpu(struct rmnet_pcpu_stats); + if (!priv->pcpu_stats) + return -ENOMEM; + + err = gro_cells_init(&priv->gro_cells, dev); + if (err) { + free_percpu(priv->pcpu_stats); + return err; + } + + return 0; +} + +static void rmnet_vnd_uninit(struct net_device *dev) +{ + struct rmnet_priv *priv = netdev_priv(dev); + void *qos; + + gro_cells_destroy(&priv->gro_cells); + free_percpu(priv->pcpu_stats); + + qos = priv->qos_info; + RCU_INIT_POINTER(priv->qos_info, NULL); + synchronize_rcu(); + qmi_rmnet_qos_exit(dev, qos); +} + +static void rmnet_get_stats64(struct net_device *dev, + struct rtnl_link_stats64 *s) +{ + struct rmnet_priv *priv = netdev_priv(dev); + struct rmnet_vnd_stats total_stats; + struct rmnet_pcpu_stats *pcpu_ptr; + unsigned int cpu, start; + + memset(&total_stats, 0, sizeof(struct rmnet_vnd_stats)); + + for_each_possible_cpu(cpu) { + pcpu_ptr = per_cpu_ptr(priv->pcpu_stats, cpu); + + do { + start = u64_stats_fetch_begin_irq(&pcpu_ptr->syncp); + total_stats.rx_pkts += pcpu_ptr->stats.rx_pkts; + total_stats.rx_bytes += pcpu_ptr->stats.rx_bytes; + total_stats.tx_pkts += pcpu_ptr->stats.tx_pkts; + total_stats.tx_bytes += pcpu_ptr->stats.tx_bytes; + } while (u64_stats_fetch_retry_irq(&pcpu_ptr->syncp, start)); + + total_stats.tx_drops += pcpu_ptr->stats.tx_drops; + } + + s->rx_packets = total_stats.rx_pkts; + s->rx_bytes = total_stats.rx_bytes; + s->tx_packets = total_stats.tx_pkts; + s->tx_bytes = total_stats.tx_bytes; + s->tx_dropped = total_stats.tx_drops; +} + +static u16 rmnet_vnd_select_queue(struct net_device *dev, + struct sk_buff *skb, + struct net_device *sb_dev) +{ + struct rmnet_priv *priv = netdev_priv(dev); + int txq = 0; + + if (priv->real_dev) + txq = qmi_rmnet_get_queue(dev, skb); + + return (txq < dev->real_num_tx_queues) ? txq : 0; +} + +static const struct net_device_ops rmnet_vnd_ops = { + .ndo_start_xmit = rmnet_vnd_start_xmit, + .ndo_change_mtu = rmnet_vnd_change_mtu, + .ndo_get_iflink = rmnet_vnd_get_iflink, + .ndo_add_slave = rmnet_add_bridge, + .ndo_del_slave = rmnet_del_bridge, + .ndo_init = rmnet_vnd_init, + .ndo_uninit = rmnet_vnd_uninit, + .ndo_get_stats64 = rmnet_get_stats64, + .ndo_select_queue = rmnet_vnd_select_queue, +}; + +static const char rmnet_gstrings_stats[][ETH_GSTRING_LEN] = { + "Checksum ok", + "Checksum valid bit not set", + "Checksum validation failed", + "Checksum error bad buffer", + "Checksum error bad ip version", + "Checksum error bad transport", + "Checksum skipped on ip fragment", + "Checksum skipped", + "Checksum computed in software", + "Checksum computed in hardware", + "Coalescing packets received", + "Coalesced packets", + "Coalescing header NLO errors", + "Coalescing header pcount errors", + "Coalescing checksum errors", + "Coalescing packet reconstructs", + "Coalescing IP version invalid", + "Coalescing L4 header invalid", + "Coalescing close Non-coalescable", + "Coalescing close L3 mismatch", + "Coalescing close L4 mismatch", + "Coalescing close HW NLO limit", + "Coalescing close HW packet limit", + "Coalescing close HW byte limit", + "Coalescing close HW time limit", + "Coalescing close HW eviction", + "Coalescing close Coalescable", + "Coalescing packets over VEID0", + "Coalescing packets over VEID1", + "Coalescing packets over VEID2", + "Coalescing packets over VEID3", +}; + +static const char rmnet_port_gstrings_stats[][ETH_GSTRING_LEN] = { + "MAP Cmd last version", + "MAP Cmd last ep id", + "MAP Cmd last transaction id", + "DL header last seen sequence", + "DL header last seen bytes", + "DL header last seen packets", + "DL header last seen flows", + "DL header pkts received", + "DL header total bytes received", + "DL header total pkts received", + "DL trailer last seen sequence", + "DL trailer pkts received", +}; + +static void rmnet_get_strings(struct net_device *dev, u32 stringset, u8 *buf) +{ + switch (stringset) { + case ETH_SS_STATS: + memcpy(buf, &rmnet_gstrings_stats, + sizeof(rmnet_gstrings_stats)); + memcpy(buf + sizeof(rmnet_gstrings_stats), + &rmnet_port_gstrings_stats, + sizeof(rmnet_port_gstrings_stats)); + break; + } +} + +static int rmnet_get_sset_count(struct net_device *dev, int sset) +{ + switch (sset) { + case ETH_SS_STATS: + return ARRAY_SIZE(rmnet_gstrings_stats) + + ARRAY_SIZE(rmnet_port_gstrings_stats); + default: + return -EOPNOTSUPP; + } +} + +static void rmnet_get_ethtool_stats(struct net_device *dev, + struct ethtool_stats *stats, u64 *data) +{ + struct rmnet_priv *priv = netdev_priv(dev); + struct rmnet_priv_stats *st = &priv->stats; + struct rmnet_port_priv_stats *stp; + struct rmnet_port *port; + + port = rmnet_get_port(priv->real_dev); + + if (!data || !port) + return; + + stp = &port->stats; + + memcpy(data, st, ARRAY_SIZE(rmnet_gstrings_stats) * sizeof(u64)); + memcpy(data + ARRAY_SIZE(rmnet_gstrings_stats), stp, + ARRAY_SIZE(rmnet_port_gstrings_stats) * sizeof(u64)); +} + +static int rmnet_stats_reset(struct net_device *dev) +{ + struct rmnet_priv *priv = netdev_priv(dev); + struct rmnet_port_priv_stats *stp; + struct rmnet_port *port; + + port = rmnet_get_port(priv->real_dev); + if (!port) + return -EINVAL; + + stp = &port->stats; + + memset(stp, 0, sizeof(*stp)); + return 0; +} + +static const struct ethtool_ops rmnet_ethtool_ops = { + .get_ethtool_stats = rmnet_get_ethtool_stats, + .get_strings = rmnet_get_strings, + .get_sset_count = rmnet_get_sset_count, + .nway_reset = rmnet_stats_reset, +}; + +/* Called by kernel whenever a new rmnet device is created. Sets MTU, + * flags, ARP type, needed headroom, etc... + */ +void rmnet_vnd_setup(struct net_device *rmnet_dev) +{ + rmnet_dev->netdev_ops = &rmnet_vnd_ops; + rmnet_dev->mtu = RMNET_DFLT_PACKET_SIZE; + rmnet_dev->needed_headroom = RMNET_NEEDED_HEADROOM; + random_ether_addr(rmnet_dev->dev_addr); + rmnet_dev->tx_queue_len = RMNET_TX_QUEUE_LEN; + + /* Raw IP mode */ + rmnet_dev->header_ops = NULL; /* No header */ + rmnet_dev->type = ARPHRD_RAWIP; + rmnet_dev->hard_header_len = 0; + rmnet_dev->flags &= ~(IFF_BROADCAST | IFF_MULTICAST); + + rmnet_dev->needs_free_netdev = true; + rmnet_dev->ethtool_ops = &rmnet_ethtool_ops; +} + +/* Exposed API */ + +int rmnet_vnd_newlink(u8 id, struct net_device *rmnet_dev, + struct rmnet_port *port, + struct net_device *real_dev, + struct rmnet_endpoint *ep) +{ + struct rmnet_priv *priv = netdev_priv(rmnet_dev); + int rc; + + if (ep->egress_dev) + return -EINVAL; + + if (rmnet_get_endpoint(port, id)) + return -EBUSY; + + rmnet_dev->hw_features = NETIF_F_RXCSUM; + rmnet_dev->hw_features |= NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM; + rmnet_dev->hw_features |= NETIF_F_SG; + rmnet_dev->hw_features |= NETIF_F_GRO_HW; + + priv->real_dev = real_dev; + + rc = register_netdevice(rmnet_dev); + if (!rc) { + ep->egress_dev = rmnet_dev; + ep->mux_id = id; + port->nr_rmnet_devs++; + + rmnet_dev->rtnl_link_ops = &rmnet_link_ops; + + priv->mux_id = id; + priv->qos_info = qmi_rmnet_qos_init(real_dev, id); + + netdev_dbg(rmnet_dev, "rmnet dev created\n"); + } + + return rc; +} + +int rmnet_vnd_dellink(u8 id, struct rmnet_port *port, + struct rmnet_endpoint *ep) +{ + if (id >= RMNET_MAX_LOGICAL_EP || !ep->egress_dev) + return -EINVAL; + + ep->egress_dev = NULL; + port->nr_rmnet_devs--; + return 0; +} + +u8 rmnet_vnd_get_mux(struct net_device *rmnet_dev) +{ + struct rmnet_priv *priv; + + priv = netdev_priv(rmnet_dev); + return priv->mux_id; +} + +int rmnet_vnd_do_flow_control(struct net_device *rmnet_dev, int enable) +{ + netdev_dbg(rmnet_dev, "Setting VND TX queue state to %d\n", enable); + /* Although we expect similar number of enable/disable + * commands, optimize for the disable. That is more + * latency sensitive than enable + */ + if (unlikely(enable)) + netif_wake_queue(rmnet_dev); + else + netif_stop_queue(rmnet_dev); + + return 0; +} diff --git a/core/rmnet_vnd.h b/core/rmnet_vnd.h new file mode 100644 index 000000000000..8b26f7cb63f9 --- /dev/null +++ b/core/rmnet_vnd.h @@ -0,0 +1,30 @@ +/* Copyright (c) 2013-2018, The Linux Foundation. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 and + * only version 2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * RMNET Data Virtual Network Device APIs + * + */ + +#ifndef _RMNET_VND_H_ +#define _RMNET_VND_H_ + +int rmnet_vnd_do_flow_control(struct net_device *dev, int enable); +int rmnet_vnd_newlink(u8 id, struct net_device *rmnet_dev, + struct rmnet_port *port, + struct net_device *real_dev, + struct rmnet_endpoint *ep); +int rmnet_vnd_dellink(u8 id, struct rmnet_port *port, + struct rmnet_endpoint *ep); +void rmnet_vnd_rx_fixup(struct net_device *dev, u32 skb_len); +void rmnet_vnd_tx_fixup(struct net_device *dev, u32 skb_len); +u8 rmnet_vnd_get_mux(struct net_device *rmnet_dev); +void rmnet_vnd_setup(struct net_device *dev); +#endif /* _RMNET_VND_H_ */ diff --git a/core/wda.h b/core/wda.h new file mode 100644 index 000000000000..b6038ad819af --- /dev/null +++ b/core/wda.h @@ -0,0 +1,77 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* Copyright (c) 2018, The Linux Foundation. All rights reserved. + */ + +#undef TRACE_SYSTEM +#define TRACE_SYSTEM wda +#undef TRACE_INCLUDE_PATH +#define TRACE_INCLUDE_PATH ../../../../vendor/qcom/opensource/datarmnet/core +#define TRACE_INCLUDE_FILE wda + +#if !defined(_TRACE_WDA_H) || defined(TRACE_HEADER_MULTI_READ) +#define _TRACE_WDA_H + +#include + +TRACE_EVENT(wda_set_powersave_mode, + + TP_PROTO(int enable), + + TP_ARGS(enable), + + TP_STRUCT__entry( + __field(int, enable) + ), + + TP_fast_assign( + __entry->enable = enable; + ), + + TP_printk("set powersave mode to %s", + __entry->enable ? "enable" : "disable") +); + +TRACE_EVENT(wda_client_state_up, + + TP_PROTO(u32 instance, u32 ep_type, u32 iface), + + TP_ARGS(instance, ep_type, iface), + + TP_STRUCT__entry( + __field(u32, instance) + __field(u32, ep_type) + __field(u32, iface) + ), + + TP_fast_assign( + __entry->instance = instance; + __entry->ep_type = ep_type; + __entry->iface = iface; + ), + + TP_printk("Client: Connected with WDA instance=%u ep_type=%u i_id=%u", + __entry->instance, __entry->ep_type, __entry->iface) +); + +TRACE_EVENT(wda_client_state_down, + + TP_PROTO(int from_cb), + + TP_ARGS(from_cb), + + TP_STRUCT__entry( + __field(int, from_cb) + ), + + TP_fast_assign( + __entry->from_cb = from_cb; + ), + + TP_printk("Client: Connection with WDA lost Exit by callback %d", + __entry->from_cb) +); + +#endif /* _TRACE_WDA_H */ + +/* This part must be outside protection */ +#include diff --git a/core/wda_qmi.c b/core/wda_qmi.c new file mode 100644 index 000000000000..7e6bc947bbff --- /dev/null +++ b/core/wda_qmi.c @@ -0,0 +1,458 @@ +/* Copyright (c) 2018-2019, The Linux Foundation. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 and + * only version 2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + */ + +#include +#include +#include "rmnet_qmi.h" +#define CREATE_TRACE_POINTS +#include "wda.h" +#undef CREATE_TRACE_POINTS +#include "qmi_rmnet_i.h" + +struct wda_qmi_data { + void *rmnet_port; + struct workqueue_struct *wda_wq; + struct work_struct svc_arrive; + struct qmi_handle handle; + struct sockaddr_qrtr ssctl; + struct svc_info svc; + int restart_state; +}; + +static void wda_svc_config(struct work_struct *work); +/* **************************************************** */ +#define WDA_SERVICE_ID_V01 0x1A +#define WDA_SERVICE_VERS_V01 0x01 +#define WDA_TIMEOUT_JF msecs_to_jiffies(1000) + +#define QMI_WDA_SET_POWERSAVE_CONFIG_REQ_V01 0x002D +#define QMI_WDA_SET_POWERSAVE_CONFIG_RESP_V01 0x002D +#define QMI_WDA_SET_POWERSAVE_CONFIG_REQ_V01_MAX_MSG_LEN 18 +#define QMI_WDA_SET_POWERSAVE_CONFIG_RESP_V01_MAX_MSG_LEN 14 + +#define QMI_WDA_SET_POWERSAVE_MODE_REQ_V01 0x002E +#define QMI_WDA_SET_POWERSAVE_MODE_RESP_V01 0x002E +#define QMI_WDA_SET_POWERSAVE_MODE_REQ_V01_MAX_MSG_LEN 4 +#define QMI_WDA_SET_POWERSAVE_MODE_RESP_V01_MAX_MSG_LEN 7 + +enum wda_powersave_config_mask_enum_v01 { + WDA_DATA_POWERSAVE_CONFIG_MASK_ENUM_MIN_ENUM_VAL_V01 = -2147483647, + WDA_DATA_POWERSAVE_CONFIG_NOT_SUPPORTED = 0x00, + WDA_DATA_POWERSAVE_CONFIG_DL_MARKER_V01 = 0x01, + WDA_DATA_POWERSAVE_CONFIG_FLOW_CTL_V01 = 0x02, + WDA_DATA_POWERSAVE_CONFIG_ALL_MASK_V01 = 0x7FFFFFFF, + WDA_DATA_POWERSAVE_CONFIG_MASK_ENUM_MAX_ENUM_VAL_V01 = 2147483647 +}; + +struct wda_set_powersave_config_req_msg_v01 { + /* Mandatory */ + struct data_ep_id_type_v01 ep_id; + /* Optional */ + uint8_t req_data_cfg_valid; + enum wda_powersave_config_mask_enum_v01 req_data_cfg; +}; + +struct wda_set_powersave_config_resp_msg_v01 { + /* Mandatory */ + struct qmi_response_type_v01 resp; + /* Optional */ + uint8_t data_cfg_valid; + enum wda_powersave_config_mask_enum_v01 data_cfg; +}; + +struct wda_set_powersave_mode_req_msg_v01 { + /* Mandatory */ + uint8_t powersave_control_flag; +}; + +struct wda_set_powersave_mode_resp_msg_v01 { + /* Mandatory */ + struct qmi_response_type_v01 resp; +}; + +static struct qmi_elem_info wda_set_powersave_config_req_msg_v01_ei[] = { + { + .data_type = QMI_STRUCT, + .elem_len = 1, + .elem_size = sizeof(struct data_ep_id_type_v01), + .array_type = NO_ARRAY, + .tlv_type = 0x01, + .offset = offsetof(struct + wda_set_powersave_config_req_msg_v01, + ep_id), + .ei_array = data_ep_id_type_v01_ei, + }, + { + .data_type = QMI_OPT_FLAG, + .elem_len = 1, + .elem_size = sizeof(uint8_t), + .array_type = NO_ARRAY, + .tlv_type = 0x10, + .offset = offsetof(struct + wda_set_powersave_config_req_msg_v01, + req_data_cfg_valid), + .ei_array = NULL, + }, + { + .data_type = QMI_SIGNED_4_BYTE_ENUM, + .elem_len = 1, + .elem_size = sizeof(enum + wda_powersave_config_mask_enum_v01), + .array_type = NO_ARRAY, + .tlv_type = 0x10, + .offset = offsetof(struct + wda_set_powersave_config_req_msg_v01, + req_data_cfg), + .ei_array = NULL, + }, + { + .data_type = QMI_EOTI, + .array_type = NO_ARRAY, + .tlv_type = QMI_COMMON_TLV_TYPE, + }, +}; + +static struct qmi_elem_info wda_set_powersave_config_resp_msg_v01_ei[] = { + { + .data_type = QMI_STRUCT, + .elem_len = 1, + .elem_size = sizeof(struct qmi_response_type_v01), + .array_type = NO_ARRAY, + .tlv_type = 0x02, + .offset = offsetof(struct + wda_set_powersave_config_resp_msg_v01, + resp), + .ei_array = qmi_response_type_v01_ei, + }, + { + .data_type = QMI_OPT_FLAG, + .elem_len = 1, + .elem_size = sizeof(uint8_t), + .array_type = NO_ARRAY, + .tlv_type = 0x10, + .offset = offsetof(struct + wda_set_powersave_config_resp_msg_v01, + data_cfg_valid), + .ei_array = NULL, + }, + { + .data_type = QMI_SIGNED_4_BYTE_ENUM, + .elem_len = 1, + .elem_size = sizeof(enum + wda_powersave_config_mask_enum_v01), + .array_type = NO_ARRAY, + .tlv_type = 0x10, + .offset = offsetof(struct + wda_set_powersave_config_resp_msg_v01, + data_cfg), + .ei_array = NULL, + }, + { + .data_type = QMI_EOTI, + .array_type = NO_ARRAY, + .tlv_type = QMI_COMMON_TLV_TYPE, + }, +}; + +static struct qmi_elem_info wda_set_powersave_mode_req_msg_v01_ei[] = { + { + .data_type = QMI_UNSIGNED_1_BYTE, + .elem_len = 1, + .elem_size = sizeof(uint8_t), + .array_type = NO_ARRAY, + .tlv_type = 0x01, + .offset = offsetof(struct + wda_set_powersave_mode_req_msg_v01, + powersave_control_flag), + .ei_array = NULL, + }, + { + .data_type = QMI_EOTI, + .array_type = NO_ARRAY, + .tlv_type = QMI_COMMON_TLV_TYPE, + }, +}; + +static struct qmi_elem_info wda_set_powersave_mode_resp_msg_v01_ei[] = { + { + .data_type = QMI_STRUCT, + .elem_len = 1, + .elem_size = sizeof(struct qmi_response_type_v01), + .array_type = NO_ARRAY, + .tlv_type = 0x02, + .offset = offsetof(struct + wda_set_powersave_mode_resp_msg_v01, + resp), + .ei_array = qmi_response_type_v01_ei, + }, + { + .data_type = QMI_EOTI, + .array_type = NO_ARRAY, + .tlv_type = QMI_COMMON_TLV_TYPE, + }, +}; + +static int wda_set_powersave_mode_req(void *wda_data, uint8_t enable) +{ + struct wda_qmi_data *data = (struct wda_qmi_data *)wda_data; + struct wda_set_powersave_mode_resp_msg_v01 *resp; + struct wda_set_powersave_mode_req_msg_v01 *req; + struct qmi_txn txn; + int ret; + + if (!data || !data->rmnet_port) + return -EINVAL; + + req = kzalloc(sizeof(*req), GFP_ATOMIC); + if (!req) + return -ENOMEM; + + resp = kzalloc(sizeof(*resp), GFP_ATOMIC); + if (!resp) { + kfree(req); + return -ENOMEM; + } + + ret = qmi_txn_init(&data->handle, &txn, + wda_set_powersave_mode_resp_msg_v01_ei, resp); + if (ret < 0) { + pr_err("%s() Failed init for response, err: %d\n", + __func__, ret); + goto out; + } + + req->powersave_control_flag = enable; + ret = qmi_send_request(&data->handle, &data->ssctl, &txn, + QMI_WDA_SET_POWERSAVE_MODE_REQ_V01, + QMI_WDA_SET_POWERSAVE_MODE_REQ_V01_MAX_MSG_LEN, + wda_set_powersave_mode_req_msg_v01_ei, req); + if (ret < 0) { + qmi_txn_cancel(&txn); + pr_err("%s() Failed sending request, err: %d\n", + __func__, ret); + goto out; + } + + ret = qmi_txn_wait(&txn, WDA_TIMEOUT_JF); + if (ret < 0) { + pr_err("%s() Response waiting failed, err: %d\n", + __func__, ret); + } else if (resp->resp.result != QMI_RESULT_SUCCESS_V01) { + pr_err("%s() Request rejected, result: %d, err: %d\n", + __func__, resp->resp.result, resp->resp.error); + ret = -resp->resp.result; + } + +out: + kfree(resp); + kfree(req); + return ret; +} + +static int wda_set_powersave_config_req(struct qmi_handle *wda_handle) +{ + struct wda_qmi_data *data = container_of(wda_handle, + struct wda_qmi_data, handle); + struct wda_set_powersave_config_resp_msg_v01 *resp; + struct wda_set_powersave_config_req_msg_v01 *req; + struct qmi_txn txn; + int ret; + + req = kzalloc(sizeof(*req), GFP_ATOMIC); + if (!req) + return -ENOMEM; + + resp = kzalloc(sizeof(*resp), GFP_ATOMIC); + if (!resp) { + kfree(req); + return -ENOMEM; + } + + ret = qmi_txn_init(wda_handle, &txn, + wda_set_powersave_config_resp_msg_v01_ei, resp); + if (ret < 0) { + pr_err("%s() Failed init for response, err: %d\n", + __func__, ret); + goto out; + } + + req->ep_id.ep_type = data->svc.ep_type; + req->ep_id.iface_id = data->svc.iface_id; + req->req_data_cfg_valid = 1; + req->req_data_cfg = WDA_DATA_POWERSAVE_CONFIG_ALL_MASK_V01; + ret = qmi_send_request(wda_handle, &data->ssctl, &txn, + QMI_WDA_SET_POWERSAVE_CONFIG_REQ_V01, + QMI_WDA_SET_POWERSAVE_CONFIG_REQ_V01_MAX_MSG_LEN, + wda_set_powersave_config_req_msg_v01_ei, req); + if (ret < 0) { + qmi_txn_cancel(&txn); + pr_err("%s() Failed sending request, err: %d\n", __func__, ret); + goto out; + } + + ret = qmi_txn_wait(&txn, WDA_TIMEOUT_JF); + if (ret < 0) { + pr_err("%s() Response waiting failed, err: %d\n", + __func__, ret); + } else if (resp->resp.result != QMI_RESULT_SUCCESS_V01) { + pr_err("%s() Request rejected, result: %d, error: %d\n", + __func__, resp->resp.result, resp->resp.error); + ret = -resp->resp.result; + } + +out: + kfree(resp); + kfree(req); + return ret; +} + +static void wda_svc_config(struct work_struct *work) +{ + struct wda_qmi_data *data = container_of(work, struct wda_qmi_data, + svc_arrive); + struct qmi_info *qmi; + int rc; + + if (data->restart_state == 1) + return; + rc = wda_set_powersave_config_req(&data->handle); + if (rc < 0) { + pr_err("%s Failed to init service, err[%d]\n", __func__, rc); + return; + } + + if (data->restart_state == 1) + return; + while (!rtnl_trylock()) { + if (!data->restart_state) + cond_resched(); + else + return; + } + qmi = (struct qmi_info *)rmnet_get_qmi_pt(data->rmnet_port); + if (!qmi) { + rtnl_unlock(); + return; + } + + qmi->wda_pending = NULL; + qmi->wda_client = (void *)data; + trace_wda_client_state_up(data->svc.instance, + data->svc.ep_type, + data->svc.iface_id); + + rtnl_unlock(); + + pr_info("Connection established with the WDA Service\n"); +} + +static int wda_svc_arrive(struct qmi_handle *qmi, struct qmi_service *svc) +{ + struct wda_qmi_data *data = container_of(qmi, struct wda_qmi_data, + handle); + + data->ssctl.sq_family = AF_QIPCRTR; + data->ssctl.sq_node = svc->node; + data->ssctl.sq_port = svc->port; + + queue_work(data->wda_wq, &data->svc_arrive); + + return 0; +} + +static void wda_svc_exit(struct qmi_handle *qmi, struct qmi_service *svc) +{ + struct wda_qmi_data *data = container_of(qmi, struct wda_qmi_data, + handle); + + if (!data) + pr_info("%s() data is null\n", __func__); +} + +static struct qmi_ops server_ops = { + .new_server = wda_svc_arrive, + .del_server = wda_svc_exit, +}; + +int +wda_qmi_client_init(void *port, struct svc_info *psvc, struct qmi_info *qmi) +{ + struct wda_qmi_data *data; + int rc = -ENOMEM; + + if (!port || !qmi) + return -EINVAL; + + data = kzalloc(sizeof(*data), GFP_KERNEL); + if (!data) + return -ENOMEM; + + data->wda_wq = create_singlethread_workqueue("wda_wq"); + if (!data->wda_wq) { + pr_err("%s Could not create workqueue\n", __func__); + goto err0; + } + + data->rmnet_port = port; + data->restart_state = 0; + memcpy(&data->svc, psvc, sizeof(data->svc)); + INIT_WORK(&data->svc_arrive, wda_svc_config); + + rc = qmi_handle_init(&data->handle, + QMI_WDA_SET_POWERSAVE_CONFIG_RESP_V01_MAX_MSG_LEN, + &server_ops, NULL); + if (rc < 0) { + pr_err("%s: Failed qmi_handle_init, err: %d\n", __func__, rc); + goto err1; + } + + rc = qmi_add_lookup(&data->handle, WDA_SERVICE_ID_V01, + WDA_SERVICE_VERS_V01, psvc->instance); + if (rc < 0) { + pr_err("%s(): Failed qmi_add_lookup, err: %d\n", __func__, rc); + goto err2; + } + + qmi->wda_pending = (void *)data; + return 0; + +err2: + qmi_handle_release(&data->handle); +err1: + destroy_workqueue(data->wda_wq); +err0: + kfree(data); + return rc; +} + +void wda_qmi_client_exit(void *wda_data) +{ + struct wda_qmi_data *data = (struct wda_qmi_data *)wda_data; + + if (!data) { + pr_info("%s() data is null\n", __func__); + return; + } + + data->restart_state = 1; + trace_wda_client_state_down(0); + qmi_handle_release(&data->handle); + destroy_workqueue(data->wda_wq); + kfree(data); +} + +int wda_set_powersave_mode(void *wda_data, uint8_t enable) +{ + trace_wda_set_powersave_mode(enable); + return wda_set_powersave_mode_req(wda_data, enable); +} diff --git a/datarmnet_dlkm_vendor_board.mk b/datarmnet_dlkm_vendor_board.mk new file mode 100644 index 000000000000..303c6b81865b --- /dev/null +++ b/datarmnet_dlkm_vendor_board.mk @@ -0,0 +1,8 @@ +#Build rmnet core +DATA_DLKM_BOARD_PLATFORMS_LIST := lahaina +ifneq ($(TARGET_BOARD_AUTO),true) +ifeq ($(call is-board-platform-in-list,$(DATA_DLKM_BOARD_PLATFORMS_LIST)),true) +BOARD_VENDOR_KERNEL_MODULES += $(KERNEL_MODULES_OUT)/rmnet_core.ko +endif +endif + From eeb49449642daf1bdc4bb989ba16b11e69a586ed Mon Sep 17 00:00:00 2001 From: Sean Tranchetti Date: Mon, 13 Jan 2020 12:21:14 -0700 Subject: [PATCH 03/54] core: rmnet: Fastforward to 4.19 tip This brings the RmNet and DFC modules up to date with the 4.19 tip as of commit 9b38611ea527 ("rmnet: Reduce synchronize_rcu calls"). As part of this, the rmnet_ctl driver was also incorporated, using commit 4ceee3aafb7d ("rmnet_ctl: Add IPC logging and optimizations") Change-Id: Ic45d46074c7401dfed408c769cfb6462dac0d4ee Signed-off-by: Sean Tranchetti --- core/Android.mk | 13 +- core/Kbuild | 5 +- core/dfc.h | 68 +++-- core/dfc_qmap.c | 12 +- core/dfc_qmi.c | 218 +++++----------- core/qmi_rmnet.c | 545 +++++++++++++++++++++++++++------------- core/qmi_rmnet.h | 23 +- core/qmi_rmnet_i.h | 81 +++--- core/rmnet_config.c | 77 ++++-- core/rmnet_config.h | 22 +- core/rmnet_ctl.h | 66 +++++ core/rmnet_ctl_client.c | 173 +++++++++++++ core/rmnet_ctl_client.h | 29 +++ core/rmnet_ctl_mhi.c | 206 +++++++++++++++ core/rmnet_descriptor.c | 302 ++++++++++++++++------ core/rmnet_descriptor.h | 6 +- core/rmnet_genl.c | 412 ++++++++++++++++++++++++++++++ core/rmnet_genl.h | 96 +++++++ core/rmnet_handlers.c | 42 ++-- core/rmnet_handlers.h | 3 +- core/rmnet_map.h | 26 +- core/rmnet_map_data.c | 478 ++++++++++++++++++++++++++++++----- core/rmnet_private.h | 8 +- core/rmnet_qmi.h | 68 ++++- core/rmnet_vnd.c | 27 +- core/wda_qmi.c | 27 +- 26 files changed, 2433 insertions(+), 600 deletions(-) create mode 100644 core/rmnet_ctl.h create mode 100644 core/rmnet_ctl_client.c create mode 100644 core/rmnet_ctl_client.h create mode 100644 core/rmnet_ctl_mhi.c create mode 100644 core/rmnet_genl.c create mode 100644 core/rmnet_genl.h diff --git a/core/Android.mk b/core/Android.mk index 3c9aaa72ec57..b1211d666d64 100644 --- a/core/Android.mk +++ b/core/Android.mk @@ -12,7 +12,18 @@ LOCAL_CLANG :=true LOCAL_MODULE_PATH := $(KERNEL_MODULES_OUT) LOCAL_MODULE := rmnet_core.ko -LOCAL_SRC_FILES := rmnet_config.c rmnet_descriptor.c rmnet_handlers.c rmnet_map_command.c rmnet_map_data.c rmnet_vnd.c dfc_qmi.c qmi_rmnet.c wda_qmi.c +LOCAL_SRC_FILES := \ + rmnet_config.c \ + rmnet_descriptor.c \ + rmnet_genl.c \ + rmnet_handlers.c \ + rmnet_map_command.c \ + rmnet_map_data.c \ + rmnet_vnd.c \ + dfc_qmap.c \ + dfc_qmi.c \ + qmi_rmnet.c \ + wda_qmi.c RMNET_BLD_DIR := ../../vendor/qcom/opensource/datarmnet/core DLKM_DIR := $(TOP)/device/qcom/common/dlkm diff --git a/core/Kbuild b/core/Kbuild index d63e0ab4f690..e9d19b16dc90 100644 --- a/core/Kbuild +++ b/core/Kbuild @@ -1,5 +1,6 @@ obj-m += rmnet_core.o -rmnet_core-y := rmnet_config.o rmnet_handlers.o rmnet_descriptor.o rmnet_map_command.o rmnet_map_data.o rmnet_vnd.o rmnet_descriptor.o\ - qmi_rmnet.o wda_qmi.o dfc_qmi.o +rmnet_core-y := rmnet_config.o rmnet_handlers.o rmnet_descriptor.o \ + rmnet_genl.o rmnet_map_command.o rmnet_map_data.o rmnet_vnd.o\ + qmi_rmnet.o wda_qmi.o dfc_qmi.o dfc_qmap.o diff --git a/core/dfc.h b/core/dfc.h index 48a20935fb42..e962ddd0492e 100644 --- a/core/dfc.h +++ b/core/dfc.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2018-2019, The Linux Foundation. All rights reserved. +/* Copyright (c) 2018-2020, The Linux Foundation. All rights reserved. * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 and @@ -23,35 +23,25 @@ TRACE_EVENT(dfc_qmi_tc, - TP_PROTO(const char *name, u8 bearer_id, u32 flow_id, u32 grant, - int qlen, u32 tcm_handle, int enable), + TP_PROTO(const char *name, u32 txq, int enable), - TP_ARGS(name, bearer_id, flow_id, grant, qlen, tcm_handle, enable), + TP_ARGS(name, txq, enable), TP_STRUCT__entry( __string(dev_name, name) - __field(u8, bid) - __field(u32, fid) - __field(u32, grant) - __field(int, qlen) - __field(u32, tcm_handle) + __field(u32, txq) __field(int, enable) ), TP_fast_assign( __assign_str(dev_name, name); - __entry->bid = bearer_id; - __entry->fid = flow_id; - __entry->grant = grant; - __entry->qlen = qlen; - __entry->tcm_handle = tcm_handle; + __entry->txq = txq; __entry->enable = enable; ), - TP_printk("dev=%s bearer_id=%u grant=%u len=%d flow_id=%u q=%d %s", + TP_printk("dev=%s txq=%u %s", __get_str(dev_name), - __entry->bid, __entry->grant, __entry->qlen, __entry->fid, - __entry->tcm_handle, + __entry->txq, __entry->enable ? "enable" : "disable") ); @@ -93,14 +83,16 @@ TRACE_EVENT(dfc_flow_ind, TRACE_EVENT(dfc_flow_check, - TP_PROTO(const char *name, u8 bearer_id, unsigned int len, u32 grant), + TP_PROTO(const char *name, u8 bearer_id, unsigned int len, + u32 mark, u32 grant), - TP_ARGS(name, bearer_id, len, grant), + TP_ARGS(name, bearer_id, len, mark, grant), TP_STRUCT__entry( __string(dev_name, name) __field(u8, bearer_id) __field(unsigned int, len) + __field(u32, mark) __field(u32, grant) ), @@ -108,12 +100,13 @@ TRACE_EVENT(dfc_flow_check, __assign_str(dev_name, name) __entry->bearer_id = bearer_id; __entry->len = len; + __entry->mark = mark; __entry->grant = grant; ), - TP_printk("dev=%s bearer_id=%u skb_len=%u current_grant=%u", - __get_str(dev_name), - __entry->bearer_id, __entry->len, __entry->grant) + TP_printk("dev=%s bearer_id=%u skb_len=%u mark=%u current_grant=%u", + __get_str(dev_name), __entry->bearer_id, + __entry->len, __entry->mark, __entry->grant) ); TRACE_EVENT(dfc_flow_info, @@ -141,7 +134,7 @@ TRACE_EVENT(dfc_flow_info, __entry->action = add; ), - TP_printk("%s: dev=%s bearer_id=%u flow_id=%u ip_type=%d q=%d", + TP_printk("%s: dev=%s bearer_id=%u flow_id=%u ip_type=%d txq=%d", __entry->action ? "add flow" : "delete flow", __get_str(dev_name), __entry->bid, __entry->fid, __entry->ip, __entry->handle) @@ -167,8 +160,7 @@ TRACE_EVENT(dfc_client_state_up, __entry->iface = iface; ), - TP_printk("Client[%d]: Connection established with DFC Service " - "instance=%u ep_type=%u iface_id=%u", + TP_printk("DFC Client[%d] connect: instance=%u ep_type=%u iface_id=%u", __entry->idx, __entry->instance, __entry->ep_type, __entry->iface) ); @@ -189,8 +181,7 @@ TRACE_EVENT(dfc_client_state_down, __entry->from_cb = from_cb; ), - TP_printk("Client[%d]: Connection with DFC service lost. " - "Exit by callback %d", + TP_printk("DFC Client[%d] exit: callback %d", __entry->idx, __entry->from_cb) ); @@ -248,6 +239,29 @@ TRACE_EVENT(dfc_tx_link_status_ind, __entry->mid, __entry->bid) ); +TRACE_EVENT(dfc_qmap, + + TP_PROTO(const void *data, size_t len, bool in), + + TP_ARGS(data, len, in), + + TP_STRUCT__entry( + __field(bool, in) + __field(size_t, len) + __dynamic_array(u8, data, len) + ), + + TP_fast_assign( + __entry->in = in; + __entry->len = len; + memcpy(__get_dynamic_array(data), data, len); + ), + + TP_printk("%s [%s]", + __entry->in ? "<--" : "-->", + __print_hex(__get_dynamic_array(data), __entry->len)) +); + #endif /* _TRACE_DFC_H */ /* This part must be outside protection */ diff --git a/core/dfc_qmap.c b/core/dfc_qmap.c index a4b2095b9ec5..d5fe172b6f8f 100644 --- a/core/dfc_qmap.c +++ b/core/dfc_qmap.c @@ -1,14 +1,14 @@ // SPDX-License-Identifier: GPL-2.0-only /* - * Copyright (c) 2019, The Linux Foundation. All rights reserved. + * Copyright (c) 2019-2020, The Linux Foundation. All rights reserved. */ #include -#include -#include -#include -#include +#include "rmnet_ctl.h" #include "dfc_defs.h" +#include "rmnet_qmi.h" +#include "qmi_rmnet.h" +#include "dfc.h" #define QMAP_DFC_VER 1 @@ -148,6 +148,7 @@ static void dfc_qmap_send_inband_ack(struct dfc_qmi_data *dfc, skb->protocol = htons(ETH_P_MAP); skb->dev = rmnet_get_real_dev(dfc->rmnet_port); + rmnet_ctl_log_debug("TXI", skb->data, skb->len); trace_dfc_qmap(skb->data, skb->len, false); dev_queue_xmit(skb); } @@ -433,6 +434,7 @@ static void dfc_qmap_send_end_marker_cnf(struct qos_info *qos, skb->dev = qos->real_dev; /* This cmd needs to be sent in-band */ + rmnet_ctl_log_info("TXI", skb->data, skb->len); trace_dfc_qmap(skb->data, skb->len, false); rmnet_map_tx_qmap_cmd(skb); } diff --git a/core/dfc_qmi.c b/core/dfc_qmi.c index 7d3e46b849fd..e91433b12bf8 100644 --- a/core/dfc_qmi.c +++ b/core/dfc_qmi.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018-2019, The Linux Foundation. All rights reserved. + * Copyright (c) 2018-2020, The Linux Foundation. All rights reserved. * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 and @@ -11,27 +11,14 @@ * GNU General Public License for more details. */ -#include #include -#include #include "rmnet_qmi.h" #include "qmi_rmnet.h" +#include "dfc_defs.h" -#include "qmi_rmnet_i.h" +#define CREATE_TRACE_POINTS #include "dfc.h" -#define DFC_MASK_TCP_BIDIR 0x1 -#define DFC_MASK_RAT_SWITCH 0x2 -#define DFC_IS_TCP_BIDIR(r) (bool)((r) & DFC_MASK_TCP_BIDIR) -#define DFC_IS_RAT_SWITCH(r) (bool)((r) & DFC_MASK_RAT_SWITCH) - -#define DFC_IS_ANCILLARY(type) ((type) != AF_INET && (type) != AF_INET6) - -#define DFC_MAX_QOS_ID_V01 2 - -#define DFC_ACK_TYPE_DISABLE 1 -#define DFC_ACK_TYPE_THRESHOLD 2 - struct dfc_qmap_header { u8 pad_len:6; u8 reserved_bit:1; @@ -56,20 +43,6 @@ struct dfc_ack_cmd { u8 bearer_id; } __aligned(1); -struct dfc_qmi_data { - void *rmnet_port; - struct workqueue_struct *dfc_wq; - struct work_struct svc_arrive; - struct qmi_handle handle; - struct sockaddr_qrtr ssctl; - struct svc_info svc; - struct work_struct qmi_ind_work; - struct list_head qmi_ind_q; - spinlock_t qmi_ind_lock; - int index; - int restart_state; -}; - static void dfc_svc_init(struct work_struct *work); /* **************************************************** */ @@ -115,28 +88,6 @@ struct dfc_indication_register_resp_msg_v01 { struct qmi_response_type_v01 resp; }; -enum dfc_ip_type_enum_v01 { - DFC_IP_TYPE_ENUM_MIN_ENUM_VAL_V01 = -2147483647, - DFC_IPV4_TYPE_V01 = 0x4, - DFC_IPV6_TYPE_V01 = 0x6, - DFC_IP_TYPE_ENUM_MAX_ENUM_VAL_V01 = 2147483647 -}; - -struct dfc_qos_id_type_v01 { - u32 qos_id; - enum dfc_ip_type_enum_v01 ip_type; -}; - -struct dfc_flow_status_info_type_v01 { - u8 subs_id; - u8 mux_id; - u8 bearer_id; - u32 num_bytes; - u16 seq_num; - u8 qos_ids_len; - struct dfc_qos_id_type_v01 qos_ids[DFC_MAX_QOS_ID_V01]; -}; - static struct qmi_elem_info dfc_qos_id_type_v01_ei[] = { { .data_type = QMI_UNSIGNED_4_BYTE, @@ -250,13 +201,6 @@ static struct qmi_elem_info dfc_flow_status_info_type_v01_ei[] = { }, }; -struct dfc_ancillary_info_type_v01 { - u8 subs_id; - u8 mux_id; - u8 bearer_id; - u32 reserved; -}; - static struct qmi_elem_info dfc_ancillary_info_type_v01_ei[] = { { .data_type = QMI_UNSIGNED_1_BYTE, @@ -309,31 +253,6 @@ static struct qmi_elem_info dfc_ancillary_info_type_v01_ei[] = { }, }; -struct dfc_flow_status_ind_msg_v01 { - u8 flow_status_valid; - u8 flow_status_len; - struct dfc_flow_status_info_type_v01 flow_status[DFC_MAX_BEARERS_V01]; - u8 eod_ack_reqd_valid; - u8 eod_ack_reqd; - u8 ancillary_info_valid; - u8 ancillary_info_len; - struct dfc_ancillary_info_type_v01 ancillary_info[DFC_MAX_BEARERS_V01]; -}; - -struct dfc_bearer_info_type_v01 { - u8 subs_id; - u8 mux_id; - u8 bearer_id; - enum dfc_ip_type_enum_v01 ip_type; -}; - -struct dfc_tx_link_status_ind_msg_v01 { - u8 tx_status; - u8 bearer_info_valid; - u8 bearer_info_len; - struct dfc_bearer_info_type_v01 bearer_info[DFC_MAX_BEARERS_V01]; -}; - struct dfc_get_flow_status_req_msg_v01 { u8 bearer_id_list_valid; u8 bearer_id_list_len; @@ -963,6 +882,11 @@ dfc_send_ack(struct net_device *dev, u8 bearer_id, u16 seq, u8 mux_id, u8 type) if (!qos) return; + if (dfc_qmap) { + dfc_qmap_send_ack(qos, bearer_id, seq, type); + return; + } + skb = alloc_skb(data_size, GFP_ATOMIC); if (!skb) return; @@ -995,75 +919,45 @@ int dfc_bearer_flow_ctl(struct net_device *dev, struct rmnet_bearer_map *bearer, struct qos_info *qos) { - struct rmnet_flow_map *itm; - int rc = 0, qlen; - int enable; + bool enable; - enable = bearer->grant_size ? 1 : 0; + enable = bearer->grant_size ? true : false; - list_for_each_entry(itm, &qos->flow_head, list) { - if (itm->bearer_id == bearer->bearer_id) { - /* - * Do not flow disable ancillary q if ancillary is true - */ - if (bearer->tcp_bidir && enable == 0 && - DFC_IS_ANCILLARY(itm->ip_type)) - continue; + qmi_rmnet_flow_control(dev, bearer->mq_idx, enable); - qlen = qmi_rmnet_flow_control(dev, itm->tcm_handle, - enable); - trace_dfc_qmi_tc(dev->name, itm->bearer_id, - itm->flow_id, bearer->grant_size, - qlen, itm->tcm_handle, enable); - rc++; - } - } + /* Do not flow disable tcp ack q in tcp bidir */ + if (bearer->ack_mq_idx != INVALID_MQ && + (enable || !bearer->tcp_bidir)) + qmi_rmnet_flow_control(dev, bearer->ack_mq_idx, enable); - if (enable == 0 && bearer->ack_req) + if (!enable && bearer->ack_req) dfc_send_ack(dev, bearer->bearer_id, bearer->seq, qos->mux_id, DFC_ACK_TYPE_DISABLE); - return rc; + return 0; } static int dfc_all_bearer_flow_ctl(struct net_device *dev, struct qos_info *qos, u8 ack_req, u32 ancillary, struct dfc_flow_status_info_type_v01 *fc_info) { - struct rmnet_bearer_map *bearer_itm; - struct rmnet_flow_map *flow_itm; - int rc = 0, qlen; - bool enable; + struct rmnet_bearer_map *bearer; - enable = fc_info->num_bytes > 0 ? 1 : 0; + list_for_each_entry(bearer, &qos->bearer_head, list) { + bearer->grant_size = fc_info->num_bytes; + bearer->grant_thresh = + qmi_rmnet_grant_per(bearer->grant_size); + bearer->seq = fc_info->seq_num; + bearer->ack_req = ack_req; + bearer->tcp_bidir = DFC_IS_TCP_BIDIR(ancillary); + bearer->last_grant = fc_info->num_bytes; + bearer->last_seq = fc_info->seq_num; - list_for_each_entry(bearer_itm, &qos->bearer_head, list) { - bearer_itm->grant_size = fc_info->num_bytes; - bearer_itm->grant_thresh = - qmi_rmnet_grant_per(bearer_itm->grant_size); - bearer_itm->seq = fc_info->seq_num; - bearer_itm->ack_req = ack_req; - bearer_itm->tcp_bidir = DFC_IS_TCP_BIDIR(ancillary); - bearer_itm->last_grant = fc_info->num_bytes; - bearer_itm->last_seq = fc_info->seq_num; + dfc_bearer_flow_ctl(dev, bearer, qos); } - list_for_each_entry(flow_itm, &qos->flow_head, list) { - qlen = qmi_rmnet_flow_control(dev, flow_itm->tcm_handle, - enable); - trace_dfc_qmi_tc(dev->name, flow_itm->bearer_id, - flow_itm->flow_id, fc_info->num_bytes, - qlen, flow_itm->tcm_handle, enable); - rc++; - } - - if (enable == 0 && ack_req) - dfc_send_ack(dev, fc_info->bearer_id, - fc_info->seq_num, fc_info->mux_id, - DFC_ACK_TYPE_DISABLE); - - return rc; + return 0; } static int dfc_update_fc_map(struct net_device *dev, struct qos_info *qos, @@ -1075,6 +969,9 @@ static int dfc_update_fc_map(struct net_device *dev, struct qos_info *qos, bool action = false; itm = qmi_rmnet_get_bearer_map(qos, fc_info->bearer_id); + if (!itm) + itm = qmi_rmnet_get_bearer_noref(qos, fc_info->bearer_id); + if (itm) { /* The RAT switch flag indicates the start and end of * the switch. Ignore indications in between. @@ -1093,6 +990,11 @@ static int dfc_update_fc_map(struct net_device *dev, struct qos_info *qos, (itm->grant_size > 0 && fc_info->num_bytes == 0)) action = true; + /* This is needed by qmap */ + if (dfc_qmap && itm->ack_req && !ack_req && itm->grant_size) + dfc_qmap_send_ack(qos, itm->bearer_id, + itm->seq, DFC_ACK_TYPE_DISABLE); + itm->grant_size = fc_info->num_bytes; itm->grant_thresh = qmi_rmnet_grant_per(itm->grant_size); itm->seq = fc_info->seq_num; @@ -1103,17 +1005,14 @@ static int dfc_update_fc_map(struct net_device *dev, struct qos_info *qos, if (action) rc = dfc_bearer_flow_ctl(dev, itm, qos); - } else { - pr_debug("grant %u before flow activate", fc_info->num_bytes); - qos->default_grant = fc_info->num_bytes; } + return rc; } -static void dfc_do_burst_flow_control(struct dfc_qmi_data *dfc, - struct dfc_svc_ind *svc_ind) +void dfc_do_burst_flow_control(struct dfc_qmi_data *dfc, + struct dfc_flow_status_ind_msg_v01 *ind) { - struct dfc_flow_status_ind_msg_v01 *ind = &svc_ind->d.dfc_info; struct net_device *dev; struct qos_info *qos; struct dfc_flow_status_info_type_v01 *flow_status; @@ -1187,13 +1086,17 @@ static void dfc_update_tx_link_status(struct net_device *dev, if (!itm) return; + /* If no change in tx status, ignore */ + if (itm->tx_off == !tx_status) + return; + if (itm->grant_size && !tx_status) { itm->grant_size = 0; itm->tcp_bidir = false; dfc_bearer_flow_ctl(dev, itm, qos); } else if (itm->grant_size == 0 && tx_status && !itm->rat_switch) { itm->grant_size = DEFAULT_GRANT; - itm->grant_thresh = DEFAULT_GRANT; + itm->grant_thresh = qmi_rmnet_grant_per(DEFAULT_GRANT); itm->seq = 0; itm->ack_req = 0; dfc_bearer_flow_ctl(dev, itm, qos); @@ -1202,10 +1105,9 @@ static void dfc_update_tx_link_status(struct net_device *dev, itm->tx_off = !tx_status; } -static void dfc_handle_tx_link_status_ind(struct dfc_qmi_data *dfc, - struct dfc_svc_ind *svc_ind) +void dfc_handle_tx_link_status_ind(struct dfc_qmi_data *dfc, + struct dfc_tx_link_status_ind_msg_v01 *ind) { - struct dfc_tx_link_status_ind_msg_v01 *ind = &svc_ind->d.tx_status; struct net_device *dev; struct qos_info *qos; struct dfc_bearer_info_type_v01 *bearer_info; @@ -1267,10 +1169,12 @@ static void dfc_qmi_ind_work(struct work_struct *work) if (!dfc->restart_state) { if (svc_ind->msg_id == QMI_DFC_FLOW_STATUS_IND_V01) - dfc_do_burst_flow_control(dfc, svc_ind); + dfc_do_burst_flow_control( + dfc, &svc_ind->d.dfc_info); else if (svc_ind->msg_id == QMI_DFC_TX_LINK_STATUS_IND_V01) - dfc_handle_tx_link_status_ind(dfc, svc_ind); + dfc_handle_tx_link_status_ind( + dfc, &svc_ind->d.tx_status); } kfree(svc_ind); } while (1); @@ -1518,22 +1422,28 @@ void dfc_qmi_client_exit(void *dfc_data) void dfc_qmi_burst_check(struct net_device *dev, struct qos_info *qos, int ip_type, u32 mark, unsigned int len) { - struct rmnet_bearer_map *bearer; + struct rmnet_bearer_map *bearer = NULL; struct rmnet_flow_map *itm; u32 start_grant; spin_lock_bh(&qos->qos_lock); - itm = qmi_rmnet_get_flow_map(qos, mark, ip_type); - if (unlikely(!itm)) - goto out; + if (dfc_mode == DFC_MODE_MQ_NUM) { + /* Mark is mq num */ + if (likely(mark < MAX_MQ_NUM)) + bearer = qos->mq[mark].bearer; + } else { + /* Mark is flow_id */ + itm = qmi_rmnet_get_flow_map(qos, mark, ip_type); + if (likely(itm)) + bearer = itm->bearer; + } - bearer = qmi_rmnet_get_bearer_map(qos, itm->bearer_id); if (unlikely(!bearer)) goto out; trace_dfc_flow_check(dev->name, bearer->bearer_id, - len, bearer->grant_size); + len, mark, bearer->grant_size); if (!bearer->grant_size) goto out; @@ -1588,7 +1498,7 @@ void dfc_qmi_query_flow(void *dfc_data) svc_ind->d.dfc_info.flow_status_len = resp->flow_status_len; memcpy(&svc_ind->d.dfc_info.flow_status, resp->flow_status, sizeof(resp->flow_status[0]) * resp->flow_status_len); - dfc_do_burst_flow_control(data, svc_ind); + dfc_do_burst_flow_control(data, &svc_ind->d.dfc_info); done: kfree(svc_ind); diff --git a/core/qmi_rmnet.c b/core/qmi_rmnet.c index 9de6769011c8..5818ebda32ca 100644 --- a/core/qmi_rmnet.c +++ b/core/qmi_rmnet.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018-2019, The Linux Foundation. All rights reserved. + * Copyright (c) 2018-2020, The Linux Foundation. All rights reserved. * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 and @@ -15,40 +15,56 @@ #include "qmi_rmnet_i.h" #include "qmi_rmnet.h" #include "rmnet_qmi.h" +#include "dfc.h" #include #include #include -#define CREATE_TRACE_POINTS -#include "dfc.h" -#undef CREATE_TRACE_POINTS #include #include #include #include - +#include #define NLMSG_FLOW_ACTIVATE 1 #define NLMSG_FLOW_DEACTIVATE 2 #define NLMSG_CLIENT_SETUP 4 #define NLMSG_CLIENT_DELETE 5 +#define NLMSG_SCALE_FACTOR 6 +#define NLMSG_WQ_FREQUENCY 7 #define FLAG_DFC_MASK 0x000F #define FLAG_POWERSAVE_MASK 0x0010 -#define DFC_MODE_MULTIQ 2 +#define FLAG_QMAP_MASK 0x0020 + +#define FLAG_TO_MODE(f) ((f) & FLAG_DFC_MASK) + +#define DFC_SUPPORTED_MODE(m) \ + ((m) == DFC_MODE_FLOW_ID || (m) == DFC_MODE_MQ_NUM || \ + (m) == DFC_MODE_SA) + +#define FLAG_TO_QMAP(f) ((f) & FLAG_QMAP_MASK) + +int dfc_mode; +int dfc_qmap; +#define IS_ANCILLARY(type) ((type) != AF_INET && (type) != AF_INET6) unsigned int rmnet_wq_frequency __read_mostly = 1000; -module_param(rmnet_wq_frequency, uint, 0644); -MODULE_PARM_DESC(rmnet_wq_frequency, "Frequency of PS check in ms"); #define PS_WORK_ACTIVE_BIT 0 #define PS_INTERVAL (((!rmnet_wq_frequency) ? \ 1 : rmnet_wq_frequency/10) * (HZ/100)) #define NO_DELAY (0x0000 * HZ) +#define PS_INTERVAL_KT (ms_to_ktime(1000)) #ifdef CONFIG_QTI_QMI_DFC static unsigned int qmi_rmnet_scale_factor = 5; +static LIST_HEAD(qos_cleanup_list); #endif +static int +qmi_rmnet_del_flow(struct net_device *dev, struct tcmsg *tcm, + struct qmi_info *qmi); + struct qmi_elem_info data_ep_id_type_v01_ei[] = { { .data_type = QMI_SIGNED_4_BYTE_ENUM, @@ -86,7 +102,7 @@ void *qmi_rmnet_has_dfc_client(struct qmi_info *qmi) { int i; - if (!qmi || ((qmi->flag & FLAG_DFC_MASK) != DFC_MODE_MULTIQ)) + if (!qmi) return NULL; for (i = 0; i < MAX_CLIENT_NUM; i++) { @@ -124,8 +140,7 @@ qmi_rmnet_has_pending(struct qmi_info *qmi) #ifdef CONFIG_QTI_QMI_DFC static void -qmi_rmnet_clean_flow_list(struct qmi_info *qmi, struct net_device *dev, - struct qos_info *qos) +qmi_rmnet_clean_flow_list(struct qos_info *qos) { struct rmnet_bearer_map *bearer, *br_tmp; struct rmnet_flow_map *itm, *fl_tmp; @@ -141,6 +156,8 @@ qmi_rmnet_clean_flow_list(struct qmi_info *qmi, struct net_device *dev, list_del(&bearer->list); kfree(bearer); } + + memset(qos->mq, 0, sizeof(qos->mq)); } struct rmnet_flow_map * @@ -179,17 +196,17 @@ static void qmi_rmnet_update_flow_map(struct rmnet_flow_map *itm, itm->bearer_id = new_map->bearer_id; itm->flow_id = new_map->flow_id; itm->ip_type = new_map->ip_type; - itm->tcm_handle = new_map->tcm_handle; + itm->mq_idx = new_map->mq_idx; } -int qmi_rmnet_flow_control(struct net_device *dev, u32 tcm_handle, int enable) +int qmi_rmnet_flow_control(struct net_device *dev, u32 mq_idx, int enable) { struct netdev_queue *q; - if (unlikely(tcm_handle >= dev->num_tx_queues)) + if (unlikely(mq_idx >= dev->num_tx_queues)) return 0; - q = netdev_get_tx_queue(dev, tcm_handle); + q = netdev_get_tx_queue(dev, mq_idx); if (unlikely(!q)) return 0; @@ -198,6 +215,8 @@ int qmi_rmnet_flow_control(struct net_device *dev, u32 tcm_handle, int enable) else netif_tx_stop_queue(q); + trace_dfc_qmi_tc(dev->name, mq_idx, enable); + return 0; } @@ -216,76 +235,192 @@ static void qmi_rmnet_reset_txq(struct net_device *dev, unsigned int txq) } } +static struct rmnet_bearer_map *__qmi_rmnet_bearer_get( + struct qos_info *qos_info, u8 bearer_id) +{ + struct rmnet_bearer_map *bearer; + + bearer = qmi_rmnet_get_bearer_map(qos_info, bearer_id); + if (bearer) { + bearer->flow_ref++; + } else { + bearer = kzalloc(sizeof(*bearer), GFP_ATOMIC); + if (!bearer) + return NULL; + + bearer->bearer_id = bearer_id; + bearer->flow_ref = 1; + bearer->grant_size = DEFAULT_CALL_GRANT; + bearer->grant_thresh = qmi_rmnet_grant_per(bearer->grant_size); + bearer->mq_idx = INVALID_MQ; + bearer->ack_mq_idx = INVALID_MQ; + list_add(&bearer->list, &qos_info->bearer_head); + } + + return bearer; +} + +static void __qmi_rmnet_bearer_put(struct net_device *dev, + struct qos_info *qos_info, + struct rmnet_bearer_map *bearer, + bool reset) +{ + struct mq_map *mq; + int i, j; + + if (bearer && --bearer->flow_ref == 0) { + for (i = 0; i < MAX_MQ_NUM; i++) { + mq = &qos_info->mq[i]; + if (mq->bearer != bearer) + continue; + + mq->bearer = NULL; + if (reset) { + qmi_rmnet_reset_txq(dev, i); + qmi_rmnet_flow_control(dev, i, 1); + + if (dfc_mode == DFC_MODE_SA) { + j = i + ACK_MQ_OFFSET; + qmi_rmnet_reset_txq(dev, j); + qmi_rmnet_flow_control(dev, j, 1); + } + } + } + + /* Remove from bearer map */ + list_del(&bearer->list); + kfree(bearer); + } +} + +static void __qmi_rmnet_update_mq(struct net_device *dev, + struct qos_info *qos_info, + struct rmnet_bearer_map *bearer, + struct rmnet_flow_map *itm) +{ + struct mq_map *mq; + + /* In SA mode default mq is not associated with any bearer */ + if (dfc_mode == DFC_MODE_SA && itm->mq_idx == DEFAULT_MQ_NUM) + return; + + mq = &qos_info->mq[itm->mq_idx]; + if (!mq->bearer) { + mq->bearer = bearer; + + if (dfc_mode == DFC_MODE_SA) { + bearer->mq_idx = itm->mq_idx; + bearer->ack_mq_idx = itm->mq_idx + ACK_MQ_OFFSET; + } else { + if (IS_ANCILLARY(itm->ip_type)) + bearer->ack_mq_idx = itm->mq_idx; + else + bearer->mq_idx = itm->mq_idx; + } + + qmi_rmnet_flow_control(dev, itm->mq_idx, + bearer->grant_size > 0 ? 1 : 0); + + if (dfc_mode == DFC_MODE_SA) + qmi_rmnet_flow_control(dev, bearer->ack_mq_idx, + bearer->grant_size > 0 ? 1 : 0); + } +} + +static int __qmi_rmnet_rebind_flow(struct net_device *dev, + struct qos_info *qos_info, + struct rmnet_flow_map *itm, + struct rmnet_flow_map *new_map) +{ + struct rmnet_bearer_map *bearer; + + __qmi_rmnet_bearer_put(dev, qos_info, itm->bearer, false); + + bearer = __qmi_rmnet_bearer_get(qos_info, new_map->bearer_id); + if (!bearer) + return -ENOMEM; + + qmi_rmnet_update_flow_map(itm, new_map); + itm->bearer = bearer; + + __qmi_rmnet_update_mq(dev, qos_info, bearer, itm); + + return 0; +} + static int qmi_rmnet_add_flow(struct net_device *dev, struct tcmsg *tcm, struct qmi_info *qmi) { struct qos_info *qos_info = (struct qos_info *)rmnet_get_qos_pt(dev); struct rmnet_flow_map new_map, *itm; struct rmnet_bearer_map *bearer; + struct tcmsg tmp_tcm; + int rc = 0; - if (!qos_info) + if (!qos_info || !tcm || tcm->tcm_handle >= MAX_MQ_NUM) return -EINVAL; ASSERT_RTNL(); /* flow activate * tcm->tcm__pad1 - bearer_id, tcm->tcm_parent - flow_id, - * tcm->tcm_ifindex - ip_type, tcm->tcm_handle - tcm_handle + * tcm->tcm_ifindex - ip_type, tcm->tcm_handle - mq_idx */ new_map.bearer_id = tcm->tcm__pad1; new_map.flow_id = tcm->tcm_parent; new_map.ip_type = tcm->tcm_ifindex; - new_map.tcm_handle = tcm->tcm_handle; + new_map.mq_idx = tcm->tcm_handle; trace_dfc_flow_info(dev->name, new_map.bearer_id, new_map.flow_id, - new_map.ip_type, new_map.tcm_handle, 1); + new_map.ip_type, new_map.mq_idx, 1); +again: spin_lock_bh(&qos_info->qos_lock); itm = qmi_rmnet_get_flow_map(qos_info, new_map.flow_id, new_map.ip_type); if (itm) { - qmi_rmnet_update_flow_map(itm, &new_map); - } else { - itm = kzalloc(sizeof(*itm), GFP_ATOMIC); - if (!itm) { + if (itm->bearer_id != new_map.bearer_id) { + rc = __qmi_rmnet_rebind_flow( + dev, qos_info, itm, &new_map); + goto done; + } else if (itm->mq_idx != new_map.mq_idx) { + tmp_tcm.tcm__pad1 = itm->bearer_id; + tmp_tcm.tcm_parent = itm->flow_id; + tmp_tcm.tcm_ifindex = itm->ip_type; + tmp_tcm.tcm_handle = itm->mq_idx; spin_unlock_bh(&qos_info->qos_lock); - return -ENOMEM; - } - - qmi_rmnet_update_flow_map(itm, &new_map); - list_add(&itm->list, &qos_info->flow_head); - - bearer = qmi_rmnet_get_bearer_map(qos_info, new_map.bearer_id); - if (bearer) { - bearer->flow_ref++; + qmi_rmnet_del_flow(dev, &tmp_tcm, qmi); + goto again; } else { - bearer = kzalloc(sizeof(*bearer), GFP_ATOMIC); - if (!bearer) { - spin_unlock_bh(&qos_info->qos_lock); - return -ENOMEM; - } - - bearer->bearer_id = new_map.bearer_id; - bearer->flow_ref = 1; - bearer->grant_size = qos_info->default_grant; - bearer->grant_thresh = - qmi_rmnet_grant_per(bearer->grant_size); - qos_info->default_grant = DEFAULT_GRANT; - list_add(&bearer->list, &qos_info->bearer_head); + goto done; } - - qmi_rmnet_flow_control(dev, itm->tcm_handle, - bearer->grant_size > 0 ? 1 : 0); - - trace_dfc_qmi_tc(dev->name, itm->bearer_id, itm->flow_id, - bearer->grant_size, 0, itm->tcm_handle, - bearer->grant_size > 0 ? 1 : 0); } - spin_unlock_bh(&qos_info->qos_lock); + /* Create flow map */ + itm = kzalloc(sizeof(*itm), GFP_ATOMIC); + if (!itm) { + spin_unlock_bh(&qos_info->qos_lock); + return -ENOMEM; + } - return 0; + qmi_rmnet_update_flow_map(itm, &new_map); + list_add(&itm->list, &qos_info->flow_head); + + /* Create or update bearer map */ + bearer = __qmi_rmnet_bearer_get(qos_info, new_map.bearer_id); + if (!bearer) { + rc = -ENOMEM; + goto done; + } + + itm->bearer = bearer; + + __qmi_rmnet_update_mq(dev, qos_info, bearer, itm); + +done: + spin_unlock_bh(&qos_info->qos_lock); + return rc; } static int @@ -294,7 +429,6 @@ qmi_rmnet_del_flow(struct net_device *dev, struct tcmsg *tcm, { struct qos_info *qos_info = (struct qos_info *)rmnet_get_qos_pt(dev); struct rmnet_flow_map new_map, *itm; - struct rmnet_bearer_map *bearer; if (!qos_info) return -EINVAL; @@ -316,35 +450,19 @@ qmi_rmnet_del_flow(struct net_device *dev, struct tcmsg *tcm, if (itm) { trace_dfc_flow_info(dev->name, new_map.bearer_id, new_map.flow_id, new_map.ip_type, - itm->tcm_handle, 0); + itm->mq_idx, 0); + + __qmi_rmnet_bearer_put(dev, qos_info, itm->bearer, true); + + /* Remove from flow map */ list_del(&itm->list); - - /*clear bearer map*/ - bearer = qmi_rmnet_get_bearer_map(qos_info, new_map.bearer_id); - if (bearer && --bearer->flow_ref == 0) { - list_del(&bearer->list); - kfree(bearer); - - /* Purge pending packets for dedicated flow */ - if (itm->flow_id) - qmi_rmnet_reset_txq(dev, itm->tcm_handle); - } - - /* Enable flow to allow new flow setup */ - qmi_rmnet_flow_control(dev, itm->tcm_handle, 1); - trace_dfc_qmi_tc(dev->name, itm->bearer_id, itm->flow_id, - 0, 0, itm->tcm_handle, 1); - kfree(itm); } - if (list_empty(&qos_info->flow_head)) { + if (list_empty(&qos_info->flow_head)) netif_tx_wake_all_queues(dev); - trace_dfc_qmi_tc(dev->name, 0xFF, 0, DEFAULT_GRANT, 0, 0, 1); - } spin_unlock_bh(&qos_info->qos_lock); - return 0; } @@ -353,42 +471,24 @@ static void qmi_rmnet_query_flows(struct qmi_info *qmi) int i; for (i = 0; i < MAX_CLIENT_NUM; i++) { - if (qmi->dfc_clients[i]) + if (qmi->dfc_clients[i] && !dfc_qmap) dfc_qmi_query_flow(qmi->dfc_clients[i]); } } -static int qmi_rmnet_set_scale_factor(const char *val, - const struct kernel_param *kp) +struct rmnet_bearer_map *qmi_rmnet_get_bearer_noref(struct qos_info *qos_info, + u8 bearer_id) { - int ret; - unsigned int num = 0; + struct rmnet_bearer_map *bearer; - ret = kstrtouint(val, 10, &num); - if (ret != 0 || num == 0) - return -EINVAL; + bearer = __qmi_rmnet_bearer_get(qos_info, bearer_id); + if (bearer) + bearer->flow_ref--; - return param_set_uint(val, kp); + return bearer; } -static const struct kernel_param_ops qmi_rmnet_scale_ops = { - .set = qmi_rmnet_set_scale_factor, - .get = param_get_uint, -}; - -module_param_cb(qmi_rmnet_scale_factor, &qmi_rmnet_scale_ops, - &qmi_rmnet_scale_factor, 0664); #else -static inline void -qmi_rmnet_update_flow_link(struct qmi_info *qmi, struct net_device *dev, - struct rmnet_flow_map *itm, int add_flow) -{ -} - -static inline void qmi_rmnet_clean_flow_list(struct qos_info *qos) -{ -} - static inline void qmi_rmnet_update_flow_map(struct rmnet_flow_map *itm, struct rmnet_flow_map *new_map) @@ -417,7 +517,7 @@ static inline void qmi_rmnet_query_flows(struct qmi_info *qmi) static int qmi_rmnet_setup_client(void *port, struct qmi_info *qmi, struct tcmsg *tcm) { - int idx, rc, err = 0; + int idx, err = 0; struct svc_info svc; ASSERT_RTNL(); @@ -441,18 +541,17 @@ qmi_rmnet_setup_client(void *port, struct qmi_info *qmi, struct tcmsg *tcm) svc.ep_type = tcm->tcm_info; svc.iface_id = tcm->tcm_parent; - if (((tcm->tcm_ifindex & FLAG_DFC_MASK) == DFC_MODE_MULTIQ) && + if (DFC_SUPPORTED_MODE(dfc_mode) && !qmi->dfc_clients[idx] && !qmi->dfc_pending[idx]) { - rc = dfc_qmi_client_init(port, idx, &svc, qmi); - if (rc < 0) - err = rc; + if (dfc_qmap) + err = dfc_qmap_client_init(port, idx, &svc, qmi); + else + err = dfc_qmi_client_init(port, idx, &svc, qmi); } if ((tcm->tcm_ifindex & FLAG_POWERSAVE_MASK) && (idx == 0) && !qmi->wda_client && !qmi->wda_pending) { - rc = wda_qmi_client_init(port, &svc, qmi); - if (rc < 0) - err = rc; + err = wda_qmi_client_init(port, &svc, qmi); } return err; @@ -471,7 +570,10 @@ __qmi_rmnet_delete_client(void *port, struct qmi_info *qmi, int idx) data = qmi->dfc_pending[idx]; if (data) { - dfc_qmi_client_exit(data); + if (dfc_qmap) + dfc_qmap_client_exit(data); + else + dfc_qmi_client_exit(data); qmi->dfc_clients[idx] = NULL; qmi->dfc_pending[idx] = NULL; } @@ -518,20 +620,23 @@ void qmi_rmnet_change_link(struct net_device *dev, void *port, void *tcm_pt) switch (tcm->tcm_family) { case NLMSG_FLOW_ACTIVATE: - if (!qmi || ((qmi->flag & FLAG_DFC_MASK) != DFC_MODE_MULTIQ) || + if (!qmi || !DFC_SUPPORTED_MODE(dfc_mode) || !qmi_rmnet_has_dfc_client(qmi)) return; qmi_rmnet_add_flow(dev, tcm, qmi); break; case NLMSG_FLOW_DEACTIVATE: - if (!qmi || ((qmi->flag & FLAG_DFC_MASK) != DFC_MODE_MULTIQ)) + if (!qmi || !DFC_SUPPORTED_MODE(dfc_mode)) return; qmi_rmnet_del_flow(dev, tcm, qmi); break; case NLMSG_CLIENT_SETUP: - if (((tcm->tcm_ifindex & FLAG_DFC_MASK) != DFC_MODE_MULTIQ) && + dfc_mode = FLAG_TO_MODE(tcm->tcm_ifindex); + dfc_qmap = FLAG_TO_QMAP(tcm->tcm_ifindex); + + if (!DFC_SUPPORTED_MODE(dfc_mode) && !(tcm->tcm_ifindex & FLAG_POWERSAVE_MASK)) return; @@ -558,6 +663,14 @@ void qmi_rmnet_change_link(struct net_device *dev, void *port, void *tcm_pt) } qmi_rmnet_delete_client(port, qmi, tcm); break; + case NLMSG_SCALE_FACTOR: + if (!tcm->tcm_ifindex) + return; + qmi_rmnet_scale_factor = tcm->tcm_ifindex; + break; + case NLMSG_WQ_FREQUENCY: + rmnet_wq_frequency = tcm->tcm_ifindex; + break; default: pr_debug("%s(): No handler\n", __func__); break; @@ -613,7 +726,7 @@ void qmi_rmnet_enable_all_flows(struct net_device *dev) continue; do_wake = !bearer->grant_size; bearer->grant_size = DEFAULT_GRANT; - bearer->grant_thresh = DEFAULT_GRANT; + bearer->grant_thresh = qmi_rmnet_grant_per(DEFAULT_GRANT); bearer->seq = 0; bearer->ack_req = 0; bearer->tcp_bidir = false; @@ -665,43 +778,93 @@ void qmi_rmnet_burst_fc_check(struct net_device *dev, } EXPORT_SYMBOL(qmi_rmnet_burst_fc_check); +static bool qmi_rmnet_is_tcp_ack(struct sk_buff *skb) +{ + unsigned int len = skb->len; + + switch (skb->protocol) { + /* TCPv4 ACKs */ + case htons(ETH_P_IP): + if ((ip_hdr(skb)->protocol == IPPROTO_TCP) && + (ip_hdr(skb)->ihl == 5) && + (len == 40 || len == 52) && + ((tcp_flag_word(tcp_hdr(skb)) & + cpu_to_be32(0x00FF0000)) == TCP_FLAG_ACK)) + return true; + break; + + /* TCPv6 ACKs */ + case htons(ETH_P_IPV6): + if ((ipv6_hdr(skb)->nexthdr == IPPROTO_TCP) && + (len == 60 || len == 72) && + ((tcp_flag_word(tcp_hdr(skb)) & + cpu_to_be32(0x00FF0000)) == TCP_FLAG_ACK)) + return true; + break; + } + + return false; +} + +static int qmi_rmnet_get_queue_sa(struct qos_info *qos, struct sk_buff *skb) +{ + struct rmnet_flow_map *itm; + int ip_type; + int txq = DEFAULT_MQ_NUM; + + /* Put RS/NS in default mq */ + if (skb->protocol == htons(ETH_P_IPV6) && + ipv6_hdr(skb)->nexthdr == IPPROTO_ICMPV6 && + (icmp6_hdr(skb)->icmp6_type == 133 || + icmp6_hdr(skb)->icmp6_type == 135)) { + return DEFAULT_MQ_NUM; + } + + ip_type = (skb->protocol == htons(ETH_P_IPV6)) ? AF_INET6 : AF_INET; + + spin_lock_bh(&qos->qos_lock); + + itm = qmi_rmnet_get_flow_map(qos, skb->mark, ip_type); + if (unlikely(!itm)) + goto done; + + /* Put the packet in the assigned mq except TCP ack */ + if (likely(itm->bearer) && qmi_rmnet_is_tcp_ack(skb)) + txq = itm->bearer->ack_mq_idx; + else + txq = itm->mq_idx; + +done: + spin_unlock_bh(&qos->qos_lock); + return txq; +} + int qmi_rmnet_get_queue(struct net_device *dev, struct sk_buff *skb) { struct qos_info *qos = rmnet_get_qos_pt(dev); int txq = 0, ip_type = AF_INET; - unsigned int len = skb->len; struct rmnet_flow_map *itm; u32 mark = skb->mark; if (!qos) return 0; - switch (skb->protocol) { - /* TCPv4 ACKs */ - case htons(ETH_P_IP): - ip_type = AF_INET; - if ((!mark) && - (ip_hdr(skb)->protocol == IPPROTO_TCP) && - (len == 40 || len == 52) && - (ip_hdr(skb)->ihl == 5) && - ((tcp_flag_word(tcp_hdr(skb)) & 0xFF00) == TCP_FLAG_ACK)) - return 1; - break; + /* If mark is mq num return it */ + if (dfc_mode == DFC_MODE_MQ_NUM) + return mark; - /* TCPv6 ACKs */ - case htons(ETH_P_IPV6): - ip_type = AF_INET6; - if ((!mark) && - (ipv6_hdr(skb)->nexthdr == IPPROTO_TCP) && - (len == 60 || len == 72) && - ((tcp_flag_word(tcp_hdr(skb)) & 0xFF00) == TCP_FLAG_ACK)) - return 1; - /* Fall through */ - } + if (dfc_mode == DFC_MODE_SA) + return qmi_rmnet_get_queue_sa(qos, skb); /* Default flows */ - if (!mark) - return 0; + if (!mark) { + if (qmi_rmnet_is_tcp_ack(skb)) + return 1; + else + return 0; + } + + ip_type = (skb->protocol == htons(ETH_P_IPV6)) ? AF_INET6 : AF_INET; /* Dedicated flows */ spin_lock_bh(&qos->qos_lock); @@ -710,7 +873,7 @@ int qmi_rmnet_get_queue(struct net_device *dev, struct sk_buff *skb) if (unlikely(!itm)) goto done; - txq = itm->tcm_handle; + txq = itm->mq_idx; done: spin_unlock_bh(&qos->qos_lock); @@ -728,13 +891,12 @@ void *qmi_rmnet_qos_init(struct net_device *real_dev, u8 mux_id) { struct qos_info *qos; - qos = kmalloc(sizeof(*qos), GFP_KERNEL); + qos = kzalloc(sizeof(*qos), GFP_KERNEL); if (!qos) return NULL; qos->mux_id = mux_id; qos->real_dev = real_dev; - qos->default_grant = DEFAULT_GRANT; qos->tran_num = 0; INIT_LIST_HEAD(&qos->flow_head); INIT_LIST_HEAD(&qos->bearer_head); @@ -744,28 +906,38 @@ void *qmi_rmnet_qos_init(struct net_device *real_dev, u8 mux_id) } EXPORT_SYMBOL(qmi_rmnet_qos_init); -void qmi_rmnet_qos_exit(struct net_device *dev, void *qos) +void qmi_rmnet_qos_exit_pre(void *qos) { - void *port = rmnet_get_rmnet_port(dev); - struct qmi_info *qmi = rmnet_get_qmi_pt(port); - struct qos_info *qos_info = (struct qos_info *)qos; - - if (!qmi || !qos) + if (!qos) return; - qmi_rmnet_clean_flow_list(qmi, dev, qos_info); - kfree(qos); + list_add(&((struct qos_info *)qos)->list, &qos_cleanup_list); } -EXPORT_SYMBOL(qmi_rmnet_qos_exit); +EXPORT_SYMBOL(qmi_rmnet_qos_exit_pre); + +void qmi_rmnet_qos_exit_post(void) +{ + struct qos_info *qos, *tmp; + + synchronize_rcu(); + list_for_each_entry_safe(qos, tmp, &qos_cleanup_list, list) { + list_del(&qos->list); + qmi_rmnet_clean_flow_list(qos); + kfree(qos); + } +} +EXPORT_SYMBOL(qmi_rmnet_qos_exit_post); #endif #ifdef CONFIG_QTI_QMI_POWER_COLLAPSE static struct workqueue_struct *rmnet_ps_wq; static struct rmnet_powersave_work *rmnet_work; +static bool rmnet_work_quit; static LIST_HEAD(ps_list); struct rmnet_powersave_work { struct delayed_work work; + struct alarm atimer; void *port; u64 old_rx_pkts; u64 old_tx_pkts; @@ -775,7 +947,7 @@ void qmi_rmnet_ps_on_notify(void *port) { struct qmi_rmnet_ps_ind *tmp; - list_for_each_entry(tmp, &ps_list, list) + list_for_each_entry_rcu(tmp, &ps_list, list) tmp->ps_on_handler(port); } EXPORT_SYMBOL(qmi_rmnet_ps_on_notify); @@ -784,7 +956,7 @@ void qmi_rmnet_ps_off_notify(void *port) { struct qmi_rmnet_ps_ind *tmp; - list_for_each_entry(tmp, &ps_list, list) + list_for_each_entry_rcu(tmp, &ps_list, list) tmp->ps_off_handler(port); } EXPORT_SYMBOL(qmi_rmnet_ps_off_notify); @@ -811,7 +983,7 @@ int qmi_rmnet_ps_ind_deregister(void *port, if (!port || !ps_ind) return -EINVAL; - list_for_each_entry(tmp, &ps_list, list) { + list_for_each_entry_rcu(tmp, &ps_list, list) { if (tmp == ps_ind) { list_del_rcu(&ps_ind->list); goto done; @@ -844,9 +1016,20 @@ EXPORT_SYMBOL(qmi_rmnet_set_powersave_mode); static void qmi_rmnet_work_restart(void *port) { - if (!rmnet_ps_wq || !rmnet_work) - return; - queue_delayed_work(rmnet_ps_wq, &rmnet_work->work, NO_DELAY); + rcu_read_lock(); + if (!rmnet_work_quit) + queue_delayed_work(rmnet_ps_wq, &rmnet_work->work, NO_DELAY); + rcu_read_unlock(); +} + +static enum alarmtimer_restart qmi_rmnet_work_alarm(struct alarm *atimer, + ktime_t now) +{ + struct rmnet_powersave_work *real_work; + + real_work = container_of(atimer, struct rmnet_powersave_work, atimer); + qmi_rmnet_work_restart(real_work->port); + return ALARMTIMER_NORESTART; } static void qmi_rmnet_check_stats(struct work_struct *work) @@ -856,6 +1039,7 @@ static void qmi_rmnet_check_stats(struct work_struct *work) u64 rxd, txd; u64 rx, tx; bool dl_msg_active; + bool use_alarm_timer = true; real_work = container_of(to_delayed_work(work), struct rmnet_powersave_work, work); @@ -873,13 +1057,9 @@ static void qmi_rmnet_check_stats(struct work_struct *work) qmi->ps_ignore_grant = false; /* Register to get QMI DFC and DL marker */ - if (qmi_rmnet_set_powersave_mode(real_work->port, 0) < 0) { - /* If this failed need to retry quickly */ - queue_delayed_work(rmnet_ps_wq, - &real_work->work, HZ / 50); - return; + if (qmi_rmnet_set_powersave_mode(real_work->port, 0) < 0) + goto end; - } qmi->ps_enabled = false; /* Do a query when coming out of powersave */ @@ -905,15 +1085,15 @@ static void qmi_rmnet_check_stats(struct work_struct *work) * (likely in RLF), no need to enter powersave */ if (!dl_msg_active && - !rmnet_all_flows_enabled(real_work->port)) + !rmnet_all_flows_enabled(real_work->port)) { + use_alarm_timer = false; goto end; + } /* Deregister to suppress QMI DFC and DL marker */ - if (qmi_rmnet_set_powersave_mode(real_work->port, 1) < 0) { - queue_delayed_work(rmnet_ps_wq, - &real_work->work, PS_INTERVAL); - return; - } + if (qmi_rmnet_set_powersave_mode(real_work->port, 1) < 0) + goto end; + qmi->ps_enabled = true; /* Ignore grant after going into powersave */ @@ -931,7 +1111,16 @@ static void qmi_rmnet_check_stats(struct work_struct *work) return; } end: - queue_delayed_work(rmnet_ps_wq, &real_work->work, PS_INTERVAL); + rcu_read_lock(); + if (!rmnet_work_quit) { + if (use_alarm_timer) + alarm_start_relative(&real_work->atimer, + PS_INTERVAL_KT); + else + queue_delayed_work(rmnet_ps_wq, &real_work->work, + PS_INTERVAL); + } + rcu_read_unlock(); } static void qmi_rmnet_work_set_active(void *port, int status) @@ -956,17 +1145,22 @@ void qmi_rmnet_work_init(void *port) rmnet_ps_wq = alloc_workqueue("rmnet_powersave_work", WQ_MEM_RECLAIM | WQ_CPU_INTENSIVE, 1); - rmnet_work = kmalloc(sizeof(*rmnet_work), GFP_ATOMIC); + if (!rmnet_ps_wq) + return; + + rmnet_work = kzalloc(sizeof(*rmnet_work), GFP_ATOMIC); if (!rmnet_work) { destroy_workqueue(rmnet_ps_wq); rmnet_ps_wq = NULL; return; } INIT_DEFERRABLE_WORK(&rmnet_work->work, qmi_rmnet_check_stats); + alarm_init(&rmnet_work->atimer, ALARM_BOOTTIME, qmi_rmnet_work_alarm); rmnet_work->port = port; rmnet_get_packets(rmnet_work->port, &rmnet_work->old_rx_pkts, &rmnet_work->old_tx_pkts); + rmnet_work_quit = false; qmi_rmnet_work_set_active(rmnet_work->port, 1); queue_delayed_work(rmnet_ps_wq, &rmnet_work->work, PS_INTERVAL); } @@ -989,6 +1183,11 @@ void qmi_rmnet_work_exit(void *port) { if (!rmnet_ps_wq || !rmnet_work) return; + + rmnet_work_quit = true; + synchronize_rcu(); + + alarm_cancel(&rmnet_work->atimer); cancel_delayed_work_sync(&rmnet_work->work); destroy_workqueue(rmnet_ps_wq); qmi_rmnet_work_set_active(port, 0); @@ -1027,6 +1226,4 @@ bool qmi_rmnet_ignore_grant(void *port) return qmi->ps_ignore_grant; } EXPORT_SYMBOL(qmi_rmnet_ignore_grant); - #endif -MODULE_LICENSE("GPL v2"); diff --git a/core/qmi_rmnet.h b/core/qmi_rmnet.h index 6fb284e2b937..856295a94d26 100644 --- a/core/qmi_rmnet.h +++ b/core/qmi_rmnet.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018-2019, The Linux Foundation. All rights reserved. + * Copyright (c) 2018-2020, The Linux Foundation. All rights reserved. * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 and @@ -21,8 +21,8 @@ #define CONFIG_QTI_QMI_POWER_COLLAPSE 1 struct qmi_rmnet_ps_ind { - void (*ps_on_handler)(void *); - void (*ps_off_handler)(void *); + void (*ps_on_handler)(void *port); + void (*ps_off_handler)(void *port); struct list_head list; }; @@ -56,7 +56,8 @@ qmi_rmnet_all_flows_enabled(struct net_device *dev) #ifdef CONFIG_QTI_QMI_DFC void *qmi_rmnet_qos_init(struct net_device *real_dev, u8 mux_id); -void qmi_rmnet_qos_exit(struct net_device *dev, void *qos); +void qmi_rmnet_qos_exit_pre(void *qos); +void qmi_rmnet_qos_exit_post(void); void qmi_rmnet_burst_fc_check(struct net_device *dev, int ip_type, u32 mark, unsigned int len); int qmi_rmnet_get_queue(struct net_device *dev, struct sk_buff *skb); @@ -67,7 +68,11 @@ qmi_rmnet_qos_init(struct net_device *real_dev, u8 mux_id) return NULL; } -static inline void qmi_rmnet_qos_exit(struct net_device *dev, void *qos) +static inline void qmi_rmnet_qos_exit_pre(void *qos) +{ +} + +static inline void qmi_rmnet_qos_exit_post(void) { } @@ -122,23 +127,23 @@ static inline bool qmi_rmnet_ignore_grant(void *port) return false; } -static inline int qmi_rmnet_ps_ind_register(void *port, +static inline int qmi_rmnet_ps_ind_register(struct rmnet_port *port, struct qmi_rmnet_ps_ind *ps_ind) { return 0; } -static inline int qmi_rmnet_ps_ind_deregister(void *port, +static inline int qmi_rmnet_ps_ind_deregister(struct rmnet_port *port, struct qmi_rmnet_ps_ind *ps_ind) { return 0; } -static inline void qmi_rmnet_ps_off_notify(void *port) +static inline void qmi_rmnet_ps_off_notify(struct rmnet_port *port) { } -static inline void qmi_rmnet_ps_on_notify(void *port) +static inline void qmi_rmnet_ps_on_notify(struct rmnet_port *port) { } diff --git a/core/qmi_rmnet_i.h b/core/qmi_rmnet_i.h index 27a34df98a36..9d7c09c82e14 100644 --- a/core/qmi_rmnet_i.h +++ b/core/qmi_rmnet_i.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018-2019, The Linux Foundation. All rights reserved. + * Copyright (c) 2018-2020, The Linux Foundation. All rights reserved. * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 and @@ -17,25 +17,26 @@ #include #include -#define IP_VER_4 4 -#define IP_VER_6 6 - +#define MAX_MQ_NUM 16 #define MAX_CLIENT_NUM 2 #define MAX_FLOW_NUM 32 #define DEFAULT_GRANT 1 +#define DEFAULT_CALL_GRANT 20480 #define DFC_MAX_BEARERS_V01 16 +#define DEFAULT_MQ_NUM 0 +#define ACK_MQ_OFFSET (MAX_MQ_NUM - 1) +#define INVALID_MQ 0xFF + +#define DFC_MODE_FLOW_ID 2 +#define DFC_MODE_MQ_NUM 3 +#define DFC_MODE_SA 4 #define CONFIG_QTI_QMI_RMNET 1 #define CONFIG_QTI_QMI_DFC 1 #define CONFIG_QTI_QMI_POWER_COLLAPSE 1 -struct rmnet_flow_map { - struct list_head list; - u8 bearer_id; - u32 flow_id; - int ip_type; - u32 tcm_handle; -}; +extern int dfc_mode; +extern int dfc_qmap; struct rmnet_bearer_map { struct list_head list; @@ -50,6 +51,18 @@ struct rmnet_bearer_map { bool tcp_bidir; bool rat_switch; bool tx_off; + u32 ack_txid; + u32 mq_idx; + u32 ack_mq_idx; +}; + +struct rmnet_flow_map { + struct list_head list; + u8 bearer_id; + u32 flow_id; + int ip_type; + u32 mq_idx; + struct rmnet_bearer_map *bearer; }; struct svc_info { @@ -58,21 +71,21 @@ struct svc_info { u32 iface_id; }; +struct mq_map { + struct rmnet_bearer_map *bearer; +}; + struct qos_info { + struct list_head list; u8 mux_id; struct net_device *real_dev; struct list_head flow_head; struct list_head bearer_head; - u32 default_grant; + struct mq_map mq[MAX_MQ_NUM]; u32 tran_num; spinlock_t qos_lock; }; -struct flow_info { - struct net_device *dev; - struct rmnet_flow_map *itm; -}; - struct qmi_info { int flag; void *wda_client; @@ -123,13 +136,23 @@ void dfc_qmi_client_exit(void *dfc_data); void dfc_qmi_burst_check(struct net_device *dev, struct qos_info *qos, int ip_type, u32 mark, unsigned int len); -int qmi_rmnet_flow_control(struct net_device *dev, u32 tcm_handle, int enable); +int qmi_rmnet_flow_control(struct net_device *dev, u32 mq_idx, int enable); void dfc_qmi_query_flow(void *dfc_data); int dfc_bearer_flow_ctl(struct net_device *dev, struct rmnet_bearer_map *bearer, struct qos_info *qos); + +int dfc_qmap_client_init(void *port, int index, struct svc_info *psvc, + struct qmi_info *qmi); + +void dfc_qmap_client_exit(void *dfc_data); + +void dfc_qmap_send_ack(struct qos_info *qos, u8 bearer_id, u16 seq, u8 type); + +struct rmnet_bearer_map *qmi_rmnet_get_bearer_noref(struct qos_info *qos_info, + u8 bearer_id); #else static inline struct rmnet_flow_map * qmi_rmnet_get_flow_map(struct qos_info *qos_info, @@ -155,17 +178,6 @@ static inline void dfc_qmi_client_exit(void *dfc_data) { } -static inline void -dfc_qmi_burst_check(struct net_device *dev, struct qos_info *qos, - int ip_type, u32 mark, unsigned int len) -{ -} - -static inline void -dfc_qmi_query_flow(void *dfc_data) -{ -} - static inline int dfc_bearer_flow_ctl(struct net_device *dev, struct rmnet_bearer_map *bearer, @@ -173,6 +185,17 @@ dfc_bearer_flow_ctl(struct net_device *dev, { return 0; } + +static inline int +dfc_qmap_client_init(void *port, int index, struct svc_info *psvc, + struct qmi_info *qmi) +{ + return -EINVAL; +} + +static inline void dfc_qmap_client_exit(void *dfc_data) +{ +} #endif #ifdef CONFIG_QTI_QMI_POWER_COLLAPSE diff --git a/core/rmnet_config.c b/core/rmnet_config.c index 907acd305cd2..1c562203502f 100644 --- a/core/rmnet_config.c +++ b/core/rmnet_config.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2013-2019, The Linux Foundation. All rights reserved. +/* Copyright (c) 2013-2020, The Linux Foundation. All rights reserved. * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 and @@ -23,6 +23,7 @@ #include "rmnet_private.h" #include "rmnet_map.h" #include "rmnet_descriptor.h" +#include "rmnet_genl.h" #include "rmnet_qmi.h" #include "qmi_rmnet.h" #define CONFIG_QTI_QMI_RMNET 1 @@ -90,6 +91,8 @@ static int rmnet_unregister_real_device(struct net_device *real_dev, if (port->nr_rmnet_devs) return -EINVAL; + netdev_rx_handler_unregister(real_dev); + rmnet_map_cmd_exit(port); rmnet_map_tx_aggregate_exit(port); @@ -97,8 +100,6 @@ static int rmnet_unregister_real_device(struct net_device *real_dev, kfree(port); - netdev_rx_handler_unregister(real_dev); - /* release reference on real_dev */ dev_put(real_dev); @@ -173,11 +174,11 @@ static int rmnet_newlink(struct net *src_net, struct net_device *dev, struct nlattr *tb[], struct nlattr *data[], struct netlink_ext_ack *extack) { - u32 data_format = RMNET_FLAGS_INGRESS_DEAGGREGATION; struct net_device *real_dev; int mode = RMNET_EPMODE_VND; struct rmnet_endpoint *ep; struct rmnet_port *port; + u32 data_format; int err = 0; u16 mux_id; @@ -212,11 +213,10 @@ static int rmnet_newlink(struct net *src_net, struct net_device *dev, flags = nla_data(data[IFLA_RMNET_FLAGS]); data_format = flags->flags & flags->mask; + netdev_dbg(dev, "data format [0x%08X]\n", data_format); + port->data_format = data_format; } - netdev_dbg(dev, "data format [0x%08X]\n", data_format); - port->data_format = data_format; - if (data[IFLA_RMNET_UL_AGG_PARAMS]) { void *agg_params; unsigned long irq_flags; @@ -266,9 +266,11 @@ static void rmnet_dellink(struct net_device *dev, struct list_head *head) if (!port->nr_rmnet_devs) qmi_rmnet_qmi_exit(port->qmi_info, port); - rmnet_unregister_real_device(real_dev, port); + unregister_netdevice(dev); - unregister_netdevice_queue(dev, head); + qmi_rmnet_qos_exit_post(); + + rmnet_unregister_real_device(real_dev, port); } static void rmnet_force_unassociate_device(struct net_device *dev) @@ -279,6 +281,7 @@ static void rmnet_force_unassociate_device(struct net_device *dev) struct rmnet_port *port; unsigned long bkt_ep; LIST_HEAD(list); + HLIST_HEAD(cleanup_list); if (!rmnet_is_real_dev_registered(real_dev)) return; @@ -295,12 +298,23 @@ static void rmnet_force_unassociate_device(struct net_device *dev) rmnet_vnd_dellink(ep->mux_id, port, ep); hlist_del_init_rcu(&ep->hlnode); - synchronize_rcu(); + hlist_add_head(&ep->hlnode, &cleanup_list); + } + + synchronize_rcu(); + + hlist_for_each_entry_safe(ep, tmp_ep, &cleanup_list, hlnode) { + hlist_del(&ep->hlnode); kfree(ep); } + /* Unregistering devices in context before freeing port. + * If this API becomes non-context their order should switch. + */ unregister_netdevice_many(&list); + qmi_rmnet_qos_exit_post(); + rmnet_unregister_real_device(real_dev, port); } @@ -400,14 +414,13 @@ static int rmnet_changelink(struct net_device *dev, struct nlattr *tb[], } if (data[IFLA_RMNET_UL_AGG_PARAMS]) { - void *agg_params; - unsigned long irq_flags; + struct rmnet_egress_agg_params *agg_params; agg_params = nla_data(data[IFLA_RMNET_UL_AGG_PARAMS]); - spin_lock_irqsave(&port->agg_lock, irq_flags); - memcpy(&port->egress_agg_params, agg_params, - sizeof(port->egress_agg_params)); - spin_unlock_irqrestore(&port->agg_lock, irq_flags); + rmnet_map_update_ul_agg_config(port, agg_params->agg_size, + agg_params->agg_count, + agg_params->agg_features, + agg_params->agg_time); } return 0; @@ -564,9 +577,12 @@ EXPORT_SYMBOL(rmnet_get_qmi_pt); void *rmnet_get_qos_pt(struct net_device *dev) { - if (dev) - return rcu_dereference( - ((struct rmnet_priv *)netdev_priv(dev))->qos_info); + struct rmnet_priv *priv; + + if (dev) { + priv = netdev_priv(dev); + return rcu_dereference(priv->qos_info); + } return NULL; } @@ -708,6 +724,26 @@ int rmnet_get_powersave_notif(void *port) } EXPORT_SYMBOL(rmnet_get_powersave_notif); +struct net_device *rmnet_get_real_dev(void *port) +{ + if (port) + return ((struct rmnet_port *)port)->dev; + + return NULL; +} +EXPORT_SYMBOL(rmnet_get_real_dev); + +int rmnet_get_dlmarker_info(void *port) +{ + if (!port) + return 0; + + return ((struct rmnet_port *)port)->data_format & + (RMNET_INGRESS_FORMAT_DL_MARKER_V1 | + RMNET_INGRESS_FORMAT_DL_MARKER_V2); +} +EXPORT_SYMBOL(rmnet_get_dlmarker_info); + /* Startup/Shutdown */ static int __init rmnet_init(void) @@ -724,6 +760,8 @@ static int __init rmnet_init(void) unregister_netdevice_notifier(&rmnet_dev_notifier); return rc; } + + rmnet_core_genl_init(); return rc; } @@ -731,6 +769,7 @@ static void __exit rmnet_exit(void) { unregister_netdevice_notifier(&rmnet_dev_notifier); rtnl_link_unregister(&rmnet_link_ops); + rmnet_core_genl_deinit(); } module_init(rmnet_init) diff --git a/core/rmnet_config.h b/core/rmnet_config.h index f2f20a913468..46d4352ab885 100644 --- a/core/rmnet_config.h +++ b/core/rmnet_config.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2013-2014, 2016-2019 The Linux Foundation. All rights reserved. +/* Copyright (c) 2013-2014, 2016-2020 The Linux Foundation. All rights reserved. * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 and @@ -28,6 +28,11 @@ struct rmnet_endpoint { struct hlist_node hlnode; }; +struct rmnet_agg_stats { + u64 ul_agg_reuse; + u64 ul_agg_alloc; +}; + struct rmnet_port_priv_stats { u64 dl_hdr_last_qmap_vers; u64 dl_hdr_last_ep_id; @@ -41,14 +46,22 @@ struct rmnet_port_priv_stats { u64 dl_hdr_total_pkts; u64 dl_trl_last_seq; u64 dl_trl_count; + struct rmnet_agg_stats agg; }; struct rmnet_egress_agg_params { u16 agg_size; - u16 agg_count; + u8 agg_count; + u8 agg_features; u32 agg_time; }; +struct rmnet_agg_page { + struct list_head list; + struct page *page; +}; + + /* One instance of this structure is instantiated for each real_dev associated * with rmnet. */ @@ -73,12 +86,16 @@ struct rmnet_port { struct timespec64 agg_last; struct hrtimer hrtimer; struct work_struct agg_wq; + u8 agg_size_order; + struct list_head agg_list; + struct rmnet_agg_page *agg_head; void *qmi_info; /* dl marker elements */ struct list_head dl_list; struct rmnet_port_priv_stats stats; + int dl_marker_flush; /* Descriptor pool */ spinlock_t desc_pool_lock; @@ -137,6 +154,7 @@ struct rmnet_priv_stats { u64 csum_sw; u64 csum_hw; struct rmnet_coal_stats coal; + u64 ul_prio; }; struct rmnet_priv { diff --git a/core/rmnet_ctl.h b/core/rmnet_ctl.h new file mode 100644 index 000000000000..0c7061837c12 --- /dev/null +++ b/core/rmnet_ctl.h @@ -0,0 +1,66 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* Copyright (c) 2019-2020, The Linux Foundation. All rights reserved. + * + * RMNET_CTL header + * + */ + +#ifndef _RMNET_CTL_H_ +#define _RMNET_CTL_H_ + +#include + +enum rmnet_ctl_log_lvl { + RMNET_CTL_LOG_CRIT, + RMNET_CTL_LOG_ERR, + RMNET_CTL_LOG_INFO, + RMNET_CTL_LOG_DEBUG, +}; + +#define rmnet_ctl_log_err(msg, rc, data, len) \ + rmnet_ctl_log(RMNET_CTL_LOG_ERR, msg, rc, data, len) + +#define rmnet_ctl_log_info(msg, data, len) \ + rmnet_ctl_log(RMNET_CTL_LOG_INFO, msg, 0, data, len) + +#define rmnet_ctl_log_debug(msg, data, len) \ + rmnet_ctl_log(RMNET_CTL_LOG_DEBUG, msg, 0, data, len) + +struct rmnet_ctl_client_hooks { + void (*ctl_dl_client_hook)(struct sk_buff *skb); +}; + +#ifdef CONFIG_RMNET_CTL + +void *rmnet_ctl_register_client(struct rmnet_ctl_client_hooks *hook); +int rmnet_ctl_unregister_client(void *handle); +int rmnet_ctl_send_client(void *handle, struct sk_buff *skb); +void rmnet_ctl_log(enum rmnet_ctl_log_lvl lvl, const char *msg, + int rc, const void *data, unsigned int len); + +#else + +static inline void *rmnet_ctl_register_client( + struct rmnet_ctl_client_hooks *hook) +{ + return NULL; +} + +static inline int rmnet_ctl_unregister_client(void *handle) +{ + return -EINVAL; +} + +static inline int rmnet_ctl_send_client(void *handle, struct sk_buff *skb) +{ + return -EINVAL; +} + +static inline void rmnet_ctl_log(enum rmnet_ctl_log_lvl lvl, const char *msg, + int rc, const void *data, unsigned int len) +{ +} + +#endif /* CONFIG_RMNET_CTL */ + +#endif /* _RMNET_CTL_H_ */ diff --git a/core/rmnet_ctl_client.c b/core/rmnet_ctl_client.c new file mode 100644 index 000000000000..f082101972ed --- /dev/null +++ b/core/rmnet_ctl_client.c @@ -0,0 +1,173 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright (c) 2019-2020, The Linux Foundation. All rights reserved. + * + * RMNET_CTL client handlers + * + */ + +#include +#include +#include "rmnet_ctl.h" +#include "rmnet_ctl_client.h" + +#define RMNET_CTL_LOG_PAGE 10 +#define RMNET_CTL_LOG_NAME "rmnet_ctl" +#define RMNET_CTL_LOG_LVL "ipc_log_lvl" + +struct rmnet_ctl_client { + struct rmnet_ctl_client_hooks hooks; +}; + +struct rmnet_ctl_endpoint { + struct rmnet_ctl_dev __rcu *dev; + struct rmnet_ctl_client __rcu *client; + struct dentry *dbgfs_dir; + struct dentry *dbgfs_loglvl; + void *ipc_log; +}; + +#ifdef CONFIG_RMNET_CTL_DEBUG +static u8 ipc_log_lvl = RMNET_CTL_LOG_DEBUG; +#else +static u8 ipc_log_lvl = RMNET_CTL_LOG_ERR; +#endif + +static DEFINE_SPINLOCK(client_lock); +static struct rmnet_ctl_endpoint ctl_ep; + +void rmnet_ctl_endpoint_setdev(const struct rmnet_ctl_dev *dev) +{ + rcu_assign_pointer(ctl_ep.dev, dev); + + if (dev) { + ctl_ep.dbgfs_dir = debugfs_create_dir( + RMNET_CTL_LOG_NAME, NULL); + if (!IS_ERR_OR_NULL(ctl_ep.dbgfs_dir)) + ctl_ep.dbgfs_loglvl = debugfs_create_u8( + RMNET_CTL_LOG_LVL, 0644, ctl_ep.dbgfs_dir, + &ipc_log_lvl); + + if (!ctl_ep.ipc_log) + ctl_ep.ipc_log = ipc_log_context_create( + RMNET_CTL_LOG_PAGE, RMNET_CTL_LOG_NAME, 0); + } else { + debugfs_remove_recursive(ctl_ep.dbgfs_dir); + } +} + +void rmnet_ctl_endpoint_post(const void *data, size_t len) +{ + struct rmnet_ctl_client *client; + struct sk_buff *skb; + + if (unlikely(!data || !len)) + return; + + rmnet_ctl_log_info("RX", data, len); + + rcu_read_lock(); + + client = rcu_dereference(ctl_ep.client); + + if (client && client->hooks.ctl_dl_client_hook) { + skb = alloc_skb(len, GFP_ATOMIC); + if (skb) { + skb_put_data(skb, data, len); + skb->protocol = htons(ETH_P_MAP); + client->hooks.ctl_dl_client_hook(skb); + } + } + + rcu_read_unlock(); +} + +void *rmnet_ctl_register_client(struct rmnet_ctl_client_hooks *hook) +{ + struct rmnet_ctl_client *client; + + if (!hook) + return NULL; + + client = kzalloc(sizeof(*client), GFP_KERNEL); + if (!client) + return NULL; + client->hooks = *hook; + + spin_lock(&client_lock); + + /* Only support one client for now */ + if (rcu_dereference(ctl_ep.client)) { + spin_unlock(&client_lock); + kfree(client); + return NULL; + } + + rcu_assign_pointer(ctl_ep.client, client); + + spin_unlock(&client_lock); + + return client; +} +EXPORT_SYMBOL(rmnet_ctl_register_client); + +int rmnet_ctl_unregister_client(void *handle) +{ + struct rmnet_ctl_client *client = (struct rmnet_ctl_client *)handle; + + spin_lock(&client_lock); + + if (rcu_dereference(ctl_ep.client) != client) { + spin_unlock(&client_lock); + return -EINVAL; + } + + RCU_INIT_POINTER(ctl_ep.client, NULL); + + spin_unlock(&client_lock); + + synchronize_rcu(); + kfree(client); + + return 0; +} +EXPORT_SYMBOL(rmnet_ctl_unregister_client); + +int rmnet_ctl_send_client(void *handle, struct sk_buff *skb) +{ + struct rmnet_ctl_client *client = (struct rmnet_ctl_client *)handle; + struct rmnet_ctl_dev *dev; + int rc = -EINVAL; + + if (client != rcu_dereference(ctl_ep.client)) + return rc; + + rmnet_ctl_log_info("TX", skb->data, skb->len); + + rcu_read_lock(); + + dev = rcu_dereference(ctl_ep.dev); + if (dev && dev->xmit) + rc = dev->xmit(dev, skb); + + rcu_read_unlock(); + + if (rc) + rmnet_ctl_log_err("TXE", rc, skb->data, skb->len); + + return rc; +} +EXPORT_SYMBOL(rmnet_ctl_send_client); + +void rmnet_ctl_log(enum rmnet_ctl_log_lvl lvl, const char *msg, + int rc, const void *data, unsigned int len) +{ + if (lvl <= ipc_log_lvl && ctl_ep.ipc_log) { + if (data == NULL || len == 0) + ipc_log_string(ctl_ep.ipc_log, "%3s(%d): (null)\n", + msg, rc); + else + ipc_log_string(ctl_ep.ipc_log, "%3s(%d): %*ph\n", + msg, rc, len > 32 ? 32 : len, data); + } +} +EXPORT_SYMBOL(rmnet_ctl_log); diff --git a/core/rmnet_ctl_client.h b/core/rmnet_ctl_client.h new file mode 100644 index 000000000000..384064d134df --- /dev/null +++ b/core/rmnet_ctl_client.h @@ -0,0 +1,29 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* Copyright (c) 2019-2020, The Linux Foundation. All rights reserved. + * + * RMNET_CTL client handlers + * + */ + +#ifndef _RMNET_CTL_CLIENT_H_ +#define _RMNET_CTL_CLIENT_H_ + +#include + +struct rmnet_ctl_stats { + u64 rx_pkts; + u64 rx_err; + u64 tx_pkts; + u64 tx_err; + u64 tx_complete; +}; + +struct rmnet_ctl_dev { + int (*xmit)(struct rmnet_ctl_dev *dev, struct sk_buff *skb); + struct rmnet_ctl_stats stats; +}; + +void rmnet_ctl_endpoint_post(const void *data, size_t len); +void rmnet_ctl_endpoint_setdev(const struct rmnet_ctl_dev *dev); + +#endif /* _RMNET_CTL_CLIENT_H_ */ diff --git a/core/rmnet_ctl_mhi.c b/core/rmnet_ctl_mhi.c new file mode 100644 index 000000000000..e22c137df01e --- /dev/null +++ b/core/rmnet_ctl_mhi.c @@ -0,0 +1,206 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright (c) 2019-2020, The Linux Foundation. All rights reserved. + * + * RMNET_CTL mhi handler + * + */ + +#include +#include +#include +#include +#include +#include "rmnet_ctl.h" +#include "rmnet_ctl_client.h" + +#define RMNET_CTL_DEFAULT_MRU 256 + +struct rmnet_ctl_mhi_dev { + struct mhi_device *mhi_dev; + struct rmnet_ctl_dev dev; + u32 mru; + spinlock_t rx_lock; /* rx lock */ + spinlock_t tx_lock; /* tx lock */ + atomic_t in_reset; +}; + +static int rmnet_ctl_send_mhi(struct rmnet_ctl_dev *dev, struct sk_buff *skb) +{ + struct rmnet_ctl_mhi_dev *ctl_dev = container_of( + dev, struct rmnet_ctl_mhi_dev, dev); + int rc; + + spin_lock_bh(&ctl_dev->tx_lock); + + rc = mhi_queue_transfer(ctl_dev->mhi_dev, + DMA_TO_DEVICE, skb, skb->len, MHI_EOT); + if (rc) + dev->stats.tx_err++; + else + dev->stats.tx_pkts++; + + spin_unlock_bh(&ctl_dev->tx_lock); + + return rc; +} + +static void rmnet_ctl_alloc_buffers(struct rmnet_ctl_mhi_dev *ctl_dev, + gfp_t gfp, void *free_buf) +{ + struct mhi_device *mhi_dev = ctl_dev->mhi_dev; + void *buf; + int no_tre, i, rc; + + no_tre = mhi_get_no_free_descriptors(mhi_dev, DMA_FROM_DEVICE); + + if (!no_tre && free_buf) { + kfree(free_buf); + return; + } + + for (i = 0; i < no_tre; i++) { + if (free_buf) { + buf = free_buf; + free_buf = NULL; + } else { + buf = kmalloc(ctl_dev->mru, gfp); + } + + if (!buf) + return; + + spin_lock_bh(&ctl_dev->rx_lock); + rc = mhi_queue_transfer(mhi_dev, DMA_FROM_DEVICE, + buf, ctl_dev->mru, MHI_EOT); + spin_unlock_bh(&ctl_dev->rx_lock); + + if (rc) { + kfree(buf); + return; + } + } +} + +static void rmnet_ctl_dl_callback(struct mhi_device *mhi_dev, + struct mhi_result *mhi_res) +{ + struct rmnet_ctl_mhi_dev *ctl_dev = dev_get_drvdata(&mhi_dev->dev); + + if (mhi_res->transaction_status == -ENOTCONN) { + kfree(mhi_res->buf_addr); + return; + } else if (mhi_res->transaction_status || + !mhi_res->buf_addr || !mhi_res->bytes_xferd) { + rmnet_ctl_log_err("RXE", mhi_res->transaction_status, NULL, 0); + ctl_dev->dev.stats.rx_err++; + } else { + ctl_dev->dev.stats.rx_pkts++; + rmnet_ctl_endpoint_post(mhi_res->buf_addr, + mhi_res->bytes_xferd); + } + + /* Re-supply receive buffers */ + rmnet_ctl_alloc_buffers(ctl_dev, GFP_ATOMIC, mhi_res->buf_addr); +} + +static void rmnet_ctl_ul_callback(struct mhi_device *mhi_dev, + struct mhi_result *mhi_res) +{ + struct rmnet_ctl_mhi_dev *ctl_dev = dev_get_drvdata(&mhi_dev->dev); + struct sk_buff *skb = (struct sk_buff *)mhi_res->buf_addr; + + if (skb) { + if (mhi_res->transaction_status) { + rmnet_ctl_log_err("TXE", mhi_res->transaction_status, + skb->data, skb->len); + ctl_dev->dev.stats.tx_err++; + } else { + rmnet_ctl_log_debug("TXC", skb->data, skb->len); + ctl_dev->dev.stats.tx_complete++; + } + kfree_skb(skb); + } +} + +static void rmnet_ctl_status_callback(struct mhi_device *mhi_dev, + enum MHI_CB mhi_cb) +{ + struct rmnet_ctl_mhi_dev *ctl_dev = dev_get_drvdata(&mhi_dev->dev); + + if (mhi_cb != MHI_CB_FATAL_ERROR) + return; + + atomic_inc(&ctl_dev->in_reset); +} + +static int rmnet_ctl_probe(struct mhi_device *mhi_dev, + const struct mhi_device_id *id) +{ + struct rmnet_ctl_mhi_dev *ctl_dev; + struct device_node *of_node = mhi_dev->dev.of_node; + int rc; + + ctl_dev = devm_kzalloc(&mhi_dev->dev, sizeof(*ctl_dev), GFP_KERNEL); + if (!ctl_dev) + return -ENOMEM; + + ctl_dev->mhi_dev = mhi_dev; + ctl_dev->dev.xmit = rmnet_ctl_send_mhi; + + spin_lock_init(&ctl_dev->rx_lock); + spin_lock_init(&ctl_dev->tx_lock); + atomic_set(&ctl_dev->in_reset, 0); + dev_set_drvdata(&mhi_dev->dev, ctl_dev); + + rc = of_property_read_u32(of_node, "mhi,mru", &ctl_dev->mru); + if (rc || !ctl_dev->mru) + ctl_dev->mru = RMNET_CTL_DEFAULT_MRU; + + rc = mhi_prepare_for_transfer(mhi_dev); + if (rc) { + pr_err("%s(): Failed to prep for transfer %d\n", __func__, rc); + return -EINVAL; + } + + /* Post receive buffers */ + rmnet_ctl_alloc_buffers(ctl_dev, GFP_KERNEL, NULL); + + rmnet_ctl_endpoint_setdev(&ctl_dev->dev); + + pr_info("rmnet_ctl driver probed\n"); + + return 0; +} + +static void rmnet_ctl_remove(struct mhi_device *mhi_dev) +{ + rmnet_ctl_endpoint_setdev(NULL); + synchronize_rcu(); + dev_set_drvdata(&mhi_dev->dev, NULL); + + pr_info("rmnet_ctl driver removed\n"); +} + +static const struct mhi_device_id rmnet_ctl_mhi_match[] = { + { .chan = "RMNET_CTL" }, + {} +}; + +static struct mhi_driver rmnet_ctl_driver = { + .probe = rmnet_ctl_probe, + .remove = rmnet_ctl_remove, + .dl_xfer_cb = rmnet_ctl_dl_callback, + .ul_xfer_cb = rmnet_ctl_ul_callback, + .status_cb = rmnet_ctl_status_callback, + .id_table = rmnet_ctl_mhi_match, + .driver = { + .name = "rmnet_ctl", + .owner = THIS_MODULE, + }, +}; + +module_driver(rmnet_ctl_driver, + mhi_driver_register, mhi_driver_unregister); + +MODULE_DESCRIPTION("RmNet Control Driver"); +MODULE_LICENSE("GPL v2"); diff --git a/core/rmnet_descriptor.c b/core/rmnet_descriptor.c index ddc9f12fa689..618f453e70e5 100644 --- a/core/rmnet_descriptor.c +++ b/core/rmnet_descriptor.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2013-2019, The Linux Foundation. All rights reserved. +/* Copyright (c) 2013-2020, The Linux Foundation. All rights reserved. * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 and @@ -42,8 +42,9 @@ rmnet_get_frag_descriptor(struct rmnet_port *port) { struct rmnet_frag_descriptor_pool *pool = port->frag_desc_pool; struct rmnet_frag_descriptor *frag_desc; + unsigned long flags; - spin_lock(&port->desc_pool_lock); + spin_lock_irqsave(&port->desc_pool_lock, flags); if (!list_empty(&pool->free_list)) { frag_desc = list_first_entry(&pool->free_list, struct rmnet_frag_descriptor, @@ -60,7 +61,7 @@ rmnet_get_frag_descriptor(struct rmnet_port *port) } out: - spin_unlock(&port->desc_pool_lock); + spin_unlock_irqrestore(&port->desc_pool_lock, flags); return frag_desc; } EXPORT_SYMBOL(rmnet_get_frag_descriptor); @@ -69,16 +70,19 @@ void rmnet_recycle_frag_descriptor(struct rmnet_frag_descriptor *frag_desc, struct rmnet_port *port) { struct rmnet_frag_descriptor_pool *pool = port->frag_desc_pool; + struct page *page = skb_frag_page(&frag_desc->frag); + unsigned long flags; list_del(&frag_desc->list); - put_page(skb_frag_page(&frag_desc->frag)); + if (page) + put_page(page); memset(frag_desc, 0, sizeof(*frag_desc)); INIT_LIST_HEAD(&frag_desc->list); INIT_LIST_HEAD(&frag_desc->sub_frags); - spin_lock(&port->desc_pool_lock); + spin_lock_irqsave(&port->desc_pool_lock, flags); list_add_tail(&frag_desc->list, &pool->free_list); - spin_unlock(&port->desc_pool_lock); + spin_unlock_irqrestore(&port->desc_pool_lock, flags); } EXPORT_SYMBOL(rmnet_recycle_frag_descriptor); @@ -381,12 +385,28 @@ static void rmnet_frag_gso_stamp(struct sk_buff *skb, struct rmnet_frag_descriptor *frag_desc) { struct skb_shared_info *shinfo = skb_shinfo(skb); + + if (frag_desc->trans_proto == IPPROTO_TCP) + shinfo->gso_type = (frag_desc->ip_proto == 4) ? + SKB_GSO_TCPV4 : SKB_GSO_TCPV6; + else + shinfo->gso_type = SKB_GSO_UDP_L4; + + shinfo->gso_size = frag_desc->gso_size; + shinfo->gso_segs = frag_desc->gso_segs; +} + +/* Set the partial checksum information. Sets the transport checksum to the + * pseudoheader checksum and sets the offload metadata. + */ +static void rmnet_frag_partial_csum(struct sk_buff *skb, + struct rmnet_frag_descriptor *frag_desc) +{ struct iphdr *iph = (struct iphdr *)skb->data; __sum16 pseudo; u16 pkt_len = skb->len - frag_desc->ip_len; - bool ipv4 = frag_desc->ip_proto == 4; - if (ipv4) { + if (frag_desc->ip_proto == 4) { iph->tot_len = htons(skb->len); iph->check = 0; iph->check = ip_fast_csum(iph, iph->ihl); @@ -407,7 +427,6 @@ static void rmnet_frag_gso_stamp(struct sk_buff *skb, ((u8 *)iph + frag_desc->ip_len); tp->check = pseudo; - shinfo->gso_type = (ipv4) ? SKB_GSO_TCPV4 : SKB_GSO_TCPV6; skb->csum_offset = offsetof(struct tcphdr, check); } else { struct udphdr *up = (struct udphdr *) @@ -415,14 +434,11 @@ static void rmnet_frag_gso_stamp(struct sk_buff *skb, up->len = htons(pkt_len); up->check = pseudo; - shinfo->gso_type = SKB_GSO_UDP_L4; skb->csum_offset = offsetof(struct udphdr, check); } skb->ip_summed = CHECKSUM_PARTIAL; skb->csum_start = (u8 *)iph + frag_desc->ip_len - skb->head; - shinfo->gso_size = frag_desc->gso_size; - shinfo->gso_segs = frag_desc->gso_segs; } /* Allocate and populate an skb to contain the packet represented by the @@ -450,15 +466,19 @@ static struct sk_buff *rmnet_alloc_skb(struct rmnet_frag_descriptor *frag_desc, if (frag_desc->trans_len) skb_set_transport_header(head_skb, frag_desc->ip_len); - /* Packets that have no data portion don't need any frags */ - if (hdr_len == skb_frag_size(&frag_desc->frag)) - goto skip_frags; - /* If the headers we added are the start of the page, * we don't want to add them twice */ - if (frag_desc->hdr_ptr == rmnet_frag_data_ptr(frag_desc)) - rmnet_frag_pull(frag_desc, port, hdr_len); + if (frag_desc->hdr_ptr == rmnet_frag_data_ptr(frag_desc)) { + /* "Header only" packets can be fast-forwarded */ + if (hdr_len == skb_frag_size(&frag_desc->frag)) + goto skip_frags; + + if (!rmnet_frag_pull(frag_desc, port, hdr_len)) { + kfree_skb(head_skb); + return NULL; + } + } } else { /* Allocate enough space to avoid penalties in the stack * from __pskb_pull_tail() @@ -542,8 +562,64 @@ skip_frags: } /* Handle csum offloading */ - if (frag_desc->csum_valid) + if (frag_desc->csum_valid && frag_desc->hdrs_valid) { + /* Set the partial checksum information */ + rmnet_frag_partial_csum(head_skb, frag_desc); + } else if (frag_desc->csum_valid) { + /* Non-RSB/RSC/perf packet. The current checksum is fine */ head_skb->ip_summed = CHECKSUM_UNNECESSARY; + } else if (frag_desc->hdrs_valid && + (frag_desc->trans_proto == IPPROTO_TCP || + frag_desc->trans_proto == IPPROTO_UDP)) { + /* Unfortunately, we have to fake a bad checksum here, since + * the original bad value is lost by the hardware. The only + * reliable way to do it is to calculate the actual checksum + * and corrupt it. + */ + __sum16 *check; + __wsum csum; + unsigned int offset = skb_transport_offset(head_skb); + __sum16 pseudo; + + /* Calculate pseudo header and update header fields */ + if (frag_desc->ip_proto == 4) { + struct iphdr *iph = ip_hdr(head_skb); + __be16 tot_len = htons(head_skb->len); + + csum_replace2(&iph->check, iph->tot_len, tot_len); + iph->tot_len = tot_len; + pseudo = ~csum_tcpudp_magic(iph->saddr, iph->daddr, + head_skb->len - + frag_desc->ip_len, + frag_desc->trans_proto, 0); + } else { + struct ipv6hdr *ip6h = ipv6_hdr(head_skb); + + ip6h->payload_len = htons(head_skb->len - + sizeof(*ip6h)); + pseudo = ~csum_ipv6_magic(&ip6h->saddr, &ip6h->daddr, + head_skb->len - + frag_desc->ip_len, + frag_desc->trans_proto, 0); + } + + if (frag_desc->trans_proto == IPPROTO_TCP) { + check = &tcp_hdr(head_skb)->check; + } else { + udp_hdr(head_skb)->len = htons(head_skb->len - + frag_desc->ip_len); + check = &udp_hdr(head_skb)->check; + } + + *check = pseudo; + csum = skb_checksum(head_skb, offset, head_skb->len - offset, + 0); + /* Add 1 to corrupt. This cannot produce a final value of 0 + * since csum_fold() can't return a value of 0xFFFF + */ + *check = csum16_add(csum_fold(csum), htons(1)); + head_skb->ip_summed = CHECKSUM_NONE; + } /* Handle any rmnet_perf metadata */ if (frag_desc->hash) { @@ -576,8 +652,8 @@ EXPORT_SYMBOL(rmnet_frag_deliver); static void __rmnet_frag_segment_data(struct rmnet_frag_descriptor *coal_desc, struct rmnet_port *port, - struct list_head *list, - u8 pkt_id) + struct list_head *list, u8 pkt_id, + bool csum_valid) { struct rmnet_priv *priv = netdev_priv(coal_desc->dev); struct rmnet_frag_descriptor *new_frag; @@ -606,6 +682,12 @@ static void __rmnet_frag_segment_data(struct rmnet_frag_descriptor *coal_desc, new_frag->tcp_seq_set = 1; new_frag->tcp_seq = htonl(ntohl(th->seq) + coal_desc->data_offset); + } else if (coal_desc->trans_proto == IPPROTO_UDP) { + struct udphdr *uh; + + uh = (struct udphdr *)(hdr_start + coal_desc->ip_len); + if (coal_desc->ip_proto == 4 && !uh->check) + csum_valid = true; } if (coal_desc->ip_proto == 4) { @@ -617,7 +699,7 @@ static void __rmnet_frag_segment_data(struct rmnet_frag_descriptor *coal_desc, } new_frag->hdr_ptr = hdr_start; - new_frag->csum_valid = true; + new_frag->csum_valid = csum_valid; priv->stats.coal.coal_reconstruct++; /* Update meta information to move past the data we just segmented */ @@ -628,6 +710,33 @@ static void __rmnet_frag_segment_data(struct rmnet_frag_descriptor *coal_desc, list_add_tail(&new_frag->list, list); } +static bool rmnet_frag_validate_csum(struct rmnet_frag_descriptor *frag_desc) +{ + u8 *data = rmnet_frag_data_ptr(frag_desc); + unsigned int datagram_len; + __wsum csum; + __sum16 pseudo; + + datagram_len = skb_frag_size(&frag_desc->frag) - frag_desc->ip_len; + if (frag_desc->ip_proto == 4) { + struct iphdr *iph = (struct iphdr *)data; + + pseudo = ~csum_tcpudp_magic(iph->saddr, iph->daddr, + datagram_len, + frag_desc->trans_proto, 0); + } else { + struct ipv6hdr *ip6h = (struct ipv6hdr *)data; + + pseudo = ~csum_ipv6_magic(&ip6h->saddr, &ip6h->daddr, + datagram_len, frag_desc->trans_proto, + 0); + } + + csum = csum_partial(data + frag_desc->ip_len, datagram_len, + csum_unfold(pseudo)); + return !csum_fold(csum); +} + /* Converts the coalesced frame into a list of descriptors. * NLOs containing csum erros will not be included. */ @@ -643,12 +752,16 @@ rmnet_frag_segment_coal_data(struct rmnet_frag_descriptor *coal_desc, u8 pkt, total_pkt = 0; u8 nlo; bool gro = coal_desc->dev->features & NETIF_F_GRO_HW; + bool zero_csum = false; /* Pull off the headers we no longer need */ - rmnet_frag_pull(coal_desc, port, sizeof(struct rmnet_map_header)); + if (!rmnet_frag_pull(coal_desc, port, sizeof(struct rmnet_map_header))) + return; + coal_hdr = (struct rmnet_map_v5_coal_header *) rmnet_frag_data_ptr(coal_desc); - rmnet_frag_pull(coal_desc, port, sizeof(*coal_hdr)); + if (!rmnet_frag_pull(coal_desc, port, sizeof(*coal_hdr))) + return; iph = (struct iphdr *)rmnet_frag_data_ptr(coal_desc); @@ -662,24 +775,28 @@ rmnet_frag_segment_coal_data(struct rmnet_frag_descriptor *coal_desc, gro = false; } else if (iph->version == 6) { struct ipv6hdr *ip6h = (struct ipv6hdr *)iph; + int ip_len; __be16 frag_off; u8 protocol = ip6h->nexthdr; coal_desc->ip_proto = 6; - coal_desc->ip_len = rmnet_frag_ipv6_skip_exthdr(coal_desc, - sizeof(*ip6h), - &protocol, - &frag_off); + ip_len = rmnet_frag_ipv6_skip_exthdr(coal_desc, + sizeof(*ip6h), + &protocol, + &frag_off); coal_desc->trans_proto = protocol; /* If we run into a problem, or this has a fragment header * (which should technically not be possible, if the HW * works as intended...), bail. */ - if (coal_desc->ip_len < 0 || frag_off) { + if (ip_len < 0 || frag_off) { priv->stats.coal.coal_ip_invalid++; return; - } else if (coal_desc->ip_len > sizeof(*ip6h)) { + } + + coal_desc->ip_len = (u16)ip_len; + if (coal_desc->ip_len > sizeof(*ip6h)) { /* Don't allow coalescing of any packets with IPv6 * extension headers. */ @@ -696,44 +813,87 @@ rmnet_frag_segment_coal_data(struct rmnet_frag_descriptor *coal_desc, th = (struct tcphdr *)((u8 *)iph + coal_desc->ip_len); coal_desc->trans_len = th->doff * 4; } else if (coal_desc->trans_proto == IPPROTO_UDP) { - coal_desc->trans_len = sizeof(struct udphdr); + struct udphdr *uh; + + uh = (struct udphdr *)((u8 *)iph + coal_desc->ip_len); + coal_desc->trans_len = sizeof(*uh); + if (coal_desc->ip_proto == 4 && !uh->check) + zero_csum = true; } else { priv->stats.coal.coal_trans_invalid++; return; } coal_desc->hdrs_valid = 1; + + if (rmnet_map_v5_csum_buggy(coal_hdr) && !zero_csum) { + /* Mark the checksum as valid if it checks out */ + if (rmnet_frag_validate_csum(coal_desc)) + coal_desc->csum_valid = true; + + coal_desc->hdr_ptr = rmnet_frag_data_ptr(coal_desc); + coal_desc->gso_size = ntohs(coal_hdr->nl_pairs[0].pkt_len); + coal_desc->gso_size -= coal_desc->ip_len + coal_desc->trans_len; + coal_desc->gso_segs = coal_hdr->nl_pairs[0].num_packets; + list_add_tail(&coal_desc->list, list); + return; + } + + /* Fast-forward the case where we have 1 NLO (i.e. 1 packet length), + * no checksum errors, and are allowing GRO. We can just reuse this + * descriptor unchanged. + */ + if (gro && coal_hdr->num_nlos == 1 && coal_hdr->csum_valid) { + coal_desc->csum_valid = true; + coal_desc->hdr_ptr = rmnet_frag_data_ptr(coal_desc); + coal_desc->gso_size = ntohs(coal_hdr->nl_pairs[0].pkt_len); + coal_desc->gso_size -= coal_desc->ip_len + coal_desc->trans_len; + coal_desc->gso_segs = coal_hdr->nl_pairs[0].num_packets; + list_add_tail(&coal_desc->list, list); + return; + } + + /* Segment the coalesced descriptor into new packets */ for (nlo = 0; nlo < coal_hdr->num_nlos; nlo++) { pkt_len = ntohs(coal_hdr->nl_pairs[nlo].pkt_len); pkt_len -= coal_desc->ip_len + coal_desc->trans_len; coal_desc->gso_size = pkt_len; for (pkt = 0; pkt < coal_hdr->nl_pairs[nlo].num_packets; - pkt++, total_pkt++) { - nlo_err_mask <<= 1; - if (nlo_err_mask & (1ULL << 63)) { + pkt++, total_pkt++, nlo_err_mask >>= 1) { + bool csum_err = nlo_err_mask & 1; + + /* Segment the packet if we're not sending the larger + * packet up the stack. + */ + if (!gro) { + coal_desc->gso_segs = 1; + if (csum_err) + priv->stats.coal.coal_csum_err++; + + __rmnet_frag_segment_data(coal_desc, port, + list, total_pkt, + !csum_err); + continue; + } + + if (csum_err) { priv->stats.coal.coal_csum_err++; /* Segment out the good data */ - if (gro && coal_desc->gso_segs) + if (coal_desc->gso_segs) __rmnet_frag_segment_data(coal_desc, port, list, - total_pkt); + total_pkt, + true); - /* skip over bad packet */ - coal_desc->data_offset += pkt_len; - coal_desc->pkt_id = total_pkt + 1; + /* Segment out the bad checksum */ + coal_desc->gso_segs = 1; + __rmnet_frag_segment_data(coal_desc, port, + list, total_pkt, + false); } else { coal_desc->gso_segs++; - - /* Segment the packet if we aren't sending the - * larger packet up the stack. - */ - if (!gro) - __rmnet_frag_segment_data(coal_desc, - port, - list, - total_pkt); } } @@ -741,25 +901,9 @@ rmnet_frag_segment_coal_data(struct rmnet_frag_descriptor *coal_desc, * the previous one, if we haven't done so. NLOs only switch * when the packet length changes. */ - if (gro && coal_desc->gso_segs) { - /* Fast forward the (hopefully) common case. - * Frames with only one NLO (i.e. one packet length) and - * no checksum errors don't need to be segmented here. - * We can just pass off the original skb. - */ - if (coal_desc->gso_size * coal_desc->gso_segs == - skb_frag_size(&coal_desc->frag) - - coal_desc->ip_len - coal_desc->trans_len) { - coal_desc->hdr_ptr = - rmnet_frag_data_ptr(coal_desc); - coal_desc->csum_valid = true; - list_add_tail(&coal_desc->list, list); - return; - } - + if (coal_desc->gso_segs) __rmnet_frag_segment_data(coal_desc, port, list, - total_pkt); - } + total_pkt, true); } } @@ -840,7 +984,7 @@ rmnet_frag_data_check_coal_header(struct rmnet_frag_descriptor *frag_desc, u8 err = coal_hdr->nl_pairs[i].csum_error_bitmap; u8 pkt = coal_hdr->nl_pairs[i].num_packets; - mask |= ((u64)err) << (7 - i) * 8; + mask |= ((u64)err) << (8 * i); /* Track total packets in frame */ pkts += pkt; @@ -897,15 +1041,23 @@ int rmnet_frag_process_next_hdr_packet(struct rmnet_frag_descriptor *frag_desc, priv->stats.csum_valid_unset++; } - rmnet_frag_pull(frag_desc, port, - sizeof(struct rmnet_map_header) + - sizeof(struct rmnet_map_v5_csum_header)); + if (!rmnet_frag_pull(frag_desc, port, + sizeof(struct rmnet_map_header) + + sizeof(struct rmnet_map_v5_csum_header))) { + rc = -EINVAL; + break; + } + frag_desc->hdr_ptr = rmnet_frag_data_ptr(frag_desc); /* Remove padding only for csum offload packets. * Coalesced packets should never have padding. */ - rmnet_frag_trim(frag_desc, port, len); + if (!rmnet_frag_trim(frag_desc, port, len)) { + rc = -EINVAL; + break; + } + list_del_init(&frag_desc->list); list_add_tail(&frag_desc->list, list); break; @@ -969,10 +1121,14 @@ __rmnet_frag_ingress_handler(struct rmnet_frag_descriptor *frag_desc, goto recycle; } else { /* We only have the main QMAP header to worry about */ - rmnet_frag_pull(frag_desc, port, sizeof(*qmap)); + if (!rmnet_frag_pull(frag_desc, port, sizeof(*qmap))) + return; + frag_desc->hdr_ptr = rmnet_frag_data_ptr(frag_desc); - rmnet_frag_trim(frag_desc, port, len); + if (!rmnet_frag_trim(frag_desc, port, len)) + return; + list_add_tail(&frag_desc->list, &segs); } diff --git a/core/rmnet_descriptor.h b/core/rmnet_descriptor.h index 996a6caa1dd8..06b9c9dd231b 100644 --- a/core/rmnet_descriptor.h +++ b/core/rmnet_descriptor.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2013-2019, The Linux Foundation. All rights reserved. +/* Copyright (c) 2013-2020, The Linux Foundation. All rights reserved. * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 and @@ -92,6 +92,8 @@ static inline void *rmnet_frag_pull(struct rmnet_frag_descriptor *frag_desc, unsigned int size) { if (size >= skb_frag_size(&frag_desc->frag)) { + pr_info("%s(): Pulling %u bytes from %u byte pkt. Dropping\n", + __func__, size, skb_frag_size(&frag_desc->frag)); rmnet_recycle_frag_descriptor(frag_desc, port); return NULL; } @@ -107,6 +109,8 @@ static inline void *rmnet_frag_trim(struct rmnet_frag_descriptor *frag_desc, unsigned int size) { if (!size) { + pr_info("%s(): Trimming %u byte pkt to 0. Dropping\n", + __func__, skb_frag_size(&frag_desc->frag)); rmnet_recycle_frag_descriptor(frag_desc, port); return NULL; } diff --git a/core/rmnet_genl.c b/core/rmnet_genl.c new file mode 100644 index 000000000000..fa45ef43ca3a --- /dev/null +++ b/core/rmnet_genl.c @@ -0,0 +1,412 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright (c) 2019, The Linux Foundation. All rights reserved. + * + * RMNET Data Generic Netlink + * + */ + +#include "rmnet_genl.h" +#include +#include + +/* Static Functions and Definitions */ +static struct nla_policy rmnet_genl_attr_policy[RMNET_CORE_GENL_ATTR_MAX + + 1] = { + [RMNET_CORE_GENL_ATTR_INT] = { .type = NLA_S32 }, + [RMNET_CORE_GENL_ATTR_PID_BPS] = { .len = + sizeof(struct rmnet_core_pid_bps_resp) }, + [RMNET_CORE_GENL_ATTR_PID_BOOST] = { .len = + sizeof(struct rmnet_core_pid_boost_req) }, + [RMNET_CORE_GENL_ATTR_STR] = { .type = NLA_NUL_STRING }, +}; + +#define RMNET_CORE_GENL_OP(_cmd, _func) \ + { \ + .cmd = _cmd, \ + .doit = _func, \ + .dumpit = NULL, \ + .flags = 0, \ + } + +static const struct genl_ops rmnet_core_genl_ops[] = { + RMNET_CORE_GENL_OP(RMNET_CORE_GENL_CMD_PID_BPS_REQ, + rmnet_core_genl_pid_bps_req_hdlr), + RMNET_CORE_GENL_OP(RMNET_CORE_GENL_CMD_PID_BOOST_REQ, + rmnet_core_genl_pid_boost_req_hdlr), +}; + +struct genl_family rmnet_core_genl_family = { + .hdrsize = 0, + .name = RMNET_CORE_GENL_FAMILY_NAME, + .version = RMNET_CORE_GENL_VERSION, + .maxattr = RMNET_CORE_GENL_ATTR_MAX, + .policy = rmnet_genl_attr_policy, + .ops = rmnet_core_genl_ops, + .n_ops = ARRAY_SIZE(rmnet_core_genl_ops), +}; + +#define RMNET_PID_STATS_HT_SIZE (8) +#define RMNET_PID_STATS_HT rmnet_pid_ht +DEFINE_HASHTABLE(rmnet_pid_ht, RMNET_PID_STATS_HT_SIZE); + +/* Spinlock definition for pid hash table */ +static DEFINE_SPINLOCK(rmnet_pid_ht_splock); + +#define RMNET_GENL_SEC_TO_MSEC(x) ((x) * 1000) +#define RMNET_GENL_SEC_TO_NSEC(x) ((x) * 1000000000) +#define RMNET_GENL_BYTES_TO_BITS(x) ((x) * 8) +#define RMNET_GENL_NSEC_TO_SEC(x) ({\ + u64 __quotient = (x); \ + do_div(__quotient, 1000000000); \ + __quotient; \ +}) + +int rmnet_core_userspace_connected; +#define RMNET_QUERY_PERIOD_SEC (1) /* Period of pid/bps queries */ + +struct rmnet_pid_node_s { + struct hlist_node list; + time_t timstamp_last_query; + u64 tx_bytes; + u64 tx_bytes_last_query; + u64 tx_bps; + u64 sched_boost_period_ms; + int sched_boost_remaining_ms; + int sched_boost_enable; + pid_t pid; +}; + +void rmnet_update_pid_and_check_boost(pid_t pid, unsigned int len, + int *boost_enable, u64 *boost_period) +{ + struct hlist_node *tmp; + struct rmnet_pid_node_s *node_p; + unsigned long ht_flags; + u8 is_match_found = 0; + u64 tx_bytes = 0; + + *boost_enable = 0; + *boost_period = 0; + + /* Using do while to spin lock and unlock only once */ + spin_lock_irqsave(&rmnet_pid_ht_splock, ht_flags); + do { + hash_for_each_possible_safe(RMNET_PID_STATS_HT, node_p, tmp, + list, pid) { + if (pid != node_p->pid) + continue; + + /* PID Match found */ + is_match_found = 1; + node_p->tx_bytes += len; + tx_bytes = node_p->tx_bytes; + + if (node_p->sched_boost_enable) { + rm_err("boost triggered for pid %d", + pid); + /* Just triggered boost, dont re-trigger */ + node_p->sched_boost_enable = 0; + *boost_enable = 1; + *boost_period = node_p->sched_boost_period_ms; + node_p->sched_boost_remaining_ms = + (int)*boost_period; + } + + break; + } + + if (is_match_found) + break; + + /* No PID match */ + node_p = kzalloc(sizeof(*node_p), GFP_ATOMIC); + if (!node_p) + break; + + node_p->pid = pid; + node_p->tx_bytes = len; + node_p->sched_boost_enable = 0; + node_p->sched_boost_period_ms = 0; + node_p->sched_boost_remaining_ms = 0; + hash_add_rcu(RMNET_PID_STATS_HT, &node_p->list, pid); + break; + } while (0); + spin_unlock_irqrestore(&rmnet_pid_ht_splock, ht_flags); +} + +void rmnet_boost_for_pid(pid_t pid, int boost_enable, + u64 boost_period) +{ + struct hlist_node *tmp; + struct rmnet_pid_node_s *node_p; + unsigned long ht_flags; + + /* Using do while to spin lock and unlock only once */ + spin_lock_irqsave(&rmnet_pid_ht_splock, ht_flags); + do { + hash_for_each_possible_safe(RMNET_PID_STATS_HT, node_p, tmp, + list, pid) { + if (pid != node_p->pid) + continue; + + /* PID Match found */ + rm_err("CORE_BOOST: enable boost for pid %d for %d ms", + pid, boost_period); + node_p->sched_boost_enable = boost_enable; + node_p->sched_boost_period_ms = boost_period; + break; + } + + break; + } while (0); + spin_unlock_irqrestore(&rmnet_pid_ht_splock, ht_flags); +} + +static void rmnet_create_pid_bps_resp(struct rmnet_core_pid_bps_resp + *pid_bps_resp_ptr) +{ + struct timespec time; + struct hlist_node *tmp; + struct rmnet_pid_node_s *node_p; + unsigned long ht_flags; + u64 tx_bytes_cur, byte_diff, time_diff_ns, tmp_bits; + int i; + u16 bkt; + + (void)getnstimeofday(&time); + pid_bps_resp_ptr->timestamp = RMNET_GENL_SEC_TO_NSEC(time.tv_sec) + + time.tv_nsec; + + /* Using do while to spin lock and unlock only once */ + spin_lock_irqsave(&rmnet_pid_ht_splock, ht_flags); + do { + i = 0; + + hash_for_each_safe(RMNET_PID_STATS_HT, bkt, tmp, + node_p, list) { + tx_bytes_cur = node_p->tx_bytes; + if (tx_bytes_cur <= node_p->tx_bytes_last_query) { + /* Dont send inactive pids to userspace */ + /* TODO: can remove from hash table probably */ + node_p->tx_bps = 0; + node_p->timstamp_last_query = + pid_bps_resp_ptr->timestamp; + node_p->sched_boost_remaining_ms = 0; + continue; + } + + /* Compute bits per second */ + byte_diff = (node_p->tx_bytes - + node_p->tx_bytes_last_query); + time_diff_ns = (pid_bps_resp_ptr->timestamp - + node_p->timstamp_last_query); + + tmp_bits = RMNET_GENL_BYTES_TO_BITS(byte_diff); + /* Note that do_div returns remainder and the */ + /* numerator gets assigned the quotient */ + /* Since do_div takes the numerator as a reference, */ + /* a tmp_bits is used*/ + do_div(tmp_bits, RMNET_GENL_NSEC_TO_SEC(time_diff_ns)); + node_p->tx_bps = tmp_bits; + + if (node_p->sched_boost_remaining_ms >= + RMNET_GENL_SEC_TO_MSEC(RMNET_QUERY_PERIOD_SEC)) { + node_p->sched_boost_remaining_ms -= + RMNET_GENL_SEC_TO_MSEC(RMNET_QUERY_PERIOD_SEC); + + rm_err("CORE_BOOST: enabling boost for pid %d\n" + "sched boost remaining = %d ms", + node_p->pid, + node_p->sched_boost_remaining_ms); + } else { + node_p->sched_boost_remaining_ms = 0; + } + + pid_bps_resp_ptr->list[i].pid = node_p->pid; + pid_bps_resp_ptr->list[i].tx_bps = node_p->tx_bps; + pid_bps_resp_ptr->list[i].boost_remaining_ms = + node_p->sched_boost_remaining_ms; + + node_p->timstamp_last_query = + pid_bps_resp_ptr->timestamp; + node_p->tx_bytes_last_query = tx_bytes_cur; + i++; + + /* Support copying up to 32 active pids */ + if (i >= RMNET_CORE_GENL_MAX_PIDS) + break; + } + break; + } while (0); + spin_unlock_irqrestore(&rmnet_pid_ht_splock, ht_flags); + + pid_bps_resp_ptr->list_len = i; +} + +int rmnet_core_genl_send_resp(struct genl_info *info, + struct rmnet_core_pid_bps_resp *pid_bps_resp) +{ + struct sk_buff *skb; + void *msg_head; + int rc; + + if (!info || !pid_bps_resp) { + rm_err("%s", "SHS_GNL: Invalid params\n"); + goto out; + } + + skb = genlmsg_new(sizeof(struct rmnet_core_pid_bps_resp), GFP_KERNEL); + if (!skb) + goto out; + + msg_head = genlmsg_put(skb, 0, info->snd_seq + 1, + &rmnet_core_genl_family, + 0, RMNET_CORE_GENL_CMD_PID_BPS_REQ); + if (!msg_head) { + rc = -ENOMEM; + goto out; + } + rc = nla_put(skb, RMNET_CORE_GENL_ATTR_PID_BPS, + sizeof(struct rmnet_core_pid_bps_resp), + pid_bps_resp); + if (rc != 0) + goto out; + + genlmsg_end(skb, msg_head); + + rc = genlmsg_unicast(genl_info_net(info), skb, info->snd_portid); + if (rc != 0) + goto out; + + rm_err("%s", "SHS_GNL: Successfully sent pid/bytes info\n"); + return RMNET_GENL_SUCCESS; + +out: + /* TODO: Need to free skb?? */ + rm_err("%s", "SHS_GNL: FAILED to send pid/bytes info\n"); + rmnet_core_userspace_connected = 0; + return RMNET_GENL_FAILURE; +} + +int rmnet_core_genl_pid_bps_req_hdlr(struct sk_buff *skb_2, + struct genl_info *info) +{ + struct nlattr *na; + struct rmnet_core_pid_bps_req pid_bps_req; + struct rmnet_core_pid_bps_resp pid_bps_resp; + int is_req_valid = 0; + + rm_err("CORE_GNL: %s connected = %d", __func__, + rmnet_core_userspace_connected); + + if (!info) { + rm_err("%s", "CORE_GNL: error - info is null"); + pid_bps_resp.valid = 0; + } else { + na = info->attrs[RMNET_CORE_GENL_ATTR_PID_BPS]; + if (na) { + if (nla_memcpy(&pid_bps_req, na, + sizeof(pid_bps_req)) > 0) { + is_req_valid = 1; + } else { + rm_err("CORE_GNL: nla_memcpy failed %d\n", + RMNET_CORE_GENL_ATTR_PID_BPS); + } + } else { + rm_err("CORE_GNL: no info->attrs %d\n", + RMNET_CORE_GENL_ATTR_PID_BPS); + } + } + + if (!rmnet_core_userspace_connected) + rmnet_core_userspace_connected = 1; + + /* Copy to pid/byte list to the payload */ + if (is_req_valid) { + memset(&pid_bps_resp, 0x0, + sizeof(pid_bps_resp)); + rmnet_create_pid_bps_resp(&pid_bps_resp); + } + pid_bps_resp.valid = 1; + + rmnet_core_genl_send_resp(info, &pid_bps_resp); + + return RMNET_GENL_SUCCESS; +} + +int rmnet_core_genl_pid_boost_req_hdlr(struct sk_buff *skb_2, + struct genl_info *info) +{ + struct nlattr *na; + struct rmnet_core_pid_boost_req pid_boost_req; + int is_req_valid = 0; + u16 boost_pid_cnt = RMNET_CORE_GENL_MAX_PIDS; + u16 i = 0; + + rm_err("%s", "CORE_GNL: %s", __func__); + + if (!info) { + rm_err("%s", "CORE_GNL: error - info is null"); + return RMNET_GENL_FAILURE; + } + + na = info->attrs[RMNET_CORE_GENL_ATTR_PID_BOOST]; + if (na) { + if (nla_memcpy(&pid_boost_req, na, sizeof(pid_boost_req)) > 0) { + is_req_valid = 1; + } else { + rm_err("CORE_GNL: nla_memcpy failed %d\n", + RMNET_CORE_GENL_ATTR_PID_BOOST); + return RMNET_GENL_FAILURE; + } + } else { + rm_err("CORE_GNL: no info->attrs %d\n", + RMNET_CORE_GENL_ATTR_PID_BOOST); + return RMNET_GENL_FAILURE; + } + + if (pid_boost_req.list_len < RMNET_CORE_GENL_MAX_PIDS) + boost_pid_cnt = pid_boost_req.list_len; + + if (!pid_boost_req.valid) + boost_pid_cnt = 0; + + for (i = 0; i < boost_pid_cnt; i++) { + if (pid_boost_req.list[i].boost_enabled) { + rmnet_boost_for_pid(pid_boost_req.list[i].pid, 1, + pid_boost_req.list[i].boost_period); + } + } + + return RMNET_GENL_SUCCESS; +} + +/* register new rmnet core driver generic netlink family */ +int rmnet_core_genl_init(void) +{ + int ret; + + ret = genl_register_family(&rmnet_core_genl_family); + if (ret != 0) { + rm_err("CORE_GNL: register family failed: %i", ret); + genl_unregister_family(&rmnet_core_genl_family); + return RMNET_GENL_FAILURE; + } + + rm_err("CORE_GNL: successfully registered generic netlink family: %s", + RMNET_CORE_GENL_FAMILY_NAME); + + return RMNET_GENL_SUCCESS; +} + +/* Unregister the generic netlink family */ +int rmnet_core_genl_deinit(void) +{ + int ret; + + ret = genl_unregister_family(&rmnet_core_genl_family); + if (ret != 0) + rm_err("CORE_GNL: unregister family failed: %i\n", ret); + + return RMNET_GENL_SUCCESS; +} diff --git a/core/rmnet_genl.h b/core/rmnet_genl.h new file mode 100644 index 000000000000..2a735fa2ff1e --- /dev/null +++ b/core/rmnet_genl.h @@ -0,0 +1,96 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* Copyright (c) 2019, The Linux Foundation. All rights reserved. + * + * RMNET Data Generic Netlink + * + */ + +#ifndef _RMNET_GENL_H_ +#define _RMNET_GENL_H_ + +#include + +#define RMNET_CORE_DEBUG 0 + +#define rm_err(fmt, ...) \ + do { if (RMNET_CORE_DEBUG) pr_err(fmt, __VA_ARGS__); } while (0) + +/* Generic Netlink Definitions */ +#define RMNET_CORE_GENL_VERSION 1 +#define RMNET_CORE_GENL_FAMILY_NAME "RMNET_CORE" + +#define RMNET_CORE_GENL_MAX_PIDS 32 + +#define RMNET_GENL_SUCCESS (0) +#define RMNET_GENL_FAILURE (-1) + +extern int rmnet_core_userspace_connected; + +enum { + RMNET_CORE_GENL_CMD_UNSPEC, + RMNET_CORE_GENL_CMD_PID_BPS_REQ, + RMNET_CORE_GENL_CMD_PID_BOOST_REQ, + __RMNET_CORE_GENL_CMD_MAX, +}; + +enum { + RMNET_CORE_GENL_ATTR_UNSPEC, + RMNET_CORE_GENL_ATTR_STR, + RMNET_CORE_GENL_ATTR_INT, + RMNET_CORE_GENL_ATTR_PID_BPS, + RMNET_CORE_GENL_ATTR_PID_BOOST, + __RMNET_CORE_GENL_ATTR_MAX, +}; + +#define RMNET_CORE_GENL_ATTR_MAX (__RMNET_CORE_GENL_ATTR_MAX - 1) + +struct rmnet_core_pid_bps_info { + u64 tx_bps; + u32 pid; + u32 boost_remaining_ms; +}; + +struct rmnet_core_pid_boost_info { + u32 boost_enabled; + /* Boost period in ms */ + u32 boost_period; + u32 pid; +}; + +struct rmnet_core_pid_bps_req { + struct rmnet_core_pid_bps_info list[RMNET_CORE_GENL_MAX_PIDS]; + u64 timestamp; + u16 list_len; + u8 valid; +}; + +struct rmnet_core_pid_bps_resp { + struct rmnet_core_pid_bps_info list[RMNET_CORE_GENL_MAX_PIDS]; + u64 timestamp; + u16 list_len; + u8 valid; +}; + +struct rmnet_core_pid_boost_req { + struct rmnet_core_pid_boost_info list[RMNET_CORE_GENL_MAX_PIDS]; + u64 timestamp; + u16 list_len; + u8 valid; +}; + +/* Function Prototypes */ +int rmnet_core_genl_pid_bps_req_hdlr(struct sk_buff *skb_2, + struct genl_info *info); + +int rmnet_core_genl_pid_boost_req_hdlr(struct sk_buff *skb_2, + struct genl_info *info); + +/* Called by vnd select queue */ +void rmnet_update_pid_and_check_boost(pid_t pid, unsigned int len, + int *boost_enable, u64 *boost_period); + +int rmnet_core_genl_init(void); + +int rmnet_core_genl_deinit(void); + +#endif /*_RMNET_CORE_GENL_H_*/ diff --git a/core/rmnet_handlers.c b/core/rmnet_handlers.c index a4e5d36488b9..a640710c70ad 100644 --- a/core/rmnet_handlers.c +++ b/core/rmnet_handlers.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2013-2019, The Linux Foundation. All rights reserved. +/* Copyright (c) 2013-2020, The Linux Foundation. All rights reserved. * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 and @@ -69,6 +69,21 @@ void rmnet_set_skb_proto(struct sk_buff *skb) } EXPORT_SYMBOL(rmnet_set_skb_proto); +bool (*rmnet_shs_slow_start_detect)(u32 hash_key) __rcu __read_mostly; +EXPORT_SYMBOL(rmnet_shs_slow_start_detect); + +bool rmnet_slow_start_on(u32 hash_key) +{ + bool (*rmnet_shs_slow_start_on)(u32 hash_key); + + rmnet_shs_slow_start_on = rcu_dereference(rmnet_shs_slow_start_detect); + if (rmnet_shs_slow_start_on) + return rmnet_shs_slow_start_on(hash_key); + + return false; +} +EXPORT_SYMBOL(rmnet_slow_start_on); + /* Shs hook handler */ int (*rmnet_shs_skb_entry)(struct sk_buff *skb, struct rmnet_port *port) __rcu __read_mostly; @@ -245,7 +260,7 @@ rmnet_map_ingress_handler(struct sk_buff *skb, struct rmnet_port *port); if (skb->dev->type == ARPHRD_ETHER) { - if (pskb_expand_head(skb, ETH_HLEN, 0, GFP_KERNEL)) { + if (pskb_expand_head(skb, ETH_HLEN, 0, GFP_ATOMIC)) { kfree_skb(skb); return; } @@ -311,7 +326,8 @@ static int rmnet_map_egress_handler(struct sk_buff *skb, if (port->data_format & RMNET_FLAGS_EGRESS_MAP_CKSUMV4) { additional_header_len = sizeof(struct rmnet_map_ul_csum_header); csum_type = RMNET_FLAGS_EGRESS_MAP_CKSUMV4; - } else if (port->data_format & RMNET_FLAGS_EGRESS_MAP_CKSUMV5) { + } else if ((port->data_format & RMNET_FLAGS_EGRESS_MAP_CKSUMV5) || + (port->data_format & RMNET_EGRESS_FORMAT_PRIORITY)) { additional_header_len = sizeof(struct rmnet_map_v5_csum_header); csum_type = RMNET_FLAGS_EGRESS_MAP_CKSUMV5; } @@ -323,8 +339,12 @@ static int rmnet_map_egress_handler(struct sk_buff *skb, return -ENOMEM; } + if (port->data_format & RMNET_INGRESS_FORMAT_PS) + qmi_rmnet_work_maybe_restart(port); + if (csum_type) - rmnet_map_checksum_uplink_packet(skb, orig_dev, csum_type); + rmnet_map_checksum_uplink_packet(skb, port, orig_dev, + csum_type); map_header = rmnet_map_add_map_header(skb, additional_header_len, 0, port); @@ -334,19 +354,9 @@ static int rmnet_map_egress_handler(struct sk_buff *skb, map_header->mux_id = mux_id; if (port->data_format & RMNET_EGRESS_FORMAT_AGGREGATION) { - int non_linear_skb; - if (rmnet_map_tx_agg_skip(skb, required_headroom)) goto done; - non_linear_skb = (orig_dev->features & NETIF_F_GSO) && - skb_is_nonlinear(skb); - - if (non_linear_skb) { - if (unlikely(__skb_linearize(skb))) - goto done; - } - rmnet_map_tx_aggregate(skb, port); return -EINPROGRESS; } @@ -430,9 +440,9 @@ void rmnet_egress_handler(struct sk_buff *skb) skb_len = skb->len; err = rmnet_map_egress_handler(skb, port, mux_id, orig_dev); - if (err == -ENOMEM) + if (err == -ENOMEM) { goto drop; - else if (err == -EINPROGRESS) { + } else if (err == -EINPROGRESS) { rmnet_vnd_tx_fixup(orig_dev, skb_len); return; } diff --git a/core/rmnet_handlers.h b/core/rmnet_handlers.h index f821b764a909..4cc4cff2de72 100644 --- a/core/rmnet_handlers.h +++ b/core/rmnet_handlers.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2013, 2016-2017, 2019, The Linux Foundation. All rights reserved. +/* Copyright (c) 2013, 2016-2017, 2019-2020, The Linux Foundation. All rights reserved. * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 and @@ -28,6 +28,7 @@ void rmnet_deliver_skb(struct sk_buff *skb, struct rmnet_port *port); void rmnet_deliver_skb_wq(struct sk_buff *skb, struct rmnet_port *port, enum rmnet_packet_context ctx); void rmnet_set_skb_proto(struct sk_buff *skb); +bool rmnet_slow_start_on(u32 hash_key); rx_handler_result_t _rmnet_map_ingress_handler(struct sk_buff *skb, struct rmnet_port *port); rx_handler_result_t rmnet_rx_handler(struct sk_buff **pskb); diff --git a/core/rmnet_map.h b/core/rmnet_map.h index 07164d952384..f7116d348a0a 100644 --- a/core/rmnet_map.h +++ b/core/rmnet_map.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2013-2019, The Linux Foundation. All rights reserved. +/* Copyright (c) 2013-2020, The Linux Foundation. All rights reserved. * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 and @@ -80,7 +80,9 @@ struct rmnet_map_header { struct rmnet_map_v5_csum_header { u8 next_hdr:1; u8 header_type:7; - u8 hw_reserved:7; + u8 hw_reserved:5; + u8 priority:1; + u8 hw_reserved_bit:1; u8 csum_valid_required:1; __be16 reserved; } __aligned(1); @@ -122,7 +124,7 @@ struct rmnet_map_dl_csum_trailer { struct rmnet_map_ul_csum_header { __be16 csum_start_offset; u16 csum_insert_offset:14; - u16 udp_ip4_ind:1; + u16 udp_ind:1; u16 csum_enabled:1; } __aligned(1); @@ -179,16 +181,18 @@ struct rmnet_map_dl_ind_trl { struct rmnet_map_dl_ind { u8 priority; union { - void (*dl_hdr_handler)(struct rmnet_map_dl_ind_hdr *); - void (*dl_hdr_handler_v2)(struct rmnet_map_dl_ind_hdr *, + void (*dl_hdr_handler)(struct rmnet_map_dl_ind_hdr *dlhdr); + void (*dl_hdr_handler_v2)(struct rmnet_map_dl_ind_hdr *dlhdr, struct - rmnet_map_control_command_header *); + rmnet_map_control_command_header + * qcmd); } __aligned(1); union { - void (*dl_trl_handler)(struct rmnet_map_dl_ind_trl *); - void (*dl_trl_handler_v2)(struct rmnet_map_dl_ind_trl *, + void (*dl_trl_handler)(struct rmnet_map_dl_ind_trl *dltrl); + void (*dl_trl_handler_v2)(struct rmnet_map_dl_ind_trl *dltrl, struct - rmnet_map_control_command_header *); + rmnet_map_control_command_header + * qcmd); } __aligned(1); struct list_head list; }; @@ -258,8 +262,10 @@ struct rmnet_map_header *rmnet_map_add_map_header(struct sk_buff *skb, void rmnet_map_command(struct sk_buff *skb, struct rmnet_port *port); int rmnet_map_checksum_downlink_packet(struct sk_buff *skb, u16 len); void rmnet_map_checksum_uplink_packet(struct sk_buff *skb, + struct rmnet_port *port, struct net_device *orig_dev, int csum_type); +bool rmnet_map_v5_csum_buggy(struct rmnet_map_v5_coal_header *coal_hdr); int rmnet_map_process_next_hdr_packet(struct sk_buff *skb, struct sk_buff_head *list, u16 len); @@ -267,6 +273,8 @@ int rmnet_map_tx_agg_skip(struct sk_buff *skb, int offset); void rmnet_map_tx_aggregate(struct sk_buff *skb, struct rmnet_port *port); void rmnet_map_tx_aggregate_init(struct rmnet_port *port); void rmnet_map_tx_aggregate_exit(struct rmnet_port *port); +void rmnet_map_update_ul_agg_config(struct rmnet_port *port, u16 size, + u8 count, u8 features, u32 time); void rmnet_map_dl_hdr_notify(struct rmnet_port *port, struct rmnet_map_dl_ind_hdr *dl_hdr); void rmnet_map_dl_hdr_notify_v2(struct rmnet_port *port, diff --git a/core/rmnet_map_data.c b/core/rmnet_map_data.c index 505fcc33de2f..34e815e4f103 100644 --- a/core/rmnet_map_data.c +++ b/core/rmnet_map_data.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2013-2019, The Linux Foundation. All rights reserved. +/* Copyright (c) 2013-2020, The Linux Foundation. All rights reserved. * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 and @@ -230,9 +230,9 @@ rmnet_map_ipv4_ul_csum_header(void *iphdr, ul_header->csum_insert_offset = skb->csum_offset; ul_header->csum_enabled = 1; if (ip4h->protocol == IPPROTO_UDP) - ul_header->udp_ip4_ind = 1; + ul_header->udp_ind = 1; else - ul_header->udp_ip4_ind = 0; + ul_header->udp_ind = 0; /* Changing remaining fields to network order */ hdr++; @@ -263,6 +263,7 @@ rmnet_map_ipv6_ul_csum_header(void *ip6hdr, struct rmnet_map_ul_csum_header *ul_header, struct sk_buff *skb) { + struct ipv6hdr *ip6h = (struct ipv6hdr *)ip6hdr; __be16 *hdr = (__be16 *)ul_header, offset; offset = htons((__force u16)(skb_transport_header(skb) - @@ -270,7 +271,11 @@ rmnet_map_ipv6_ul_csum_header(void *ip6hdr, ul_header->csum_start_offset = offset; ul_header->csum_insert_offset = skb->csum_offset; ul_header->csum_enabled = 1; - ul_header->udp_ip4_ind = 0; + + if (ip6h->nexthdr == IPPROTO_UDP) + ul_header->udp_ind = 1; + else + ul_header->udp_ind = 0; /* Changing remaining fields to network order */ hdr++; @@ -479,12 +484,25 @@ sw_csum: ul_header->csum_start_offset = 0; ul_header->csum_insert_offset = 0; ul_header->csum_enabled = 0; - ul_header->udp_ip4_ind = 0; + ul_header->udp_ind = 0; priv->stats.csum_sw++; } +static void rmnet_map_v5_check_priority(struct sk_buff *skb, + struct net_device *orig_dev, + struct rmnet_map_v5_csum_header *hdr) +{ + struct rmnet_priv *priv = netdev_priv(orig_dev); + + if (skb->priority) { + priv->stats.ul_prio++; + hdr->priority = 1; + } +} + void rmnet_map_v5_checksum_uplink_packet(struct sk_buff *skb, + struct rmnet_port *port, struct net_device *orig_dev) { struct rmnet_priv *priv = netdev_priv(orig_dev); @@ -495,6 +513,13 @@ void rmnet_map_v5_checksum_uplink_packet(struct sk_buff *skb, memset(ul_header, 0, sizeof(*ul_header)); ul_header->header_type = RMNET_MAP_HEADER_TYPE_CSUM_OFFLOAD; + if (port->data_format & RMNET_EGRESS_FORMAT_PRIORITY) + rmnet_map_v5_check_priority(skb, orig_dev, ul_header); + + /* Allow priority w/o csum offload */ + if (!(port->data_format & RMNET_FLAGS_EGRESS_MAP_CKSUMV5)) + return; + if (skb->ip_summed == CHECKSUM_PARTIAL) { void *iph = (char *)ul_header + sizeof(*ul_header); void *trans; @@ -535,6 +560,7 @@ sw_csum: * packets that are supported for UL checksum offload. */ void rmnet_map_checksum_uplink_packet(struct sk_buff *skb, + struct rmnet_port *port, struct net_device *orig_dev, int csum_type) { @@ -543,13 +569,36 @@ void rmnet_map_checksum_uplink_packet(struct sk_buff *skb, rmnet_map_v4_checksum_uplink_packet(skb, orig_dev); break; case RMNET_FLAGS_EGRESS_MAP_CKSUMV5: - rmnet_map_v5_checksum_uplink_packet(skb, orig_dev); + rmnet_map_v5_checksum_uplink_packet(skb, port, orig_dev); break; default: break; } } +bool rmnet_map_v5_csum_buggy(struct rmnet_map_v5_coal_header *coal_hdr) +{ + /* Only applies to frames with a single packet */ + if (coal_hdr->num_nlos != 1 || coal_hdr->nl_pairs[0].num_packets != 1) + return false; + + /* TCP header has FIN or PUSH set */ + if (coal_hdr->close_type == RMNET_MAP_COAL_CLOSE_COAL) + return true; + + /* Hit packet limit, byte limit, or time limit/EOF on DMA */ + if (coal_hdr->close_type == RMNET_MAP_COAL_CLOSE_HW) { + switch (coal_hdr->close_value) { + case RMNET_MAP_COAL_CLOSE_HW_PKT: + case RMNET_MAP_COAL_CLOSE_HW_BYTE: + case RMNET_MAP_COAL_CLOSE_HW_TIME: + return true; + } + } + + return false; +} + static void rmnet_map_move_headers(struct sk_buff *skb) { struct iphdr *iph; @@ -636,50 +685,67 @@ static void rmnet_map_gso_stamp(struct sk_buff *skb, struct rmnet_map_coal_metadata *coal_meta) { struct skb_shared_info *shinfo = skb_shinfo(skb); - struct iphdr *iph = ip_hdr(skb); + + if (coal_meta->trans_proto == IPPROTO_TCP) + shinfo->gso_type = (coal_meta->ip_proto == 4) ? + SKB_GSO_TCPV4 : SKB_GSO_TCPV6; + else + shinfo->gso_type = SKB_GSO_UDP_L4; + + shinfo->gso_size = coal_meta->data_len; + shinfo->gso_segs = coal_meta->pkt_count; +} + +/* Handles setting up the partial checksum in the skb. Sets the transport + * checksum to the pseudoheader checksum and sets the csum offload metadata + */ +static void rmnet_map_partial_csum(struct sk_buff *skb, + struct rmnet_map_coal_metadata *coal_meta) +{ + unsigned char *data = skb->data; __sum16 pseudo; u16 pkt_len = skb->len - coal_meta->ip_len; - bool ipv4 = coal_meta->ip_proto == 4; - if (ipv4) { + if (coal_meta->ip_proto == 4) { + struct iphdr *iph = (struct iphdr *)data; + pseudo = ~csum_tcpudp_magic(iph->saddr, iph->daddr, pkt_len, coal_meta->trans_proto, 0); } else { - struct ipv6hdr *ip6h = ipv6_hdr(skb); + struct ipv6hdr *ip6h = (struct ipv6hdr *)data; pseudo = ~csum_ipv6_magic(&ip6h->saddr, &ip6h->daddr, pkt_len, coal_meta->trans_proto, 0); } if (coal_meta->trans_proto == IPPROTO_TCP) { - struct tcphdr *tp = tcp_hdr(skb); + struct tcphdr *tp = (struct tcphdr *)(data + coal_meta->ip_len); tp->check = pseudo; - shinfo->gso_type = (ipv4) ? SKB_GSO_TCPV4 : SKB_GSO_TCPV6; skb->csum_offset = offsetof(struct tcphdr, check); } else { - struct udphdr *up = udp_hdr(skb); + struct udphdr *up = (struct udphdr *)(data + coal_meta->ip_len); up->check = pseudo; - shinfo->gso_type = SKB_GSO_UDP_L4; skb->csum_offset = offsetof(struct udphdr, check); } skb->ip_summed = CHECKSUM_PARTIAL; skb->csum_start = skb->data + coal_meta->ip_len - skb->head; - shinfo->gso_size = coal_meta->data_len; - shinfo->gso_segs = coal_meta->pkt_count; } static void __rmnet_map_segment_coal_skb(struct sk_buff *coal_skb, struct rmnet_map_coal_metadata *coal_meta, - struct sk_buff_head *list, u8 pkt_id) + struct sk_buff_head *list, u8 pkt_id, + bool csum_valid) { struct sk_buff *skbn; struct rmnet_priv *priv = netdev_priv(coal_skb->dev); + __sum16 *check = NULL; u32 alloc_len; + bool zero_csum = false; /* We can avoid copying the data if the SKB we got from the lower-level * drivers was nonlinear. @@ -688,7 +754,7 @@ __rmnet_map_segment_coal_skb(struct sk_buff *coal_skb, alloc_len = coal_meta->ip_len + coal_meta->trans_len; else alloc_len = coal_meta->ip_len + coal_meta->trans_len + - coal_meta->data_len; + (coal_meta->data_len * coal_meta->pkt_count); skbn = alloc_skb(alloc_len, GFP_ATOMIC); if (!skbn) @@ -705,8 +771,14 @@ __rmnet_map_segment_coal_skb(struct sk_buff *coal_skb, struct tcphdr *th = tcp_hdr(skbn); th->seq = htonl(ntohl(th->seq) + coal_meta->data_offset); + check = &th->check; } else if (coal_meta->trans_proto == IPPROTO_UDP) { - udp_hdr(skbn)->len = htons(skbn->len); + struct udphdr *uh = udp_hdr(skbn); + + uh->len = htons(skbn->len); + check = &uh->check; + if (coal_meta->ip_proto == 4 && !uh->check) + zero_csum = true; } /* Push IP header and update necessary fields */ @@ -726,7 +798,45 @@ __rmnet_map_segment_coal_skb(struct sk_buff *coal_skb, sizeof(struct ipv6hdr)); } - skbn->ip_summed = CHECKSUM_UNNECESSARY; + /* Handle checksum status */ + if (likely(csum_valid) || zero_csum) { + /* Set the partial checksum information */ + rmnet_map_partial_csum(skbn, coal_meta); + } else if (check) { + /* Unfortunately, we have to fake a bad checksum here, since + * the original bad value is lost by the hardware. The only + * reliable way to do it is to calculate the actual checksum + * and corrupt it. + */ + __wsum csum; + unsigned int offset = skb_transport_offset(skbn); + __sum16 pseudo; + + /* Calculate pseudo header */ + if (coal_meta->ip_proto == 4) { + struct iphdr *iph = ip_hdr(skbn); + + pseudo = ~csum_tcpudp_magic(iph->saddr, iph->daddr, + skbn->len - + coal_meta->ip_len, + coal_meta->trans_proto, 0); + } else { + struct ipv6hdr *ip6h = ipv6_hdr(skbn); + + pseudo = ~csum_ipv6_magic(&ip6h->saddr, &ip6h->daddr, + skbn->len - coal_meta->ip_len, + coal_meta->trans_proto, 0); + } + + *check = pseudo; + csum = skb_checksum(skbn, offset, skbn->len - offset, 0); + /* Add 1 to corrupt. This cannot produce a final value of 0 + * since csum_fold() can't return a value of 0xFFFF. + */ + *check = csum16_add(csum_fold(csum), htons(1)); + skbn->ip_summed = CHECKSUM_NONE; + } + skbn->dev = coal_skb->dev; priv->stats.coal.coal_reconstruct++; @@ -742,6 +852,34 @@ __rmnet_map_segment_coal_skb(struct sk_buff *coal_skb, coal_meta->pkt_count = 0; } +static bool rmnet_map_validate_csum(struct sk_buff *skb, + struct rmnet_map_coal_metadata *meta) +{ + u8 *data = rmnet_map_data_ptr(skb); + unsigned int datagram_len; + __wsum csum; + __sum16 pseudo; + + datagram_len = skb->len - meta->ip_len; + if (meta->ip_proto == 4) { + struct iphdr *iph = (struct iphdr *)data; + + pseudo = ~csum_tcpudp_magic(iph->saddr, iph->daddr, + datagram_len, + meta->trans_proto, 0); + } else { + struct ipv6hdr *ip6h = (struct ipv6hdr *)data; + + pseudo = ~csum_ipv6_magic(&ip6h->saddr, &ip6h->daddr, + datagram_len, meta->trans_proto, + 0); + } + + csum = skb_checksum(skb, meta->ip_len, datagram_len, + csum_unfold(pseudo)); + return !csum_fold(csum); +} + /* Converts the coalesced SKB into a list of SKBs. * NLOs containing csum erros will not be included. * The original coalesced SKB should be treated as invalid and @@ -759,6 +897,7 @@ static void rmnet_map_segment_coal_skb(struct sk_buff *coal_skb, u8 pkt, total_pkt = 0; u8 nlo; bool gro = coal_skb->dev->features & NETIF_F_GRO_HW; + bool zero_csum = false; memset(&coal_meta, 0, sizeof(coal_meta)); @@ -820,43 +959,85 @@ static void rmnet_map_segment_coal_skb(struct sk_buff *coal_skb, uh = (struct udphdr *)((u8 *)iph + coal_meta.ip_len); coal_meta.trans_len = sizeof(*uh); coal_meta.trans_header = uh; + /* Check for v4 zero checksum */ + if (coal_meta.ip_proto == 4 && !uh->check) + zero_csum = true; } else { priv->stats.coal.coal_trans_invalid++; return; } + if (rmnet_map_v5_csum_buggy(coal_hdr) && !zero_csum) { + rmnet_map_move_headers(coal_skb); + /* Mark as valid if it checks out */ + if (rmnet_map_validate_csum(coal_skb, &coal_meta)) + coal_skb->ip_summed = CHECKSUM_UNNECESSARY; + + __skb_queue_tail(list, coal_skb); + return; + } + + /* Fast-forward the case where we have 1 NLO (i.e. 1 packet length), + * no checksum errors, and are allowing GRO. We can just reuse this + * SKB unchanged. + */ + if (gro && coal_hdr->num_nlos == 1 && coal_hdr->csum_valid) { + rmnet_map_move_headers(coal_skb); + coal_skb->ip_summed = CHECKSUM_UNNECESSARY; + coal_meta.data_len = ntohs(coal_hdr->nl_pairs[0].pkt_len); + coal_meta.data_len -= coal_meta.ip_len + coal_meta.trans_len; + coal_meta.pkt_count = coal_hdr->nl_pairs[0].num_packets; + if (coal_meta.pkt_count > 1) { + rmnet_map_partial_csum(coal_skb, &coal_meta); + rmnet_map_gso_stamp(coal_skb, &coal_meta); + } + + __skb_queue_tail(list, coal_skb); + return; + } + + /* Segment the coalesced SKB into new packets */ for (nlo = 0; nlo < coal_hdr->num_nlos; nlo++) { pkt_len = ntohs(coal_hdr->nl_pairs[nlo].pkt_len); pkt_len -= coal_meta.ip_len + coal_meta.trans_len; coal_meta.data_len = pkt_len; for (pkt = 0; pkt < coal_hdr->nl_pairs[nlo].num_packets; - pkt++, total_pkt++) { - nlo_err_mask <<= 1; - if (nlo_err_mask & (1ULL << 63)) { + pkt++, total_pkt++, nlo_err_mask >>= 1) { + bool csum_err = nlo_err_mask & 1; + + /* Segment the packet if we're not sending the larger + * packet up the stack. + */ + if (!gro) { + coal_meta.pkt_count = 1; + if (csum_err) + priv->stats.coal.coal_csum_err++; + + __rmnet_map_segment_coal_skb(coal_skb, + &coal_meta, list, + total_pkt, + !csum_err); + continue; + } + + if (csum_err) { priv->stats.coal.coal_csum_err++; /* Segment out the good data */ - if (gro && coal_meta.pkt_count) { + if (gro && coal_meta.pkt_count) __rmnet_map_segment_coal_skb(coal_skb, &coal_meta, list, - total_pkt); - } + total_pkt, + true); - /* skip over bad packet */ - coal_meta.data_offset += pkt_len; - coal_meta.pkt_id = total_pkt + 1; + /* Segment out the bad checksum */ + coal_meta.pkt_count = 1; + __rmnet_map_segment_coal_skb(coal_skb, + &coal_meta, list, + total_pkt, false); } else { coal_meta.pkt_count++; - - /* Segment the packet if we aren't sending the - * larger packet up the stack. - */ - if (!gro) - __rmnet_map_segment_coal_skb(coal_skb, - &coal_meta, - list, - total_pkt); } } @@ -864,27 +1045,9 @@ static void rmnet_map_segment_coal_skb(struct sk_buff *coal_skb, * the previous one, if we haven't done so. NLOs only switch * when the packet length changes. */ - if (gro && coal_meta.pkt_count) { - /* Fast forward the (hopefully) common case. - * Frames with only one NLO (i.e. one packet length) and - * no checksum errors don't need to be segmented here. - * We can just pass off the original skb. - */ - if (pkt_len * coal_meta.pkt_count == - coal_skb->len - coal_meta.ip_len - - coal_meta.trans_len) { - rmnet_map_move_headers(coal_skb); - coal_skb->ip_summed = CHECKSUM_UNNECESSARY; - if (coal_meta.pkt_count > 1) - rmnet_map_gso_stamp(coal_skb, - &coal_meta); - __skb_queue_tail(list, coal_skb); - return; - } - + if (coal_meta.pkt_count) __rmnet_map_segment_coal_skb(coal_skb, &coal_meta, list, - total_pkt); - } + total_pkt, true); } } @@ -964,7 +1127,7 @@ static int rmnet_map_data_check_coal_header(struct sk_buff *skb, u8 err = coal_hdr->nl_pairs[i].csum_error_bitmap; u8 pkt = coal_hdr->nl_pairs[i].num_packets; - mask |= ((u64)err) << (7 - i) * 8; + mask |= ((u64)err) << (8 * i); /* Track total packets in frame */ pkts += pkt; @@ -1105,13 +1268,154 @@ enum hrtimer_restart rmnet_map_flush_tx_packet_queue(struct hrtimer *t) return HRTIMER_NORESTART; } +static void rmnet_map_linearize_copy(struct sk_buff *dst, struct sk_buff *src) +{ + unsigned int linear = src->len - src->data_len, target = src->len; + unsigned char *src_buf; + struct sk_buff *skb; + + src_buf = src->data; + skb_put_data(dst, src_buf, linear); + target -= linear; + + skb = src; + + while (target) { + unsigned int i = 0, non_linear = 0; + + for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { + non_linear = skb_frag_size(&skb_shinfo(skb)->frags[i]); + src_buf = skb_frag_address(&skb_shinfo(skb)->frags[i]); + + skb_put_data(dst, src_buf, non_linear); + target -= non_linear; + } + + if (skb_shinfo(skb)->frag_list) { + skb = skb_shinfo(skb)->frag_list; + continue; + } + + if (skb->next) + skb = skb->next; + } +} + +static void rmnet_free_agg_pages(struct rmnet_port *port) +{ + struct rmnet_agg_page *agg_page, *idx; + + list_for_each_entry_safe(agg_page, idx, &port->agg_list, list) { + list_del(&agg_page->list); + put_page(agg_page->page); + kfree(agg_page); + } + + port->agg_head = NULL; +} + +static struct page *rmnet_get_agg_pages(struct rmnet_port *port) +{ + struct rmnet_agg_page *agg_page; + struct page *page = NULL; + int i = 0; + + if (!(port->egress_agg_params.agg_features & RMNET_PAGE_RECYCLE)) + goto alloc; + + do { + agg_page = port->agg_head; + if (unlikely(!agg_page)) + break; + + if (page_ref_count(agg_page->page) == 1) { + page = agg_page->page; + page_ref_inc(agg_page->page); + + port->stats.agg.ul_agg_reuse++; + port->agg_head = list_next_entry(agg_page, list); + break; + } + + port->agg_head = list_next_entry(agg_page, list); + i++; + } while (i <= 5); + +alloc: + if (!page) { + page = __dev_alloc_pages(GFP_ATOMIC, port->agg_size_order); + port->stats.agg.ul_agg_alloc++; + } + + return page; +} + +static struct rmnet_agg_page *__rmnet_alloc_agg_pages(struct rmnet_port *port) +{ + struct rmnet_agg_page *agg_page; + struct page *page; + + agg_page = kzalloc(sizeof(*agg_page), GFP_ATOMIC); + if (!agg_page) + return NULL; + + page = __dev_alloc_pages(GFP_ATOMIC, port->agg_size_order); + if (!page) { + kfree(agg_page); + return NULL; + } + + agg_page->page = page; + INIT_LIST_HEAD(&agg_page->list); + + return agg_page; +} + +static void rmnet_alloc_agg_pages(struct rmnet_port *port) +{ + struct rmnet_agg_page *agg_page = NULL; + int i = 0; + + for (i = 0; i < 512; i++) { + agg_page = __rmnet_alloc_agg_pages(port); + + if (agg_page) + list_add_tail(&agg_page->list, &port->agg_list); + } + + port->agg_head = list_first_entry_or_null(&port->agg_list, + struct rmnet_agg_page, list); +} + +static struct sk_buff *rmnet_map_build_skb(struct rmnet_port *port) +{ + struct sk_buff *skb; + unsigned int size; + struct page *page; + void *vaddr; + + page = rmnet_get_agg_pages(port); + if (!page) + return NULL; + + vaddr = page_address(page); + size = PAGE_SIZE << port->agg_size_order; + + skb = build_skb(vaddr, size); + if (!skb) { + put_page(page); + return NULL; + } + + return skb; +} + void rmnet_map_tx_aggregate(struct sk_buff *skb, struct rmnet_port *port) { struct timespec64 diff, last; int size, agg_count = 0; struct sk_buff *agg_skb; unsigned long flags; - u8 *dest_buff; new_packet: spin_lock_irqsave(&port->agg_lock, flags); @@ -1133,7 +1437,7 @@ new_packet: return; } - port->agg_skb = skb_copy_expand(skb, 0, size, GFP_ATOMIC); + port->agg_skb = rmnet_map_build_skb(port); if (!port->agg_skb) { port->agg_skb = 0; port->agg_count = 0; @@ -1143,6 +1447,9 @@ new_packet: dev_queue_xmit(skb); return; } + + rmnet_map_linearize_copy(port->agg_skb, skb); + port->agg_skb->dev = skb->dev; port->agg_skb->protocol = htons(ETH_P_MAP); port->agg_count = 1; ktime_get_real_ts64(&port->agg_time); @@ -1167,8 +1474,7 @@ new_packet: goto new_packet; } - dest_buff = skb_put(port->agg_skb, skb->len); - memcpy(dest_buff, skb->data, skb->len); + rmnet_map_linearize_copy(port->agg_skb, skb); port->agg_count++; dev_kfree_skb_any(skb); @@ -1182,14 +1488,51 @@ schedule: spin_unlock_irqrestore(&port->agg_lock, flags); } +void rmnet_map_update_ul_agg_config(struct rmnet_port *port, u16 size, + u8 count, u8 features, u32 time) +{ + unsigned long irq_flags; + + spin_lock_irqsave(&port->agg_lock, irq_flags); + port->egress_agg_params.agg_count = count; + port->egress_agg_params.agg_time = time; + port->egress_agg_params.agg_size = size; + port->egress_agg_params.agg_features = features; + + rmnet_free_agg_pages(port); + + /* This effectively disables recycling in case the UL aggregation + * size is lesser than PAGE_SIZE. + */ + if (size < PAGE_SIZE) + goto done; + + port->agg_size_order = get_order(size); + + size = PAGE_SIZE << port->agg_size_order; + size -= SKB_DATA_ALIGN(sizeof(struct skb_shared_info)); + port->egress_agg_params.agg_size = size; + + if (port->egress_agg_params.agg_features == RMNET_PAGE_RECYCLE) + rmnet_alloc_agg_pages(port); + +done: + spin_unlock_irqrestore(&port->agg_lock, irq_flags); +} + void rmnet_map_tx_aggregate_init(struct rmnet_port *port) { hrtimer_init(&port->hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); port->hrtimer.function = rmnet_map_flush_tx_packet_queue; - port->egress_agg_params.agg_size = 8192; - port->egress_agg_params.agg_count = 20; - port->egress_agg_params.agg_time = 3000000; spin_lock_init(&port->agg_lock); + INIT_LIST_HEAD(&port->agg_list); + + /* Since PAGE_SIZE - 1 is specified here, no pages are pre-allocated. + * This is done to reduce memory usage in cases where + * UL aggregation is disabled. + * Additionally, the features flag is also set to 0. + */ + rmnet_map_update_ul_agg_config(port, PAGE_SIZE - 1, 20, 0, 3000000); INIT_WORK(&port->agg_wq, rmnet_map_flush_tx_packet_work); } @@ -1213,6 +1556,7 @@ void rmnet_map_tx_aggregate_exit(struct rmnet_port *port) port->agg_state = 0; } + rmnet_free_agg_pages(port); spin_unlock_irqrestore(&port->agg_lock, flags); } @@ -1224,7 +1568,7 @@ void rmnet_map_tx_qmap_cmd(struct sk_buff *qmap_skb) port = rmnet_get_port(qmap_skb->dev); - if (port->data_format & RMNET_EGRESS_FORMAT_AGGREGATION) { + if (port && (port->data_format & RMNET_EGRESS_FORMAT_AGGREGATION)) { spin_lock_irqsave(&port->agg_lock, flags); if (port->agg_skb) { agg_skb = port->agg_skb; diff --git a/core/rmnet_private.h b/core/rmnet_private.h index 420c65683650..03d297a9ed71 100644 --- a/core/rmnet_private.h +++ b/core/rmnet_private.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2013-2014, 2016-2019 The Linux Foundation. All rights reserved. +/* Copyright (c) 2013-2014, 2016-2020 The Linux Foundation. All rights reserved. * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 and @@ -30,10 +30,16 @@ #define RMNET_INGRESS_FORMAT_DL_MARKER (RMNET_INGRESS_FORMAT_DL_MARKER_V1 |\ RMNET_INGRESS_FORMAT_DL_MARKER_V2) +/* UL Packet prioritization */ +#define RMNET_EGRESS_FORMAT_PRIORITY BIT(28) + /* Power save feature*/ #define RMNET_INGRESS_FORMAT_PS BIT(27) #define RMNET_FORMAT_PS_NOTIF BIT(26) +/* UL Aggregation parameters */ +#define RMNET_PAGE_RECYCLE BIT(0) + /* Replace skb->dev to a virtual rmnet device and pass up the stack */ #define RMNET_EPMODE_VND (1) /* Pass the frame directly to another device with dev_queue_xmit() */ diff --git a/core/rmnet_qmi.h b/core/rmnet_qmi.h index 06844879ec0c..5c2a4953e75e 100644 --- a/core/rmnet_qmi.h +++ b/core/rmnet_qmi.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018-2019, The Linux Foundation. All rights reserved. + * Copyright (c) 2018-2020, The Linux Foundation. All rights reserved. * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 and @@ -16,9 +16,11 @@ #include #include +#define CONFIG_QTI_QMI_RMNET 1 void rmnet_map_tx_qmap_cmd(struct sk_buff *qmap_skb); +#ifdef CONFIG_QTI_QMI_RMNET void *rmnet_get_qmi_pt(void *port); void *rmnet_get_qos_pt(struct net_device *dev); void *rmnet_get_rmnet_port(struct net_device *dev); @@ -31,4 +33,68 @@ void rmnet_set_powersave_format(void *port); void rmnet_clear_powersave_format(void *port); void rmnet_get_packets(void *port, u64 *rx, u64 *tx); int rmnet_get_powersave_notif(void *port); +struct net_device *rmnet_get_real_dev(void *port); +int rmnet_get_dlmarker_info(void *port); +#else +static inline void *rmnet_get_qmi_pt(void *port) +{ + return NULL; +} + +static inline void *rmnet_get_qos_pt(struct net_device *dev) +{ + return NULL; +} + +static inline void *rmnet_get_rmnet_port(struct net_device *dev) +{ + return NULL; +} + +static inline struct net_device *rmnet_get_rmnet_dev(void *port, + u8 mux_id) +{ + return NULL; +} + +static inline void rmnet_reset_qmi_pt(void *port) +{ +} + +static inline void rmnet_init_qmi_pt(void *port, void *qmi) +{ +} + +static inline void rmnet_enable_all_flows(void *port) +{ +} + +static inline bool rmnet_all_flows_enabled(void *port) +{ + return true; +} + +static inline void rmnet_set_port_format(void *port) +{ +} + +static inline void rmnet_get_packets(void *port, u64 *rx, u64 *tx) +{ +} + +static inline int rmnet_get_powersave_notif(void *port) +{ + return 0; +} + +static inline struct net_device *rmnet_get_real_dev(void *port) +{ + return NULL; +} + +static inline int rmnet_get_dlmarker_info(void *port) +{ + return 0; +} +#endif /* CONFIG_QTI_QMI_RMNET */ #endif /*_RMNET_QMI_H*/ diff --git a/core/rmnet_vnd.c b/core/rmnet_vnd.c index c340f327d589..096b19ffcc10 100644 --- a/core/rmnet_vnd.c +++ b/core/rmnet_vnd.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2013-2019, The Linux Foundation. All rights reserved. +/* Copyright (c) 2013-2020, The Linux Foundation. All rights reserved. * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 and @@ -23,6 +23,7 @@ #include "rmnet_private.h" #include "rmnet_map.h" #include "rmnet_vnd.h" +#include "rmnet_genl.h" #include "qmi_rmnet.h" #include "rmnet_qmi.h" @@ -127,8 +128,7 @@ static void rmnet_vnd_uninit(struct net_device *dev) qos = priv->qos_info; RCU_INIT_POINTER(priv->qos_info, NULL); - synchronize_rcu(); - qmi_rmnet_qos_exit(dev, qos); + qmi_rmnet_qos_exit_pre(qos); } static void rmnet_get_stats64(struct net_device *dev, @@ -167,11 +167,23 @@ static u16 rmnet_vnd_select_queue(struct net_device *dev, struct net_device *sb_dev) { struct rmnet_priv *priv = netdev_priv(dev); + u64 boost_period = 0; + int boost_trigger = 0; int txq = 0; if (priv->real_dev) txq = qmi_rmnet_get_queue(dev, skb); + if (rmnet_core_userspace_connected) { + rmnet_update_pid_and_check_boost(task_pid_nr(current), + skb->len, + &boost_trigger, + &boost_period); + + if (boost_trigger) + (void) boost_period; + } + return (txq < dev->real_num_tx_queues) ? txq : 0; } @@ -219,6 +231,7 @@ static const char rmnet_gstrings_stats[][ETH_GSTRING_LEN] = { "Coalescing packets over VEID1", "Coalescing packets over VEID2", "Coalescing packets over VEID3", + "Uplink priority packets", }; static const char rmnet_port_gstrings_stats[][ETH_GSTRING_LEN] = { @@ -234,6 +247,8 @@ static const char rmnet_port_gstrings_stats[][ETH_GSTRING_LEN] = { "DL header total pkts received", "DL trailer last seen sequence", "DL trailer pkts received", + "UL agg reuse", + "UL agg alloc", }; static void rmnet_get_strings(struct net_device *dev, u32 stringset, u8 *buf) @@ -284,6 +299,7 @@ static int rmnet_stats_reset(struct net_device *dev) { struct rmnet_priv *priv = netdev_priv(dev); struct rmnet_port_priv_stats *stp; + struct rmnet_priv_stats *st; struct rmnet_port *port; port = rmnet_get_port(priv->real_dev); @@ -293,6 +309,11 @@ static int rmnet_stats_reset(struct net_device *dev) stp = &port->stats; memset(stp, 0, sizeof(*stp)); + + st = &priv->stats; + + memset(st, 0, sizeof(*st)); + return 0; } diff --git a/core/wda_qmi.c b/core/wda_qmi.c index 7e6bc947bbff..5e3cc106b878 100644 --- a/core/wda_qmi.c +++ b/core/wda_qmi.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2018-2019, The Linux Foundation. All rights reserved. +/* Copyright (c) 2018-2020, The Linux Foundation. All rights reserved. * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 and @@ -259,7 +259,8 @@ out: return ret; } -static int wda_set_powersave_config_req(struct qmi_handle *wda_handle) +static int wda_set_powersave_config_req(struct qmi_handle *wda_handle, + int dl_marker) { struct wda_qmi_data *data = container_of(wda_handle, struct wda_qmi_data, handle); @@ -289,7 +290,8 @@ static int wda_set_powersave_config_req(struct qmi_handle *wda_handle) req->ep_id.ep_type = data->svc.ep_type; req->ep_id.iface_id = data->svc.iface_id; req->req_data_cfg_valid = 1; - req->req_data_cfg = WDA_DATA_POWERSAVE_CONFIG_ALL_MASK_V01; + req->req_data_cfg = dl_marker ? WDA_DATA_POWERSAVE_CONFIG_ALL_MASK_V01 : + WDA_DATA_POWERSAVE_CONFIG_FLOW_CTL_V01; ret = qmi_send_request(wda_handle, &data->ssctl, &txn, QMI_WDA_SET_POWERSAVE_CONFIG_REQ_V01, QMI_WDA_SET_POWERSAVE_CONFIG_REQ_V01_MAX_MSG_LEN, @@ -321,11 +323,22 @@ static void wda_svc_config(struct work_struct *work) struct wda_qmi_data *data = container_of(work, struct wda_qmi_data, svc_arrive); struct qmi_info *qmi; - int rc; + int rc, dl_marker = 0; + + while (!rtnl_trylock()) { + if (!data->restart_state) + cond_resched(); + else + return; + } + + dl_marker = rmnet_get_dlmarker_info(data->rmnet_port); + rtnl_unlock(); if (data->restart_state == 1) return; - rc = wda_set_powersave_config_req(&data->handle); + + rc = wda_set_powersave_config_req(&data->handle, dl_marker); if (rc < 0) { pr_err("%s Failed to init service, err[%d]\n", __func__, rc); return; @@ -339,6 +352,7 @@ static void wda_svc_config(struct work_struct *work) else return; } + qmi = (struct qmi_info *)rmnet_get_qmi_pt(data->rmnet_port); if (!qmi) { rtnl_unlock(); @@ -353,7 +367,8 @@ static void wda_svc_config(struct work_struct *work) rtnl_unlock(); - pr_info("Connection established with the WDA Service\n"); + pr_info("Connection established with the WDA Service, DL Marker %s\n", + dl_marker ? "enabled" : "disabled"); } static int wda_svc_arrive(struct qmi_handle *qmi, struct qmi_service *svc) From 9caf0590281f920642eaea4ce138e56e6b7cc965 Mon Sep 17 00:00:00 2001 From: Subash Abhinov Kasiviswanathan Date: Wed, 5 Feb 2020 14:24:02 -0700 Subject: [PATCH 04/54] core: Add to product packages This is needed to install the module at the preferred location. CRs-Fixed: 2616023 Change-Id: I480ece2dac0a2169c7f8c7187a5a51e1d637e6c2 Signed-off-by: Subash Abhinov Kasiviswanathan --- datarmnet_dlkm_vendor_board.mk | 1 + 1 file changed, 1 insertion(+) diff --git a/datarmnet_dlkm_vendor_board.mk b/datarmnet_dlkm_vendor_board.mk index 303c6b81865b..040675c61518 100644 --- a/datarmnet_dlkm_vendor_board.mk +++ b/datarmnet_dlkm_vendor_board.mk @@ -3,6 +3,7 @@ DATA_DLKM_BOARD_PLATFORMS_LIST := lahaina ifneq ($(TARGET_BOARD_AUTO),true) ifeq ($(call is-board-platform-in-list,$(DATA_DLKM_BOARD_PLATFORMS_LIST)),true) BOARD_VENDOR_KERNEL_MODULES += $(KERNEL_MODULES_OUT)/rmnet_core.ko +PRODUCT_PACKAGES += rmnet_core.ko endif endif From 19bd91be77b75f4aa7b1c9f74e15fad47a04f970 Mon Sep 17 00:00:00 2001 From: Subash Abhinov Kasiviswanathan Date: Wed, 12 Feb 2020 10:15:19 -0700 Subject: [PATCH 05/54] core: Split product packages Add logical separation between board and product makefiles. CRs-Fixed: 2620634 Change-Id: I84a43e7a4e13367918efcdb760f6991fc5cced9d Signed-off-by: Subash Abhinov Kasiviswanathan --- datarmnet_dlkm_vendor_board.mk | 1 - datarmnet_dlkm_vendor_product.mk | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) create mode 100644 datarmnet_dlkm_vendor_product.mk diff --git a/datarmnet_dlkm_vendor_board.mk b/datarmnet_dlkm_vendor_board.mk index 040675c61518..303c6b81865b 100644 --- a/datarmnet_dlkm_vendor_board.mk +++ b/datarmnet_dlkm_vendor_board.mk @@ -3,7 +3,6 @@ DATA_DLKM_BOARD_PLATFORMS_LIST := lahaina ifneq ($(TARGET_BOARD_AUTO),true) ifeq ($(call is-board-platform-in-list,$(DATA_DLKM_BOARD_PLATFORMS_LIST)),true) BOARD_VENDOR_KERNEL_MODULES += $(KERNEL_MODULES_OUT)/rmnet_core.ko -PRODUCT_PACKAGES += rmnet_core.ko endif endif diff --git a/datarmnet_dlkm_vendor_product.mk b/datarmnet_dlkm_vendor_product.mk new file mode 100644 index 000000000000..fe147f060629 --- /dev/null +++ b/datarmnet_dlkm_vendor_product.mk @@ -0,0 +1 @@ +PRODUCT_PACKAGES += rmnet_core.ko From c9eea21709f95b0f8f2efd0ed44a350f5c8e41aa Mon Sep 17 00:00:00 2001 From: Subash Abhinov Kasiviswanathan Date: Thu, 20 Feb 2020 17:00:03 -0700 Subject: [PATCH 06/54] core: Do not zero out transport checksum Hardware does not use the value populated in the transport checksum field for checksum computation in MAPv5 format if checksum required bit is set. The partial checksum can be left as is in these cases. Explicitly marking the field as 0 could potentially cause incorrect verdicts in hardware in translation scenarios for UDPv4. CRs-Fixed: 2626450 Change-Id: I8d9e38e6af0df344ceebd630a895b1d7f467b1db Signed-off-by: Subash Abhinov Kasiviswanathan --- core/rmnet_map_data.c | 1 - 1 file changed, 1 deletion(-) diff --git a/core/rmnet_map_data.c b/core/rmnet_map_data.c index 34e815e4f103..d7da68240d47 100644 --- a/core/rmnet_map_data.c +++ b/core/rmnet_map_data.c @@ -543,7 +543,6 @@ void rmnet_map_v5_checksum_uplink_packet(struct sk_buff *skb, check = rmnet_map_get_csum_field(proto, trans); if (check) { - *check = 0; skb->ip_summed = CHECKSUM_NONE; /* Ask for checksum offloading */ ul_header->csum_valid_required = 1; From 1d2f235dc4b274e144fe1a6133499c4488b1ce9b Mon Sep 17 00:00:00 2001 From: Subash Abhinov Kasiviswanathan Date: Fri, 21 Feb 2020 16:12:11 -0700 Subject: [PATCH 07/54] dfc: Not flow control NDP packets Allow all NDP packets to be sent regardless of flow control state. These packets are needed for address configuration even if the default flow is disabled from the beginning of the call such as in 5G SA. Change-Id: Ia10e41c5ec8e163b9682c6b4f553f9b102020042 Acked-by: Weiyi Chen Signed-off-by: Subash Abhinov Kasiviswanathan --- core/qmi_rmnet.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/core/qmi_rmnet.c b/core/qmi_rmnet.c index 5818ebda32ca..7c1a6857500f 100644 --- a/core/qmi_rmnet.c +++ b/core/qmi_rmnet.c @@ -812,11 +812,11 @@ static int qmi_rmnet_get_queue_sa(struct qos_info *qos, struct sk_buff *skb) int ip_type; int txq = DEFAULT_MQ_NUM; - /* Put RS/NS in default mq */ + /* Put NDP in default mq */ if (skb->protocol == htons(ETH_P_IPV6) && ipv6_hdr(skb)->nexthdr == IPPROTO_ICMPV6 && - (icmp6_hdr(skb)->icmp6_type == 133 || - icmp6_hdr(skb)->icmp6_type == 135)) { + icmp6_hdr(skb)->icmp6_type >= 133 && + icmp6_hdr(skb)->icmp6_type <= 137) { return DEFAULT_MQ_NUM; } From 1ea6d3f0119bc48a1095e659d699b6622bddd255 Mon Sep 17 00:00:00 2001 From: Sean Tranchetti Date: Mon, 24 Feb 2020 13:55:04 -0700 Subject: [PATCH 08/54] net: ethernet: qualcomm: rmnet: Track RSB/RSC byte counts Allows calculation of the average buffer utilization for RSB/RSC packets. Change-Id: Id719b97ceffc62b1b9ce28bfab8ec32c6604529c Acked-by: Conner Huff Signed-off-by: Sean Tranchetti --- core/rmnet_config.h | 4 ++++ core/rmnet_descriptor.c | 6 ++++++ core/rmnet_vnd.c | 4 ++++ 3 files changed, 14 insertions(+) diff --git a/core/rmnet_config.h b/core/rmnet_config.h index 46d4352ab885..7d8fb8ef7f72 100644 --- a/core/rmnet_config.h +++ b/core/rmnet_config.h @@ -140,6 +140,10 @@ struct rmnet_coal_stats { u64 coal_trans_invalid; struct rmnet_coal_close_stats close; u64 coal_veid[RMNET_MAX_VEID]; + u64 coal_tcp; + u64 coal_tcp_bytes; + u64 coal_udp; + u64 coal_udp_bytes; }; struct rmnet_priv_stats { diff --git a/core/rmnet_descriptor.c b/core/rmnet_descriptor.c index 618f453e70e5..1df9855ef961 100644 --- a/core/rmnet_descriptor.c +++ b/core/rmnet_descriptor.c @@ -812,11 +812,17 @@ rmnet_frag_segment_coal_data(struct rmnet_frag_descriptor *coal_desc, th = (struct tcphdr *)((u8 *)iph + coal_desc->ip_len); coal_desc->trans_len = th->doff * 4; + priv->stats.coal.coal_tcp++; + priv->stats.coal.coal_tcp_bytes += + skb_frag_size(&coal_desc->frag); } else if (coal_desc->trans_proto == IPPROTO_UDP) { struct udphdr *uh; uh = (struct udphdr *)((u8 *)iph + coal_desc->ip_len); coal_desc->trans_len = sizeof(*uh); + priv->stats.coal.coal_udp++; + priv->stats.coal.coal_udp_bytes += + skb_frag_size(&coal_desc->frag); if (coal_desc->ip_proto == 4 && !uh->check) zero_csum = true; } else { diff --git a/core/rmnet_vnd.c b/core/rmnet_vnd.c index 096b19ffcc10..5b36fe88e764 100644 --- a/core/rmnet_vnd.c +++ b/core/rmnet_vnd.c @@ -231,6 +231,10 @@ static const char rmnet_gstrings_stats[][ETH_GSTRING_LEN] = { "Coalescing packets over VEID1", "Coalescing packets over VEID2", "Coalescing packets over VEID3", + "Coalescing TCP frames", + "Coalescing TCP bytes", + "Coalescing UDP frames", + "Coalescing UDP bytes", "Uplink priority packets", }; From 609413920b05a9dda668d9948c28ede58e1135e6 Mon Sep 17 00:00:00 2001 From: Conner Huff Date: Thu, 5 Mar 2020 16:41:07 -0800 Subject: [PATCH 09/54] core: Update netlink policy structure Newer versions of kernel require attribute structs to define lengths exactly. Change-Id: I7195b4974cf18ff3b3cbeb4d2ed863ff97ab7b92 Acked-by: Ryan Chapman Signed-off-by: Conner Huff --- core/rmnet_genl.c | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/core/rmnet_genl.c b/core/rmnet_genl.c index fa45ef43ca3a..9020e265e080 100644 --- a/core/rmnet_genl.c +++ b/core/rmnet_genl.c @@ -9,15 +9,18 @@ #include #include +#define RMNET_CORE_GENL_MAX_STR_LEN 255 + /* Static Functions and Definitions */ static struct nla_policy rmnet_genl_attr_policy[RMNET_CORE_GENL_ATTR_MAX + 1] = { [RMNET_CORE_GENL_ATTR_INT] = { .type = NLA_S32 }, - [RMNET_CORE_GENL_ATTR_PID_BPS] = { .len = + [RMNET_CORE_GENL_ATTR_PID_BPS] = { .type = NLA_EXACT_LEN, .len = sizeof(struct rmnet_core_pid_bps_resp) }, - [RMNET_CORE_GENL_ATTR_PID_BOOST] = { .len = + [RMNET_CORE_GENL_ATTR_PID_BOOST] = { .type = NLA_EXACT_LEN, .len = sizeof(struct rmnet_core_pid_boost_req) }, - [RMNET_CORE_GENL_ATTR_STR] = { .type = NLA_NUL_STRING }, + [RMNET_CORE_GENL_ATTR_STR] = { .type = NLA_NUL_STRING, .len = + RMNET_CORE_GENL_MAX_STR_LEN }, }; #define RMNET_CORE_GENL_OP(_cmd, _func) \ From 0dfa55e06d18098330d93af9d8424a0440a54a4d Mon Sep 17 00:00:00 2001 From: Subash Abhinov Kasiviswanathan Date: Tue, 17 Mar 2020 21:08:30 -0600 Subject: [PATCH 10/54] core: rmnet: Remove unnecessary warning Remove warning from module load. Fixes the following- WARNING: CPU: 2 PID: 735 at core/rmnet_config.c:753 init_module+0x18/0x64 [rmnet_core] pstate: 60400005 (nZCv daif +PAN -UAO) pc : init_module+0x18/0x64 [rmnet_core] lr : do_one_initcall+0x190/0x380 Call trace: init_module+0x18/0x64 [rmnet_core] do_one_initcall+0x190/0x380 do_init_module+0x5c/0x210 load_module+0x45d8/0x480c __arm64_sys_finit_module+0xec/0x120 el0_svc_common+0xbc/0x1c0 el0_svc_handler+0x6c/0x88 el0_svc+0x8/0xc CRs-Fixed: 2644088 Change-Id: Icab7f3b3021a94e4376374b9cb3b905178f9894a Signed-off-by: Subash Abhinov Kasiviswanathan --- core/rmnet_config.c | 1 - 1 file changed, 1 deletion(-) diff --git a/core/rmnet_config.c b/core/rmnet_config.c index 1c562203502f..4e8910e747ba 100644 --- a/core/rmnet_config.c +++ b/core/rmnet_config.c @@ -750,7 +750,6 @@ static int __init rmnet_init(void) { int rc; - WARN_ON(1); rc = register_netdevice_notifier(&rmnet_dev_notifier); if (rc != 0) return rc; From 2204dee91bd1560285b7f0ce8f51b73ffbb364e8 Mon Sep 17 00:00:00 2001 From: Subash Abhinov Kasiviswanathan Date: Sun, 22 Mar 2020 21:02:58 -0600 Subject: [PATCH 11/54] core: Remove dl marker v1 usage This removes the unused dl marker v1 definitions, handlers and users since these are no longer used. This also fixes the following CFI warning- CFI failure (target: rmnet_perf_core_handle_map_control_end.cfi_jt+0x0/0x4 [rmnet_perf]): WARNING: CPU: 1 PID: 0 at kernel/cfi.c:29 __ubsan_handle_cfi_check_fail+0x4c/0x54 pstate: 60400005 (nZCv daif +PAN -UAO) pc: __ubsan_handle_cfi_check_fail+0x4c/0x54 lr: __ubsan_handle_cfi_check_fail+0x4c/0x54 Call trace: __ubsan_handle_cfi_check_fail+0x4c/0x54 __cfi_check+0x204/0x220 [rmnet_perf] rmnet_map_dl_trl_notify_v2+0x58/0x88 [rmnet_core] rmnet_frag_flow_command+0x110/0x120 [rmnet_core] rmnet_frag_ingress_handler+0xe0/0x3bc [rmnet_core] rmnet_rx_handler+0x1cc/0x2a4 [rmnet_core] __netif_receive_skb_core+0x554/0xdc4 process_backlog$4cc8cf18b485f47de9fc54109f04daea+0x1a4/0x314 net_rx_action$4cc8cf18b485f47de9fc54109f04daea+0x144/0x578 __do_softirq+0x250/0x580 irq_exit+0xcc/0xd0 handle_IPI+0x228/0x3b0 efi_header_end+0x148/0x17c el1_irq+0x108/0x200 lpm_cpuidle_enter$a9941074ca35bb1f25355cf2ff310eae+0x57c/0x5c8 cpuidle_enter_state+0x130/0x334 cpuidle_enter+0x38/0x50 do_idle.llvm.4847091502713502628+0x1e4/0x2ec cpu_startup_entry+0x24/0x28 __cpu_disable+0x0/0xbc CRs-Fixed: 2647192 Change-Id: I5dc625f0791aff9738b2128f99c79d7e0dadf26d Signed-off-by: Subash Abhinov Kasiviswanathan --- core/rmnet_descriptor.c | 4 ---- core/rmnet_map.h | 22 ++++------------------ core/rmnet_map_command.c | 32 ++------------------------------ 3 files changed, 6 insertions(+), 52 deletions(-) diff --git a/core/rmnet_descriptor.c b/core/rmnet_descriptor.c index 1df9855ef961..34f8ac123f38 100644 --- a/core/rmnet_descriptor.c +++ b/core/rmnet_descriptor.c @@ -237,8 +237,6 @@ rmnet_frag_process_flow_start(struct rmnet_map_control_command_header *cmd, */ if (is_dl_mark_v2) rmnet_map_dl_hdr_notify_v2(port, dlhdr, cmd); - else - rmnet_map_dl_hdr_notify(port, dlhdr); } static void @@ -265,8 +263,6 @@ rmnet_frag_process_flow_end(struct rmnet_map_control_command_header *cmd, */ if (is_dl_mark_v2) rmnet_map_dl_trl_notify_v2(port, dltrl, cmd); - else - rmnet_map_dl_trl_notify(port, dltrl); } /* Process MAP command frame and send N/ACK message as appropriate. Message cmd diff --git a/core/rmnet_map.h b/core/rmnet_map.h index f7116d348a0a..853318da8f2c 100644 --- a/core/rmnet_map.h +++ b/core/rmnet_map.h @@ -180,20 +180,10 @@ struct rmnet_map_dl_ind_trl { struct rmnet_map_dl_ind { u8 priority; - union { - void (*dl_hdr_handler)(struct rmnet_map_dl_ind_hdr *dlhdr); - void (*dl_hdr_handler_v2)(struct rmnet_map_dl_ind_hdr *dlhdr, - struct - rmnet_map_control_command_header - * qcmd); - } __aligned(1); - union { - void (*dl_trl_handler)(struct rmnet_map_dl_ind_trl *dltrl); - void (*dl_trl_handler_v2)(struct rmnet_map_dl_ind_trl *dltrl, - struct - rmnet_map_control_command_header - * qcmd); - } __aligned(1); + void (*dl_hdr_handler_v2)(struct rmnet_map_dl_ind_hdr *dlhdr, + struct rmnet_map_control_command_header *qcmd); + void (*dl_trl_handler_v2)(struct rmnet_map_dl_ind_trl *dltrl, + struct rmnet_map_control_command_header *qcmd); struct list_head list; }; @@ -275,13 +265,9 @@ void rmnet_map_tx_aggregate_init(struct rmnet_port *port); void rmnet_map_tx_aggregate_exit(struct rmnet_port *port); void rmnet_map_update_ul_agg_config(struct rmnet_port *port, u16 size, u8 count, u8 features, u32 time); -void rmnet_map_dl_hdr_notify(struct rmnet_port *port, - struct rmnet_map_dl_ind_hdr *dl_hdr); void rmnet_map_dl_hdr_notify_v2(struct rmnet_port *port, struct rmnet_map_dl_ind_hdr *dl_hdr, struct rmnet_map_control_command_header *qcmd); -void rmnet_map_dl_trl_notify(struct rmnet_port *port, - struct rmnet_map_dl_ind_trl *dltrl); void rmnet_map_dl_trl_notify_v2(struct rmnet_port *port, struct rmnet_map_dl_ind_trl *dltrl, struct rmnet_map_control_command_header *qcmd); diff --git a/core/rmnet_map_command.c b/core/rmnet_map_command.c index ce058d2efedd..be60422c37a9 100644 --- a/core/rmnet_map_command.c +++ b/core/rmnet_map_command.c @@ -107,16 +107,6 @@ rmnet_map_dl_hdr_notify_v2(struct rmnet_port *port, tmp->dl_hdr_handler_v2(dlhdr, qcmd); } -void -rmnet_map_dl_hdr_notify(struct rmnet_port *port, - struct rmnet_map_dl_ind_hdr *dlhdr) -{ - struct rmnet_map_dl_ind *tmp; - - list_for_each_entry(tmp, &port->dl_list, list) - tmp->dl_hdr_handler(dlhdr); -} - void rmnet_map_dl_trl_notify_v2(struct rmnet_port *port, struct rmnet_map_dl_ind_trl *dltrl, @@ -128,16 +118,6 @@ rmnet_map_dl_trl_notify_v2(struct rmnet_port *port, tmp->dl_trl_handler_v2(dltrl, qcmd); } -void -rmnet_map_dl_trl_notify(struct rmnet_port *port, - struct rmnet_map_dl_ind_trl *dltrl) -{ - struct rmnet_map_dl_ind *tmp; - - list_for_each_entry(tmp, &port->dl_list, list) - tmp->dl_trl_handler(dltrl); -} - static void rmnet_map_process_flow_start(struct sk_buff *skb, struct rmnet_port *port, bool rmnet_perf) @@ -160,8 +140,6 @@ static void rmnet_map_process_flow_start(struct sk_buff *skb, port->stats.dl_hdr_last_qmap_vers = qcmd->reserved; port->stats.dl_hdr_last_trans_id = qcmd->transaction_id; pskb_pull(skb, sizeof(struct rmnet_map_control_command_header)); - } else { - pskb_pull(skb, RMNET_MAP_CMD_SIZE); } dlhdr = (struct rmnet_map_dl_ind_hdr *)rmnet_map_data_ptr(skb); @@ -176,8 +154,6 @@ static void rmnet_map_process_flow_start(struct sk_buff *skb, if (is_dl_mark_v2) rmnet_map_dl_hdr_notify_v2(port, dlhdr, qcmd); - else - rmnet_map_dl_hdr_notify(port, dlhdr); if (rmnet_perf) { unsigned int pull_size; @@ -208,8 +184,6 @@ static void rmnet_map_process_flow_end(struct sk_buff *skb, qcmd = (struct rmnet_map_control_command_header *) rmnet_map_data_ptr(skb); pskb_pull(skb, sizeof(struct rmnet_map_control_command_header)); - } else { - pskb_pull(skb, RMNET_MAP_CMD_SIZE); } dltrl = (struct rmnet_map_dl_ind_trl *)rmnet_map_data_ptr(skb); @@ -219,8 +193,6 @@ static void rmnet_map_process_flow_end(struct sk_buff *skb, if (is_dl_mark_v2) rmnet_map_dl_trl_notify_v2(port, dltrl, qcmd); - else - rmnet_map_dl_trl_notify(port, dltrl); if (rmnet_perf) { unsigned int pull_size; @@ -311,8 +283,8 @@ int rmnet_map_dl_ind_register(struct rmnet_port *port, struct rmnet_map_dl_ind *dl_ind_iterator; bool empty_ind_list = true; - if (!port || !dl_ind || !dl_ind->dl_hdr_handler || - !dl_ind->dl_trl_handler) + if (!port || !dl_ind || !dl_ind->dl_hdr_handler_v2 || + !dl_ind->dl_trl_handler_v2) return -EINVAL; list_for_each_entry_rcu(dl_ind_iterator, &port->dl_list, list) { From d6d43874afa8a360c1b13ddc1d858410536c4d60 Mon Sep 17 00:00:00 2001 From: Sean Tranchetti Date: Wed, 4 Mar 2020 16:32:43 -0700 Subject: [PATCH 12/54] net: qualcomm: rmnet: flush uplink aggregation on priority tx If the RmNet driver needs to transmit a packet marked as needing prioritization, it should skip uplink aggregation of the packet to avoid adding additional delay. Additionally, any aggregated packets should be flushed prior to sending the prioritized packet to avoid any ordering issues. Change-Id: Ia3063ec3f0553693a0f66c8eb5bbb88d250f7523 Signed-off-by: Sean Tranchetti --- core/rmnet_map_data.c | 43 ++++++++++++++++++++++++++++++++----------- 1 file changed, 32 insertions(+), 11 deletions(-) diff --git a/core/rmnet_map_data.c b/core/rmnet_map_data.c index d7da68240d47..cc1ecd16c21c 100644 --- a/core/rmnet_map_data.c +++ b/core/rmnet_map_data.c @@ -1409,11 +1409,30 @@ static struct sk_buff *rmnet_map_build_skb(struct rmnet_port *port) return skb; } +static void rmnet_map_send_agg_skb(struct rmnet_port *port, unsigned long flags) +{ + struct sk_buff *agg_skb; + + if (!port->agg_skb) { + spin_unlock_irqrestore(&port->agg_lock, flags); + return; + } + + agg_skb = port->agg_skb; + /* Reset the aggregation state */ + port->agg_skb = NULL; + port->agg_count = 0; + memset(&port->agg_time, 0, sizeof(port->agg_time)); + port->agg_state = 0; + spin_unlock_irqrestore(&port->agg_lock, flags); + hrtimer_cancel(&port->hrtimer); + dev_queue_xmit(agg_skb); +} + void rmnet_map_tx_aggregate(struct sk_buff *skb, struct rmnet_port *port) { struct timespec64 diff, last; - int size, agg_count = 0; - struct sk_buff *agg_skb; + int size; unsigned long flags; new_packet: @@ -1421,6 +1440,16 @@ new_packet: memcpy(&last, &port->agg_last, sizeof(struct timespec)); ktime_get_real_ts64(&port->agg_last); + if ((port->data_format & RMNET_EGRESS_FORMAT_PRIORITY) && + skb->priority) { + /* Send out any aggregated SKBs we have */ + rmnet_map_send_agg_skb(port, flags); + /* Send out the priority SKB. Not holding agg_lock anymore */ + skb->protocol = htons(ETH_P_MAP); + dev_queue_xmit(skb); + return; + } + if (!port->agg_skb) { /* Check to see if we should agg first. If the traffic is very * sparse, don't aggregate. We will need to tune this later @@ -1461,15 +1490,7 @@ new_packet: if (skb->len > size || port->agg_count >= port->egress_agg_params.agg_count || diff.tv_sec > 0 || diff.tv_nsec > rmnet_agg_time_limit) { - agg_skb = port->agg_skb; - agg_count = port->agg_count; - port->agg_skb = 0; - port->agg_count = 0; - memset(&port->agg_time, 0, sizeof(struct timespec)); - port->agg_state = 0; - spin_unlock_irqrestore(&port->agg_lock, flags); - hrtimer_cancel(&port->hrtimer); - dev_queue_xmit(agg_skb); + rmnet_map_send_agg_skb(port, flags); goto new_packet; } From 5fd6051c6c62a400e10b8d7d30fb28d8b53fd1c0 Mon Sep 17 00:00:00 2001 From: Sean Tranchetti Date: Thu, 2 Apr 2020 15:20:45 -0600 Subject: [PATCH 13/54] core: Update timespec API usage Kernel code should now use the timespec64 struct instead of the older struct timespec and its APIs. Change-Id: I2792b637bf1054521e3f1ff7c82d1cdbf4f3adea Signed-off-by: Sean Tranchetti --- core/rmnet_genl.c | 4 ++-- core/rmnet_map_data.c | 10 +++++----- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/core/rmnet_genl.c b/core/rmnet_genl.c index 9020e265e080..f568672db569 100644 --- a/core/rmnet_genl.c +++ b/core/rmnet_genl.c @@ -168,7 +168,7 @@ void rmnet_boost_for_pid(pid_t pid, int boost_enable, static void rmnet_create_pid_bps_resp(struct rmnet_core_pid_bps_resp *pid_bps_resp_ptr) { - struct timespec time; + struct timespec64 time; struct hlist_node *tmp; struct rmnet_pid_node_s *node_p; unsigned long ht_flags; @@ -176,7 +176,7 @@ static void rmnet_create_pid_bps_resp(struct rmnet_core_pid_bps_resp int i; u16 bkt; - (void)getnstimeofday(&time); + ktime_get_real_ts64(&time); pid_bps_resp_ptr->timestamp = RMNET_GENL_SEC_TO_NSEC(time.tv_sec) + time.tv_nsec; diff --git a/core/rmnet_map_data.c b/core/rmnet_map_data.c index cc1ecd16c21c..58244722c5a0 100644 --- a/core/rmnet_map_data.c +++ b/core/rmnet_map_data.c @@ -1247,7 +1247,7 @@ static void rmnet_map_flush_tx_packet_work(struct work_struct *work) skb = port->agg_skb; port->agg_skb = NULL; port->agg_count = 0; - memset(&port->agg_time, 0, sizeof(struct timespec)); + memset(&port->agg_time, 0, sizeof(port->agg_time)); } port->agg_state = 0; } @@ -1437,7 +1437,7 @@ void rmnet_map_tx_aggregate(struct sk_buff *skb, struct rmnet_port *port) new_packet: spin_lock_irqsave(&port->agg_lock, flags); - memcpy(&last, &port->agg_last, sizeof(struct timespec)); + memcpy(&last, &port->agg_last, sizeof(last)); ktime_get_real_ts64(&port->agg_last); if ((port->data_format & RMNET_EGRESS_FORMAT_PRIORITY) && @@ -1469,7 +1469,7 @@ new_packet: if (!port->agg_skb) { port->agg_skb = 0; port->agg_count = 0; - memset(&port->agg_time, 0, sizeof(struct timespec)); + memset(&port->agg_time, 0, sizeof(port->agg_time)); spin_unlock_irqrestore(&port->agg_lock, flags); skb->protocol = htons(ETH_P_MAP); dev_queue_xmit(skb); @@ -1570,7 +1570,7 @@ void rmnet_map_tx_aggregate_exit(struct rmnet_port *port) kfree_skb(port->agg_skb); port->agg_skb = NULL; port->agg_count = 0; - memset(&port->agg_time, 0, sizeof(struct timespec)); + memset(&port->agg_time, 0, sizeof(port->agg_time)); } port->agg_state = 0; @@ -1594,7 +1594,7 @@ void rmnet_map_tx_qmap_cmd(struct sk_buff *qmap_skb) agg_skb = port->agg_skb; port->agg_skb = 0; port->agg_count = 0; - memset(&port->agg_time, 0, sizeof(struct timespec)); + memset(&port->agg_time, 0, sizeof(port->agg_time)); port->agg_state = 0; spin_unlock_irqrestore(&port->agg_lock, flags); hrtimer_cancel(&port->hrtimer); From 5a88e586529f2f1b4e4c95aa113e5fe04a34e4bb Mon Sep 17 00:00:00 2001 From: Subash Abhinov Kasiviswanathan Date: Mon, 2 Mar 2020 12:10:49 -0800 Subject: [PATCH 14/54] rmnet_ctl: Support IPA EP Support QMAP commands from dedicated IPA EP. Change-Id: Ic5e489d2b33223b2af2fb471d20611bc5a0d7d58 Acked-by: Weiyi Chen Signed-off-by: Subash Abhinov Kasiviswanathan --- core/Android.mk | 21 ++++++ core/Kbuild | 2 + core/Kconfig | 6 ++ core/rmnet_ctl.h | 2 + core/rmnet_ctl_client.c | 59 ++++++++++++---- core/rmnet_ctl_client.h | 1 + core/rmnet_ctl_ipa.c | 114 +++++++++++++++++++++++++++++++ core/rmnet_ctl_mhi.c | 22 +++++- datarmnet_dlkm_vendor_board.mk | 1 + datarmnet_dlkm_vendor_product.mk | 1 + 10 files changed, 211 insertions(+), 18 deletions(-) create mode 100644 core/rmnet_ctl_ipa.c diff --git a/core/Android.mk b/core/Android.mk index b1211d666d64..d9e1d9714afd 100644 --- a/core/Android.mk +++ b/core/Android.mk @@ -33,5 +33,26 @@ KBUILD_OPTIONS := $(RMNET_BLD_DIR) $(warning $(DLKM_DIR)) include $(DLKM_DIR)/AndroidKernelModule.mk +######## Create RMNET_CTL DLKM ######## +include $(CLEAR_VARS) + +LOCAL_CFLAGS := -Wno-macro-redefined -Wno-unused-function -Wall -Werror +LOCAL_CLANG :=true + +LOCAL_MODULE_PATH := $(KERNEL_MODULES_OUT) +LOCAL_MODULE := rmnet_ctl.ko + +LOCAL_SRC_FILES := \ + rmnet_ctl_client.c \ + rmnet_ctl_ipa.c + +RMNET_BLD_DIR := ../../vendor/qcom/opensource/datarmnet/core +DLKM_DIR := $(TOP)/device/qcom/common/dlkm + +KBUILD_OPTIONS := $(RMNET_BLD_DIR) + +$(warning $(DLKM_DIR)) +include $(DLKM_DIR)/AndroidKernelModule.mk + endif #End of Check for target endif #End of Check for qssi target diff --git a/core/Kbuild b/core/Kbuild index e9d19b16dc90..51ac345578dc 100644 --- a/core/Kbuild +++ b/core/Kbuild @@ -1,6 +1,8 @@ obj-m += rmnet_core.o +obj-m += rmnet_ctl.o rmnet_core-y := rmnet_config.o rmnet_handlers.o rmnet_descriptor.o \ rmnet_genl.o rmnet_map_command.o rmnet_map_data.o rmnet_vnd.o\ qmi_rmnet.o wda_qmi.o dfc_qmi.o dfc_qmap.o +rmnet_ctl-y := rmnet_ctl_client.o rmnet_ctl_ipa.o diff --git a/core/Kconfig b/core/Kconfig index b6356c1ee2a4..0fe2530349a4 100644 --- a/core/Kconfig +++ b/core/Kconfig @@ -10,3 +10,9 @@ menuconfig RMNET_CORE for handling data in the multiplexing and aggregation protocol (MAP) format in the embedded data path. RMNET devices can be attached to any IP mode physical device. + +menuconfig RMNET_CTL + default m + ---help--- + Enable the RMNET CTL module which is used for handling QMAP commands + for flow control purposes. diff --git a/core/rmnet_ctl.h b/core/rmnet_ctl.h index 0c7061837c12..54d2385c4eb9 100644 --- a/core/rmnet_ctl.h +++ b/core/rmnet_ctl.h @@ -10,6 +10,8 @@ #include +#define CONFIG_RMNET_CTL 1 + enum rmnet_ctl_log_lvl { RMNET_CTL_LOG_CRIT, RMNET_CTL_LOG_ERR, diff --git a/core/rmnet_ctl_client.c b/core/rmnet_ctl_client.c index f082101972ed..1fe348b1387d 100644 --- a/core/rmnet_ctl_client.c +++ b/core/rmnet_ctl_client.c @@ -26,6 +26,10 @@ struct rmnet_ctl_endpoint { void *ipc_log; }; +#if defined(CONFIG_IPA_DEBUG) || defined(CONFIG_MHI_DEBUG) +#define CONFIG_RMNET_CTL_DEBUG 1 +#endif + #ifdef CONFIG_RMNET_CTL_DEBUG static u8 ipc_log_lvl = RMNET_CTL_LOG_DEBUG; #else @@ -35,13 +39,13 @@ static u8 ipc_log_lvl = RMNET_CTL_LOG_ERR; static DEFINE_SPINLOCK(client_lock); static struct rmnet_ctl_endpoint ctl_ep; -void rmnet_ctl_endpoint_setdev(const struct rmnet_ctl_dev *dev) +void rmnet_ctl_set_dbgfs(bool enable) { - rcu_assign_pointer(ctl_ep.dev, dev); + if (enable) { + if (IS_ERR_OR_NULL(ctl_ep.dbgfs_dir)) + ctl_ep.dbgfs_dir = debugfs_create_dir( + RMNET_CTL_LOG_NAME, NULL); - if (dev) { - ctl_ep.dbgfs_dir = debugfs_create_dir( - RMNET_CTL_LOG_NAME, NULL); if (!IS_ERR_OR_NULL(ctl_ep.dbgfs_dir)) ctl_ep.dbgfs_loglvl = debugfs_create_u8( RMNET_CTL_LOG_LVL, 0644, ctl_ep.dbgfs_dir, @@ -52,9 +56,18 @@ void rmnet_ctl_endpoint_setdev(const struct rmnet_ctl_dev *dev) RMNET_CTL_LOG_PAGE, RMNET_CTL_LOG_NAME, 0); } else { debugfs_remove_recursive(ctl_ep.dbgfs_dir); + ipc_log_context_destroy(ctl_ep.ipc_log); + ctl_ep.dbgfs_dir = NULL; + ctl_ep.dbgfs_loglvl = NULL; + ctl_ep.ipc_log = NULL; } } +void rmnet_ctl_endpoint_setdev(const struct rmnet_ctl_dev *dev) +{ + rcu_assign_pointer(ctl_ep.dev, dev); +} + void rmnet_ctl_endpoint_post(const void *data, size_t len) { struct rmnet_ctl_client *client; @@ -63,22 +76,38 @@ void rmnet_ctl_endpoint_post(const void *data, size_t len) if (unlikely(!data || !len)) return; - rmnet_ctl_log_info("RX", data, len); + if (len == 0xFFFFFFFF) { + skb = (struct sk_buff *)data; + rmnet_ctl_log_info("RX", skb->data, skb->len); - rcu_read_lock(); + rcu_read_lock(); - client = rcu_dereference(ctl_ep.client); - - if (client && client->hooks.ctl_dl_client_hook) { - skb = alloc_skb(len, GFP_ATOMIC); - if (skb) { - skb_put_data(skb, data, len); + client = rcu_dereference(ctl_ep.client); + if (client && client->hooks.ctl_dl_client_hook) { skb->protocol = htons(ETH_P_MAP); client->hooks.ctl_dl_client_hook(skb); + } else { + kfree(skb); } - } - rcu_read_unlock(); + rcu_read_unlock(); + } else { + rmnet_ctl_log_info("RX", data, len); + + rcu_read_lock(); + + client = rcu_dereference(ctl_ep.client); + if (client && client->hooks.ctl_dl_client_hook) { + skb = alloc_skb(len, GFP_ATOMIC); + if (skb) { + skb_put_data(skb, data, len); + skb->protocol = htons(ETH_P_MAP); + client->hooks.ctl_dl_client_hook(skb); + } + } + + rcu_read_unlock(); + } } void *rmnet_ctl_register_client(struct rmnet_ctl_client_hooks *hook) diff --git a/core/rmnet_ctl_client.h b/core/rmnet_ctl_client.h index 384064d134df..f7b1b6a69e61 100644 --- a/core/rmnet_ctl_client.h +++ b/core/rmnet_ctl_client.h @@ -25,5 +25,6 @@ struct rmnet_ctl_dev { void rmnet_ctl_endpoint_post(const void *data, size_t len); void rmnet_ctl_endpoint_setdev(const struct rmnet_ctl_dev *dev); +void rmnet_ctl_set_dbgfs(bool enable); #endif /* _RMNET_CTL_CLIENT_H_ */ diff --git a/core/rmnet_ctl_ipa.c b/core/rmnet_ctl_ipa.c new file mode 100644 index 000000000000..fc4864cc12df --- /dev/null +++ b/core/rmnet_ctl_ipa.c @@ -0,0 +1,114 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright (c) 2020, The Linux Foundation. All rights reserved. + * + * RMNET_CTL mhi handler + * + */ + +#include +#include +#include +#include "rmnet_ctl.h" +#include "rmnet_ctl_client.h" + +struct rmnet_ctl_ipa_dev { + struct rmnet_ctl_dev dev; + spinlock_t rx_lock; /* rx lock */ + spinlock_t tx_lock; /* tx lock */ +}; + +static struct rmnet_ctl_ipa_dev ctl_ipa_dev; + +static int rmnet_ctl_send_ipa(struct rmnet_ctl_dev *dev, struct sk_buff *skb) +{ + struct rmnet_ctl_ipa_dev *ctl_dev = container_of( + dev, struct rmnet_ctl_ipa_dev, dev); + int rc; + + spin_lock_bh(&ctl_dev->tx_lock); + + rc = ipa_rmnet_ctl_xmit(skb); + if (rc) + dev->stats.tx_err++; + else + dev->stats.tx_pkts++; + + spin_unlock_bh(&ctl_dev->tx_lock); + + return rc; +} + +static void rmnet_ctl_dl_callback(void *user_data, void *rx_data) +{ + struct rmnet_ctl_ipa_dev *ctl_dev = user_data; + + ctl_dev->dev.stats.rx_pkts++; + rmnet_ctl_endpoint_post(rx_data, 0xFFFFFFFF); +} + +static void rmnet_ctl_probe(void *user_data) +{ + memset(&ctl_ipa_dev, 0, sizeof(ctl_ipa_dev)); + + spin_lock_init(&ctl_ipa_dev.rx_lock); + spin_lock_init(&ctl_ipa_dev.tx_lock); + + ctl_ipa_dev.dev.xmit = rmnet_ctl_send_ipa; + rmnet_ctl_endpoint_setdev(&ctl_ipa_dev.dev); + + pr_info("rmnet_ctl driver probed\n"); +} + +static void rmnet_ctl_remove(void *user_data) +{ + rmnet_ctl_endpoint_setdev(NULL); + + pr_info("rmnet_ctl driver removed\n"); +} + +static void rmnet_ctl_ipa_ready(void *user_data) +{ + int rc; + + rc = ipa_register_rmnet_ctl_cb( + rmnet_ctl_probe, + &ctl_ipa_dev, + rmnet_ctl_remove, + &ctl_ipa_dev, + rmnet_ctl_dl_callback, + &ctl_ipa_dev); + + pr_info("%s: %d\n", __func__, rc); +} + +static int __init rmnet_ctl_init(void) +{ + int rc; + + rc = ipa_register_rmnet_ctl_cb( + rmnet_ctl_probe, + &ctl_ipa_dev, + rmnet_ctl_remove, + &ctl_ipa_dev, + rmnet_ctl_dl_callback, + &ctl_ipa_dev); + + if (rc == -EAGAIN) + rc = ipa_register_ipa_ready_cb( + rmnet_ctl_ipa_ready, NULL); + + pr_info("%s: %d\n", __func__, rc); + + return 0; +} + +static void __exit rmnet_ctl_exit(void) +{ + ipa_unregister_rmnet_ctl_cb(); +} + +module_init(rmnet_ctl_init) +module_exit(rmnet_ctl_exit) + +MODULE_DESCRIPTION("RmNet control IPA Driver"); +MODULE_LICENSE("GPL v2"); diff --git a/core/rmnet_ctl_mhi.c b/core/rmnet_ctl_mhi.c index e22c137df01e..f9cc2ba2943c 100644 --- a/core/rmnet_ctl_mhi.c +++ b/core/rmnet_ctl_mhi.c @@ -199,8 +199,24 @@ static struct mhi_driver rmnet_ctl_driver = { }, }; -module_driver(rmnet_ctl_driver, - mhi_driver_register, mhi_driver_unregister); +static int __init rmnet_ctl_init(void) +{ + int rc; -MODULE_DESCRIPTION("RmNet Control Driver"); + rc = mhi_driver_register(&rmnet_ctl_driver); + rmnet_ctl_set_dbgfs(true); + + return rc; +} + +static void __exit rmnet_ctl_exit(void) +{ + mhi_driver_unregister(&rmnet_ctl_driver); + rmnet_ctl_set_dbgfs(false); +} + +module_init(rmnet_ctl_init) +module_exit(rmnet_ctl_exit) + +MODULE_DESCRIPTION("RmNet Control MHI Driver"); MODULE_LICENSE("GPL v2"); diff --git a/datarmnet_dlkm_vendor_board.mk b/datarmnet_dlkm_vendor_board.mk index 303c6b81865b..aa855f23db04 100644 --- a/datarmnet_dlkm_vendor_board.mk +++ b/datarmnet_dlkm_vendor_board.mk @@ -3,6 +3,7 @@ DATA_DLKM_BOARD_PLATFORMS_LIST := lahaina ifneq ($(TARGET_BOARD_AUTO),true) ifeq ($(call is-board-platform-in-list,$(DATA_DLKM_BOARD_PLATFORMS_LIST)),true) BOARD_VENDOR_KERNEL_MODULES += $(KERNEL_MODULES_OUT)/rmnet_core.ko +BOARD_VENDOR_KERNEL_MODULES += $(KERNEL_MODULES_OUT)/rmnet_ctl.ko endif endif diff --git a/datarmnet_dlkm_vendor_product.mk b/datarmnet_dlkm_vendor_product.mk index fe147f060629..33343754c1e6 100644 --- a/datarmnet_dlkm_vendor_product.mk +++ b/datarmnet_dlkm_vendor_product.mk @@ -1 +1,2 @@ PRODUCT_PACKAGES += rmnet_core.ko +PRODUCT_PACKAGES += rmnet_ctl.ko From 2724751ae6f40e366293b7710643249735bbf5ca Mon Sep 17 00:00:00 2001 From: Subash Abhinov Kasiviswanathan Date: Fri, 10 Apr 2020 14:30:20 -0700 Subject: [PATCH 15/54] dfc: port the latest fixes Port the latest fixes from msm-4.19. This includes: - Clear status flags when entering powersave (Ifc6457f1696ef) - Adjust QMAP query grants (I9da42a9d2425) - fix use-after-free (I76ab4a99b3f4) Change-Id: I1beb93d1f6fdf214c8756bd0e488914bf876aa5e Acked-by: Weiyi Chen Signed-off-by: Subash Abhinov Kasiviswanathan --- core/dfc.h | 30 ++++++++++++++++++++ core/dfc_defs.h | 7 +++-- core/dfc_qmap.c | 24 ++++++++++++---- core/dfc_qmi.c | 71 ++++++++++++++++++++++++++++++++++++++++------ core/qmi_rmnet.c | 17 +++++++---- core/qmi_rmnet_i.h | 3 ++ 6 files changed, 129 insertions(+), 23 deletions(-) diff --git a/core/dfc.h b/core/dfc.h index e962ddd0492e..e94c38e7b0a6 100644 --- a/core/dfc.h +++ b/core/dfc.h @@ -262,6 +262,36 @@ TRACE_EVENT(dfc_qmap, __print_hex(__get_dynamic_array(data), __entry->len)) ); +TRACE_EVENT(dfc_adjust_grant, + + TP_PROTO(u8 mux_id, u8 bearer_id, u32 grant, u32 rx_bytes, + u32 inflight, u32 a_grant), + + TP_ARGS(mux_id, bearer_id, grant, rx_bytes, inflight, a_grant), + + TP_STRUCT__entry( + __field(u8, mux_id) + __field(u8, bearer_id) + __field(u32, grant) + __field(u32, rx_bytes) + __field(u32, inflight) + __field(u32, a_grant) + ), + + TP_fast_assign( + __entry->mux_id = mux_id; + __entry->bearer_id = bearer_id; + __entry->grant = grant; + __entry->rx_bytes = rx_bytes; + __entry->inflight = inflight; + __entry->a_grant = a_grant; + ), + + TP_printk("mid=%u bid=%u grant=%u rx=%u inflight=%u adjusted_grant=%u", + __entry->mux_id, __entry->bearer_id, __entry->grant, + __entry->rx_bytes, __entry->inflight, __entry->a_grant) +); + #endif /* _TRACE_DFC_H */ /* This part must be outside protection */ diff --git a/core/dfc_defs.h b/core/dfc_defs.h index 7553707d3ab6..2a30a29f903b 100644 --- a/core/dfc_defs.h +++ b/core/dfc_defs.h @@ -1,6 +1,6 @@ /* SPDX-License-Identifier: GPL-2.0-only */ /* - * Copyright (c) 2019, The Linux Foundation. All rights reserved. + * Copyright (c) 2019-2020, The Linux Foundation. All rights reserved. */ #ifndef _DFC_DEFS_H @@ -53,6 +53,8 @@ struct dfc_flow_status_info_type_v01 { u16 seq_num; u8 qos_ids_len; struct dfc_qos_id_type_v01 qos_ids[DFC_MAX_QOS_ID_V01]; + u8 rx_bytes_valid; + u32 rx_bytes; }; struct dfc_ancillary_info_type_v01 { @@ -88,7 +90,8 @@ struct dfc_tx_link_status_ind_msg_v01 { }; void dfc_do_burst_flow_control(struct dfc_qmi_data *dfc, - struct dfc_flow_status_ind_msg_v01 *ind); + struct dfc_flow_status_ind_msg_v01 *ind, + bool is_query); void dfc_handle_tx_link_status_ind(struct dfc_qmi_data *dfc, struct dfc_tx_link_status_ind_msg_v01 *ind); diff --git a/core/dfc_qmap.c b/core/dfc_qmap.c index d5fe172b6f8f..06b49e794c58 100644 --- a/core/dfc_qmap.c +++ b/core/dfc_qmap.c @@ -65,13 +65,14 @@ struct qmap_dfc_ind { u8 reserved2; u8 tx_info_valid:1; u8 tx_info:1; - u8 reserved3:6; + u8 rx_bytes_valid:1; + u8 reserved3:5; u8 bearer_id; u8 tcp_bidir:1; u8 bearer_status:3; u8 reserved4:4; __be32 grant; - u32 reserved5; + __be32 rx_bytes; u32 reserved6; } __aligned(1); @@ -89,11 +90,12 @@ struct qmap_dfc_query_resp { u8 cmd_ver; u8 bearer_id; u8 tcp_bidir:1; - u8 reserved:7; + u8 rx_bytes_valid:1; + u8 reserved:6; u8 invalid:1; u8 reserved2:7; __be32 grant; - u32 reserved3; + __be32 rx_bytes; u32 reserved4; } __aligned(1); @@ -185,6 +187,11 @@ static int dfc_qmap_handle_ind(struct dfc_qmi_data *dfc, qmap_flow_ind.flow_status[0].num_bytes = ntohl(cmd->grant); qmap_flow_ind.flow_status[0].seq_num = ntohs(cmd->seq_num); + if (cmd->rx_bytes_valid) { + qmap_flow_ind.flow_status[0].rx_bytes_valid = 1; + qmap_flow_ind.flow_status[0].rx_bytes = ntohl(cmd->rx_bytes); + } + if (cmd->tcp_bidir) { qmap_flow_ind.ancillary_info_valid = 1; qmap_flow_ind.ancillary_info_len = 1; @@ -193,7 +200,7 @@ static int dfc_qmap_handle_ind(struct dfc_qmi_data *dfc, qmap_flow_ind.ancillary_info[0].reserved = DFC_MASK_TCP_BIDIR; } - dfc_do_burst_flow_control(dfc, &qmap_flow_ind); + dfc_do_burst_flow_control(dfc, &qmap_flow_ind, false); done: return QMAP_CMD_ACK; @@ -221,6 +228,11 @@ static int dfc_qmap_handle_query_resp(struct dfc_qmi_data *dfc, qmap_flow_ind.flow_status[0].num_bytes = ntohl(cmd->grant); qmap_flow_ind.flow_status[0].seq_num = 0xFFFF; + if (cmd->rx_bytes_valid) { + qmap_flow_ind.flow_status[0].rx_bytes_valid = 1; + qmap_flow_ind.flow_status[0].rx_bytes = ntohl(cmd->rx_bytes); + } + if (cmd->tcp_bidir) { qmap_flow_ind.ancillary_info_valid = 1; qmap_flow_ind.ancillary_info_len = 1; @@ -229,7 +241,7 @@ static int dfc_qmap_handle_query_resp(struct dfc_qmi_data *dfc, qmap_flow_ind.ancillary_info[0].reserved = DFC_MASK_TCP_BIDIR; } - dfc_do_burst_flow_control(dfc, &qmap_flow_ind); + dfc_do_burst_flow_control(dfc, &qmap_flow_ind, true); return QMAP_CMD_DONE; } diff --git a/core/dfc_qmi.c b/core/dfc_qmi.c index e91433b12bf8..2b25e7d17aab 100644 --- a/core/dfc_qmi.c +++ b/core/dfc_qmi.c @@ -953,6 +953,7 @@ static int dfc_all_bearer_flow_ctl(struct net_device *dev, bearer->tcp_bidir = DFC_IS_TCP_BIDIR(ancillary); bearer->last_grant = fc_info->num_bytes; bearer->last_seq = fc_info->seq_num; + bearer->last_adjusted_grant = fc_info->num_bytes; dfc_bearer_flow_ctl(dev, bearer, qos); } @@ -960,13 +961,40 @@ static int dfc_all_bearer_flow_ctl(struct net_device *dev, return 0; } +static u32 dfc_adjust_grant(struct rmnet_bearer_map *bearer, + struct dfc_flow_status_info_type_v01 *fc_info) +{ + u32 grant; + + if (!fc_info->rx_bytes_valid) + return fc_info->num_bytes; + + if (bearer->bytes_in_flight > fc_info->rx_bytes) + bearer->bytes_in_flight -= fc_info->rx_bytes; + else + bearer->bytes_in_flight = 0; + + /* Adjusted grant = grant - bytes_in_flight */ + if (fc_info->num_bytes > bearer->bytes_in_flight) + grant = fc_info->num_bytes - bearer->bytes_in_flight; + else + grant = 0; + + trace_dfc_adjust_grant(fc_info->mux_id, fc_info->bearer_id, + fc_info->num_bytes, fc_info->rx_bytes, + bearer->bytes_in_flight, grant); + return grant; +} + static int dfc_update_fc_map(struct net_device *dev, struct qos_info *qos, u8 ack_req, u32 ancillary, - struct dfc_flow_status_info_type_v01 *fc_info) + struct dfc_flow_status_info_type_v01 *fc_info, + bool is_query) { struct rmnet_bearer_map *itm = NULL; int rc = 0; bool action = false; + u32 adjusted_grant; itm = qmi_rmnet_get_bearer_map(qos, fc_info->bearer_id); if (!itm) @@ -986,8 +1014,16 @@ static int dfc_update_fc_map(struct net_device *dev, struct qos_info *qos, if (itm->tx_off && fc_info->num_bytes > 0) return 0; - if ((itm->grant_size == 0 && fc_info->num_bytes > 0) || - (itm->grant_size > 0 && fc_info->num_bytes == 0)) + /* Adjuste grant for query */ + if (dfc_qmap && is_query) { + adjusted_grant = dfc_adjust_grant(itm, fc_info); + } else { + adjusted_grant = fc_info->num_bytes; + itm->bytes_in_flight = 0; + } + + if ((itm->grant_size == 0 && adjusted_grant > 0) || + (itm->grant_size > 0 && adjusted_grant == 0)) action = true; /* This is needed by qmap */ @@ -995,13 +1031,24 @@ static int dfc_update_fc_map(struct net_device *dev, struct qos_info *qos, dfc_qmap_send_ack(qos, itm->bearer_id, itm->seq, DFC_ACK_TYPE_DISABLE); - itm->grant_size = fc_info->num_bytes; - itm->grant_thresh = qmi_rmnet_grant_per(itm->grant_size); + itm->grant_size = adjusted_grant; + + /* No further query if the adjusted grant is less + * than 20% of the original grant + */ + if (dfc_qmap && is_query && + itm->grant_size < (fc_info->num_bytes / 5)) + itm->grant_thresh = itm->grant_size; + else + itm->grant_thresh = + qmi_rmnet_grant_per(itm->grant_size); + itm->seq = fc_info->seq_num; itm->ack_req = ack_req; itm->tcp_bidir = DFC_IS_TCP_BIDIR(ancillary); itm->last_grant = fc_info->num_bytes; itm->last_seq = fc_info->seq_num; + itm->last_adjusted_grant = adjusted_grant; if (action) rc = dfc_bearer_flow_ctl(dev, itm, qos); @@ -1011,7 +1058,8 @@ static int dfc_update_fc_map(struct net_device *dev, struct qos_info *qos, } void dfc_do_burst_flow_control(struct dfc_qmi_data *dfc, - struct dfc_flow_status_ind_msg_v01 *ind) + struct dfc_flow_status_ind_msg_v01 *ind, + bool is_query) { struct net_device *dev; struct qos_info *qos; @@ -1067,7 +1115,8 @@ void dfc_do_burst_flow_control(struct dfc_qmi_data *dfc, dev, qos, ack_req, ancillary, flow_status); else dfc_update_fc_map( - dev, qos, ack_req, ancillary, flow_status); + dev, qos, ack_req, ancillary, flow_status, + is_query); spin_unlock_bh(&qos->qos_lock); } @@ -1093,6 +1142,7 @@ static void dfc_update_tx_link_status(struct net_device *dev, if (itm->grant_size && !tx_status) { itm->grant_size = 0; itm->tcp_bidir = false; + itm->bytes_in_flight = 0; dfc_bearer_flow_ctl(dev, itm, qos); } else if (itm->grant_size == 0 && tx_status && !itm->rat_switch) { itm->grant_size = DEFAULT_GRANT; @@ -1170,7 +1220,8 @@ static void dfc_qmi_ind_work(struct work_struct *work) if (!dfc->restart_state) { if (svc_ind->msg_id == QMI_DFC_FLOW_STATUS_IND_V01) dfc_do_burst_flow_control( - dfc, &svc_ind->d.dfc_info); + dfc, &svc_ind->d.dfc_info, + false); else if (svc_ind->msg_id == QMI_DFC_TX_LINK_STATUS_IND_V01) dfc_handle_tx_link_status_ind( @@ -1445,6 +1496,8 @@ void dfc_qmi_burst_check(struct net_device *dev, struct qos_info *qos, trace_dfc_flow_check(dev->name, bearer->bearer_id, len, mark, bearer->grant_size); + bearer->bytes_in_flight += len; + if (!bearer->grant_size) goto out; @@ -1498,7 +1551,7 @@ void dfc_qmi_query_flow(void *dfc_data) svc_ind->d.dfc_info.flow_status_len = resp->flow_status_len; memcpy(&svc_ind->d.dfc_info.flow_status, resp->flow_status, sizeof(resp->flow_status[0]) * resp->flow_status_len); - dfc_do_burst_flow_control(data, &svc_ind->d.dfc_info); + dfc_do_burst_flow_control(data, &svc_ind->d.dfc_info, true); done: kfree(svc_ind); diff --git a/core/qmi_rmnet.c b/core/qmi_rmnet.c index 7c1a6857500f..dcc02b3488dd 100644 --- a/core/qmi_rmnet.c +++ b/core/qmi_rmnet.c @@ -471,7 +471,8 @@ static void qmi_rmnet_query_flows(struct qmi_info *qmi) int i; for (i = 0; i < MAX_CLIENT_NUM; i++) { - if (qmi->dfc_clients[i] && !dfc_qmap) + if (qmi->dfc_clients[i] && !dfc_qmap && + !qmi->dfc_client_exiting[i]) dfc_qmi_query_flow(qmi->dfc_clients[i]); } } @@ -547,6 +548,7 @@ qmi_rmnet_setup_client(void *port, struct qmi_info *qmi, struct tcmsg *tcm) err = dfc_qmap_client_init(port, idx, &svc, qmi); else err = dfc_qmi_client_init(port, idx, &svc, qmi); + qmi->dfc_client_exiting[idx] = false; } if ((tcm->tcm_ifindex & FLAG_POWERSAVE_MASK) && @@ -607,6 +609,7 @@ qmi_rmnet_delete_client(void *port, struct qmi_info *qmi, struct tcmsg *tcm) qmi->wda_client = NULL; qmi->wda_pending = NULL; } else { + qmi->dfc_client_exiting[idx] = true; qmi_rmnet_flush_ps_wq(); } @@ -722,15 +725,18 @@ void qmi_rmnet_enable_all_flows(struct net_device *dev) spin_lock_bh(&qos->qos_lock); list_for_each_entry(bearer, &qos->bearer_head, list) { + bearer->seq = 0; + bearer->ack_req = 0; + bearer->bytes_in_flight = 0; + bearer->tcp_bidir = false; + bearer->rat_switch = false; + if (bearer->tx_off) continue; + do_wake = !bearer->grant_size; bearer->grant_size = DEFAULT_GRANT; bearer->grant_thresh = qmi_rmnet_grant_per(DEFAULT_GRANT); - bearer->seq = 0; - bearer->ack_req = 0; - bearer->tcp_bidir = false; - bearer->rat_switch = false; if (do_wake) dfc_bearer_flow_ctl(dev, bearer, qos); @@ -989,7 +995,6 @@ int qmi_rmnet_ps_ind_deregister(void *port, goto done; } } - done: return 0; } diff --git a/core/qmi_rmnet_i.h b/core/qmi_rmnet_i.h index 9d7c09c82e14..f478c459503d 100644 --- a/core/qmi_rmnet_i.h +++ b/core/qmi_rmnet_i.h @@ -48,6 +48,8 @@ struct rmnet_bearer_map { u8 ack_req; u32 last_grant; u16 last_seq; + u32 bytes_in_flight; + u32 last_adjusted_grant; bool tcp_bidir; bool rat_switch; bool tx_off; @@ -92,6 +94,7 @@ struct qmi_info { void *wda_pending; void *dfc_clients[MAX_CLIENT_NUM]; void *dfc_pending[MAX_CLIENT_NUM]; + bool dfc_client_exiting[MAX_CLIENT_NUM]; unsigned long ps_work_active; bool ps_enabled; bool dl_msg_active; From 05ac1acb02f4b047cb32a4b879aa7d923d9c8b0f Mon Sep 17 00:00:00 2001 From: Subash Abhinov Kasiviswanathan Date: Mon, 13 Apr 2020 17:17:39 -0700 Subject: [PATCH 16/54] rmnet_ctl: enable IPC logging Enable rmnet_ctl debugfs parameters and IPC logging. Change-Id: I2228e5d4ae86ad5ef2fb17205f14e837255d706c Acked-by: Weiyi Chen Signed-off-by: Subash Abhinov Kasiviswanathan --- core/rmnet_ctl_client.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/core/rmnet_ctl_client.c b/core/rmnet_ctl_client.c index 1fe348b1387d..4a4bc13d917e 100644 --- a/core/rmnet_ctl_client.c +++ b/core/rmnet_ctl_client.c @@ -135,6 +135,8 @@ void *rmnet_ctl_register_client(struct rmnet_ctl_client_hooks *hook) spin_unlock(&client_lock); + rmnet_ctl_set_dbgfs(true); + return client; } EXPORT_SYMBOL(rmnet_ctl_register_client); @@ -157,6 +159,8 @@ int rmnet_ctl_unregister_client(void *handle) synchronize_rcu(); kfree(client); + rmnet_ctl_set_dbgfs(false); + return 0; } EXPORT_SYMBOL(rmnet_ctl_unregister_client); From e87877cb0c2a031c8a08750a4c929a6c09eef7b0 Mon Sep 17 00:00:00 2001 From: Subash Abhinov Kasiviswanathan Date: Sat, 11 Apr 2020 15:33:27 -0700 Subject: [PATCH 17/54] rmnet_ctl: remove dependencies Removes hard dependendencies between rmnet_core and rmnet_ctl. Change-Id: I9931380fc6406eec5842a824181a0bacbb6ce73a Acked-by: Weiyi Chen Signed-off-by: Subash Abhinov Kasiviswanathan --- core/dfc_qmap.c | 36 ++++++++++++++++++++++++----- core/rmnet_ctl.h | 50 ++++++----------------------------------- core/rmnet_ctl_client.c | 8 +++++++ core/rmnet_ctl_client.h | 13 +++++++++++ 4 files changed, 59 insertions(+), 48 deletions(-) diff --git a/core/dfc_qmap.c b/core/dfc_qmap.c index 06b49e794c58..69dc6e7a0241 100644 --- a/core/dfc_qmap.c +++ b/core/dfc_qmap.c @@ -4,6 +4,7 @@ */ #include +#include #include "rmnet_ctl.h" #include "dfc_defs.h" #include "rmnet_qmi.h" @@ -127,6 +128,9 @@ static struct dfc_qmi_data __rcu *qmap_dfc_data; static atomic_t qmap_txid; static void *rmnet_ctl_handle; +extern struct rmnet_ctl_client_if rmnet_ctl_if; +static struct rmnet_ctl_client_if *rmnet_ctl; + static void dfc_qmap_send_end_marker_cnf(struct qos_info *qos, u8 bearer_id, u16 seq, u32 tx_id); @@ -134,7 +138,8 @@ static void dfc_qmap_send_cmd(struct sk_buff *skb) { trace_dfc_qmap(skb->data, skb->len, false); - if (rmnet_ctl_send_client(rmnet_ctl_handle, skb)) { + if (unlikely(!rmnet_ctl || !rmnet_ctl->send) || + rmnet_ctl->send(rmnet_ctl_handle, skb)) { pr_err("Failed to send to rmnet ctl\n"); kfree_skb(skb); } @@ -150,7 +155,9 @@ static void dfc_qmap_send_inband_ack(struct dfc_qmi_data *dfc, skb->protocol = htons(ETH_P_MAP); skb->dev = rmnet_get_real_dev(dfc->rmnet_port); - rmnet_ctl_log_debug("TXI", skb->data, skb->len); + if (likely(rmnet_ctl && rmnet_ctl->log)) + rmnet_ctl->log(RMNET_CTL_LOG_DEBUG, "TXI", 0, + skb->data, skb->len); trace_dfc_qmap(skb->data, skb->len, false); dev_queue_xmit(skb); } @@ -446,7 +453,9 @@ static void dfc_qmap_send_end_marker_cnf(struct qos_info *qos, skb->dev = qos->real_dev; /* This cmd needs to be sent in-band */ - rmnet_ctl_log_info("TXI", skb->data, skb->len); + if (likely(rmnet_ctl && rmnet_ctl->log)) + rmnet_ctl->log(RMNET_CTL_LOG_INFO, "TXI", 0, + skb->data, skb->len); trace_dfc_qmap(skb->data, skb->len, false); rmnet_map_tx_qmap_cmd(skb); } @@ -490,7 +499,16 @@ int dfc_qmap_client_init(void *port, int index, struct svc_info *psvc, atomic_set(&qmap_txid, 0); - rmnet_ctl_handle = rmnet_ctl_register_client(&cb); + if (!rmnet_ctl) + rmnet_ctl = symbol_get(rmnet_ctl_if); + if (!rmnet_ctl) { + pr_err("rmnet_ctl module not loaded\n"); + goto out; + } + + if (rmnet_ctl->reg) + rmnet_ctl_handle = rmnet_ctl->reg(&cb); + if (!rmnet_ctl_handle) pr_err("Failed to register with rmnet ctl\n"); @@ -501,6 +519,7 @@ int dfc_qmap_client_init(void *port, int index, struct svc_info *psvc, dfc_qmap_send_config(data); +out: return 0; } @@ -515,7 +534,9 @@ void dfc_qmap_client_exit(void *dfc_data) trace_dfc_client_state_down(data->index, 0); - rmnet_ctl_unregister_client(rmnet_ctl_handle); + if (rmnet_ctl && rmnet_ctl->dereg) + rmnet_ctl->dereg(rmnet_ctl_handle); + rmnet_ctl_handle = NULL; WRITE_ONCE(data->restart_state, 1); RCU_INIT_POINTER(qmap_dfc_data, NULL); @@ -523,5 +544,10 @@ void dfc_qmap_client_exit(void *dfc_data) kfree(data); + if (rmnet_ctl) { + symbol_put(rmnet_ctl_if); + rmnet_ctl = NULL; + } + pr_info("DFC QMAP exit\n"); } diff --git a/core/rmnet_ctl.h b/core/rmnet_ctl.h index 54d2385c4eb9..cb3ca722057a 100644 --- a/core/rmnet_ctl.h +++ b/core/rmnet_ctl.h @@ -10,8 +10,6 @@ #include -#define CONFIG_RMNET_CTL 1 - enum rmnet_ctl_log_lvl { RMNET_CTL_LOG_CRIT, RMNET_CTL_LOG_ERR, @@ -19,50 +17,16 @@ enum rmnet_ctl_log_lvl { RMNET_CTL_LOG_DEBUG, }; -#define rmnet_ctl_log_err(msg, rc, data, len) \ - rmnet_ctl_log(RMNET_CTL_LOG_ERR, msg, rc, data, len) - -#define rmnet_ctl_log_info(msg, data, len) \ - rmnet_ctl_log(RMNET_CTL_LOG_INFO, msg, 0, data, len) - -#define rmnet_ctl_log_debug(msg, data, len) \ - rmnet_ctl_log(RMNET_CTL_LOG_DEBUG, msg, 0, data, len) - struct rmnet_ctl_client_hooks { void (*ctl_dl_client_hook)(struct sk_buff *skb); }; -#ifdef CONFIG_RMNET_CTL - -void *rmnet_ctl_register_client(struct rmnet_ctl_client_hooks *hook); -int rmnet_ctl_unregister_client(void *handle); -int rmnet_ctl_send_client(void *handle, struct sk_buff *skb); -void rmnet_ctl_log(enum rmnet_ctl_log_lvl lvl, const char *msg, - int rc, const void *data, unsigned int len); - -#else - -static inline void *rmnet_ctl_register_client( - struct rmnet_ctl_client_hooks *hook) -{ - return NULL; -} - -static inline int rmnet_ctl_unregister_client(void *handle) -{ - return -EINVAL; -} - -static inline int rmnet_ctl_send_client(void *handle, struct sk_buff *skb) -{ - return -EINVAL; -} - -static inline void rmnet_ctl_log(enum rmnet_ctl_log_lvl lvl, const char *msg, - int rc, const void *data, unsigned int len) -{ -} - -#endif /* CONFIG_RMNET_CTL */ +struct rmnet_ctl_client_if { + void * (*reg)(struct rmnet_ctl_client_hooks *hook); + int (*dereg)(void *handle); + int (*send)(void *handle, struct sk_buff *skb); + void (*log)(enum rmnet_ctl_log_lvl lvl, const char *msg, int rc, + const void *data, unsigned int len); +}; #endif /* _RMNET_CTL_H_ */ diff --git a/core/rmnet_ctl_client.c b/core/rmnet_ctl_client.c index 1fe348b1387d..e24b7bea9463 100644 --- a/core/rmnet_ctl_client.c +++ b/core/rmnet_ctl_client.c @@ -200,3 +200,11 @@ void rmnet_ctl_log(enum rmnet_ctl_log_lvl lvl, const char *msg, } } EXPORT_SYMBOL(rmnet_ctl_log); + +struct rmnet_ctl_client_if rmnet_ctl_if = { + .reg = rmnet_ctl_register_client, + .dereg = rmnet_ctl_unregister_client, + .send = rmnet_ctl_send_client, + .log = rmnet_ctl_log, +}; +EXPORT_SYMBOL(rmnet_ctl_if); diff --git a/core/rmnet_ctl_client.h b/core/rmnet_ctl_client.h index f7b1b6a69e61..f044aa29e6d6 100644 --- a/core/rmnet_ctl_client.h +++ b/core/rmnet_ctl_client.h @@ -9,6 +9,19 @@ #define _RMNET_CTL_CLIENT_H_ #include +#include "rmnet_ctl.h" + +void rmnet_ctl_log(enum rmnet_ctl_log_lvl lvl, const char *msg, + int rc, const void *data, unsigned int len); + +#define rmnet_ctl_log_err(msg, rc, data, len) \ + rmnet_ctl_log(RMNET_CTL_LOG_ERR, msg, rc, data, len) + +#define rmnet_ctl_log_info(msg, data, len) \ + rmnet_ctl_log(RMNET_CTL_LOG_INFO, msg, 0, data, len) + +#define rmnet_ctl_log_debug(msg, data, len) \ + rmnet_ctl_log(RMNET_CTL_LOG_DEBUG, msg, 0, data, len) struct rmnet_ctl_stats { u64 rx_pkts; From fa1352fa7aae45a68948457a1f18d990e8644df0 Mon Sep 17 00:00:00 2001 From: Subash Abhinov Kasiviswanathan Date: Mon, 27 Apr 2020 15:04:59 -0700 Subject: [PATCH 18/54] dfc: remove unsupported modes Removes DFC modes that no longer supported. Change-Id: I86c5d549ca02b0313dc89ed9e12d770323e55a9e Acked-by: Weiyi Chen Signed-off-by: Subash Abhinov Kasiviswanathan --- core/dfc_qmap.c | 8 ++++++-- core/dfc_qmi.c | 14 ++++---------- core/qmi_rmnet.c | 32 ++++++-------------------------- core/qmi_rmnet_i.h | 2 -- 4 files changed, 16 insertions(+), 40 deletions(-) diff --git a/core/dfc_qmap.c b/core/dfc_qmap.c index 69dc6e7a0241..e2a7d18620ce 100644 --- a/core/dfc_qmap.c +++ b/core/dfc_qmap.c @@ -486,6 +486,11 @@ int dfc_qmap_client_init(void *port, int index, struct svc_info *psvc, if (!port || !qmi) return -EINVAL; + /* Prevent double init */ + data = rcu_dereference(qmap_dfc_data); + if (data) + return -EINVAL; + data = kzalloc(sizeof(struct dfc_qmi_data), GFP_KERNEL); if (!data) return -ENOMEM; @@ -499,8 +504,7 @@ int dfc_qmap_client_init(void *port, int index, struct svc_info *psvc, atomic_set(&qmap_txid, 0); - if (!rmnet_ctl) - rmnet_ctl = symbol_get(rmnet_ctl_if); + rmnet_ctl = symbol_get(rmnet_ctl_if); if (!rmnet_ctl) { pr_err("rmnet_ctl module not loaded\n"); goto out; diff --git a/core/dfc_qmi.c b/core/dfc_qmi.c index 2b25e7d17aab..9fe29550c2d2 100644 --- a/core/dfc_qmi.c +++ b/core/dfc_qmi.c @@ -1479,16 +1479,10 @@ void dfc_qmi_burst_check(struct net_device *dev, struct qos_info *qos, spin_lock_bh(&qos->qos_lock); - if (dfc_mode == DFC_MODE_MQ_NUM) { - /* Mark is mq num */ - if (likely(mark < MAX_MQ_NUM)) - bearer = qos->mq[mark].bearer; - } else { - /* Mark is flow_id */ - itm = qmi_rmnet_get_flow_map(qos, mark, ip_type); - if (likely(itm)) - bearer = itm->bearer; - } + /* Mark is flow_id */ + itm = qmi_rmnet_get_flow_map(qos, mark, ip_type); + if (likely(itm)) + bearer = itm->bearer; if (unlikely(!bearer)) goto out; diff --git a/core/qmi_rmnet.c b/core/qmi_rmnet.c index dcc02b3488dd..6c4a7d081a0f 100644 --- a/core/qmi_rmnet.c +++ b/core/qmi_rmnet.c @@ -39,14 +39,12 @@ #define FLAG_TO_MODE(f) ((f) & FLAG_DFC_MASK) #define DFC_SUPPORTED_MODE(m) \ - ((m) == DFC_MODE_FLOW_ID || (m) == DFC_MODE_MQ_NUM || \ - (m) == DFC_MODE_SA) + ((m) == DFC_MODE_SA) #define FLAG_TO_QMAP(f) ((f) & FLAG_QMAP_MASK) int dfc_mode; int dfc_qmap; -#define IS_ANCILLARY(type) ((type) != AF_INET && (type) != AF_INET6) unsigned int rmnet_wq_frequency __read_mostly = 1000; @@ -312,10 +310,7 @@ static void __qmi_rmnet_update_mq(struct net_device *dev, bearer->mq_idx = itm->mq_idx; bearer->ack_mq_idx = itm->mq_idx + ACK_MQ_OFFSET; } else { - if (IS_ANCILLARY(itm->ip_type)) - bearer->ack_mq_idx = itm->mq_idx; - else - bearer->mq_idx = itm->mq_idx; + bearer->mq_idx = itm->mq_idx; } qmi_rmnet_flow_control(dev, itm->mq_idx, @@ -855,34 +850,19 @@ int qmi_rmnet_get_queue(struct net_device *dev, struct sk_buff *skb) if (!qos) return 0; - /* If mark is mq num return it */ - if (dfc_mode == DFC_MODE_MQ_NUM) - return mark; - - if (dfc_mode == DFC_MODE_SA) + if (likely(dfc_mode == DFC_MODE_SA)) return qmi_rmnet_get_queue_sa(qos, skb); - /* Default flows */ - if (!mark) { - if (qmi_rmnet_is_tcp_ack(skb)) - return 1; - else - return 0; - } - ip_type = (skb->protocol == htons(ETH_P_IPV6)) ? AF_INET6 : AF_INET; - /* Dedicated flows */ spin_lock_bh(&qos->qos_lock); itm = qmi_rmnet_get_flow_map(qos, mark, ip_type); - if (unlikely(!itm)) - goto done; + if (itm) + txq = itm->mq_idx; - txq = itm->mq_idx; - -done: spin_unlock_bh(&qos->qos_lock); + return txq; } EXPORT_SYMBOL(qmi_rmnet_get_queue); diff --git a/core/qmi_rmnet_i.h b/core/qmi_rmnet_i.h index f478c459503d..e4fe7825dc3a 100644 --- a/core/qmi_rmnet_i.h +++ b/core/qmi_rmnet_i.h @@ -27,8 +27,6 @@ #define ACK_MQ_OFFSET (MAX_MQ_NUM - 1) #define INVALID_MQ 0xFF -#define DFC_MODE_FLOW_ID 2 -#define DFC_MODE_MQ_NUM 3 #define DFC_MODE_SA 4 #define CONFIG_QTI_QMI_RMNET 1 From c4bd722431e77ccd665f0f1bb1f4d2e6b4c12a74 Mon Sep 17 00:00:00 2001 From: Subash Abhinov Kasiviswanathan Date: Mon, 11 May 2020 19:41:08 -0600 Subject: [PATCH 19/54] core: Hold on to module reference Do not remove module if the dependent modules are removed. CRs-Fixed: 2683697 Change-Id: I35539aff061fe57a85f0bb8eb3dcf40499eca760 Signed-off-by: Subash Abhinov Kasiviswanathan --- core/rmnet_config.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/core/rmnet_config.c b/core/rmnet_config.c index 4e8910e747ba..95208b6e6df6 100644 --- a/core/rmnet_config.c +++ b/core/rmnet_config.c @@ -761,6 +761,8 @@ static int __init rmnet_init(void) } rmnet_core_genl_init(); + + try_module_get(THIS_MODULE); return rc; } @@ -769,6 +771,8 @@ static void __exit rmnet_exit(void) unregister_netdevice_notifier(&rmnet_dev_notifier); rtnl_link_unregister(&rmnet_link_ops); rmnet_core_genl_deinit(); + + module_put(THIS_MODULE); } module_init(rmnet_init) From 0daa9b15d32d37357fdff123929e5fd630fb02e5 Mon Sep 17 00:00:00 2001 From: Subash Abhinov Kasiviswanathan Date: Fri, 15 May 2020 17:56:12 -0600 Subject: [PATCH 20/54] core: Check for aggregated skb size before copy This ensures that the target aggregated skb has sufficient tailroom to copy the incoming skb content. Checking for the configured size may not reflect the true value. Fixes the following- <2> skbuff: skb_over_panic: text:000000006faa6b25 len:4044 put:1288 head:000000007024fb17 data:000000007024fb17 tail:0xfcc end:0xec0 dev:rmnet_ipa0 <2> ------------[ cut here ]------------ <2> kernel ... at net/core/skbuff.c:108! <2> Internal error: Oops - BUG: 0 [#1] PREEMPT SMP <2> pc : skb_panic+0x4c/0x50 <2> lr : skb_panic+0x4c/0x50 <2> Call trace: <2> skb_panic+0x4c/0x50 <2> skb_push+0x0/0x44 <2> skb_copy_bits+0x0/0x264 <2> rmnet_map_linearize_copy+0xa8/0x110 [rmnet_core] <2> rmnet_map_tx_aggregate+0x358/0x45c [rmnet_core] <2> rmnet_egress_handler+0x2cc/0x2f4 [rmnet_core] <2> rmnet_vnd_start_xmit+0x1a0/0x1f8 [rmnet_core] <2> dev_hard_start_xmit+0x1e0/0x394 <2> sch_direct_xmit+0xe4/0x32c <2> __qdisc_run+0x4ec/0xb48 <2> __dev_queue_xmit.llvm.8867084656021901384+0x3ec/0x9c8 <2> neigh_direct_output+0x18/0x24 <2> ip_finish_output2+0x3f4/0x5b0 <2> __ip_finish_output+0x1d4/0x248 <2> ip_finish_output+0x50/0xe0 <2> ip_output+0x148/0x194 <2> __ip_queue_xmit+0x378/0x3d4 <2> ip_queue_xmit+0x14/0x20 <2> __tcp_transmit_skb+0x7b4/0xc4c <2> tcp_write_xmit+0x868/0x1194 <2> __tcp_push_pending_frames+0x38/0xd0 <2> tcp_push+0x124/0x154 <2> tcp_sendmsg_locked+0xae4/0xcd0 <2> tcp_sendmsg+0x38/0x5c <2> inet6_sendmsg+0x5c/0xa4 <2> sock_write_iter+0xf8/0x17c <2> __vfs_write+0x1a0/0x220 <2> vfs_write+0xe4/0x1a8 <2> ksys_write+0x7c/0xe8 <2> __arm64_sys_write+0x1c/0x28 <2> el0_svc_common+0xb8/0x1c4 <2> el0_svc_handler+0x6c/0x88 <2> el0_svc+0x8/0xc CRs-Fixed: 2687877 Change-Id: If8cf289c2629be28b890927c54cb4d4209b24bc5 Signed-off-by: Subash Abhinov Kasiviswanathan --- core/rmnet_map_data.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/rmnet_map_data.c b/core/rmnet_map_data.c index 58244722c5a0..29d7927854cf 100644 --- a/core/rmnet_map_data.c +++ b/core/rmnet_map_data.c @@ -1485,7 +1485,7 @@ new_packet: goto schedule; } diff = timespec64_sub(port->agg_last, port->agg_time); - size = port->egress_agg_params.agg_size - port->agg_skb->len; + size = skb_tailroom(port->agg_skb); if (skb->len > size || port->agg_count >= port->egress_agg_params.agg_count || From 94d3f634df81e8019611452adb90290e86212dd3 Mon Sep 17 00:00:00 2001 From: Sean Tranchetti Date: Fri, 21 Feb 2020 12:35:17 -0700 Subject: [PATCH 21/54] rmnet: Control DL csum offload with RXCSUM Similar to the UL direction, allow the hardware checksum offload support to be toggled off with the NETIF_F_RXCSUM flag though ethtool. Change-Id: I38e43cf9c13363eee340793878be7639f18254e3 Signed-off-by: Sean Tranchetti --- core/rmnet_descriptor.c | 4 +++- core/rmnet_map_data.c | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/core/rmnet_descriptor.c b/core/rmnet_descriptor.c index 34f8ac123f38..6e2ae12967ae 100644 --- a/core/rmnet_descriptor.c +++ b/core/rmnet_descriptor.c @@ -1036,7 +1036,9 @@ int rmnet_frag_process_next_hdr_packet(struct rmnet_frag_descriptor *frag_desc, rmnet_recycle_frag_descriptor(frag_desc, port); break; case RMNET_MAP_HEADER_TYPE_CSUM_OFFLOAD: - if (rmnet_frag_get_csum_valid(frag_desc)) { + if (unlikely(!(frag_desc->dev->features & NETIF_F_RXCSUM))) { + priv->stats.csum_sw++; + } else if (rmnet_frag_get_csum_valid(frag_desc)) { priv->stats.csum_ok++; frag_desc->csum_valid = true; } else { diff --git a/core/rmnet_map_data.c b/core/rmnet_map_data.c index 29d7927854cf..f5a9c357ba63 100644 --- a/core/rmnet_map_data.c +++ b/core/rmnet_map_data.c @@ -1172,7 +1172,9 @@ int rmnet_map_process_next_hdr_packet(struct sk_buff *skb, consume_skb(skb); break; case RMNET_MAP_HEADER_TYPE_CSUM_OFFLOAD: - if (rmnet_map_get_csum_valid(skb)) { + if (unlikely(!(skb->dev->features & NETIF_F_RXCSUM))) { + priv->stats.csum_sw++; + } else if (rmnet_map_get_csum_valid(skb)) { priv->stats.csum_ok++; skb->ip_summed = CHECKSUM_UNNECESSARY; } else { From 6c318682bce25de5b69986da2a24c9d507c916ac Mon Sep 17 00:00:00 2001 From: Subash Abhinov Kasiviswanathan Date: Wed, 20 May 2020 15:43:16 -0600 Subject: [PATCH 22/54] core: cleanup task boost list Cleanup entries in the task boost list when the pid becomes inactive. Change-Id: I0b1b2ef81cda470cd08b31ab4e78f81d346b9b70 Acked-by: Ryan Chapman Signed-off-by: Subash Abhinov Kasiviswanathan --- core/rmnet_genl.c | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/core/rmnet_genl.c b/core/rmnet_genl.c index f568672db569..e52364ad3b7a 100644 --- a/core/rmnet_genl.c +++ b/core/rmnet_genl.c @@ -1,5 +1,5 @@ // SPDX-License-Identifier: GPL-2.0-only -/* Copyright (c) 2019, The Linux Foundation. All rights reserved. +/* Copyright (c) 2019-2020, The Linux Foundation. All rights reserved. * * RMNET Data Generic Netlink * @@ -190,11 +190,8 @@ static void rmnet_create_pid_bps_resp(struct rmnet_core_pid_bps_resp tx_bytes_cur = node_p->tx_bytes; if (tx_bytes_cur <= node_p->tx_bytes_last_query) { /* Dont send inactive pids to userspace */ - /* TODO: can remove from hash table probably */ - node_p->tx_bps = 0; - node_p->timstamp_last_query = - pid_bps_resp_ptr->timestamp; - node_p->sched_boost_remaining_ms = 0; + hash_del(&node_p->list); + kfree(node_p); continue; } From 9a791f3de42917f38a733a4d7a74f5b8c9a86e78 Mon Sep 17 00:00:00 2001 From: Subash Abhinov Kasiviswanathan Date: Thu, 21 May 2020 12:37:36 -0600 Subject: [PATCH 23/54] net: qualcomm: rmnet: Allow UL aggregation time limit of 1ms Reduce the minimum allowed UL aggregation timeout to 1ms as a lower limit might be preferable to reduce latency for sporadic traffic scenarios. CRs-fixed: 2692360 Change-Id: Iba1c02232fa83d7cac112bd4b3f625128e2da88b Signed-off-by: Subash Abhinov Kasiviswanathan --- core/rmnet_config.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/rmnet_config.c b/core/rmnet_config.c index 95208b6e6df6..1d0d71dd0d99 100644 --- a/core/rmnet_config.c +++ b/core/rmnet_config.c @@ -360,7 +360,7 @@ static int rmnet_rtnl_validate(struct nlattr *tb[], struct nlattr *data[], if (data[IFLA_RMNET_UL_AGG_PARAMS]) { agg_params = nla_data(data[IFLA_RMNET_UL_AGG_PARAMS]); - if (agg_params->agg_time < 3000000) + if (agg_params->agg_time < 1000000) return -EINVAL; } } From 2a1b9716b17edf853ab187f9c1c418b03902902b Mon Sep 17 00:00:00 2001 From: Subash Abhinov Kasiviswanathan Date: Tue, 26 May 2020 12:50:43 -0700 Subject: [PATCH 24/54] rmnet_ctl: re-arrange ipa registration Re-arrange ipa registration sequence to handle cases where rmnet_ctl is being initialized before ipa is ready. Change-Id: Ic8416ad8f96f818e32f1320e997287ebfe755d03 Acked-by: Weiyi Chen Signed-off-by: Subash Abhinov Kasiviswanathan --- core/rmnet_ctl_ipa.c | 29 ++++++++++++++--------------- 1 file changed, 14 insertions(+), 15 deletions(-) diff --git a/core/rmnet_ctl_ipa.c b/core/rmnet_ctl_ipa.c index fc4864cc12df..39eb76ad7781 100644 --- a/core/rmnet_ctl_ipa.c +++ b/core/rmnet_ctl_ipa.c @@ -18,6 +18,7 @@ struct rmnet_ctl_ipa_dev { }; static struct rmnet_ctl_ipa_dev ctl_ipa_dev; +static bool rmnet_ctl_ipa_registered; static int rmnet_ctl_send_ipa(struct rmnet_ctl_dev *dev, struct sk_buff *skb) { @@ -78,33 +79,31 @@ static void rmnet_ctl_ipa_ready(void *user_data) rmnet_ctl_dl_callback, &ctl_ipa_dev); - pr_info("%s: %d\n", __func__, rc); + if (rc) + pr_err("%s: %d\n", __func__, rc); + else + rmnet_ctl_ipa_registered = true; } static int __init rmnet_ctl_init(void) { int rc; - rc = ipa_register_rmnet_ctl_cb( - rmnet_ctl_probe, - &ctl_ipa_dev, - rmnet_ctl_remove, - &ctl_ipa_dev, - rmnet_ctl_dl_callback, - &ctl_ipa_dev); - - if (rc == -EAGAIN) - rc = ipa_register_ipa_ready_cb( - rmnet_ctl_ipa_ready, NULL); - - pr_info("%s: %d\n", __func__, rc); + rc = ipa_register_ipa_ready_cb(rmnet_ctl_ipa_ready, NULL); + if (rc == -EEXIST) + rmnet_ctl_ipa_ready(NULL); + else if (rc) + pr_err("%s: %d\n", __func__, rc); return 0; } static void __exit rmnet_ctl_exit(void) { - ipa_unregister_rmnet_ctl_cb(); + if (rmnet_ctl_ipa_registered) { + ipa_unregister_rmnet_ctl_cb(); + rmnet_ctl_ipa_registered = false; + } } module_init(rmnet_ctl_init) From 17190b7e11a50bfefd6c5bf6ca30cab605c384ef Mon Sep 17 00:00:00 2001 From: Subash Abhinov Kasiviswanathan Date: Thu, 28 May 2020 22:14:41 -0600 Subject: [PATCH 25/54] core: Add trace point for gso verification This patch adds trace events to help with debug for gso feature by identifying the packets (and their lengths) that are using the segmentation offload feature. Adding source and destination port number info in the gso trace events to differentiate between the flows. CRs-Fixed: 2697145 Change-Id: I4f9786afa799cb1589bd07393c0922913037390d Signed-off-by: Subash Abhinov Kasiviswanathan --- core/rmnet_trace.h | 29 ++++++++++++++++++++++++++++- core/rmnet_vnd.c | 21 +++++++++++++++++++++ 2 files changed, 49 insertions(+), 1 deletion(-) diff --git a/core/rmnet_trace.h b/core/rmnet_trace.h index b1ae59b30c6a..28a2a2c6003c 100644 --- a/core/rmnet_trace.h +++ b/core/rmnet_trace.h @@ -1,5 +1,5 @@ /* SPDX-License-Identifier: GPL-2.0-only */ -/* Copyright (c) 2018-2019, The Linux Foundation. All rights reserved. +/* Copyright (c) 2018-2020, The Linux Foundation. All rights reserved. */ #undef TRACE_SYSTEM @@ -101,6 +101,33 @@ DEFINE_EVENT ); +TRACE_EVENT(print_skb_gso, + + TP_PROTO(struct sk_buff *skb, __be16 src, __be16 dest), + + TP_ARGS(skb, src, dest), + + TP_STRUCT__entry( + __field(void *, skbaddr) + __field(int, len) + __field(int, data_len) + __field(__be16, src) + __field(__be16, dest) + ), + + TP_fast_assign( + __entry->skbaddr = skb; + __entry->len = skb->len; + __entry->data_len = skb->data_len; + __entry->src = src; + __entry->dest = dest; + ), + + TP_printk("GSO: skbaddr=%pK, len=%d, data_len=%d, src=%u, dest=%u", + __entry->skbaddr, __entry->len, __entry->data_len, + be16_to_cpu(__entry->src), be16_to_cpu(__entry->dest)) +); + /*****************************************************************************/ /* Trace events for rmnet_perf module */ /*****************************************************************************/ diff --git a/core/rmnet_vnd.c b/core/rmnet_vnd.c index 5b36fe88e764..55da35c684a8 100644 --- a/core/rmnet_vnd.c +++ b/core/rmnet_vnd.c @@ -17,6 +17,8 @@ #include #include #include +#include +#include #include #include "rmnet_config.h" #include "rmnet_handlers.h" @@ -171,6 +173,25 @@ static u16 rmnet_vnd_select_queue(struct net_device *dev, int boost_trigger = 0; int txq = 0; + if (trace_print_skb_gso_enabled()) { + if (!skb_shinfo(skb)->gso_size) + goto skip_trace; + + if (skb->protocol == htons(ETH_P_IP)) { + if (ip_hdr(skb)->protocol != IPPROTO_TCP) + goto skip_trace; + } + + if (skb->protocol == htons(ETH_P_IPV6)) { + if (ipv6_hdr(skb)->nexthdr != IPPROTO_TCP) + goto skip_trace; + } + + trace_print_skb_gso(skb, tcp_hdr(skb)->source, + tcp_hdr(skb)->dest); + } + +skip_trace: if (priv->real_dev) txq = qmi_rmnet_get_queue(dev, skb); From aeca58ead5ebd2a5c68234c516579dcbc3648fff Mon Sep 17 00:00:00 2001 From: Subash Abhinov Kasiviswanathan Date: Wed, 24 Jun 2020 10:49:26 -0700 Subject: [PATCH 26/54] dfc: fix null pointer access Fix a null pointer dereference issue when data packets trigger the queuing of powersave work before the powersave workqueue is initialized. Change-Id: Ia3515a7aaa47cb41568c39462bca73ceae11ea9c Acked-by: Weiyi Chen Signed-off-by: Subash Abhinov Kasiviswanathan --- core/qmi_rmnet.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/core/qmi_rmnet.c b/core/qmi_rmnet.c index 6c4a7d081a0f..293a40efa517 100644 --- a/core/qmi_rmnet.c +++ b/core/qmi_rmnet.c @@ -919,6 +919,7 @@ EXPORT_SYMBOL(qmi_rmnet_qos_exit_post); static struct workqueue_struct *rmnet_ps_wq; static struct rmnet_powersave_work *rmnet_work; static bool rmnet_work_quit; +static bool rmnet_work_inited; static LIST_HEAD(ps_list); struct rmnet_powersave_work { @@ -1148,6 +1149,7 @@ void qmi_rmnet_work_init(void *port) rmnet_work_quit = false; qmi_rmnet_work_set_active(rmnet_work->port, 1); queue_delayed_work(rmnet_ps_wq, &rmnet_work->work, PS_INTERVAL); + rmnet_work_inited = true; } EXPORT_SYMBOL(qmi_rmnet_work_init); @@ -1156,7 +1158,7 @@ void qmi_rmnet_work_maybe_restart(void *port) struct qmi_info *qmi; qmi = (struct qmi_info *)rmnet_get_qmi_pt(port); - if (unlikely(!qmi)) + if (unlikely(!qmi || !rmnet_work_inited)) return; if (!test_and_set_bit(PS_WORK_ACTIVE_BIT, &qmi->ps_work_active)) @@ -1172,6 +1174,7 @@ void qmi_rmnet_work_exit(void *port) rmnet_work_quit = true; synchronize_rcu(); + rmnet_work_inited = false; alarm_cancel(&rmnet_work->atimer); cancel_delayed_work_sync(&rmnet_work->work); destroy_workqueue(rmnet_ps_wq); From 1842e30e8a9cee3b82837f565d918a087037fcd6 Mon Sep 17 00:00:00 2001 From: Subash Abhinov Kasiviswanathan Date: Wed, 24 Jun 2020 10:42:34 -0700 Subject: [PATCH 27/54] dfc: add stall recovery timer Add watchdog timer to recover potential data stall when data is not going to the expected DRB and no DFC indication is received. Change-Id: Iaa4b4814967cf9400c36115a083922376d23928d Acked-by: Weiyi Chen Signed-off-by: Subash Abhinov Kasiviswanathan --- core/dfc.h | 22 +++++++++ core/dfc_qmi.c | 11 +++-- core/qmi_rmnet.c | 110 +++++++++++++++++++++++++++++++++++++++++++-- core/qmi_rmnet.h | 6 ++- core/qmi_rmnet_i.h | 19 ++++++++ core/rmnet_vnd.c | 5 ++- 6 files changed, 163 insertions(+), 10 deletions(-) diff --git a/core/dfc.h b/core/dfc.h index e94c38e7b0a6..bb3631d343ee 100644 --- a/core/dfc.h +++ b/core/dfc.h @@ -292,6 +292,28 @@ TRACE_EVENT(dfc_adjust_grant, __entry->rx_bytes, __entry->inflight, __entry->a_grant) ); +TRACE_EVENT(dfc_watchdog, + + TP_PROTO(u8 mux_id, u8 bearer_id, u8 event), + + TP_ARGS(mux_id, bearer_id, event), + + TP_STRUCT__entry( + __field(u8, mux_id) + __field(u8, bearer_id) + __field(u8, event) + ), + + TP_fast_assign( + __entry->mux_id = mux_id; + __entry->bearer_id = bearer_id; + __entry->event = event; + ), + + TP_printk("mid=%u bid=%u event=%u", + __entry->mux_id, __entry->bearer_id, __entry->event) +); + #endif /* _TRACE_DFC_H */ /* This part must be outside protection */ diff --git a/core/dfc_qmi.c b/core/dfc_qmi.c index 9fe29550c2d2..f1bb16b82713 100644 --- a/core/dfc_qmi.c +++ b/core/dfc_qmi.c @@ -1034,14 +1034,18 @@ static int dfc_update_fc_map(struct net_device *dev, struct qos_info *qos, itm->grant_size = adjusted_grant; /* No further query if the adjusted grant is less - * than 20% of the original grant + * than 20% of the original grant. Add to watch to + * recover if no indication is received. */ if (dfc_qmap && is_query && - itm->grant_size < (fc_info->num_bytes / 5)) + itm->grant_size < (fc_info->num_bytes / 5)) { itm->grant_thresh = itm->grant_size; - else + qmi_rmnet_watchdog_add(itm); + } else { itm->grant_thresh = qmi_rmnet_grant_per(itm->grant_size); + qmi_rmnet_watchdog_remove(itm); + } itm->seq = fc_info->seq_num; itm->ack_req = ack_req; @@ -1143,6 +1147,7 @@ static void dfc_update_tx_link_status(struct net_device *dev, itm->grant_size = 0; itm->tcp_bidir = false; itm->bytes_in_flight = 0; + qmi_rmnet_watchdog_remove(itm); dfc_bearer_flow_ctl(dev, itm, qos); } else if (itm->grant_size == 0 && tx_status && !itm->rat_switch) { itm->grant_size = DEFAULT_GRANT; diff --git a/core/qmi_rmnet.c b/core/qmi_rmnet.c index 293a40efa517..ef429f606f50 100644 --- a/core/qmi_rmnet.c +++ b/core/qmi_rmnet.c @@ -53,6 +53,7 @@ unsigned int rmnet_wq_frequency __read_mostly = 1000; 1 : rmnet_wq_frequency/10) * (HZ/100)) #define NO_DELAY (0x0000 * HZ) #define PS_INTERVAL_KT (ms_to_ktime(1000)) +#define WATCHDOG_EXPIRE_JF (msecs_to_jiffies(50)) #ifdef CONFIG_QTI_QMI_DFC static unsigned int qmi_rmnet_scale_factor = 5; @@ -233,6 +234,89 @@ static void qmi_rmnet_reset_txq(struct net_device *dev, unsigned int txq) } } +/** + * qmi_rmnet_watchdog_fn - watchdog timer func + */ +static void qmi_rmnet_watchdog_fn(struct timer_list *t) +{ + struct rmnet_bearer_map *bearer; + + bearer = container_of(t, struct rmnet_bearer_map, watchdog); + + trace_dfc_watchdog(bearer->qos->mux_id, bearer->bearer_id, 2); + + spin_lock_bh(&bearer->qos->qos_lock); + + if (bearer->watchdog_quit) + goto done; + + /* + * Possible stall, try to recover. Enable 80% query and jumpstart + * the bearer if disabled. + */ + bearer->watchdog_expire_cnt++; + bearer->bytes_in_flight = 0; + if (!bearer->grant_size) { + bearer->grant_size = DEFAULT_CALL_GRANT; + bearer->grant_thresh = qmi_rmnet_grant_per(bearer->grant_size); + dfc_bearer_flow_ctl(bearer->qos->vnd_dev, bearer, bearer->qos); + } else { + bearer->grant_thresh = qmi_rmnet_grant_per(bearer->grant_size); + } + +done: + bearer->watchdog_started = false; + spin_unlock_bh(&bearer->qos->qos_lock); +} + +/** + * qmi_rmnet_watchdog_add - add the bearer to watch + * Needs to be called with qos_lock + */ +void qmi_rmnet_watchdog_add(struct rmnet_bearer_map *bearer) +{ + bearer->watchdog_quit = false; + + if (bearer->watchdog_started) + return; + + bearer->watchdog_started = true; + mod_timer(&bearer->watchdog, jiffies + WATCHDOG_EXPIRE_JF); + + trace_dfc_watchdog(bearer->qos->mux_id, bearer->bearer_id, 1); +} + +/** + * qmi_rmnet_watchdog_remove - remove the bearer from watch + * Needs to be called with qos_lock + */ +void qmi_rmnet_watchdog_remove(struct rmnet_bearer_map *bearer) +{ + bearer->watchdog_quit = true; + + if (!bearer->watchdog_started) + return; + + del_timer(&bearer->watchdog); + bearer->watchdog_started = false; + + trace_dfc_watchdog(bearer->qos->mux_id, bearer->bearer_id, 0); +} + +/** + * qmi_rmnet_bearer_clean - clean the removed bearer + * Needs to be called with rtn_lock but not qos_lock + */ +static void qmi_rmnet_bearer_clean(struct qos_info *qos) +{ + if (qos->removed_bearer) { + qos->removed_bearer->watchdog_quit = true; + del_timer_sync(&qos->removed_bearer->watchdog); + kfree(qos->removed_bearer); + qos->removed_bearer = NULL; + } +} + static struct rmnet_bearer_map *__qmi_rmnet_bearer_get( struct qos_info *qos_info, u8 bearer_id) { @@ -252,6 +336,8 @@ static struct rmnet_bearer_map *__qmi_rmnet_bearer_get( bearer->grant_thresh = qmi_rmnet_grant_per(bearer->grant_size); bearer->mq_idx = INVALID_MQ; bearer->ack_mq_idx = INVALID_MQ; + bearer->qos = qos_info; + timer_setup(&bearer->watchdog, qmi_rmnet_watchdog_fn, 0); list_add(&bearer->list, &qos_info->bearer_head); } @@ -287,7 +373,7 @@ static void __qmi_rmnet_bearer_put(struct net_device *dev, /* Remove from bearer map */ list_del(&bearer->list); - kfree(bearer); + qos_info->removed_bearer = bearer; } } @@ -415,6 +501,9 @@ again: done: spin_unlock_bh(&qos_info->qos_lock); + + qmi_rmnet_bearer_clean(qos_info); + return rc; } @@ -458,6 +547,9 @@ qmi_rmnet_del_flow(struct net_device *dev, struct tcmsg *tcm, netif_tx_wake_all_queues(dev); spin_unlock_bh(&qos_info->qos_lock); + + qmi_rmnet_bearer_clean(qos_info); + return 0; } @@ -726,6 +818,8 @@ void qmi_rmnet_enable_all_flows(struct net_device *dev) bearer->tcp_bidir = false; bearer->rat_switch = false; + qmi_rmnet_watchdog_remove(bearer); + if (bearer->tx_off) continue; @@ -873,7 +967,8 @@ inline unsigned int qmi_rmnet_grant_per(unsigned int grant) } EXPORT_SYMBOL(qmi_rmnet_grant_per); -void *qmi_rmnet_qos_init(struct net_device *real_dev, u8 mux_id) +void *qmi_rmnet_qos_init(struct net_device *real_dev, + struct net_device *vnd_dev, u8 mux_id) { struct qos_info *qos; @@ -883,6 +978,7 @@ void *qmi_rmnet_qos_init(struct net_device *real_dev, u8 mux_id) qos->mux_id = mux_id; qos->real_dev = real_dev; + qos->vnd_dev = vnd_dev; qos->tran_num = 0; INIT_LIST_HEAD(&qos->flow_head); INIT_LIST_HEAD(&qos->bearer_head); @@ -894,10 +990,18 @@ EXPORT_SYMBOL(qmi_rmnet_qos_init); void qmi_rmnet_qos_exit_pre(void *qos) { + struct qos_info *qosi = (struct qos_info *)qos; + struct rmnet_bearer_map *bearer; + if (!qos) return; - list_add(&((struct qos_info *)qos)->list, &qos_cleanup_list); + list_for_each_entry(bearer, &qosi->bearer_head, list) { + bearer->watchdog_quit = true; + del_timer_sync(&bearer->watchdog); + } + + list_add(&qosi->list, &qos_cleanup_list); } EXPORT_SYMBOL(qmi_rmnet_qos_exit_pre); diff --git a/core/qmi_rmnet.h b/core/qmi_rmnet.h index 856295a94d26..80689e807236 100644 --- a/core/qmi_rmnet.h +++ b/core/qmi_rmnet.h @@ -55,7 +55,8 @@ qmi_rmnet_all_flows_enabled(struct net_device *dev) #endif #ifdef CONFIG_QTI_QMI_DFC -void *qmi_rmnet_qos_init(struct net_device *real_dev, u8 mux_id); +void *qmi_rmnet_qos_init(struct net_device *real_dev, + struct net_device *vnd_dev, u8 mux_id); void qmi_rmnet_qos_exit_pre(void *qos); void qmi_rmnet_qos_exit_post(void); void qmi_rmnet_burst_fc_check(struct net_device *dev, @@ -63,7 +64,8 @@ void qmi_rmnet_burst_fc_check(struct net_device *dev, int qmi_rmnet_get_queue(struct net_device *dev, struct sk_buff *skb); #else static inline void * -qmi_rmnet_qos_init(struct net_device *real_dev, u8 mux_id) +qmi_rmnet_qos_init(struct net_device *real_dev, + struct net_device *vnd_dev, u8 mux_id) { return NULL; } diff --git a/core/qmi_rmnet_i.h b/core/qmi_rmnet_i.h index e4fe7825dc3a..924074d5c761 100644 --- a/core/qmi_rmnet_i.h +++ b/core/qmi_rmnet_i.h @@ -16,6 +16,7 @@ #include #include +#include #define MAX_MQ_NUM 16 #define MAX_CLIENT_NUM 2 @@ -36,6 +37,8 @@ extern int dfc_mode; extern int dfc_qmap; +struct qos_info; + struct rmnet_bearer_map { struct list_head list; u8 bearer_id; @@ -54,6 +57,11 @@ struct rmnet_bearer_map { u32 ack_txid; u32 mq_idx; u32 ack_mq_idx; + struct qos_info *qos; + struct timer_list watchdog; + bool watchdog_started; + bool watchdog_quit; + u32 watchdog_expire_cnt; }; struct rmnet_flow_map { @@ -79,11 +87,13 @@ struct qos_info { struct list_head list; u8 mux_id; struct net_device *real_dev; + struct net_device *vnd_dev; struct list_head flow_head; struct list_head bearer_head; struct mq_map mq[MAX_MQ_NUM]; u32 tran_num; spinlock_t qos_lock; + struct rmnet_bearer_map *removed_bearer; }; struct qmi_info { @@ -154,6 +164,11 @@ void dfc_qmap_send_ack(struct qos_info *qos, u8 bearer_id, u16 seq, u8 type); struct rmnet_bearer_map *qmi_rmnet_get_bearer_noref(struct qos_info *qos_info, u8 bearer_id); + +void qmi_rmnet_watchdog_add(struct rmnet_bearer_map *bearer); + +void qmi_rmnet_watchdog_remove(struct rmnet_bearer_map *bearer); + #else static inline struct rmnet_flow_map * qmi_rmnet_get_flow_map(struct qos_info *qos_info, @@ -197,6 +212,10 @@ dfc_qmap_client_init(void *port, int index, struct svc_info *psvc, static inline void dfc_qmap_client_exit(void *dfc_data) { } + +static inline void qmi_rmnet_watchdog_remove(struct rmnet_bearer_map *bearer) +{ +} #endif #ifdef CONFIG_QTI_QMI_POWER_COLLAPSE diff --git a/core/rmnet_vnd.c b/core/rmnet_vnd.c index 55da35c684a8..3674b3f94ef6 100644 --- a/core/rmnet_vnd.c +++ b/core/rmnet_vnd.c @@ -128,7 +128,7 @@ static void rmnet_vnd_uninit(struct net_device *dev) gro_cells_destroy(&priv->gro_cells); free_percpu(priv->pcpu_stats); - qos = priv->qos_info; + qos = rcu_dereference(priv->qos_info); RCU_INIT_POINTER(priv->qos_info, NULL); qmi_rmnet_qos_exit_pre(qos); } @@ -402,7 +402,8 @@ int rmnet_vnd_newlink(u8 id, struct net_device *rmnet_dev, rmnet_dev->rtnl_link_ops = &rmnet_link_ops; priv->mux_id = id; - priv->qos_info = qmi_rmnet_qos_init(real_dev, id); + rcu_assign_pointer(priv->qos_info, + qmi_rmnet_qos_init(real_dev, rmnet_dev, id)); netdev_dbg(rmnet_dev, "rmnet dev created\n"); } From ce95afc3612304600ccd8b29f031fdc3924542b8 Mon Sep 17 00:00:00 2001 From: Subash Abhinov Kasiviswanathan Date: Tue, 14 Jul 2020 19:20:08 -0700 Subject: [PATCH 28/54] rmnet_ctl: remove symbol_get and put usage Removes symbol_get() and symbol_put() usage. Change-Id: Ic4142635af717797ed43c3b4925e152c73c95581 Acked-by: Weiyi Chen Signed-off-by: Subash Abhinov Kasiviswanathan --- core/dfc_qmap.c | 9 ++------- core/rmnet_ctl.h | 2 ++ core/rmnet_ctl_client.c | 7 ++++++- 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/core/dfc_qmap.c b/core/dfc_qmap.c index e2a7d18620ce..fb883df98b96 100644 --- a/core/dfc_qmap.c +++ b/core/dfc_qmap.c @@ -128,7 +128,6 @@ static struct dfc_qmi_data __rcu *qmap_dfc_data; static atomic_t qmap_txid; static void *rmnet_ctl_handle; -extern struct rmnet_ctl_client_if rmnet_ctl_if; static struct rmnet_ctl_client_if *rmnet_ctl; static void dfc_qmap_send_end_marker_cnf(struct qos_info *qos, @@ -504,7 +503,7 @@ int dfc_qmap_client_init(void *port, int index, struct svc_info *psvc, atomic_set(&qmap_txid, 0); - rmnet_ctl = symbol_get(rmnet_ctl_if); + rmnet_ctl = rmnet_ctl_if(); if (!rmnet_ctl) { pr_err("rmnet_ctl module not loaded\n"); goto out; @@ -547,11 +546,7 @@ void dfc_qmap_client_exit(void *dfc_data) synchronize_rcu(); kfree(data); - - if (rmnet_ctl) { - symbol_put(rmnet_ctl_if); - rmnet_ctl = NULL; - } + rmnet_ctl = NULL; pr_info("DFC QMAP exit\n"); } diff --git a/core/rmnet_ctl.h b/core/rmnet_ctl.h index cb3ca722057a..50096140d7fa 100644 --- a/core/rmnet_ctl.h +++ b/core/rmnet_ctl.h @@ -29,4 +29,6 @@ struct rmnet_ctl_client_if { const void *data, unsigned int len); }; +struct rmnet_ctl_client_if *rmnet_ctl_if(void); + #endif /* _RMNET_CTL_H_ */ diff --git a/core/rmnet_ctl_client.c b/core/rmnet_ctl_client.c index fed99095ec45..4958deac1c75 100644 --- a/core/rmnet_ctl_client.c +++ b/core/rmnet_ctl_client.c @@ -205,10 +205,15 @@ void rmnet_ctl_log(enum rmnet_ctl_log_lvl lvl, const char *msg, } EXPORT_SYMBOL(rmnet_ctl_log); -struct rmnet_ctl_client_if rmnet_ctl_if = { +static struct rmnet_ctl_client_if client_if = { .reg = rmnet_ctl_register_client, .dereg = rmnet_ctl_unregister_client, .send = rmnet_ctl_send_client, .log = rmnet_ctl_log, }; + +struct rmnet_ctl_client_if *rmnet_ctl_if(void) +{ + return &client_if; +} EXPORT_SYMBOL(rmnet_ctl_if); From 1dda7d60a56f9340a0d99b0de3227a3804fadce3 Mon Sep 17 00:00:00 2001 From: Subash Abhinov Kasiviswanathan Date: Tue, 7 Jul 2020 13:51:34 -0700 Subject: [PATCH 29/54] dfc: TCP pure acks Detects TCP pure acks so they can be put into the dedicated TX queues even if they contains various options. Change-Id: I6a9b714ccb58616ff49a150467d33a348d88ec64 Acked-by: Weiyi Chen Signed-off-by: Subash Abhinov Kasiviswanathan --- core/qmi_rmnet.c | 25 +++++++++++++++++++------ 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/core/qmi_rmnet.c b/core/qmi_rmnet.c index ef429f606f50..864f2da0a4bf 100644 --- a/core/qmi_rmnet.c +++ b/core/qmi_rmnet.c @@ -19,6 +19,7 @@ #include #include #include +#include #include #include #include @@ -873,16 +874,14 @@ void qmi_rmnet_burst_fc_check(struct net_device *dev, } EXPORT_SYMBOL(qmi_rmnet_burst_fc_check); -static bool qmi_rmnet_is_tcp_ack(struct sk_buff *skb) +static bool _qmi_rmnet_is_tcp_ack(struct sk_buff *skb) { - unsigned int len = skb->len; - switch (skb->protocol) { /* TCPv4 ACKs */ case htons(ETH_P_IP): if ((ip_hdr(skb)->protocol == IPPROTO_TCP) && - (ip_hdr(skb)->ihl == 5) && - (len == 40 || len == 52) && + (ntohs(ip_hdr(skb)->tot_len) - (ip_hdr(skb)->ihl << 2) == + tcp_hdr(skb)->doff << 2) && ((tcp_flag_word(tcp_hdr(skb)) & cpu_to_be32(0x00FF0000)) == TCP_FLAG_ACK)) return true; @@ -891,7 +890,8 @@ static bool qmi_rmnet_is_tcp_ack(struct sk_buff *skb) /* TCPv6 ACKs */ case htons(ETH_P_IPV6): if ((ipv6_hdr(skb)->nexthdr == IPPROTO_TCP) && - (len == 60 || len == 72) && + (ntohs(ipv6_hdr(skb)->payload_len) == + (tcp_hdr(skb)->doff) << 2) && ((tcp_flag_word(tcp_hdr(skb)) & cpu_to_be32(0x00FF0000)) == TCP_FLAG_ACK)) return true; @@ -901,6 +901,19 @@ static bool qmi_rmnet_is_tcp_ack(struct sk_buff *skb) return false; } +static inline bool qmi_rmnet_is_tcp_ack(struct sk_buff *skb) +{ + /* Locally generated TCP acks */ + if (skb_is_tcp_pure_ack(skb)) + return true; + + /* Forwarded */ + if (unlikely(_qmi_rmnet_is_tcp_ack(skb))) + return true; + + return false; +} + static int qmi_rmnet_get_queue_sa(struct qos_info *qos, struct sk_buff *skb) { struct rmnet_flow_map *itm; From c583c0406215e7652fd508d61f6c1a449cff3f4e Mon Sep 17 00:00:00 2001 From: Sean Tranchetti Date: Wed, 22 Jul 2020 16:22:20 -0600 Subject: [PATCH 30/54] core: rmnet: validate ipv6 extension header lengths When calculating the length of the IPv6 header chain, lengths of the IPv6 extension headers are not checked against the overall packet lengths and thus it's possible to parse past the end of the packet when the packet is malformed. This adds the necessary bounds checking to ensure that parsing stops if the end of the packet is reached to avoid the following: Unable to handle kernel paging request at virtual address pc : rmnet_frag_ipv6_skip_exthdr+0xc0/0x108 [rmnet_core] lr : rmnet_frag_ipv6_skip_exthdr+0x68/0x108 [rmnet_core] Call trace: rmnet_frag_ipv6_skip_exthdr+0xc0/0x108 [rmnet_core] DATARMNET29e8d137c4+0x1a0/0x3e0 [rmnet_offload] rmnet_frag_ingress_handler+0x294/0x404 [rmnet_core] rmnet_rx_handler+0x1b4/0x284 [rmnet_core] __netif_receive_skb_core+0x740/0xd2c __netif_receive_skb+0x44/0x158 Change-Id: Ib2e2ebce733bd4d14a3dfc175133638b15015277 Signed-off-by: Sean Tranchetti --- core/rmnet_descriptor.c | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/core/rmnet_descriptor.c b/core/rmnet_descriptor.c index 6e2ae12967ae..8d71d89dadbe 100644 --- a/core/rmnet_descriptor.c +++ b/core/rmnet_descriptor.c @@ -103,6 +103,7 @@ EXPORT_SYMBOL(rmnet_descriptor_add_frag); int rmnet_frag_ipv6_skip_exthdr(struct rmnet_frag_descriptor *frag_desc, int start, u8 *nexthdrp, __be16 *fragp) { + u32 frag_size = skb_frag_size(&frag_desc->frag); u8 nexthdr = *nexthdrp; *fragp = 0; @@ -114,11 +115,17 @@ int rmnet_frag_ipv6_skip_exthdr(struct rmnet_frag_descriptor *frag_desc, if (nexthdr == NEXTHDR_NONE) return -EINVAL; - hp = rmnet_frag_data_ptr(frag_desc) + start; + if (start >= frag_size) + return -EINVAL; + hp = rmnet_frag_data_ptr(frag_desc) + start; if (nexthdr == NEXTHDR_FRAGMENT) { __be16 *fp; + if (start + offsetof(struct frag_hdr, frag_off) >= + frag_size) + return -EINVAL; + fp = rmnet_frag_data_ptr(frag_desc) + start + offsetof(struct frag_hdr, frag_off); *fragp = *fp; From 45a8e8f2c1187221ad085371d1ca4bf261fbb70f Mon Sep 17 00:00:00 2001 From: Subash Abhinov Kasiviswanathan Date: Wed, 22 Jul 2020 12:24:05 -0700 Subject: [PATCH 31/54] dfc: Re-send DFC_CONFIG command Re-send QMAP DFC_CONFIG command if no ack is received upon first QMAP indication. Change-Id: I33ec5cbbf3550d1df03ca1dd990bf8ad58ad9582 Acked-by: Weiyi Chen Signed-off-by: Subash Abhinov Kasiviswanathan --- core/dfc_qmap.c | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/core/dfc_qmap.c b/core/dfc_qmap.c index fb883df98b96..f371a0f6e7bf 100644 --- a/core/dfc_qmap.c +++ b/core/dfc_qmap.c @@ -127,9 +127,11 @@ static struct dfc_tx_link_status_ind_msg_v01 qmap_tx_ind; static struct dfc_qmi_data __rcu *qmap_dfc_data; static atomic_t qmap_txid; static void *rmnet_ctl_handle; +static bool dfc_config_acked; static struct rmnet_ctl_client_if *rmnet_ctl; +static void dfc_qmap_send_config(struct dfc_qmi_data *data); static void dfc_qmap_send_end_marker_cnf(struct qos_info *qos, u8 bearer_id, u16 seq, u32 tx_id); @@ -319,6 +321,9 @@ static void dfc_qmap_cmd_handler(struct sk_buff *skb) if (cmd->cmd_type != QMAP_CMD_ACK) goto free_skb; } else if (cmd->cmd_type != QMAP_CMD_REQUEST) { + if (cmd->cmd_type == QMAP_CMD_ACK && + cmd->cmd_name == QMAP_DFC_CONFIG) + dfc_config_acked = true; goto free_skb; } @@ -330,6 +335,12 @@ static void dfc_qmap_cmd_handler(struct sk_buff *skb) goto free_skb; } + /* Re-send DFC config once if needed */ + if (unlikely(!dfc_config_acked)) { + dfc_qmap_send_config(dfc); + dfc_config_acked = true; + } + switch (cmd->cmd_name) { case QMAP_DFC_IND: rc = dfc_qmap_handle_ind(dfc, skb); @@ -520,6 +531,7 @@ int dfc_qmap_client_init(void *port, int index, struct svc_info *psvc, pr_info("DFC QMAP init\n"); + dfc_config_acked = false; dfc_qmap_send_config(data); out: From 06bf42ec24b93523ac1fae57d32ee79fd84749c8 Mon Sep 17 00:00:00 2001 From: Sean Tranchetti Date: Tue, 16 Jun 2020 17:34:57 -0600 Subject: [PATCH 32/54] rmnet_core: Allow scatter-gather on the DL path This patch adds the necessary handling for the physical device driver to use multiple pages to hold data in the SKBs. Essentially, the following changes are implemented: - rmnet_frag_descriptor struct now hold a list of frags, instead of a single one. Pushing, pulling, and trimming APIs are updated to use this new format. - QMAP deaggregation now loops over each element in skb_shinfo->frags looking for data. Packets are allowed to be split across mutliple pages. All pages containing data for a particular packet will be added to the frag_descriptor struct representing it. - a new API, rmnet_frag_header_ptr() has been added for safely accessing packet headers. This API, modeled after skb_header_pointer(), handles the fact that headers could potentially be split across 2 pages. A pointer to the location of the header is returned in the usual case where the header is physically contiguous. If not, the header is linearized into the user-provided buffer to allow normal header struct read access. - this new header access API is used in all places on the DL path when headers are needed, including QMAP command processing, QMAPv1 handling, QMAPv5 checksum offload, and QMAPv5 coalescing. - RSB/RSC segmentation handling is updated to add all necessary pages containing packet data to the newly created descriptor. Additionally, the pages containing L3 and L4 headers are added as well, as this allows easier downstream processing, and guarantees that the header data will not be freed until all packets that need them have been converted into SKBs. - as all frag_descriptors are now guaranteed to contain the L3 and L4 header data (and because they are no longer guaranteed to be on the same page), the hdr_ptr member has been removed as it no longer serves a purpose. Change-Id: Iebb677a6ae7e442fa55e0d131af59cde1b5ce18a Signed-off-by: Sean Tranchetti --- core/rmnet_descriptor.c | 731 +++++++++++++++++++++++++++++----------- core/rmnet_descriptor.h | 95 ++---- 2 files changed, 556 insertions(+), 270 deletions(-) diff --git a/core/rmnet_descriptor.c b/core/rmnet_descriptor.c index 8d71d89dadbe..7a20aba2da2d 100644 --- a/core/rmnet_descriptor.c +++ b/core/rmnet_descriptor.c @@ -56,7 +56,7 @@ rmnet_get_frag_descriptor(struct rmnet_port *port) goto out; INIT_LIST_HEAD(&frag_desc->list); - INIT_LIST_HEAD(&frag_desc->sub_frags); + INIT_LIST_HEAD(&frag_desc->frags); pool->pool_size++; } @@ -70,64 +70,286 @@ void rmnet_recycle_frag_descriptor(struct rmnet_frag_descriptor *frag_desc, struct rmnet_port *port) { struct rmnet_frag_descriptor_pool *pool = port->frag_desc_pool; - struct page *page = skb_frag_page(&frag_desc->frag); + struct rmnet_fragment *frag, *tmp; unsigned long flags; list_del(&frag_desc->list); - if (page) - put_page(page); + + list_for_each_entry_safe(frag, tmp, &frag_desc->frags, list) { + struct page *page = skb_frag_page(&frag->frag); + + if (page) + put_page(page); + + list_del(&frag->list); + kfree(frag); + } memset(frag_desc, 0, sizeof(*frag_desc)); INIT_LIST_HEAD(&frag_desc->list); - INIT_LIST_HEAD(&frag_desc->sub_frags); + INIT_LIST_HEAD(&frag_desc->frags); spin_lock_irqsave(&port->desc_pool_lock, flags); list_add_tail(&frag_desc->list, &pool->free_list); spin_unlock_irqrestore(&port->desc_pool_lock, flags); } EXPORT_SYMBOL(rmnet_recycle_frag_descriptor); -void rmnet_descriptor_add_frag(struct rmnet_port *port, struct list_head *list, - struct page *p, u32 page_offset, u32 len) +void *rmnet_frag_pull(struct rmnet_frag_descriptor *frag_desc, + struct rmnet_port *port, unsigned int size) { - struct rmnet_frag_descriptor *frag_desc; + struct rmnet_fragment *frag, *tmp; - frag_desc = rmnet_get_frag_descriptor(port); - if (!frag_desc) - return; + if (size >= frag_desc->len) { + pr_info("%s(): Pulling %u bytes from %u byte pkt. Dropping\n", + __func__, size, frag_desc->len); + rmnet_recycle_frag_descriptor(frag_desc, port); + return NULL; + } - rmnet_frag_fill(frag_desc, p, page_offset, len); - list_add_tail(&frag_desc->list, list); + list_for_each_entry_safe(frag, tmp, &frag_desc->frags, list) { + u32 frag_size = skb_frag_size(&frag->frag); + + if (!size) + break; + + if (size >= frag_size) { + /* Remove the whole frag */ + struct page *page = skb_frag_page(&frag->frag); + + if (page) + put_page(page); + + list_del(&frag->list); + size -= frag_size; + frag_desc->len -= frag_size; + kfree(frag); + continue; + } + + /* Pull off 'size' bytes */ + skb_frag_off_add(&frag->frag, size); + skb_frag_size_sub(&frag->frag, size); + frag_desc->len -= size; + break; + } + + return rmnet_frag_data_ptr(frag_desc); } -EXPORT_SYMBOL(rmnet_descriptor_add_frag); +EXPORT_SYMBOL(rmnet_frag_pull); + +void *rmnet_frag_trim(struct rmnet_frag_descriptor *frag_desc, + struct rmnet_port *port, unsigned int size) +{ + struct rmnet_fragment *frag, *tmp; + unsigned int eat; + + if (!size) { + pr_info("%s(): Trimming %u byte pkt to 0. Dropping\n", + __func__, frag_desc->len); + rmnet_recycle_frag_descriptor(frag_desc, port); + return NULL; + } + + /* Growing bigger doesn't make sense */ + if (size >= frag_desc->len) + goto out; + + /* Compute number of bytes to remove from the end */ + eat = frag_desc->len - size; + list_for_each_entry_safe_reverse(frag, tmp, &frag_desc->frags, list) { + u32 frag_size = skb_frag_size(&frag->frag); + + if (!eat) + goto out; + + if (eat >= frag_size) { + /* Remove the whole frag */ + struct page *page = skb_frag_page(&frag->frag); + + if (page) + put_page(page); + + list_del(&frag->list); + eat -= frag_size; + frag_desc->len -= frag_size; + kfree(frag); + continue; + } + + /* Chop off 'eat' bytes from the end */ + skb_frag_size_sub(&frag->frag, eat); + frag_desc->len -= eat; + goto out; + } + +out: + return rmnet_frag_data_ptr(frag_desc); +} +EXPORT_SYMBOL(rmnet_frag_trim); + +static int rmnet_frag_copy_data(struct rmnet_frag_descriptor *frag_desc, + u32 off, u32 len, void *buf) +{ + struct rmnet_fragment *frag; + u32 frag_size, copy_len; + u32 buf_offset = 0; + + /* Don't make me do something we'd both regret */ + if (off > frag_desc->len || len > frag_desc->len || + off + len > frag_desc->len) + return -EINVAL; + + /* Copy 'len' bytes into the bufer starting from 'off' */ + list_for_each_entry(frag, &frag_desc->frags, list) { + if (!len) + break; + + frag_size = skb_frag_size(&frag->frag); + if (off < frag_size) { + copy_len = min_t(u32, len, frag_size - off); + memcpy(buf + buf_offset, + skb_frag_address(&frag->frag) + off, + copy_len); + buf_offset += copy_len; + len -= copy_len; + off = 0; + } else { + off -= frag_size; + } + } + + return 0; +} + +void *rmnet_frag_header_ptr(struct rmnet_frag_descriptor *frag_desc, u32 off, + u32 len, void *buf) +{ + struct rmnet_fragment *frag; + u8 *start; + u32 frag_size, offset; + + /* Don't take a long pointer off a short frag */ + if (off > frag_desc->len || len > frag_desc->len || + off + len > frag_desc->len) + return NULL; + + /* Find the starting fragment */ + offset = off; + list_for_each_entry(frag, &frag_desc->frags, list) { + frag_size = skb_frag_size(&frag->frag); + if (off < frag_size) { + start = skb_frag_address(&frag->frag) + off; + /* If the header is entirely on this frag, just return + * a pointer to it. + */ + if (off + len <= frag_size) + return start; + + /* Otherwise, we need to copy the data into a linear + * buffer. + */ + break; + } + + off -= frag_size; + } + + if (rmnet_frag_copy_data(frag_desc, offset, len, buf) < 0) + return NULL; + + return buf; +} +EXPORT_SYMBOL(rmnet_frag_header_ptr); + +int rmnet_frag_descriptor_add_frag(struct rmnet_frag_descriptor *frag_desc, + struct page *p, u32 page_offset, u32 len) +{ + struct rmnet_fragment *frag; + + frag = kzalloc(sizeof(*frag), GFP_ATOMIC); + if (!frag) + return -ENOMEM; + + INIT_LIST_HEAD(&frag->list); + get_page(p); + __skb_frag_set_page(&frag->frag, p); + skb_frag_size_set(&frag->frag, len); + skb_frag_off_set(&frag->frag, page_offset); + list_add_tail(&frag->list, &frag_desc->frags); + frag_desc->len += len; + return 0; +} +EXPORT_SYMBOL(rmnet_frag_descriptor_add_frag); + +int rmnet_frag_descriptor_add_frags_from(struct rmnet_frag_descriptor *to, + struct rmnet_frag_descriptor *from, + u32 off, u32 len) +{ + struct rmnet_fragment *frag; + int rc; + + /* Sanity check the lengths */ + if (off > from->len || len > from->len || off + len > from->len) + return -EINVAL; + + list_for_each_entry(frag, &from->frags, list) { + u32 frag_size; + + if (!len) + break; + + frag_size = skb_frag_size(&frag->frag); + if (off < frag_size) { + struct page *p = skb_frag_page(&frag->frag); + u32 page_off = skb_frag_off(&frag->frag); + u32 copy_len = min_t(u32, len, frag_size - off); + + rc = rmnet_frag_descriptor_add_frag(to, p, + page_off + off, + copy_len); + if (rc < 0) + return rc; + + len -= copy_len; + off = 0; + } else { + off -= frag_size; + } + } + + return 0; +} +EXPORT_SYMBOL(rmnet_frag_descriptor_add_frags_from); int rmnet_frag_ipv6_skip_exthdr(struct rmnet_frag_descriptor *frag_desc, int start, u8 *nexthdrp, __be16 *fragp) { - u32 frag_size = skb_frag_size(&frag_desc->frag); u8 nexthdr = *nexthdrp; *fragp = 0; while (ipv6_ext_hdr(nexthdr)) { - struct ipv6_opt_hdr *hp; + struct ipv6_opt_hdr *hp, __hp; int hdrlen; if (nexthdr == NEXTHDR_NONE) return -EINVAL; - if (start >= frag_size) + hp = rmnet_frag_header_ptr(frag_desc, (u32)start, sizeof(*hp), + &__hp); + if (!hp) return -EINVAL; hp = rmnet_frag_data_ptr(frag_desc) + start; if (nexthdr == NEXTHDR_FRAGMENT) { - __be16 *fp; + u32 off = offsetof(struct frag_hdr, frag_off); + __be16 *fp, __fp; - if (start + offsetof(struct frag_hdr, frag_off) >= - frag_size) + fp = rmnet_frag_header_ptr(frag_desc, (u32)start + off, + sizeof(*fp), &__fp); + if (!fp) return -EINVAL; - fp = rmnet_frag_data_ptr(frag_desc) + start + - offsetof(struct frag_hdr, frag_off); *fragp = *fp; if (ntohs(*fragp) & ~0x7) break; @@ -148,10 +370,10 @@ int rmnet_frag_ipv6_skip_exthdr(struct rmnet_frag_descriptor *frag_desc, EXPORT_SYMBOL(rmnet_frag_ipv6_skip_exthdr); static u8 rmnet_frag_do_flow_control(struct rmnet_map_header *qmap, + struct rmnet_map_control_command *cmd, struct rmnet_port *port, int enable) { - struct rmnet_map_control_command *cmd; struct rmnet_endpoint *ep; struct net_device *vnd; u16 ip_family; @@ -161,9 +383,6 @@ static u8 rmnet_frag_do_flow_control(struct rmnet_map_header *qmap, int r; mux_id = qmap->mux_id; - cmd = (struct rmnet_map_control_command *) - ((char *)qmap + sizeof(*qmap)); - if (mux_id >= RMNET_MAX_LOGICAL_EP) return RX_HANDLER_CONSUMED; @@ -213,20 +432,25 @@ static void rmnet_frag_send_ack(struct rmnet_map_header *qmap, } static void -rmnet_frag_process_flow_start(struct rmnet_map_control_command_header *cmd, +rmnet_frag_process_flow_start(struct rmnet_frag_descriptor *frag_desc, + struct rmnet_map_control_command_header *cmd, struct rmnet_port *port, u16 cmd_len) { - struct rmnet_map_dl_ind_hdr *dlhdr; + struct rmnet_map_dl_ind_hdr *dlhdr, __dlhdr; + u32 offset = sizeof(struct rmnet_map_header); u32 data_format; bool is_dl_mark_v2; - if (cmd_len + sizeof(struct rmnet_map_header) < RMNET_DL_IND_HDR_SIZE) + if (cmd_len + offset < RMNET_DL_IND_HDR_SIZE) return; data_format = port->data_format; is_dl_mark_v2 = data_format & RMNET_INGRESS_FORMAT_DL_MARKER_V2; - dlhdr = (struct rmnet_map_dl_ind_hdr *)((char *)cmd + sizeof(*cmd)); + dlhdr = rmnet_frag_header_ptr(frag_desc, offset + sizeof(*cmd), + sizeof(*dlhdr), &__dlhdr); + if (!dlhdr) + return; port->stats.dl_hdr_last_ep_id = cmd->source_id; port->stats.dl_hdr_last_qmap_vers = cmd->reserved; @@ -247,20 +471,25 @@ rmnet_frag_process_flow_start(struct rmnet_map_control_command_header *cmd, } static void -rmnet_frag_process_flow_end(struct rmnet_map_control_command_header *cmd, +rmnet_frag_process_flow_end(struct rmnet_frag_descriptor *frag_desc, + struct rmnet_map_control_command_header *cmd, struct rmnet_port *port, u16 cmd_len) { - struct rmnet_map_dl_ind_trl *dltrl; + struct rmnet_map_dl_ind_trl *dltrl, __dltrl; + u32 offset = sizeof(struct rmnet_map_header); u32 data_format; bool is_dl_mark_v2; - if (cmd_len + sizeof(struct rmnet_map_header) < RMNET_DL_IND_TRL_SIZE) + if (cmd_len + offset < RMNET_DL_IND_TRL_SIZE) return; data_format = port->data_format; is_dl_mark_v2 = data_format & RMNET_INGRESS_FORMAT_DL_MARKER_V2; - dltrl = (struct rmnet_map_dl_ind_trl *)((char *)cmd + sizeof(*cmd)); + dltrl = rmnet_frag_header_ptr(frag_desc, offset + sizeof(*cmd), + sizeof(*dltrl), &__dltrl); + if (!dltrl) + return; port->stats.dl_trl_last_seq = dltrl->seq_le; port->stats.dl_trl_count++; @@ -275,23 +504,24 @@ rmnet_frag_process_flow_end(struct rmnet_map_control_command_header *cmd, /* Process MAP command frame and send N/ACK message as appropriate. Message cmd * name is decoded here and appropriate handler is called. */ -void rmnet_frag_command(struct rmnet_map_header *qmap, struct rmnet_port *port) +void rmnet_frag_command(struct rmnet_frag_descriptor *frag_desc, + struct rmnet_map_header *qmap, struct rmnet_port *port) { - struct rmnet_map_control_command *cmd; - unsigned char command_name; + struct rmnet_map_control_command *cmd, __cmd; unsigned char rc = 0; - cmd = (struct rmnet_map_control_command *) - ((char *)qmap + sizeof(*qmap)); - command_name = cmd->command_name; + cmd = rmnet_frag_header_ptr(frag_desc, sizeof(*qmap), sizeof(*cmd), + &__cmd); + if (!cmd) + return; - switch (command_name) { + switch (cmd->command_name) { case RMNET_MAP_COMMAND_FLOW_ENABLE: - rc = rmnet_frag_do_flow_control(qmap, port, 1); + rc = rmnet_frag_do_flow_control(qmap, cmd, port, 1); break; case RMNET_MAP_COMMAND_FLOW_DISABLE: - rc = rmnet_frag_do_flow_control(qmap, port, 0); + rc = rmnet_frag_do_flow_control(qmap, cmd, port, 0); break; default: @@ -302,23 +532,23 @@ void rmnet_frag_command(struct rmnet_map_header *qmap, struct rmnet_port *port) rmnet_frag_send_ack(qmap, rc, port); } -int rmnet_frag_flow_command(struct rmnet_map_header *qmap, +int rmnet_frag_flow_command(struct rmnet_frag_descriptor *frag_desc, struct rmnet_port *port, u16 pkt_len) { - struct rmnet_map_control_command_header *cmd; - unsigned char command_name; + struct rmnet_map_control_command_header *cmd, __cmd; - cmd = (struct rmnet_map_control_command_header *) - ((char *)qmap + sizeof(*qmap)); - command_name = cmd->command_name; + cmd = rmnet_frag_header_ptr(frag_desc, sizeof(struct rmnet_map_header), + sizeof(*cmd), &__cmd); + if (!cmd) + return -1; - switch (command_name) { + switch (cmd->command_name) { case RMNET_MAP_COMMAND_FLOW_START: - rmnet_frag_process_flow_start(cmd, port, pkt_len); + rmnet_frag_process_flow_start(frag_desc, cmd, port, pkt_len); break; case RMNET_MAP_COMMAND_FLOW_END: - rmnet_frag_process_flow_end(cmd, port, pkt_len); + rmnet_frag_process_flow_end(frag_desc, cmd, port, pkt_len); break; default: @@ -329,55 +559,100 @@ int rmnet_frag_flow_command(struct rmnet_map_header *qmap, } EXPORT_SYMBOL(rmnet_frag_flow_command); -void rmnet_frag_deaggregate(skb_frag_t *frag, struct rmnet_port *port, - struct list_head *list) +static int rmnet_frag_deaggregate_one(struct skb_shared_info *shinfo, + struct rmnet_port *port, + struct list_head *list, + u32 start_frag) { + struct rmnet_frag_descriptor *frag_desc; struct rmnet_map_header *maph; - u8 *data = skb_frag_address(frag); - u32 offset = 0; - u32 packet_len; + skb_frag_t *frag; + u32 i; + u32 pkt_len; + int rc; - while (offset < skb_frag_size(frag)) { - maph = (struct rmnet_map_header *)data; - packet_len = ntohs(maph->pkt_len); + frag = &shinfo->frags[start_frag]; + maph = skb_frag_address(frag); + pkt_len = ntohs(maph->pkt_len); + /* Catch empty frames */ + if (!pkt_len) + return -1; - /* Some hardware can send us empty frames. Catch them */ - if (packet_len == 0) - return; + frag_desc = rmnet_get_frag_descriptor(port); + if (!frag_desc) + return -1; - packet_len += sizeof(*maph); + pkt_len += sizeof(*maph); + if (port->data_format & RMNET_FLAGS_INGRESS_MAP_CKSUMV4) { + pkt_len += sizeof(struct rmnet_map_dl_csum_trailer); + } else if ((port->data_format & (RMNET_FLAGS_INGRESS_MAP_CKSUMV5 | + RMNET_FLAGS_INGRESS_COALESCE)) && + !maph->cd_bit) { + u32 hsize = 0; + u8 type; - if (port->data_format & RMNET_FLAGS_INGRESS_MAP_CKSUMV4) { - packet_len += sizeof(struct rmnet_map_dl_csum_trailer); - } else if (port->data_format & - (RMNET_FLAGS_INGRESS_MAP_CKSUMV5 | - RMNET_FLAGS_INGRESS_COALESCE) && !maph->cd_bit) { - u32 hsize = 0; - u8 type; - - type = ((struct rmnet_map_v5_coal_header *) - (data + sizeof(*maph)))->header_type; - switch (type) { - case RMNET_MAP_HEADER_TYPE_COALESCING: - hsize = sizeof(struct rmnet_map_v5_coal_header); - break; - case RMNET_MAP_HEADER_TYPE_CSUM_OFFLOAD: - hsize = sizeof(struct rmnet_map_v5_csum_header); - break; - } - - packet_len += hsize; + type = ((struct rmnet_map_v5_coal_header *) + (maph + 1))->header_type; + switch (type) { + case RMNET_MAP_HEADER_TYPE_COALESCING: + hsize = sizeof(struct rmnet_map_v5_coal_header); + break; + case RMNET_MAP_HEADER_TYPE_CSUM_OFFLOAD: + hsize = sizeof(struct rmnet_map_v5_csum_header); + break; } - if ((int)skb_frag_size(frag) - (int)packet_len < 0) + pkt_len += hsize; + } + + /* Add all frags containing the packet data to the descriptor */ + for (i = start_frag; pkt_len > 0 && i < shinfo->nr_frags; ) { + u32 frag_size; + u32 copy_len; + + frag = &shinfo->frags[i]; + frag_size = skb_frag_size(frag); + copy_len = min_t(u32, frag_size, pkt_len); + rc = rmnet_frag_descriptor_add_frag(frag_desc, + skb_frag_page(frag), + skb_frag_off(frag), + copy_len); + if (rc < 0) { + rmnet_recycle_frag_descriptor(frag_desc, port); + return -1; + } + + pkt_len -= copy_len; + skb_frag_off_add(frag, copy_len); + skb_frag_size_sub(frag, copy_len); + /* If the fragment is exhausted, we can move to the next one */ + if (!skb_frag_size(frag)) + i++; + } + + if (pkt_len) { + /* Packet length is larger than the amount of data we have */ + rmnet_recycle_frag_descriptor(frag_desc, port); + return -1; + } + + list_add_tail(&frag_desc->list, list); + return (int)(i - start_frag); +} + +void rmnet_frag_deaggregate(struct sk_buff *skb, struct rmnet_port *port, + struct list_head *list) +{ + struct skb_shared_info *shinfo = skb_shinfo(skb); + u32 i = 0; + int rc; + + while (i < shinfo->nr_frags) { + rc = rmnet_frag_deaggregate_one(shinfo, port, list, i); + if (rc < 0) return; - rmnet_descriptor_add_frag(port, list, skb_frag_page(frag), - frag->bv_offset + offset, - packet_len); - - offset += packet_len; - data += packet_len; + i += (u32)rc; } } @@ -452,7 +727,7 @@ static struct sk_buff *rmnet_alloc_skb(struct rmnet_frag_descriptor *frag_desc, { struct sk_buff *head_skb, *current_skb, *skb; struct skb_shared_info *shinfo; - struct rmnet_frag_descriptor *sub_frag, *tmp; + struct rmnet_fragment *frag, *tmp; /* Use the exact sizes if we know them (i.e. RSB/RSC, rmnet_perf) */ if (frag_desc->hdrs_valid) { @@ -464,23 +739,20 @@ static struct sk_buff *rmnet_alloc_skb(struct rmnet_frag_descriptor *frag_desc, return NULL; skb_reserve(head_skb, RMNET_MAP_DEAGGR_HEADROOM); - skb_put_data(head_skb, frag_desc->hdr_ptr, hdr_len); + rmnet_frag_copy_data(frag_desc, 0, hdr_len, + skb_put(head_skb, hdr_len)); skb_reset_network_header(head_skb); if (frag_desc->trans_len) skb_set_transport_header(head_skb, frag_desc->ip_len); - /* If the headers we added are the start of the page, - * we don't want to add them twice - */ - if (frag_desc->hdr_ptr == rmnet_frag_data_ptr(frag_desc)) { - /* "Header only" packets can be fast-forwarded */ - if (hdr_len == skb_frag_size(&frag_desc->frag)) - goto skip_frags; + /* Pull the headers off carefully */ + if (hdr_len == frag_desc->len) + /* Fast forward "header only" packets */ + goto skip_frags; - if (!rmnet_frag_pull(frag_desc, port, hdr_len)) { - kfree_skb(head_skb); - return NULL; - } + if (!rmnet_frag_pull(frag_desc, port, hdr_len)) { + kfree(head_skb); + return NULL; } } else { /* Allocate enough space to avoid penalties in the stack @@ -494,30 +766,20 @@ static struct sk_buff *rmnet_alloc_skb(struct rmnet_frag_descriptor *frag_desc, skb_reserve(head_skb, RMNET_MAP_DEAGGR_HEADROOM); } - /* Add main fragment */ - get_page(skb_frag_page(&frag_desc->frag)); - skb_add_rx_frag(head_skb, 0, skb_frag_page(&frag_desc->frag), - frag_desc->frag.bv_offset, - skb_frag_size(&frag_desc->frag), - skb_frag_size(&frag_desc->frag)); - shinfo = skb_shinfo(head_skb); current_skb = head_skb; - /* Add in any frags from rmnet_perf */ - list_for_each_entry_safe(sub_frag, tmp, &frag_desc->sub_frags, list) { - skb_frag_t *frag; - u32 frag_size; - - frag = &sub_frag->frag; - frag_size = skb_frag_size(frag); + /* Add in the page fragments */ + list_for_each_entry_safe(frag, tmp, &frag_desc->frags, list) { + struct page *p = skb_frag_page(&frag->frag); + u32 frag_size = skb_frag_size(&frag->frag); add_frag: if (shinfo->nr_frags < MAX_SKB_FRAGS) { - get_page(skb_frag_page(frag)); - skb_add_rx_frag(current_skb, shinfo->nr_frags, - skb_frag_page(frag), frag->bv_offset, - frag_size, frag_size); + get_page(p); + skb_add_rx_frag(current_skb, shinfo->nr_frags, p, + skb_frag_off(&frag->frag), frag_size, + frag_size); if (current_skb != head_skb) { head_skb->len += frag_size; head_skb->data_len += frag_size; @@ -537,8 +799,6 @@ add_frag: shinfo = skb_shinfo(current_skb); goto add_frag; } - - rmnet_recycle_frag_descriptor(sub_frag, port); } skip_frags: @@ -659,58 +919,83 @@ static void __rmnet_frag_segment_data(struct rmnet_frag_descriptor *coal_desc, bool csum_valid) { struct rmnet_priv *priv = netdev_priv(coal_desc->dev); - struct rmnet_frag_descriptor *new_frag; - u8 *hdr_start = rmnet_frag_data_ptr(coal_desc); - u32 offset; + struct rmnet_frag_descriptor *new_desc; + u32 dlen = coal_desc->gso_size * coal_desc->gso_segs; + u32 hlen = coal_desc->ip_len + coal_desc->trans_len; + u32 offset = hlen + coal_desc->data_offset; + int rc; - new_frag = rmnet_get_frag_descriptor(port); - if (!new_frag) + new_desc = rmnet_get_frag_descriptor(port); + if (!new_desc) return; - /* Account for header lengths to access the data start */ - offset = coal_desc->frag.bv_offset + coal_desc->ip_len + - coal_desc->trans_len + coal_desc->data_offset; - /* Header information and most metadata is the same as the original */ - memcpy(new_frag, coal_desc, sizeof(*coal_desc)); - INIT_LIST_HEAD(&new_frag->list); - INIT_LIST_HEAD(&new_frag->sub_frags); - rmnet_frag_fill(new_frag, skb_frag_page(&coal_desc->frag), offset, - coal_desc->gso_size * coal_desc->gso_segs); + memcpy(new_desc, coal_desc, sizeof(*coal_desc)); + INIT_LIST_HEAD(&new_desc->list); + INIT_LIST_HEAD(&new_desc->frags); + new_desc->len = 0; + /* Add the header fragments */ + rc = rmnet_frag_descriptor_add_frags_from(new_desc, coal_desc, 0, + hlen); + if (rc < 0) + goto recycle; + + /* Add in the data fragments */ + rc = rmnet_frag_descriptor_add_frags_from(new_desc, coal_desc, offset, + dlen); + if (rc < 0) + goto recycle; + + /* Update protocol-specific metadata */ if (coal_desc->trans_proto == IPPROTO_TCP) { - struct tcphdr *th; + struct tcphdr *th, __th; - th = (struct tcphdr *)(hdr_start + coal_desc->ip_len); - new_frag->tcp_seq_set = 1; - new_frag->tcp_seq = htonl(ntohl(th->seq) + + th = rmnet_frag_header_ptr(coal_desc, coal_desc->ip_len, + sizeof(*th), &__th); + if (!th) + goto recycle; + + new_desc->tcp_seq_set = 1; + new_desc->tcp_seq = htonl(ntohl(th->seq) + coal_desc->data_offset); } else if (coal_desc->trans_proto == IPPROTO_UDP) { - struct udphdr *uh; + struct udphdr *uh, __uh; + + uh = rmnet_frag_header_ptr(coal_desc, coal_desc->ip_len, + sizeof(*uh), &__uh); + if (!uh) + goto recycle; - uh = (struct udphdr *)(hdr_start + coal_desc->ip_len); if (coal_desc->ip_proto == 4 && !uh->check) csum_valid = true; } if (coal_desc->ip_proto == 4) { - struct iphdr *iph; + struct iphdr *iph, __iph; - iph = (struct iphdr *)hdr_start; - new_frag->ip_id_set = 1; - new_frag->ip_id = htons(ntohs(iph->id) + coal_desc->pkt_id); + iph = rmnet_frag_header_ptr(coal_desc, 0, sizeof(*iph), + &__iph); + if (!iph) + goto recycle; + + new_desc->ip_id_set = 1; + new_desc->ip_id = htons(ntohs(iph->id) + coal_desc->pkt_id); } - new_frag->hdr_ptr = hdr_start; - new_frag->csum_valid = csum_valid; + new_desc->csum_valid = csum_valid; priv->stats.coal.coal_reconstruct++; /* Update meta information to move past the data we just segmented */ - coal_desc->data_offset += coal_desc->gso_size * coal_desc->gso_segs; + coal_desc->data_offset += dlen; coal_desc->pkt_id = pkt_id + 1; coal_desc->gso_segs = 0; - list_add_tail(&new_frag->list, list); + list_add_tail(&new_desc->list, list); + return; + +recycle: + rmnet_recycle_frag_descriptor(new_desc, port); } static bool rmnet_frag_validate_csum(struct rmnet_frag_descriptor *frag_desc) @@ -720,7 +1005,7 @@ static bool rmnet_frag_validate_csum(struct rmnet_frag_descriptor *frag_desc) __wsum csum; __sum16 pseudo; - datagram_len = skb_frag_size(&frag_desc->frag) - frag_desc->ip_len; + datagram_len = frag_desc->len - frag_desc->ip_len; if (frag_desc->ip_proto == 4) { struct iphdr *iph = (struct iphdr *)data; @@ -748,27 +1033,42 @@ rmnet_frag_segment_coal_data(struct rmnet_frag_descriptor *coal_desc, u64 nlo_err_mask, struct rmnet_port *port, struct list_head *list) { - struct iphdr *iph; struct rmnet_priv *priv = netdev_priv(coal_desc->dev); - struct rmnet_map_v5_coal_header *coal_hdr; + struct rmnet_map_v5_coal_header coal_hdr; + u8 *version; u16 pkt_len; u8 pkt, total_pkt = 0; u8 nlo; bool gro = coal_desc->dev->features & NETIF_F_GRO_HW; bool zero_csum = false; + /* Copy the coal header into our local storage before pulling it. It's + * possible that this header (or part of it) is the last port of a page + * a pulling it off would cause it to be freed. Referring back to the + * header would be invalid in that case. + */ + if (rmnet_frag_copy_data(coal_desc, sizeof(struct rmnet_map_header), + sizeof(coal_hdr), &coal_hdr) < 0) + return; + /* Pull off the headers we no longer need */ - if (!rmnet_frag_pull(coal_desc, port, sizeof(struct rmnet_map_header))) + if (!rmnet_frag_pull(coal_desc, port, sizeof(struct rmnet_map_header) + + sizeof(coal_hdr))) return; - coal_hdr = (struct rmnet_map_v5_coal_header *) - rmnet_frag_data_ptr(coal_desc); - if (!rmnet_frag_pull(coal_desc, port, sizeof(*coal_hdr))) - return; + /* By definition, this byte is linear, and the first byte on the + * first fragment. ;) Hence why no header_ptr() call is needed + * for it. + */ + version = rmnet_frag_data_ptr(coal_desc); + if ((*version & 0xF0) == 0x40) { + struct iphdr *iph, __iph; - iph = (struct iphdr *)rmnet_frag_data_ptr(coal_desc); + iph = rmnet_frag_header_ptr(coal_desc, 0, sizeof(*iph), + &__iph); + if (!iph) + return; - if (iph->version == 4) { coal_desc->ip_proto = 4; coal_desc->ip_len = iph->ihl * 4; coal_desc->trans_proto = iph->protocol; @@ -776,13 +1076,19 @@ rmnet_frag_segment_coal_data(struct rmnet_frag_descriptor *coal_desc, /* Don't allow coalescing of any packets with IP options */ if (iph->ihl != 5) gro = false; - } else if (iph->version == 6) { - struct ipv6hdr *ip6h = (struct ipv6hdr *)iph; + } else if ((*version & 0xF0) == 0x60) { + struct ipv6hdr *ip6h, __ip6h; int ip_len; __be16 frag_off; - u8 protocol = ip6h->nexthdr; + u8 protocol; + + ip6h = rmnet_frag_header_ptr(coal_desc, 0, sizeof(*ip6h), + &__ip6h); + if (!ip6h) + return; coal_desc->ip_proto = 6; + protocol = ip6h->nexthdr; ip_len = rmnet_frag_ipv6_skip_exthdr(coal_desc, sizeof(*ip6h), &protocol, @@ -811,21 +1117,29 @@ rmnet_frag_segment_coal_data(struct rmnet_frag_descriptor *coal_desc, } if (coal_desc->trans_proto == IPPROTO_TCP) { - struct tcphdr *th; + struct tcphdr *th, __th; + + th = rmnet_frag_header_ptr(coal_desc, + coal_desc->ip_len, sizeof(*th), + &__th); + if (!th) + return; - th = (struct tcphdr *)((u8 *)iph + coal_desc->ip_len); coal_desc->trans_len = th->doff * 4; priv->stats.coal.coal_tcp++; - priv->stats.coal.coal_tcp_bytes += - skb_frag_size(&coal_desc->frag); + priv->stats.coal.coal_tcp_bytes += coal_desc->len; } else if (coal_desc->trans_proto == IPPROTO_UDP) { - struct udphdr *uh; + struct udphdr *uh, __uh; + + uh = rmnet_frag_header_ptr(coal_desc, + coal_desc->ip_len, sizeof(*uh), + &__uh); + if (!uh) + return; - uh = (struct udphdr *)((u8 *)iph + coal_desc->ip_len); coal_desc->trans_len = sizeof(*uh); priv->stats.coal.coal_udp++; - priv->stats.coal.coal_udp_bytes += - skb_frag_size(&coal_desc->frag); + priv->stats.coal.coal_udp_bytes += coal_desc->len; if (coal_desc->ip_proto == 4 && !uh->check) zero_csum = true; } else { @@ -835,15 +1149,14 @@ rmnet_frag_segment_coal_data(struct rmnet_frag_descriptor *coal_desc, coal_desc->hdrs_valid = 1; - if (rmnet_map_v5_csum_buggy(coal_hdr) && !zero_csum) { + if (rmnet_map_v5_csum_buggy(&coal_hdr) && !zero_csum) { /* Mark the checksum as valid if it checks out */ if (rmnet_frag_validate_csum(coal_desc)) coal_desc->csum_valid = true; - coal_desc->hdr_ptr = rmnet_frag_data_ptr(coal_desc); - coal_desc->gso_size = ntohs(coal_hdr->nl_pairs[0].pkt_len); + coal_desc->gso_size = ntohs(coal_hdr.nl_pairs[0].pkt_len); coal_desc->gso_size -= coal_desc->ip_len + coal_desc->trans_len; - coal_desc->gso_segs = coal_hdr->nl_pairs[0].num_packets; + coal_desc->gso_segs = coal_hdr.nl_pairs[0].num_packets; list_add_tail(&coal_desc->list, list); return; } @@ -852,22 +1165,21 @@ rmnet_frag_segment_coal_data(struct rmnet_frag_descriptor *coal_desc, * no checksum errors, and are allowing GRO. We can just reuse this * descriptor unchanged. */ - if (gro && coal_hdr->num_nlos == 1 && coal_hdr->csum_valid) { + if (gro && coal_hdr.num_nlos == 1 && coal_hdr.csum_valid) { coal_desc->csum_valid = true; - coal_desc->hdr_ptr = rmnet_frag_data_ptr(coal_desc); - coal_desc->gso_size = ntohs(coal_hdr->nl_pairs[0].pkt_len); + coal_desc->gso_size = ntohs(coal_hdr.nl_pairs[0].pkt_len); coal_desc->gso_size -= coal_desc->ip_len + coal_desc->trans_len; - coal_desc->gso_segs = coal_hdr->nl_pairs[0].num_packets; + coal_desc->gso_segs = coal_hdr.nl_pairs[0].num_packets; list_add_tail(&coal_desc->list, list); return; } /* Segment the coalesced descriptor into new packets */ - for (nlo = 0; nlo < coal_hdr->num_nlos; nlo++) { - pkt_len = ntohs(coal_hdr->nl_pairs[nlo].pkt_len); + for (nlo = 0; nlo < coal_hdr.num_nlos; nlo++) { + pkt_len = ntohs(coal_hdr.nl_pairs[nlo].pkt_len); pkt_len -= coal_desc->ip_len + coal_desc->trans_len; coal_desc->gso_size = pkt_len; - for (pkt = 0; pkt < coal_hdr->nl_pairs[nlo].num_packets; + for (pkt = 0; pkt < coal_hdr.nl_pairs[nlo].num_packets; pkt++, total_pkt++, nlo_err_mask >>= 1) { bool csum_err = nlo_err_mask & 1; @@ -969,15 +1281,18 @@ static int rmnet_frag_data_check_coal_header(struct rmnet_frag_descriptor *frag_desc, u64 *nlo_err_mask) { - struct rmnet_map_v5_coal_header *coal_hdr; - unsigned char *data = rmnet_frag_data_ptr(frag_desc); + struct rmnet_map_v5_coal_header *coal_hdr, __coal_hdr; struct rmnet_priv *priv = netdev_priv(frag_desc->dev); u64 mask = 0; int i; u8 veid, pkts = 0; - coal_hdr = (struct rmnet_map_v5_coal_header *) - (data + sizeof(struct rmnet_map_header)); + coal_hdr = rmnet_frag_header_ptr(frag_desc, + sizeof(struct rmnet_map_header), + sizeof(*coal_hdr), &__coal_hdr); + if (!coal_hdr) + return -EINVAL; + veid = coal_hdr->virtual_channel_id; if (coal_hdr->num_nlos == 0 || @@ -1024,11 +1339,24 @@ int rmnet_frag_process_next_hdr_packet(struct rmnet_frag_descriptor *frag_desc, struct list_head *list, u16 len) { + struct rmnet_map_v5_csum_header *csum_hdr, __csum_hdr; struct rmnet_priv *priv = netdev_priv(frag_desc->dev); u64 nlo_err_mask; + u32 offset = sizeof(struct rmnet_map_header); int rc = 0; - switch (rmnet_frag_get_next_hdr_type(frag_desc)) { + /* Grab the header type. It's easier to grab enough for a full csum + * offload header here since it's only 8 bytes and then check the + * header type using that. This also doubles as a check to make sure + * there's enough data after the QMAP header to ensure that another + * header is present. + */ + csum_hdr = rmnet_frag_header_ptr(frag_desc, offset, sizeof(*csum_hdr), + &__csum_hdr); + if (!csum_hdr) + return -EINVAL; + + switch (csum_hdr->header_type) { case RMNET_MAP_HEADER_TYPE_COALESCING: priv->stats.coal.coal_rx++; rc = rmnet_frag_data_check_coal_header(frag_desc, @@ -1045,7 +1373,7 @@ int rmnet_frag_process_next_hdr_packet(struct rmnet_frag_descriptor *frag_desc, case RMNET_MAP_HEADER_TYPE_CSUM_OFFLOAD: if (unlikely(!(frag_desc->dev->features & NETIF_F_RXCSUM))) { priv->stats.csum_sw++; - } else if (rmnet_frag_get_csum_valid(frag_desc)) { + } else if (csum_hdr->csum_valid_required) { priv->stats.csum_ok++; frag_desc->csum_valid = true; } else { @@ -1053,14 +1381,11 @@ int rmnet_frag_process_next_hdr_packet(struct rmnet_frag_descriptor *frag_desc, } if (!rmnet_frag_pull(frag_desc, port, - sizeof(struct rmnet_map_header) + - sizeof(struct rmnet_map_v5_csum_header))) { + offset + sizeof(*csum_hdr))) { rc = -EINVAL; break; } - frag_desc->hdr_ptr = rmnet_frag_data_ptr(frag_desc); - /* Remove padding only for csum offload packets. * Coalesced packets should never have padding. */ @@ -1089,14 +1414,17 @@ __rmnet_frag_ingress_handler(struct rmnet_frag_descriptor *frag_desc, struct rmnet_port *port) { rmnet_perf_desc_hook_t rmnet_perf_ingress; - struct rmnet_map_header *qmap; + struct rmnet_map_header *qmap, __qmap; struct rmnet_endpoint *ep; struct rmnet_frag_descriptor *frag, *tmp; LIST_HEAD(segs); u16 len, pad; u8 mux_id; - qmap = (struct rmnet_map_header *)skb_frag_address(&frag_desc->frag); + qmap = rmnet_frag_header_ptr(frag_desc, 0, sizeof(*qmap), &__qmap); + if (!qmap) + goto recycle; + mux_id = qmap->mux_id; pad = qmap->pad_len; len = ntohs(qmap->pkt_len) - pad; @@ -1104,12 +1432,12 @@ __rmnet_frag_ingress_handler(struct rmnet_frag_descriptor *frag_desc, if (qmap->cd_bit) { qmi_rmnet_set_dl_msg_active(port); if (port->data_format & RMNET_INGRESS_FORMAT_DL_MARKER) { - rmnet_frag_flow_command(qmap, port, len); + rmnet_frag_flow_command(frag_desc, port, len); goto recycle; } if (port->data_format & RMNET_FLAGS_INGRESS_MAP_COMMANDS) - rmnet_frag_command(qmap, port); + rmnet_frag_command(frag_desc, qmap, port); goto recycle; } @@ -1135,8 +1463,6 @@ __rmnet_frag_ingress_handler(struct rmnet_frag_descriptor *frag_desc, if (!rmnet_frag_pull(frag_desc, port, sizeof(*qmap))) return; - frag_desc->hdr_ptr = rmnet_frag_data_ptr(frag_desc); - if (!rmnet_frag_trim(frag_desc, port, len)) return; @@ -1184,8 +1510,7 @@ void rmnet_frag_ingress_handler(struct sk_buff *skb, while (skb) { struct sk_buff *skb_frag; - rmnet_frag_deaggregate(skb_shinfo(skb)->frags, port, - &desc_list); + rmnet_frag_deaggregate(skb, port, &desc_list); if (!list_empty(&desc_list)) { struct rmnet_frag_descriptor *frag_desc, *tmp; @@ -1245,7 +1570,7 @@ int rmnet_descriptor_init(struct rmnet_port *port) return -ENOMEM; INIT_LIST_HEAD(&frag_desc->list); - INIT_LIST_HEAD(&frag_desc->sub_frags); + INIT_LIST_HEAD(&frag_desc->frags); list_add_tail(&frag_desc->list, &pool->free_list); pool->pool_size++; } diff --git a/core/rmnet_descriptor.h b/core/rmnet_descriptor.h index 06b9c9dd231b..142a14a5be1d 100644 --- a/core/rmnet_descriptor.h +++ b/core/rmnet_descriptor.h @@ -27,12 +27,16 @@ struct rmnet_frag_descriptor_pool { u32 pool_size; }; +struct rmnet_fragment { + struct list_head list; + skb_frag_t frag; +}; + struct rmnet_frag_descriptor { struct list_head list; - struct list_head sub_frags; - skb_frag_t frag; - u8 *hdr_ptr; + struct list_head frags; struct net_device *dev; + u32 len; u32 hash; __be32 tcp_seq; __be16 ip_id; @@ -57,18 +61,28 @@ struct rmnet_frag_descriptor * rmnet_get_frag_descriptor(struct rmnet_port *port); void rmnet_recycle_frag_descriptor(struct rmnet_frag_descriptor *frag_desc, struct rmnet_port *port); -void rmnet_descriptor_add_frag(struct rmnet_port *port, struct list_head *list, - struct page *p, u32 page_offset, u32 len); +void *rmnet_frag_pull(struct rmnet_frag_descriptor *frag_desc, + struct rmnet_port *port, unsigned int size); +void *rmnet_frag_trim(struct rmnet_frag_descriptor *frag_desc, + struct rmnet_port *port, unsigned int size); +void *rmnet_frag_header_ptr(struct rmnet_frag_descriptor *frag_desc, u32 off, + u32 len, void *buf); +int rmnet_frag_descriptor_add_frag(struct rmnet_frag_descriptor *frag_desc, + struct page *p, u32 page_offset, u32 len); +int rmnet_frag_descriptor_add_frags_from(struct rmnet_frag_descriptor *to, + struct rmnet_frag_descriptor *from, + u32 off, u32 len); int rmnet_frag_ipv6_skip_exthdr(struct rmnet_frag_descriptor *frag_desc, int start, u8 *nexthdrp, __be16 *fragp); /* QMAP command packets */ -void rmnet_frag_command(struct rmnet_map_header *qmap, struct rmnet_port *port); -int rmnet_frag_flow_command(struct rmnet_map_header *qmap, +void rmnet_frag_command(struct rmnet_frag_descriptor *frag_desc, + struct rmnet_map_header *qmap, struct rmnet_port *port); +int rmnet_frag_flow_command(struct rmnet_frag_descriptor *frag_desc, struct rmnet_port *port, u16 pkt_len); /* Ingress data handlers */ -void rmnet_frag_deaggregate(skb_frag_t *frag, struct rmnet_port *port, +void rmnet_frag_deaggregate(struct sk_buff *skb, struct rmnet_port *port, struct list_head *list); void rmnet_frag_deliver(struct rmnet_frag_descriptor *frag_desc, struct rmnet_port *port); @@ -84,68 +98,15 @@ void rmnet_descriptor_deinit(struct rmnet_port *port); static inline void *rmnet_frag_data_ptr(struct rmnet_frag_descriptor *frag_desc) { - return skb_frag_address(&frag_desc->frag); -} + struct rmnet_fragment *frag; -static inline void *rmnet_frag_pull(struct rmnet_frag_descriptor *frag_desc, - struct rmnet_port *port, - unsigned int size) -{ - if (size >= skb_frag_size(&frag_desc->frag)) { - pr_info("%s(): Pulling %u bytes from %u byte pkt. Dropping\n", - __func__, size, skb_frag_size(&frag_desc->frag)); - rmnet_recycle_frag_descriptor(frag_desc, port); + frag = list_first_entry_or_null(&frag_desc->frags, + struct rmnet_fragment, list); + + if (!frag) return NULL; - } - frag_desc->frag.bv_offset += size; - skb_frag_size_sub(&frag_desc->frag, size); - - return rmnet_frag_data_ptr(frag_desc); -} - -static inline void *rmnet_frag_trim(struct rmnet_frag_descriptor *frag_desc, - struct rmnet_port *port, - unsigned int size) -{ - if (!size) { - pr_info("%s(): Trimming %u byte pkt to 0. Dropping\n", - __func__, skb_frag_size(&frag_desc->frag)); - rmnet_recycle_frag_descriptor(frag_desc, port); - return NULL; - } - - if (size < skb_frag_size(&frag_desc->frag)) - skb_frag_size_set(&frag_desc->frag, size); - - return rmnet_frag_data_ptr(frag_desc); -} - -static inline void rmnet_frag_fill(struct rmnet_frag_descriptor *frag_desc, - struct page *p, u32 page_offset, u32 len) -{ - get_page(p); - __skb_frag_set_page(&frag_desc->frag, p); - skb_frag_size_set(&frag_desc->frag, len); - frag_desc->frag.bv_offset = page_offset; -} - -static inline u8 -rmnet_frag_get_next_hdr_type(struct rmnet_frag_descriptor *frag_desc) -{ - unsigned char *data = rmnet_frag_data_ptr(frag_desc); - - data += sizeof(struct rmnet_map_header); - return ((struct rmnet_map_v5_coal_header *)data)->header_type; -} - -static inline bool -rmnet_frag_get_csum_valid(struct rmnet_frag_descriptor *frag_desc) -{ - unsigned char *data = rmnet_frag_data_ptr(frag_desc); - - data += sizeof(struct rmnet_map_header); - return ((struct rmnet_map_v5_csum_header *)data)->csum_valid_required; + return skb_frag_address(&frag->frag); } #endif /* _RMNET_DESCRIPTOR_H_ */ From 9bb11086953263539919e13db77aff25353b74c9 Mon Sep 17 00:00:00 2001 From: Kaustubh Pandey Date: Thu, 20 Aug 2020 02:37:10 +0530 Subject: [PATCH 33/54] datarmnet: Enable rmnet_core and rmnet_ctl for holi Change-Id: Ide8d3c4d849c7c45207baba77dd0153f00bf4295 --- core/Android.mk | 1 + datarmnet_dlkm_vendor_board.mk | 1 + 2 files changed, 2 insertions(+) diff --git a/core/Android.mk b/core/Android.mk index d9e1d9714afd..50ef9864dbe3 100644 --- a/core/Android.mk +++ b/core/Android.mk @@ -1,5 +1,6 @@ ifneq ($(TARGET_PRODUCT),qssi) RMNET_CORE_DLKM_PLATFORMS_LIST := lahaina +RMNET_CORE_DLKM_PLATFORMS_LIST += holi ifeq ($(call is-board-platform-in-list, $(RMNET_CORE_DLKM_PLATFORMS_LIST)),true) #Make file to create RMNET_CORE DLKM diff --git a/datarmnet_dlkm_vendor_board.mk b/datarmnet_dlkm_vendor_board.mk index aa855f23db04..44a2c8ba6878 100644 --- a/datarmnet_dlkm_vendor_board.mk +++ b/datarmnet_dlkm_vendor_board.mk @@ -1,5 +1,6 @@ #Build rmnet core DATA_DLKM_BOARD_PLATFORMS_LIST := lahaina +DATA_DLKM_BOARD_PLATFORMS_LIST += holi ifneq ($(TARGET_BOARD_AUTO),true) ifeq ($(call is-board-platform-in-list,$(DATA_DLKM_BOARD_PLATFORMS_LIST)),true) BOARD_VENDOR_KERNEL_MODULES += $(KERNEL_MODULES_OUT)/rmnet_core.ko From 8a3c6765f1504f339db161ab3f46d071a9b0dbcc Mon Sep 17 00:00:00 2001 From: Sean Tranchetti Date: Mon, 31 Aug 2020 15:14:23 -0600 Subject: [PATCH 34/54] rmnet_core: fix static analysis errors Fix warnings introduced by the introduction of scatter-gather support. Change-Id: I0a12f19a1572989d259e6d8b5b12146a7c5c7cd1 Signed-off-by: Sean Tranchetti --- core/rmnet_descriptor.c | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/core/rmnet_descriptor.c b/core/rmnet_descriptor.c index 7a20aba2da2d..c0b95f330c4c 100644 --- a/core/rmnet_descriptor.c +++ b/core/rmnet_descriptor.c @@ -340,7 +340,6 @@ int rmnet_frag_ipv6_skip_exthdr(struct rmnet_frag_descriptor *frag_desc, if (!hp) return -EINVAL; - hp = rmnet_frag_data_ptr(frag_desc) + start; if (nexthdr == NEXTHDR_FRAGMENT) { u32 off = offsetof(struct frag_hdr, frag_off); __be16 *fp, __fp; @@ -1005,6 +1004,13 @@ static bool rmnet_frag_validate_csum(struct rmnet_frag_descriptor *frag_desc) __wsum csum; __sum16 pseudo; + /* Keep analysis tools happy, since they will see that + * rmnet_frag_data_ptr() could return NULL. It can't in this case, + * since we can't get this far otherwise... + */ + if (unlikely(!data)) + return false; + datagram_len = frag_desc->len - frag_desc->ip_len; if (frag_desc->ip_proto == 4) { struct iphdr *iph = (struct iphdr *)data; @@ -1061,6 +1067,9 @@ rmnet_frag_segment_coal_data(struct rmnet_frag_descriptor *coal_desc, * for it. */ version = rmnet_frag_data_ptr(coal_desc); + if (unlikely(!version)) + return; + if ((*version & 0xF0) == 0x40) { struct iphdr *iph, __iph; From 7360fe16c5e4eee3fd629614e95e44e1b0431d8d Mon Sep 17 00:00:00 2001 From: Subash Abhinov Kasiviswanathan Date: Thu, 10 Sep 2020 21:32:55 -0700 Subject: [PATCH 35/54] core: Update IID every time netdevice is brought down This is done to randomize the IPv6 address generation. This also assigns a different IID to every rmnet interface. CRs-Fixed: 2771389 Change-Id: I427481f1eeda098a2ebbd0c8ce88030eb0cd6cfd Signed-off-by: Subash Abhinov Kasiviswanathan --- core/rmnet_config.c | 4 +++- core/rmnet_vnd.c | 10 +++++++++- core/rmnet_vnd.h | 3 ++- 3 files changed, 14 insertions(+), 3 deletions(-) diff --git a/core/rmnet_config.c b/core/rmnet_config.c index 1d0d71dd0d99..4448a8efaf3b 100644 --- a/core/rmnet_config.c +++ b/core/rmnet_config.c @@ -331,7 +331,9 @@ static int rmnet_config_notify_cb(struct notifier_block *nb, netdev_dbg(dev, "Kernel unregister\n"); rmnet_force_unassociate_device(dev); break; - + case NETDEV_DOWN: + rmnet_vnd_reset_mac_addr(dev); + break; default: break; } diff --git a/core/rmnet_vnd.c b/core/rmnet_vnd.c index 3674b3f94ef6..d5187be0497d 100644 --- a/core/rmnet_vnd.c +++ b/core/rmnet_vnd.c @@ -357,7 +357,7 @@ void rmnet_vnd_setup(struct net_device *rmnet_dev) rmnet_dev->netdev_ops = &rmnet_vnd_ops; rmnet_dev->mtu = RMNET_DFLT_PACKET_SIZE; rmnet_dev->needed_headroom = RMNET_NEEDED_HEADROOM; - random_ether_addr(rmnet_dev->dev_addr); + random_ether_addr(rmnet_dev->perm_addr); rmnet_dev->tx_queue_len = RMNET_TX_QUEUE_LEN; /* Raw IP mode */ @@ -444,3 +444,11 @@ int rmnet_vnd_do_flow_control(struct net_device *rmnet_dev, int enable) return 0; } + +void rmnet_vnd_reset_mac_addr(struct net_device *dev) +{ + if (dev->netdev_ops != &rmnet_vnd_ops) + return; + + random_ether_addr(dev->perm_addr); +} diff --git a/core/rmnet_vnd.h b/core/rmnet_vnd.h index 8b26f7cb63f9..7fa9500a2aca 100644 --- a/core/rmnet_vnd.h +++ b/core/rmnet_vnd.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2013-2018, The Linux Foundation. All rights reserved. +/* Copyright (c) 2013-2018, 2020 The Linux Foundation. All rights reserved. * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 and @@ -27,4 +27,5 @@ void rmnet_vnd_rx_fixup(struct net_device *dev, u32 skb_len); void rmnet_vnd_tx_fixup(struct net_device *dev, u32 skb_len); u8 rmnet_vnd_get_mux(struct net_device *rmnet_dev); void rmnet_vnd_setup(struct net_device *dev); +void rmnet_vnd_reset_mac_addr(struct net_device *dev); #endif /* _RMNET_VND_H_ */ From 4ede88477be4dda2e8a9b8b35b0453b0ecd82b14 Mon Sep 17 00:00:00 2001 From: Sean Tranchetti Date: Mon, 14 Sep 2020 17:12:50 -0600 Subject: [PATCH 36/54] rmnet_core: Check for contiguous QMAP headers during deaggregation After the introduction of scatter gather support, it's possible that the QMAP headers may be split across pages. This patch adds proper handling for the headers during deaggregation from the original SKB from the physical device when building the descriptors used elsewhere in the rmnet driver. Change-Id: If62cf8b4993fe01b6f43e9f16f08c134d18437b8 Signed-off-by: Sean Tranchetti --- core/rmnet_descriptor.c | 40 +++++++++++++++++++++++++++++++++------- 1 file changed, 33 insertions(+), 7 deletions(-) diff --git a/core/rmnet_descriptor.c b/core/rmnet_descriptor.c index c0b95f330c4c..79612ec49cd3 100644 --- a/core/rmnet_descriptor.c +++ b/core/rmnet_descriptor.c @@ -558,20 +558,36 @@ int rmnet_frag_flow_command(struct rmnet_frag_descriptor *frag_desc, } EXPORT_SYMBOL(rmnet_frag_flow_command); -static int rmnet_frag_deaggregate_one(struct skb_shared_info *shinfo, +static int rmnet_frag_deaggregate_one(struct sk_buff *skb, struct rmnet_port *port, struct list_head *list, u32 start_frag) { + struct skb_shared_info *shinfo = skb_shinfo(skb); struct rmnet_frag_descriptor *frag_desc; - struct rmnet_map_header *maph; + struct rmnet_map_header *maph, __maph; skb_frag_t *frag; u32 i; u32 pkt_len; int rc; frag = &shinfo->frags[start_frag]; - maph = skb_frag_address(frag); + /* Grab the QMAP header. Careful, as there's no guarantee that it's + * continugous! + */ + if (likely(skb_frag_size(frag) >= sizeof(*maph))) { + maph = skb_frag_address(frag); + } else { + /* The header's split across pages. We can rebuild it. + * Probably not faster or stronger than before. But certainly + * more linear. + */ + if (skb_copy_bits(skb, 0, &__maph, sizeof(&__maph)) < 0) + return -1; + + maph = &__maph; + } + pkt_len = ntohs(maph->pkt_len); /* Catch empty frames */ if (!pkt_len) @@ -590,9 +606,19 @@ static int rmnet_frag_deaggregate_one(struct skb_shared_info *shinfo, u32 hsize = 0; u8 type; - type = ((struct rmnet_map_v5_coal_header *) - (maph + 1))->header_type; - switch (type) { + /* Check the type. This seems like should be overkill for less + * than a single byte, doesn't it? + */ + if (likely(skb_frag_size(frag) >= sizeof(*maph) + 1)) { + type = *((u8 *)maph + sizeof(*maph)); + } else { + if (skb_copy_bits(skb, sizeof(*maph), &type, + sizeof(type)) < 0) + return -1; + } + + /* Type only uses the first 7 bits */ + switch ((type & 0xFE) >> 1) { case RMNET_MAP_HEADER_TYPE_COALESCING: hsize = sizeof(struct rmnet_map_v5_coal_header); break; @@ -647,7 +673,7 @@ void rmnet_frag_deaggregate(struct sk_buff *skb, struct rmnet_port *port, int rc; while (i < shinfo->nr_frags) { - rc = rmnet_frag_deaggregate_one(shinfo, port, list, i); + rc = rmnet_frag_deaggregate_one(skb, port, list, i); if (rc < 0) return; From df2d62bed079908c127051de3385f3560c5f8c90 Mon Sep 17 00:00:00 2001 From: Sean Tranchetti Date: Mon, 14 Sep 2020 17:12:50 -0600 Subject: [PATCH 37/54] rmnet_core: Check for contiguous QMAP headers during deaggregation After the introduction of scatter gather support, it's possible that the QMAP headers may be split across pages. This patch adds proper handling for the headers during deaggregation from the original SKB from the physical device when building the descriptors used elsewhere in the rmnet driver. Change-Id: If62cf8b4993fe01b6f43e9f16f08c134d18437b8 Signed-off-by: Sean Tranchetti --- core/rmnet_descriptor.c | 40 +++++++++++++++++++++++++++++++++------- 1 file changed, 33 insertions(+), 7 deletions(-) diff --git a/core/rmnet_descriptor.c b/core/rmnet_descriptor.c index c0b95f330c4c..79612ec49cd3 100644 --- a/core/rmnet_descriptor.c +++ b/core/rmnet_descriptor.c @@ -558,20 +558,36 @@ int rmnet_frag_flow_command(struct rmnet_frag_descriptor *frag_desc, } EXPORT_SYMBOL(rmnet_frag_flow_command); -static int rmnet_frag_deaggregate_one(struct skb_shared_info *shinfo, +static int rmnet_frag_deaggregate_one(struct sk_buff *skb, struct rmnet_port *port, struct list_head *list, u32 start_frag) { + struct skb_shared_info *shinfo = skb_shinfo(skb); struct rmnet_frag_descriptor *frag_desc; - struct rmnet_map_header *maph; + struct rmnet_map_header *maph, __maph; skb_frag_t *frag; u32 i; u32 pkt_len; int rc; frag = &shinfo->frags[start_frag]; - maph = skb_frag_address(frag); + /* Grab the QMAP header. Careful, as there's no guarantee that it's + * continugous! + */ + if (likely(skb_frag_size(frag) >= sizeof(*maph))) { + maph = skb_frag_address(frag); + } else { + /* The header's split across pages. We can rebuild it. + * Probably not faster or stronger than before. But certainly + * more linear. + */ + if (skb_copy_bits(skb, 0, &__maph, sizeof(&__maph)) < 0) + return -1; + + maph = &__maph; + } + pkt_len = ntohs(maph->pkt_len); /* Catch empty frames */ if (!pkt_len) @@ -590,9 +606,19 @@ static int rmnet_frag_deaggregate_one(struct skb_shared_info *shinfo, u32 hsize = 0; u8 type; - type = ((struct rmnet_map_v5_coal_header *) - (maph + 1))->header_type; - switch (type) { + /* Check the type. This seems like should be overkill for less + * than a single byte, doesn't it? + */ + if (likely(skb_frag_size(frag) >= sizeof(*maph) + 1)) { + type = *((u8 *)maph + sizeof(*maph)); + } else { + if (skb_copy_bits(skb, sizeof(*maph), &type, + sizeof(type)) < 0) + return -1; + } + + /* Type only uses the first 7 bits */ + switch ((type & 0xFE) >> 1) { case RMNET_MAP_HEADER_TYPE_COALESCING: hsize = sizeof(struct rmnet_map_v5_coal_header); break; @@ -647,7 +673,7 @@ void rmnet_frag_deaggregate(struct sk_buff *skb, struct rmnet_port *port, int rc; while (i < shinfo->nr_frags) { - rc = rmnet_frag_deaggregate_one(shinfo, port, list, i); + rc = rmnet_frag_deaggregate_one(skb, port, list, i); if (rc < 0) return; From f5ad52623809b9f9a677159cf521b8fd805d7116 Mon Sep 17 00:00:00 2001 From: Subash Abhinov Kasiviswanathan Date: Thu, 10 Sep 2020 21:32:55 -0700 Subject: [PATCH 38/54] core: Update IID every time netdevice is brought down This is done to randomize the IPv6 address generation. This also assigns a different IID to every rmnet interface. CRs-Fixed: 2771389 Change-Id: I427481f1eeda098a2ebbd0c8ce88030eb0cd6cfd Signed-off-by: Subash Abhinov Kasiviswanathan --- core/rmnet_config.c | 4 +++- core/rmnet_vnd.c | 10 +++++++++- core/rmnet_vnd.h | 3 ++- 3 files changed, 14 insertions(+), 3 deletions(-) diff --git a/core/rmnet_config.c b/core/rmnet_config.c index 1d0d71dd0d99..4448a8efaf3b 100644 --- a/core/rmnet_config.c +++ b/core/rmnet_config.c @@ -331,7 +331,9 @@ static int rmnet_config_notify_cb(struct notifier_block *nb, netdev_dbg(dev, "Kernel unregister\n"); rmnet_force_unassociate_device(dev); break; - + case NETDEV_DOWN: + rmnet_vnd_reset_mac_addr(dev); + break; default: break; } diff --git a/core/rmnet_vnd.c b/core/rmnet_vnd.c index 3674b3f94ef6..d5187be0497d 100644 --- a/core/rmnet_vnd.c +++ b/core/rmnet_vnd.c @@ -357,7 +357,7 @@ void rmnet_vnd_setup(struct net_device *rmnet_dev) rmnet_dev->netdev_ops = &rmnet_vnd_ops; rmnet_dev->mtu = RMNET_DFLT_PACKET_SIZE; rmnet_dev->needed_headroom = RMNET_NEEDED_HEADROOM; - random_ether_addr(rmnet_dev->dev_addr); + random_ether_addr(rmnet_dev->perm_addr); rmnet_dev->tx_queue_len = RMNET_TX_QUEUE_LEN; /* Raw IP mode */ @@ -444,3 +444,11 @@ int rmnet_vnd_do_flow_control(struct net_device *rmnet_dev, int enable) return 0; } + +void rmnet_vnd_reset_mac_addr(struct net_device *dev) +{ + if (dev->netdev_ops != &rmnet_vnd_ops) + return; + + random_ether_addr(dev->perm_addr); +} diff --git a/core/rmnet_vnd.h b/core/rmnet_vnd.h index 8b26f7cb63f9..7fa9500a2aca 100644 --- a/core/rmnet_vnd.h +++ b/core/rmnet_vnd.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2013-2018, The Linux Foundation. All rights reserved. +/* Copyright (c) 2013-2018, 2020 The Linux Foundation. All rights reserved. * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 and @@ -27,4 +27,5 @@ void rmnet_vnd_rx_fixup(struct net_device *dev, u32 skb_len); void rmnet_vnd_tx_fixup(struct net_device *dev, u32 skb_len); u8 rmnet_vnd_get_mux(struct net_device *rmnet_dev); void rmnet_vnd_setup(struct net_device *dev); +void rmnet_vnd_reset_mac_addr(struct net_device *dev); #endif /* _RMNET_VND_H_ */ From 1b7901268f1350a6558c2b16e6e1644500b79513 Mon Sep 17 00:00:00 2001 From: Sean Tranchetti Date: Mon, 21 Sep 2020 13:19:07 -0600 Subject: [PATCH 39/54] rmnet_core: Pass correct size to skb_copy_bits When copying the QMAP header during deaggregation, the size of the data to copy was incorrectly passed to skb_copy_bits() as the size of the pointer instead of the struct. This lead to copying more data onto the stack than there was room for. Change-Id: I19208a8df2d8ce25a22fff95b71f1c56dcddec89 Signed-off-by: Sean Tranchetti --- core/rmnet_descriptor.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/rmnet_descriptor.c b/core/rmnet_descriptor.c index 79612ec49cd3..c71d44c63d2e 100644 --- a/core/rmnet_descriptor.c +++ b/core/rmnet_descriptor.c @@ -582,7 +582,7 @@ static int rmnet_frag_deaggregate_one(struct sk_buff *skb, * Probably not faster or stronger than before. But certainly * more linear. */ - if (skb_copy_bits(skb, 0, &__maph, sizeof(&__maph)) < 0) + if (skb_copy_bits(skb, 0, &__maph, sizeof(__maph)) < 0) return -1; maph = &__maph; From bbc6122efa62835923a3c387d97635d5d29d5042 Mon Sep 17 00:00:00 2001 From: Sean Tranchetti Date: Mon, 21 Sep 2020 13:19:07 -0600 Subject: [PATCH 40/54] rmnet_core: Pass correct size to skb_copy_bits When copying the QMAP header during deaggregation, the size of the data to copy was incorrectly passed to skb_copy_bits() as the size of the pointer instead of the struct. This lead to copying more data onto the stack than there was room for. Change-Id: I19208a8df2d8ce25a22fff95b71f1c56dcddec89 Signed-off-by: Sean Tranchetti --- core/rmnet_descriptor.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/rmnet_descriptor.c b/core/rmnet_descriptor.c index 79612ec49cd3..c71d44c63d2e 100644 --- a/core/rmnet_descriptor.c +++ b/core/rmnet_descriptor.c @@ -582,7 +582,7 @@ static int rmnet_frag_deaggregate_one(struct sk_buff *skb, * Probably not faster or stronger than before. But certainly * more linear. */ - if (skb_copy_bits(skb, 0, &__maph, sizeof(&__maph)) < 0) + if (skb_copy_bits(skb, 0, &__maph, sizeof(__maph)) < 0) return -1; maph = &__maph; From 7ce883f15f07b38c8d07a6db3a159ecae523719d Mon Sep 17 00:00:00 2001 From: Divya Sharma Date: Tue, 6 Oct 2020 08:15:02 -0700 Subject: [PATCH 41/54] Revert "rmnet_core: Pass correct size to skb_copy_bits" This reverts commit bbc6122efa62835923a3c387d97635d5d29d5042. Change-Id: I2268cf469b6746b6c84274b7e1b0f403f730c1fc --- core/rmnet_descriptor.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/rmnet_descriptor.c b/core/rmnet_descriptor.c index c71d44c63d2e..79612ec49cd3 100644 --- a/core/rmnet_descriptor.c +++ b/core/rmnet_descriptor.c @@ -582,7 +582,7 @@ static int rmnet_frag_deaggregate_one(struct sk_buff *skb, * Probably not faster or stronger than before. But certainly * more linear. */ - if (skb_copy_bits(skb, 0, &__maph, sizeof(__maph)) < 0) + if (skb_copy_bits(skb, 0, &__maph, sizeof(&__maph)) < 0) return -1; maph = &__maph; From ef2b4fd504291a533dcbe6dc96eca58b3c8fe8ea Mon Sep 17 00:00:00 2001 From: Divya Sharma Date: Tue, 6 Oct 2020 08:15:16 -0700 Subject: [PATCH 42/54] Revert "core: Update IID every time netdevice is brought down" This reverts commit f5ad52623809b9f9a677159cf521b8fd805d7116. Change-Id: I22f0b9d24567b032f153e30f02946b9568a4d40b --- core/rmnet_config.c | 4 +--- core/rmnet_vnd.c | 10 +--------- core/rmnet_vnd.h | 3 +-- 3 files changed, 3 insertions(+), 14 deletions(-) diff --git a/core/rmnet_config.c b/core/rmnet_config.c index 4448a8efaf3b..1d0d71dd0d99 100644 --- a/core/rmnet_config.c +++ b/core/rmnet_config.c @@ -331,9 +331,7 @@ static int rmnet_config_notify_cb(struct notifier_block *nb, netdev_dbg(dev, "Kernel unregister\n"); rmnet_force_unassociate_device(dev); break; - case NETDEV_DOWN: - rmnet_vnd_reset_mac_addr(dev); - break; + default: break; } diff --git a/core/rmnet_vnd.c b/core/rmnet_vnd.c index d5187be0497d..3674b3f94ef6 100644 --- a/core/rmnet_vnd.c +++ b/core/rmnet_vnd.c @@ -357,7 +357,7 @@ void rmnet_vnd_setup(struct net_device *rmnet_dev) rmnet_dev->netdev_ops = &rmnet_vnd_ops; rmnet_dev->mtu = RMNET_DFLT_PACKET_SIZE; rmnet_dev->needed_headroom = RMNET_NEEDED_HEADROOM; - random_ether_addr(rmnet_dev->perm_addr); + random_ether_addr(rmnet_dev->dev_addr); rmnet_dev->tx_queue_len = RMNET_TX_QUEUE_LEN; /* Raw IP mode */ @@ -444,11 +444,3 @@ int rmnet_vnd_do_flow_control(struct net_device *rmnet_dev, int enable) return 0; } - -void rmnet_vnd_reset_mac_addr(struct net_device *dev) -{ - if (dev->netdev_ops != &rmnet_vnd_ops) - return; - - random_ether_addr(dev->perm_addr); -} diff --git a/core/rmnet_vnd.h b/core/rmnet_vnd.h index 7fa9500a2aca..8b26f7cb63f9 100644 --- a/core/rmnet_vnd.h +++ b/core/rmnet_vnd.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2013-2018, 2020 The Linux Foundation. All rights reserved. +/* Copyright (c) 2013-2018, The Linux Foundation. All rights reserved. * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 and @@ -27,5 +27,4 @@ void rmnet_vnd_rx_fixup(struct net_device *dev, u32 skb_len); void rmnet_vnd_tx_fixup(struct net_device *dev, u32 skb_len); u8 rmnet_vnd_get_mux(struct net_device *rmnet_dev); void rmnet_vnd_setup(struct net_device *dev); -void rmnet_vnd_reset_mac_addr(struct net_device *dev); #endif /* _RMNET_VND_H_ */ From 31bf9bae378fe6fc37e03f00805eea9300376006 Mon Sep 17 00:00:00 2001 From: Divya Sharma Date: Tue, 6 Oct 2020 08:15:33 -0700 Subject: [PATCH 43/54] Revert "rmnet_core: Check for contiguous QMAP headers during deaggregation" This reverts commit df2d62bed079908c127051de3385f3560c5f8c90. Change-Id: Icc987d5f1ba74199f420e2ee65902d8e60e4fda9 --- core/rmnet_descriptor.c | 40 +++++++--------------------------------- 1 file changed, 7 insertions(+), 33 deletions(-) diff --git a/core/rmnet_descriptor.c b/core/rmnet_descriptor.c index 79612ec49cd3..c0b95f330c4c 100644 --- a/core/rmnet_descriptor.c +++ b/core/rmnet_descriptor.c @@ -558,36 +558,20 @@ int rmnet_frag_flow_command(struct rmnet_frag_descriptor *frag_desc, } EXPORT_SYMBOL(rmnet_frag_flow_command); -static int rmnet_frag_deaggregate_one(struct sk_buff *skb, +static int rmnet_frag_deaggregate_one(struct skb_shared_info *shinfo, struct rmnet_port *port, struct list_head *list, u32 start_frag) { - struct skb_shared_info *shinfo = skb_shinfo(skb); struct rmnet_frag_descriptor *frag_desc; - struct rmnet_map_header *maph, __maph; + struct rmnet_map_header *maph; skb_frag_t *frag; u32 i; u32 pkt_len; int rc; frag = &shinfo->frags[start_frag]; - /* Grab the QMAP header. Careful, as there's no guarantee that it's - * continugous! - */ - if (likely(skb_frag_size(frag) >= sizeof(*maph))) { - maph = skb_frag_address(frag); - } else { - /* The header's split across pages. We can rebuild it. - * Probably not faster or stronger than before. But certainly - * more linear. - */ - if (skb_copy_bits(skb, 0, &__maph, sizeof(&__maph)) < 0) - return -1; - - maph = &__maph; - } - + maph = skb_frag_address(frag); pkt_len = ntohs(maph->pkt_len); /* Catch empty frames */ if (!pkt_len) @@ -606,19 +590,9 @@ static int rmnet_frag_deaggregate_one(struct sk_buff *skb, u32 hsize = 0; u8 type; - /* Check the type. This seems like should be overkill for less - * than a single byte, doesn't it? - */ - if (likely(skb_frag_size(frag) >= sizeof(*maph) + 1)) { - type = *((u8 *)maph + sizeof(*maph)); - } else { - if (skb_copy_bits(skb, sizeof(*maph), &type, - sizeof(type)) < 0) - return -1; - } - - /* Type only uses the first 7 bits */ - switch ((type & 0xFE) >> 1) { + type = ((struct rmnet_map_v5_coal_header *) + (maph + 1))->header_type; + switch (type) { case RMNET_MAP_HEADER_TYPE_COALESCING: hsize = sizeof(struct rmnet_map_v5_coal_header); break; @@ -673,7 +647,7 @@ void rmnet_frag_deaggregate(struct sk_buff *skb, struct rmnet_port *port, int rc; while (i < shinfo->nr_frags) { - rc = rmnet_frag_deaggregate_one(skb, port, list, i); + rc = rmnet_frag_deaggregate_one(shinfo, port, list, i); if (rc < 0) return; From 398da54894d88521bf46d7dfe024e248c86d0b07 Mon Sep 17 00:00:00 2001 From: Subash Abhinov Kasiviswanathan Date: Tue, 27 Oct 2020 13:44:13 -0700 Subject: [PATCH 44/54] dfc: release WDA client eariler During SSR, release QMI WDA client before waiting for transaction to expire to reduce wait time. Change-Id: I1e60e3830e82f59a86d9d088840f8e1df80e6065 Acked-by: Weiyi Chen Signed-off-by: Subash Abhinov Kasiviswanathan --- core/qmi_rmnet.c | 11 +++++++++-- core/qmi_rmnet_i.h | 4 ++++ core/wda_qmi.c | 8 +++++++- 3 files changed, 20 insertions(+), 3 deletions(-) diff --git a/core/qmi_rmnet.c b/core/qmi_rmnet.c index 864f2da0a4bf..3c74fd09dcff 100644 --- a/core/qmi_rmnet.c +++ b/core/qmi_rmnet.c @@ -708,6 +708,7 @@ void qmi_rmnet_change_link(struct net_device *dev, void *port, void *tcm_pt) { struct qmi_info *qmi = (struct qmi_info *)rmnet_get_qmi_pt(port); struct tcmsg *tcm = (struct tcmsg *)tcm_pt; + void *wda_data = NULL; switch (tcm->tcm_family) { case NLMSG_FLOW_ACTIVATE: @@ -750,6 +751,11 @@ void qmi_rmnet_change_link(struct net_device *dev, void *port, void *tcm_pt) return; if (tcm->tcm_handle == 0) { /* instance 0 */ rmnet_clear_powersave_format(port); + if (qmi->wda_client) + wda_data = qmi->wda_client; + else if (qmi->wda_pending) + wda_data = qmi->wda_pending; + wda_qmi_client_release(wda_data); qmi_rmnet_work_exit(port); } qmi_rmnet_delete_client(port, qmi, tcm); @@ -780,13 +786,14 @@ void qmi_rmnet_qmi_exit(void *qmi_pt, void *port) ASSERT_RTNL(); - qmi_rmnet_work_exit(port); - if (qmi->wda_client) data = qmi->wda_client; else if (qmi->wda_pending) data = qmi->wda_pending; + wda_qmi_client_release(data); + qmi_rmnet_work_exit(port); + if (data) { wda_qmi_client_exit(data); qmi->wda_client = NULL; diff --git a/core/qmi_rmnet_i.h b/core/qmi_rmnet_i.h index 924074d5c761..636a5fe966b3 100644 --- a/core/qmi_rmnet_i.h +++ b/core/qmi_rmnet_i.h @@ -224,6 +224,7 @@ wda_qmi_client_init(void *port, struct svc_info *psvc, struct qmi_info *qmi); void wda_qmi_client_exit(void *wda_data); int wda_set_powersave_mode(void *wda_data, u8 enable); void qmi_rmnet_flush_ps_wq(void); +void wda_qmi_client_release(void *wda_data); #else static inline int wda_qmi_client_init(void *port, struct svc_info *psvc, struct qmi_info *qmi) @@ -242,5 +243,8 @@ static inline int wda_set_powersave_mode(void *wda_data, u8 enable) static inline void qmi_rmnet_flush_ps_wq(void) { } +static inline void wda_qmi_client_release(void *wda_data) +{ +} #endif #endif /*_RMNET_QMI_I_H*/ diff --git a/core/wda_qmi.c b/core/wda_qmi.c index 5e3cc106b878..ab0313a3a782 100644 --- a/core/wda_qmi.c +++ b/core/wda_qmi.c @@ -461,7 +461,6 @@ void wda_qmi_client_exit(void *wda_data) data->restart_state = 1; trace_wda_client_state_down(0); - qmi_handle_release(&data->handle); destroy_workqueue(data->wda_wq); kfree(data); } @@ -471,3 +470,10 @@ int wda_set_powersave_mode(void *wda_data, uint8_t enable) trace_wda_set_powersave_mode(enable); return wda_set_powersave_mode_req(wda_data, enable); } + +void wda_qmi_client_release(void *wda_data) +{ + if (!wda_data) + return; + qmi_handle_release(&((struct wda_qmi_data *)wda_data)->handle); +} From 253b60e402039ce8b67715b40328a92087e1eeef Mon Sep 17 00:00:00 2001 From: Sean Tranchetti Date: Thu, 29 Oct 2020 10:59:47 -0700 Subject: [PATCH 45/54] rmnet_core: check tcp flags when segmenting packets Now that the hardware will coalesce additional flags into the RSC frame, we must check the TCP flags explicitly and remove ones that make no sense when segmenting the RSC frame for any reason. In particular, we must check for TCP_FIN, and clear this flag from any segmetents of the RSSC frame except the last to avoid breaking the TCP stream integrity. Change-Id: I7f80437991821ab29cd190a9d5bd45e9cdb4c333 Signed-off-by: Sean Tranchetti --- core/rmnet_descriptor.c | 28 ++++++++++++++++++++++++++++ core/rmnet_descriptor.h | 4 +++- core/rmnet_map_data.c | 22 +++++++++++++++++----- 3 files changed, 48 insertions(+), 6 deletions(-) diff --git a/core/rmnet_descriptor.c b/core/rmnet_descriptor.c index c71d44c63d2e..f7818d90a20c 100644 --- a/core/rmnet_descriptor.c +++ b/core/rmnet_descriptor.c @@ -849,6 +849,16 @@ skip_frags: th->seq = frag_desc->tcp_seq; } + if (frag_desc->tcp_flags_set) { + struct tcphdr *th; + __be16 *flags; + + th = (struct tcphdr *) + (rmnet_map_data_ptr(head_skb) + frag_desc->ip_len); + flags = (__be16 *)&tcp_flag_word(th); + *flags = frag_desc->tcp_flags; + } + /* Handle csum offloading */ if (frag_desc->csum_valid && frag_desc->hdrs_valid) { /* Set the partial checksum information */ @@ -984,6 +994,24 @@ static void __rmnet_frag_segment_data(struct rmnet_frag_descriptor *coal_desc, new_desc->tcp_seq_set = 1; new_desc->tcp_seq = htonl(ntohl(th->seq) + coal_desc->data_offset); + + /* Don't allow any dangerous flags to appear in any segments + * other than the last. + */ + if (th->fin || th->psh) { + if (offset + dlen < coal_desc->len) { + __be32 flag_word = tcp_flag_word(th); + + /* Clear the FIN and PSH flags from this + * segment. + */ + flag_word &= ~TCP_FLAG_FIN; + flag_word &= ~TCP_FLAG_PSH; + + new_desc->tcp_flags_set = 1; + new_desc->tcp_flags = *((__be16 *)&flag_word); + } + } } else if (coal_desc->trans_proto == IPPROTO_UDP) { struct udphdr *uh, __uh; diff --git a/core/rmnet_descriptor.h b/core/rmnet_descriptor.h index 142a14a5be1d..f7bfa55bffe7 100644 --- a/core/rmnet_descriptor.h +++ b/core/rmnet_descriptor.h @@ -40,6 +40,7 @@ struct rmnet_frag_descriptor { u32 hash; __be32 tcp_seq; __be16 ip_id; + __be16 tcp_flags; u16 data_offset; u16 gso_size; u16 gso_segs; @@ -53,7 +54,8 @@ struct rmnet_frag_descriptor { ip_id_set:1, tcp_seq_set:1, flush_shs:1, - reserved:3; + tcp_flags_set:1, + reserved:2; }; /* Descriptor management */ diff --git a/core/rmnet_map_data.c b/core/rmnet_map_data.c index f5a9c357ba63..8388ce31c446 100644 --- a/core/rmnet_map_data.c +++ b/core/rmnet_map_data.c @@ -744,22 +744,23 @@ __rmnet_map_segment_coal_skb(struct sk_buff *coal_skb, struct rmnet_priv *priv = netdev_priv(coal_skb->dev); __sum16 *check = NULL; u32 alloc_len; + u32 dlen = coal_meta->data_len * coal_meta->pkt_count; + u32 hlen = coal_meta->ip_len + coal_meta->trans_len; bool zero_csum = false; /* We can avoid copying the data if the SKB we got from the lower-level * drivers was nonlinear. */ if (skb_is_nonlinear(coal_skb)) - alloc_len = coal_meta->ip_len + coal_meta->trans_len; + alloc_len = hlen; else - alloc_len = coal_meta->ip_len + coal_meta->trans_len + - (coal_meta->data_len * coal_meta->pkt_count); + alloc_len = hlen + dlen; skbn = alloc_skb(alloc_len, GFP_ATOMIC); if (!skbn) return; - skb_reserve(skbn, coal_meta->ip_len + coal_meta->trans_len); + skb_reserve(skbn, hlen); rmnet_map_nonlinear_copy(coal_skb, coal_meta, skbn); /* Push transport header and update necessary fields */ @@ -771,6 +772,17 @@ __rmnet_map_segment_coal_skb(struct sk_buff *coal_skb, th->seq = htonl(ntohl(th->seq) + coal_meta->data_offset); check = &th->check; + + /* Don't allow dangerous flags to be set in any segment but the + * last one. + */ + if (th->fin || th->psh) { + if (hlen + coal_meta->data_offset + dlen < + coal_skb->len) { + th->fin = 0; + th->psh = 0; + } + } } else if (coal_meta->trans_proto == IPPROTO_UDP) { struct udphdr *uh = udp_hdr(skbn); @@ -846,7 +858,7 @@ __rmnet_map_segment_coal_skb(struct sk_buff *coal_skb, __skb_queue_tail(list, skbn); /* Update meta information to move past the data we just segmented */ - coal_meta->data_offset += coal_meta->data_len * coal_meta->pkt_count; + coal_meta->data_offset += dlen; coal_meta->pkt_id = pkt_id + 1; coal_meta->pkt_count = 0; } From b8fea6fab41605c9d64b8a17fdcc60ac58b4bfc2 Mon Sep 17 00:00:00 2001 From: Subash Abhinov Kasiviswanathan Date: Mon, 9 Nov 2020 15:47:42 -0800 Subject: [PATCH 46/54] qmi_rmnet: Remove WQ_MEM_RECLAIM from rmnet_ps_wq Previously wq was designated as WQ_MEM_RECLAIM. Remove this as it is unnessesary and will result in all notifier queueing of work and to also be on wqs designated as WQ_MEM_RECLAIM. Change-Id: I3248e3aef1594b93c16a1ab8c7c6740d53f68886 Acked-by: Raul Martinez Signed-off-by: Subash Abhinov Kasiviswanathan --- core/qmi_rmnet.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/qmi_rmnet.c b/core/qmi_rmnet.c index 3c74fd09dcff..ee9a04ba2b60 100644 --- a/core/qmi_rmnet.c +++ b/core/qmi_rmnet.c @@ -1253,7 +1253,7 @@ void qmi_rmnet_work_init(void *port) return; rmnet_ps_wq = alloc_workqueue("rmnet_powersave_work", - WQ_MEM_RECLAIM | WQ_CPU_INTENSIVE, 1); + WQ_CPU_INTENSIVE, 1); if (!rmnet_ps_wq) return; From c859bb1d2be4d957f77d9a720f6a3a57f8e0d657 Mon Sep 17 00:00:00 2001 From: Subash Abhinov Kasiviswanathan Date: Fri, 13 Nov 2020 00:46:40 -0700 Subject: [PATCH 47/54] core: Fix incorrect receive packet handling during cleanup During rmnet unregistration, the real device rx_handler is first cleared followed by the removal of rx_handler_data after the rcu synchronization. Any packets in the receive path may observe that the rx_handler is NULL. However, there is no check when dereferencing this value to use the rmnet_port information. This fixes following splat by adding the NULL check. Unable to handle kernel NULL pointer dereference at virtual address 000000000000000d pc : rmnet_rx_handler+0x124/0x284 lr : rmnet_rx_handler+0x124/0x284 rmnet_rx_handler+0x124/0x284 __netif_receive_skb_core+0x758/0xd74 __netif_receive_skb+0x50/0x17c process_backlog+0x15c/0x1b8 napi_poll+0x88/0x284 net_rx_action+0xbc/0x23c __do_softirq+0x20c/0x48c CRs-Fixed: 2818544 Change-Id: I766db0b75469f2f049eb226daa7de9ba413d8f59 Signed-off-by: Subash Abhinov Kasiviswanathan --- core/rmnet_handlers.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/core/rmnet_handlers.c b/core/rmnet_handlers.c index a640710c70ad..4f005583c07a 100644 --- a/core/rmnet_handlers.c +++ b/core/rmnet_handlers.c @@ -397,6 +397,11 @@ rx_handler_result_t rmnet_rx_handler(struct sk_buff **pskb) 0xDEF, 0xDEF, 0xDEF, NULL, NULL); dev = skb->dev; port = rmnet_get_port(dev); + if (unlikely(!port)) { + atomic_long_inc(&skb->dev->rx_nohandler); + kfree_skb(skb); + goto done; + } switch (port->rmnet_mode) { case RMNET_EPMODE_VND: From 607125c52c9971f34e472d01d933fbf3bcfd9623 Mon Sep 17 00:00:00 2001 From: Chinmay Agarwal Date: Tue, 17 Nov 2020 11:46:07 +0530 Subject: [PATCH 48/54] datarmnet: Enable rmnet_core and rmnet_ctl for monaco Enabling rmnet_core and rmnet_ctl for monaco target. Change-Id: I55a7e50ae9b79090588d745240e139a789d302e0 Signed-off-by: Chinmay Agarwal --- core/Android.mk | 1 + datarmnet_dlkm_vendor_board.mk | 1 + 2 files changed, 2 insertions(+) diff --git a/core/Android.mk b/core/Android.mk index 50ef9864dbe3..341eaf859dc2 100644 --- a/core/Android.mk +++ b/core/Android.mk @@ -1,6 +1,7 @@ ifneq ($(TARGET_PRODUCT),qssi) RMNET_CORE_DLKM_PLATFORMS_LIST := lahaina RMNET_CORE_DLKM_PLATFORMS_LIST += holi +RMNET_CORE_DLKM_PLATFORMS_LIST += monaco ifeq ($(call is-board-platform-in-list, $(RMNET_CORE_DLKM_PLATFORMS_LIST)),true) #Make file to create RMNET_CORE DLKM diff --git a/datarmnet_dlkm_vendor_board.mk b/datarmnet_dlkm_vendor_board.mk index 44a2c8ba6878..3eb26a9fc94d 100644 --- a/datarmnet_dlkm_vendor_board.mk +++ b/datarmnet_dlkm_vendor_board.mk @@ -1,6 +1,7 @@ #Build rmnet core DATA_DLKM_BOARD_PLATFORMS_LIST := lahaina DATA_DLKM_BOARD_PLATFORMS_LIST += holi +DATA_DLKM_BOARD_PLATFORMS_LIST += monaco ifneq ($(TARGET_BOARD_AUTO),true) ifeq ($(call is-board-platform-in-list,$(DATA_DLKM_BOARD_PLATFORMS_LIST)),true) BOARD_VENDOR_KERNEL_MODULES += $(KERNEL_MODULES_OUT)/rmnet_core.ko From c6b88d0bc9b77d00cfc26ea79b49d7cac6b415c0 Mon Sep 17 00:00:00 2001 From: Subash Abhinov Kasiviswanathan Date: Tue, 5 Jan 2021 12:07:38 -0700 Subject: [PATCH 49/54] rmnet_core: memset before sending genl response Ensure that the generic netlink response has a memset payload before sending to userspace. Change-Id: Ie2fa92ce80bb3c0716e779cebeaedb2d31d759c1 Acked-by: Ryan Chapman Signed-off-by: Subash Abhinov Kasiviswanathan --- core/rmnet_genl.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/core/rmnet_genl.c b/core/rmnet_genl.c index e52364ad3b7a..409c32e437c5 100644 --- a/core/rmnet_genl.c +++ b/core/rmnet_genl.c @@ -1,5 +1,5 @@ // SPDX-License-Identifier: GPL-2.0-only -/* Copyright (c) 2019-2020, The Linux Foundation. All rights reserved. +/* Copyright (c) 2019-2021, The Linux Foundation. All rights reserved. * * RMNET Data Generic Netlink * @@ -322,9 +322,9 @@ int rmnet_core_genl_pid_bps_req_hdlr(struct sk_buff *skb_2, rmnet_core_userspace_connected = 1; /* Copy to pid/byte list to the payload */ + memset(&pid_bps_resp, 0x0, + sizeof(pid_bps_resp)); if (is_req_valid) { - memset(&pid_bps_resp, 0x0, - sizeof(pid_bps_resp)); rmnet_create_pid_bps_resp(&pid_bps_resp); } pid_bps_resp.valid = 1; From b4d76675a6feeb57f7188d5354e1cf82b7adb012 Mon Sep 17 00:00:00 2001 From: Subash Abhinov Kasiviswanathan Date: Wed, 2 Dec 2020 10:58:52 -0800 Subject: [PATCH 50/54] rmnet_core: Add support for qmap steering Change SHS hooks to allow for potential qmap steering. CR-Fixed: 2851435 Change-Id: I86c49b7b2634a66606c536166b15ce5e2e629061 Signed-off-by: Subash Abhinov Kasiviswanathan --- core/rmnet_config.c | 10 +++++++++- core/rmnet_config.h | 18 +++++++++++++++++- core/rmnet_handlers.c | 36 +++++++++++++++++++++++++++++------- 3 files changed, 55 insertions(+), 9 deletions(-) mode change 100644 => 100755 core/rmnet_config.h mode change 100644 => 100755 core/rmnet_handlers.c diff --git a/core/rmnet_config.c b/core/rmnet_config.c index 4448a8efaf3b..b341099c15b7 100644 --- a/core/rmnet_config.c +++ b/core/rmnet_config.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2013-2020, The Linux Foundation. All rights reserved. +/* Copyright (c) 2013-2021, The Linux Foundation. All rights reserved. * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 and @@ -30,6 +30,9 @@ #define CONFIG_QTI_QMI_DFC 1 #define CONFIG_QTI_QMI_POWER_COLLAPSE 1 +#define QMAP_SHS_MASK 0xFF +#define QMAP_SHS_PKT_LIMIT 200 + /* Locking scheme - * The shared resource which needs to be protected is realdev->rx_handler_data. * For the writer path, this is using rtnl_lock(). The writer paths are @@ -122,6 +125,11 @@ static int rmnet_register_real_device(struct net_device *real_dev) return -ENOMEM; port->dev = real_dev; + port->phy_shs_cfg.config = RMNET_SHS_NO_DLMKR | RMNET_SHS_NO_PSH | + RMNET_SHS_STMP_ALL; + port->phy_shs_cfg.map_mask = QMAP_SHS_MASK; + port->phy_shs_cfg.max_pkts = QMAP_SHS_PKT_LIMIT; + rc = netdev_rx_handler_register(real_dev, rmnet_rx_handler, port); if (rc) { kfree(port); diff --git a/core/rmnet_config.h b/core/rmnet_config.h old mode 100644 new mode 100755 index 7d8fb8ef7f72..4bb68eec3349 --- a/core/rmnet_config.h +++ b/core/rmnet_config.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2013-2014, 2016-2020 The Linux Foundation. All rights reserved. +/* Copyright (c) 2013-2014, 2016-2021 The Linux Foundation. All rights reserved. * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 and @@ -22,6 +22,19 @@ #define RMNET_MAX_LOGICAL_EP 255 #define RMNET_MAX_VEID 4 +#define RMNET_SHS_STMP_ALL BIT(0) +#define RMNET_SHS_NO_PSH BIT(1) +#define RMNET_SHS_NO_DLMKR BIT(2) + +struct rmnet_shs_clnt_s { + u16 config; + u16 map_mask; + u16 max_pkts; + union { + struct rmnet_port *port; + } info; +}; + struct rmnet_endpoint { u8 mux_id; struct net_device *egress_dev; @@ -96,6 +109,9 @@ struct rmnet_port { struct list_head dl_list; struct rmnet_port_priv_stats stats; int dl_marker_flush; + /* Port Config for shs */ + struct rmnet_shs_clnt_s shs_cfg; + struct rmnet_shs_clnt_s phy_shs_cfg; /* Descriptor pool */ spinlock_t desc_pool_lock; diff --git a/core/rmnet_handlers.c b/core/rmnet_handlers.c old mode 100644 new mode 100755 index 4f005583c07a..0399116017f6 --- a/core/rmnet_handlers.c +++ b/core/rmnet_handlers.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2013-2020, The Linux Foundation. All rights reserved. +/* Copyright (c) 2013-2021, The Linux Foundation. All rights reserved. * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 and @@ -51,6 +51,8 @@ EXPORT_TRACEPOINT_SYMBOL(rmnet_freq_update); EXPORT_TRACEPOINT_SYMBOL(rmnet_freq_reset); EXPORT_TRACEPOINT_SYMBOL(rmnet_freq_boost); + + /* Helper Functions */ void rmnet_set_skb_proto(struct sk_buff *skb) @@ -86,12 +88,17 @@ EXPORT_SYMBOL(rmnet_slow_start_on); /* Shs hook handler */ int (*rmnet_shs_skb_entry)(struct sk_buff *skb, - struct rmnet_port *port) __rcu __read_mostly; + struct rmnet_shs_clnt_s *cfg) __rcu __read_mostly; EXPORT_SYMBOL(rmnet_shs_skb_entry); +int (*rmnet_shs_switch)(struct sk_buff *skb, + struct rmnet_shs_clnt_s *cfg) __rcu __read_mostly; +EXPORT_SYMBOL(rmnet_shs_switch); + + /* Shs hook handler for work queue*/ int (*rmnet_shs_skb_entry_wq)(struct sk_buff *skb, - struct rmnet_port *port) __rcu __read_mostly; + struct rmnet_shs_clnt_s *cfg) __rcu __read_mostly; EXPORT_SYMBOL(rmnet_shs_skb_entry_wq); /* Generic handler */ @@ -99,7 +106,8 @@ EXPORT_SYMBOL(rmnet_shs_skb_entry_wq); void rmnet_deliver_skb(struct sk_buff *skb, struct rmnet_port *port) { - int (*rmnet_shs_stamp)(struct sk_buff *skb, struct rmnet_port *port); + int (*rmnet_shs_stamp)(struct sk_buff *skb, + struct rmnet_shs_clnt_s *cfg); trace_rmnet_low(RMNET_MODULE, RMNET_DLVR_SKB, 0xDEF, 0xDEF, 0xDEF, 0xDEF, (void *)skb, NULL); @@ -113,7 +121,7 @@ rmnet_deliver_skb(struct sk_buff *skb, struct rmnet_port *port) rcu_read_lock(); rmnet_shs_stamp = rcu_dereference(rmnet_shs_skb_entry); if (rmnet_shs_stamp) { - rmnet_shs_stamp(skb, port); + rmnet_shs_stamp(skb, &port->shs_cfg); rcu_read_unlock(); return; } @@ -128,7 +136,8 @@ void rmnet_deliver_skb_wq(struct sk_buff *skb, struct rmnet_port *port, enum rmnet_packet_context ctx) { - int (*rmnet_shs_stamp)(struct sk_buff *skb, struct rmnet_port *port); + int (*rmnet_shs_stamp)(struct sk_buff *skb, + struct rmnet_shs_clnt_s *cfg); struct rmnet_priv *priv = netdev_priv(skb->dev); trace_rmnet_low(RMNET_MODULE, RMNET_DLVR_SKB, 0xDEF, 0xDEF, @@ -147,7 +156,7 @@ rmnet_deliver_skb_wq(struct sk_buff *skb, struct rmnet_port *port, rmnet_shs_stamp = (!ctx) ? rcu_dereference(rmnet_shs_skb_entry) : rcu_dereference(rmnet_shs_skb_entry_wq); if (rmnet_shs_stamp) { - rmnet_shs_stamp(skb, port); + rmnet_shs_stamp(skb, &port->shs_cfg); rcu_read_unlock(); return; } @@ -386,6 +395,8 @@ rx_handler_result_t rmnet_rx_handler(struct sk_buff **pskb) struct sk_buff *skb = *pskb; struct rmnet_port *port; struct net_device *dev; + int (*rmnet_core_shs_switch)(struct sk_buff *skb, + struct rmnet_shs_clnt_s *cfg); if (!skb) goto done; @@ -405,6 +416,17 @@ rx_handler_result_t rmnet_rx_handler(struct sk_buff **pskb) switch (port->rmnet_mode) { case RMNET_EPMODE_VND: + + rcu_read_lock(); + rmnet_core_shs_switch = rcu_dereference(rmnet_shs_switch); + if (rmnet_core_shs_switch && !skb->cb[1]) { + skb->cb[1] = 1; + rmnet_core_shs_switch(skb, &port->phy_shs_cfg); + rcu_read_unlock(); + return RX_HANDLER_CONSUMED; + } + rcu_read_unlock(); + rmnet_map_ingress_handler(skb, port); break; case RMNET_EPMODE_BRIDGE: From 357a4b32f4e5d90638a4a4a7edae06cab6ed6e0b Mon Sep 17 00:00:00 2001 From: Kaustubh Pandey Date: Wed, 3 Mar 2021 22:21:36 +0530 Subject: [PATCH 51/54] datarmnet: Add support for monaco_go Support added for monaco_go such that modules will be compiled for monaco_go platform as well. Change-Id: I84dcb542cef7ee88dbe1d61447512230819134c4 Signed-off-by: Kaustubh Pandey --- core/Android.mk | 1 + datarmnet_dlkm_vendor_board.mk | 1 + 2 files changed, 2 insertions(+) diff --git a/core/Android.mk b/core/Android.mk index 341eaf859dc2..be4fd2d6585e 100644 --- a/core/Android.mk +++ b/core/Android.mk @@ -2,6 +2,7 @@ ifneq ($(TARGET_PRODUCT),qssi) RMNET_CORE_DLKM_PLATFORMS_LIST := lahaina RMNET_CORE_DLKM_PLATFORMS_LIST += holi RMNET_CORE_DLKM_PLATFORMS_LIST += monaco +RMNET_CORE_DLKM_PLATFORMS_LIST += monaco_go ifeq ($(call is-board-platform-in-list, $(RMNET_CORE_DLKM_PLATFORMS_LIST)),true) #Make file to create RMNET_CORE DLKM diff --git a/datarmnet_dlkm_vendor_board.mk b/datarmnet_dlkm_vendor_board.mk index 3eb26a9fc94d..41fffa769029 100644 --- a/datarmnet_dlkm_vendor_board.mk +++ b/datarmnet_dlkm_vendor_board.mk @@ -2,6 +2,7 @@ DATA_DLKM_BOARD_PLATFORMS_LIST := lahaina DATA_DLKM_BOARD_PLATFORMS_LIST += holi DATA_DLKM_BOARD_PLATFORMS_LIST += monaco +DATA_DLKM_BOARD_PLATFORMS_LIST += monaco_go ifneq ($(TARGET_BOARD_AUTO),true) ifeq ($(call is-board-platform-in-list,$(DATA_DLKM_BOARD_PLATFORMS_LIST)),true) BOARD_VENDOR_KERNEL_MODULES += $(KERNEL_MODULES_OUT)/rmnet_core.ko From 82d77d9ec40f4b29dc7b575d2740bf3c1fef4cd7 Mon Sep 17 00:00:00 2001 From: Subash Abhinov Kasiviswanathan Date: Fri, 28 May 2021 11:40:50 -0600 Subject: [PATCH 52/54] dfc_qmi: Honor tcp ancillary bit even when there is no change in grant The tcp ancillary bit has been updated to be sent in almost all cases unlike earlier where this was sent in case of TCP DL traffic. This bit will now not be set only in case of adverse scenarios, so this change ensures that the ACK queue state is updated in case the ancillary bit is unset. CRs-Fixed: 2957344 Change-Id: I4e1f26c9d3fabc64401284e36b48d82a4f3a5161 Signed-off-by: Subash Abhinov Kasiviswanathan --- core/dfc_qmi.c | 24 +++++++++++++++++------- 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/core/dfc_qmi.c b/core/dfc_qmi.c index f1bb16b82713..f52d9e3e032f 100644 --- a/core/dfc_qmi.c +++ b/core/dfc_qmi.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018-2020, The Linux Foundation. All rights reserved. + * Copyright (c) 2018-2021, The Linux Foundation. All rights reserved. * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 and @@ -923,12 +923,18 @@ int dfc_bearer_flow_ctl(struct net_device *dev, enable = bearer->grant_size ? true : false; - qmi_rmnet_flow_control(dev, bearer->mq_idx, enable); + /* Do not flow disable tcp ack q in tcp bidir + * ACK queue opened first to drain ACKs faster + * Although since tcp ancillary is true most of the time, + * this shouldn't really make a difference + * If there is non zero grant but tcp ancillary is false, + * send out ACKs anyway + */ + if (bearer->ack_mq_idx != INVALID_MQ) + qmi_rmnet_flow_control(dev, bearer->ack_mq_idx, + enable || bearer->tcp_bidir); - /* Do not flow disable tcp ack q in tcp bidir */ - if (bearer->ack_mq_idx != INVALID_MQ && - (enable || !bearer->tcp_bidir)) - qmi_rmnet_flow_control(dev, bearer->ack_mq_idx, enable); + qmi_rmnet_flow_control(dev, bearer->mq_idx, enable); if (!enable && bearer->ack_req) dfc_send_ack(dev, bearer->bearer_id, @@ -1022,8 +1028,12 @@ static int dfc_update_fc_map(struct net_device *dev, struct qos_info *qos, itm->bytes_in_flight = 0; } + /* update queue state only if there is a change in grant + * or change in ancillary tcp state + */ if ((itm->grant_size == 0 && adjusted_grant > 0) || - (itm->grant_size > 0 && adjusted_grant == 0)) + (itm->grant_size > 0 && adjusted_grant == 0) || + (itm->tcp_bidir ^ DFC_IS_TCP_BIDIR(ancillary))) action = true; /* This is needed by qmap */ From 171009e7ac330f1f109f9c92eef6caa3c4ca2b53 Mon Sep 17 00:00:00 2001 From: Subash Abhinov Kasiviswanathan Date: Fri, 28 May 2021 20:29:40 -0700 Subject: [PATCH 53/54] dfc: hold wakelock while powersave timer is running Powersave alarm timer could fail and delay suspend for 2 seconds if the timer expires in 2 seconds. Change to hold a wakelock for the actual duration of the timer so the suspend can be triggered after timer expiration without delay. Change-Id: Icc3cb835d4e174955e128c5a0c80198513213b2d Acked-by: Weiyi Chen Signed-off-by: Subash Abhinov Kasiviswanathan --- core/qmi_rmnet.c | 56 +++++++++++++++++++++++++++++++++++----------- core/qmi_rmnet_i.h | 5 ++++- 2 files changed, 47 insertions(+), 14 deletions(-) diff --git a/core/qmi_rmnet.c b/core/qmi_rmnet.c index ee9a04ba2b60..84818c9b8f57 100644 --- a/core/qmi_rmnet.c +++ b/core/qmi_rmnet.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018-2020, The Linux Foundation. All rights reserved. +* Copyright (c) 2018-2021, The Linux Foundation. All rights reserved. * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 and @@ -53,7 +53,8 @@ unsigned int rmnet_wq_frequency __read_mostly = 1000; #define PS_INTERVAL (((!rmnet_wq_frequency) ? \ 1 : rmnet_wq_frequency/10) * (HZ/100)) #define NO_DELAY (0x0000 * HZ) -#define PS_INTERVAL_KT (ms_to_ktime(1000)) +#define PS_INTERVAL_MS (1000) +#define PS_INTERVAL_KT (ms_to_ktime(PS_INTERVAL_MS)) #define WATCHDOG_EXPIRE_JF (msecs_to_jiffies(50)) #ifdef CONFIG_QTI_QMI_DFC @@ -622,6 +623,7 @@ qmi_rmnet_setup_client(void *port, struct qmi_info *qmi, struct tcmsg *tcm) if (!qmi) return -ENOMEM; + qmi->ws = wakeup_source_register(NULL, "RMNET_DFC"); rmnet_init_qmi_pt(port, qmi); } @@ -670,6 +672,7 @@ __qmi_rmnet_delete_client(void *port, struct qmi_info *qmi, int idx) if (!qmi_rmnet_has_client(qmi) && !qmi_rmnet_has_pending(qmi)) { rmnet_reset_qmi_pt(port); + wakeup_source_unregister(qmi->ws); kfree(qmi); return 0; } @@ -739,6 +742,7 @@ void qmi_rmnet_change_link(struct net_device *dev, void *port, void *tcm_pt) !qmi_rmnet_has_client(qmi) && !qmi_rmnet_has_pending(qmi)) { rmnet_reset_qmi_pt(port); + wakeup_source_unregister(qmi->ws); kfree(qmi); } } else if (tcm->tcm_ifindex & FLAG_POWERSAVE_MASK) { @@ -1142,6 +1146,22 @@ static enum alarmtimer_restart qmi_rmnet_work_alarm(struct alarm *atimer, return ALARMTIMER_NORESTART; } +static void dfc_wakelock_acquire(struct qmi_info *qmi) +{ + if (qmi && !qmi->wakelock_active) { + __pm_stay_awake(qmi->ws); + qmi->wakelock_active = true; + } +} + +static void dfc_wakelock_release(struct qmi_info *qmi) +{ + if (qmi && qmi->wakelock_active) { + __pm_relax(qmi->ws); + qmi->wakelock_active = false; + } +} + static void qmi_rmnet_check_stats(struct work_struct *work) { struct rmnet_powersave_work *real_work; @@ -1161,6 +1181,17 @@ static void qmi_rmnet_check_stats(struct work_struct *work) if (unlikely(!qmi)) return; + dfc_wakelock_release(qmi); + + rmnet_get_packets(real_work->port, &rx, &tx); + rxd = rx - real_work->old_rx_pkts; + txd = tx - real_work->old_tx_pkts; + real_work->old_rx_pkts = rx; + real_work->old_tx_pkts = tx; + + dl_msg_active = qmi->dl_msg_active; + qmi->dl_msg_active = false; + if (qmi->ps_enabled) { /* Ready to accept grant */ @@ -1181,15 +1212,6 @@ static void qmi_rmnet_check_stats(struct work_struct *work) goto end; } - rmnet_get_packets(real_work->port, &rx, &tx); - rxd = rx - real_work->old_rx_pkts; - txd = tx - real_work->old_tx_pkts; - real_work->old_rx_pkts = rx; - real_work->old_tx_pkts = tx; - - dl_msg_active = qmi->dl_msg_active; - qmi->dl_msg_active = false; - if (!rxd && !txd) { /* If no DL msg received and there is a flow disabled, * (likely in RLF), no need to enter powersave @@ -1223,12 +1245,19 @@ static void qmi_rmnet_check_stats(struct work_struct *work) end: rcu_read_lock(); if (!rmnet_work_quit) { - if (use_alarm_timer) + if (use_alarm_timer) { + /* Suspend will fail and get delayed for 2s if + * alarmtimer expires within 2s. Hold a wakelock + * for the actual timer duration to prevent suspend + */ + if (PS_INTERVAL_MS < 2000) + dfc_wakelock_acquire(qmi); alarm_start_relative(&real_work->atimer, PS_INTERVAL_KT); - else + } else { queue_delayed_work(rmnet_ps_wq, &real_work->work, PS_INTERVAL); + } } rcu_read_unlock(); } @@ -1306,6 +1335,7 @@ void qmi_rmnet_work_exit(void *port) rmnet_ps_wq = NULL; kfree(rmnet_work); rmnet_work = NULL; + dfc_wakelock_release((struct qmi_info *)rmnet_get_qmi_pt(port)); } EXPORT_SYMBOL(qmi_rmnet_work_exit); diff --git a/core/qmi_rmnet_i.h b/core/qmi_rmnet_i.h index 636a5fe966b3..082351065686 100644 --- a/core/qmi_rmnet_i.h +++ b/core/qmi_rmnet_i.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018-2020, The Linux Foundation. All rights reserved. + * Copyright (c) 2018-2021, The Linux Foundation. All rights reserved. * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 and @@ -17,6 +17,7 @@ #include #include #include +#include #define MAX_MQ_NUM 16 #define MAX_CLIENT_NUM 2 @@ -107,6 +108,8 @@ struct qmi_info { bool ps_enabled; bool dl_msg_active; bool ps_ignore_grant; + bool wakelock_active; + struct wakeup_source *ws; }; enum data_ep_type_enum_v01 { From 261c24f7ceea50241429f11be8dd4d257b29f49b Mon Sep 17 00:00:00 2001 From: shouyi Hu Date: Sun, 20 Jun 2021 15:45:26 +0800 Subject: [PATCH 54/54] monaco_go_aon: add monaco_go_aon PLATFORMS_LIST add monaco_go_aon in RMNET_CORE_DLKM_PLATFORMS_LIST add monaco_go_aon in DATA_DLKM_BOARD_PLATFORMS_LIST Change-Id: I7df542c27d3070df98d12ba136508671dda8f469 --- core/Android.mk | 1 + datarmnet_dlkm_vendor_board.mk | 1 + 2 files changed, 2 insertions(+) diff --git a/core/Android.mk b/core/Android.mk index be4fd2d6585e..9aad807d7d38 100644 --- a/core/Android.mk +++ b/core/Android.mk @@ -3,6 +3,7 @@ RMNET_CORE_DLKM_PLATFORMS_LIST := lahaina RMNET_CORE_DLKM_PLATFORMS_LIST += holi RMNET_CORE_DLKM_PLATFORMS_LIST += monaco RMNET_CORE_DLKM_PLATFORMS_LIST += monaco_go +RMNET_CORE_DLKM_PLATFORMS_LIST += monaco_go_aon ifeq ($(call is-board-platform-in-list, $(RMNET_CORE_DLKM_PLATFORMS_LIST)),true) #Make file to create RMNET_CORE DLKM diff --git a/datarmnet_dlkm_vendor_board.mk b/datarmnet_dlkm_vendor_board.mk index 41fffa769029..60a81a7d351b 100644 --- a/datarmnet_dlkm_vendor_board.mk +++ b/datarmnet_dlkm_vendor_board.mk @@ -3,6 +3,7 @@ DATA_DLKM_BOARD_PLATFORMS_LIST := lahaina DATA_DLKM_BOARD_PLATFORMS_LIST += holi DATA_DLKM_BOARD_PLATFORMS_LIST += monaco DATA_DLKM_BOARD_PLATFORMS_LIST += monaco_go +DATA_DLKM_BOARD_PLATFORMS_LIST += monaco_go_aon ifneq ($(TARGET_BOARD_AUTO),true) ifeq ($(call is-board-platform-in-list,$(DATA_DLKM_BOARD_PLATFORMS_LIST)),true) BOARD_VENDOR_KERNEL_MODULES += $(KERNEL_MODULES_OUT)/rmnet_core.ko