diff --git a/android/abi_gki_aarch64_d6430e6763.xml b/android/abi_gki_aarch64_d6430e6763.xml
index 637e934a7424..6993e484be01 100644
--- a/android/abi_gki_aarch64_d6430e6763.xml
+++ b/android/abi_gki_aarch64_d6430e6763.xml
@@ -3202,291 +3202,291 @@
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
-
+
+
-
+
-
+
-
+
-
+
@@ -3605,60 +3605,60 @@
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
@@ -21430,49 +21430,49 @@
-
+
-
-
-
-
-
+
+
+
+
+
-
-
+
+
-
-
-
-
-
-
-
+
+
+
+
+
+
+
-
-
+
+
-
-
+
+
-
-
-
-
+
+
+
+
-
-
+
+
-
-
+
+
@@ -30300,7 +30300,58 @@
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
@@ -30311,7 +30362,7 @@
-
+
@@ -30319,6 +30370,167 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
@@ -30852,24 +31064,24 @@
-
+
-
+
-
+
-
+
-
+
@@ -30878,21 +31090,21 @@
-
+
-
+
-
+
-
+
@@ -31036,7 +31248,7 @@
-
+
@@ -31455,15 +31667,15 @@
-
+
-
+
-
+
-
+
@@ -31724,26 +31936,26 @@
-
+
-
+
-
+
-
+
-
+
-
+
@@ -31798,13 +32010,13 @@
-
+
-
+
@@ -31823,7 +32035,7 @@
-
+
@@ -31966,10 +32178,10 @@
-
+
-
+
@@ -31991,10 +32203,10 @@
-
+
-
+
@@ -32368,7 +32580,7 @@
-
+
@@ -32486,7 +32698,7 @@
-
+
@@ -32507,14 +32719,14 @@
-
+
-
+
@@ -32523,17 +32735,17 @@
-
+
-
+
-
+
@@ -32546,7 +32758,7 @@
-
+
@@ -32562,17 +32774,17 @@
-
+
-
+
-
+
@@ -32595,12 +32807,12 @@
-
+
-
+
@@ -32617,12 +32829,12 @@
-
+
-
+
@@ -32632,7 +32844,7 @@
-
+
@@ -32647,12 +32859,12 @@
-
+
-
+
-
+
@@ -32670,7 +32882,7 @@
-
+
@@ -34134,11 +34346,11 @@
-
+
-
+
@@ -34146,7 +34358,7 @@
-
+
@@ -34171,11 +34383,11 @@
-
+
-
+
@@ -36116,41 +36328,6 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
@@ -39637,7 +39814,7 @@
-
+
@@ -39874,7 +40051,7 @@
-
+
@@ -39885,11 +40062,11 @@
-
+
-
+
@@ -39928,7 +40105,7 @@
-
+
@@ -39936,6 +40113,14 @@
+
+
+
+
+
+
+
+
@@ -39953,7 +40138,7 @@
-
+
@@ -40115,14 +40300,6 @@
-
-
-
-
-
-
-
-
@@ -40198,13 +40375,13 @@
-
+
-
+
@@ -41548,10 +41725,10 @@
-
+
-
+
@@ -42065,7 +42242,7 @@
-
+
@@ -42090,7 +42267,7 @@
-
+
@@ -48317,7 +48494,7 @@
-
+
@@ -50579,16 +50756,6 @@
-
-
-
-
-
-
-
-
-
-
@@ -50620,9 +50787,9 @@
-
+
-
+
@@ -50633,7 +50800,7 @@
-
+
@@ -50647,7 +50814,7 @@
-
+
@@ -51019,7 +51186,7 @@
-
+
@@ -51217,7 +51384,7 @@
-
+
@@ -51303,7 +51470,7 @@
-
+
@@ -51325,7 +51492,7 @@
-
+
@@ -51463,7 +51630,7 @@
-
+
@@ -51591,43 +51758,22 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
+
-
+
-
+
-
-
-
+
+
@@ -51681,7 +51827,7 @@
-
+
@@ -51896,7 +52042,7 @@
-
+
@@ -51911,7 +52057,7 @@
-
+
@@ -51947,7 +52093,7 @@
-
+
@@ -52373,7 +52519,7 @@
-
+
@@ -52928,7 +53074,7 @@
-
+
@@ -53323,7 +53469,7 @@
-
+
@@ -53599,10 +53745,10 @@
-
+
-
+
@@ -53615,10 +53761,10 @@
-
+
-
+
@@ -53633,10 +53779,10 @@
-
+
-
+
@@ -53745,7 +53891,7 @@
-
+
@@ -53763,7 +53909,7 @@
-
+
@@ -54531,7 +54677,7 @@
-
+
@@ -54679,13 +54825,13 @@
-
+
-
+
@@ -57357,7 +57503,7 @@
-
+
@@ -58098,7 +58244,7 @@
-
+
@@ -58110,7 +58256,7 @@
-
+
@@ -58148,7 +58294,7 @@
-
+
@@ -58156,7 +58302,7 @@
-
+
@@ -66627,7 +66773,7 @@
-
+
@@ -68952,7 +69098,7 @@
-
+
@@ -69019,7 +69165,7 @@
-
+
@@ -69046,7 +69192,7 @@
-
+
@@ -69254,7 +69400,7 @@
-
+
@@ -69270,7 +69416,7 @@
-
+
@@ -69281,12 +69427,12 @@
-
+
-
+
-
+
@@ -69326,13 +69472,13 @@
-
+
-
+
@@ -69400,7 +69546,7 @@
-
+
@@ -69408,33 +69554,33 @@
-
+
-
+
-
+
-
+
-
+
-
+
@@ -69447,7 +69593,7 @@
-
+
@@ -69501,7 +69647,7 @@
-
+
@@ -69583,11 +69729,11 @@
-
+
-
+
@@ -69655,7 +69801,7 @@
-
+
@@ -69689,7 +69835,7 @@
-
+
@@ -69941,7 +70087,7 @@
-
+
@@ -69960,7 +70106,7 @@
-
+
@@ -70280,13 +70426,13 @@
-
+
-
+
-
+
@@ -70310,7 +70456,7 @@
-
+
@@ -70567,7 +70713,7 @@
-
+
@@ -70581,23 +70727,23 @@
-
+
-
+
-
+
-
+
-
+
-
+
-
+
@@ -74762,32 +74908,32 @@
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
@@ -77666,7 +77812,7 @@
-
+
@@ -77674,7 +77820,7 @@
-
+
@@ -77682,7 +77828,7 @@
-
+
@@ -77766,7 +77912,7 @@
-
+
@@ -77819,7 +77965,7 @@
-
+
@@ -78658,7 +78804,7 @@
-
+
@@ -79461,7 +79607,7 @@
-
+
@@ -79508,7 +79654,7 @@
-
+
@@ -79556,10 +79702,10 @@
-
+
-
+
@@ -80700,7 +80846,7 @@
-
+
@@ -80721,7 +80867,7 @@
-
+
@@ -81580,7 +81726,7 @@
-
+
@@ -82381,7 +82527,7 @@
-
+
@@ -82397,7 +82543,7 @@
-
+
@@ -82483,7 +82629,7 @@
-
+
@@ -82491,7 +82637,7 @@
-
+
@@ -82809,7 +82955,7 @@
-
+
@@ -82934,7 +83080,7 @@
-
+
@@ -83082,7 +83228,7 @@
-
+
@@ -83189,7 +83335,7 @@
-
+
@@ -83771,7 +83917,7 @@
-
+
@@ -84348,7 +84494,7 @@
-
+
@@ -84372,7 +84518,7 @@
-
+
@@ -84818,7 +84964,7 @@
-
+
@@ -84882,7 +85028,7 @@
-
+
@@ -85709,7 +85855,7 @@
-
+
@@ -85795,7 +85941,7 @@
-
+
@@ -85816,7 +85962,7 @@
-
+
@@ -86291,7 +86437,7 @@
-
+
@@ -86502,7 +86648,7 @@
-
+
@@ -86625,7 +86771,7 @@
-
+
@@ -86670,7 +86816,7 @@
-
+
@@ -87661,7 +87807,7 @@
-
+
@@ -87670,7 +87816,7 @@
-
+
@@ -87769,10 +87915,10 @@
-
+
-
+
@@ -87834,7 +87980,7 @@
-
+
@@ -88171,7 +88317,7 @@
-
+
@@ -88185,7 +88331,7 @@
-
+
@@ -88344,7 +88490,7 @@
-
+
@@ -88462,6 +88608,14 @@
+
+
+
+
+
+
+
+
@@ -88472,12 +88626,12 @@
-
+
-
+
-
+
@@ -88494,7 +88648,7 @@
-
+
@@ -88524,7 +88678,7 @@
-
+
@@ -88596,9 +88750,9 @@
-
+
-
+
@@ -88609,7 +88763,7 @@
-
+
@@ -88625,7 +88779,7 @@
-
+
@@ -88661,7 +88815,7 @@
-
+
@@ -88709,7 +88863,15 @@
-
+
+
+
+
+
+
+
+
+
@@ -89164,7 +89326,7 @@
-
+
@@ -89180,7 +89342,7 @@
-
+
@@ -89199,7 +89361,7 @@
-
+
@@ -89223,129 +89385,129 @@
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
@@ -89374,14 +89536,14 @@
-
-
-
+
+
+
-
-
-
+
+
+
@@ -89488,46 +89650,46 @@
-
-
+
+
-
-
-
-
+
+
+
+
-
-
-
-
-
-
+
+
+
+
+
+
-
-
-
+
+
+
-
-
+
+
-
-
-
+
+
+
-
-
+
+
-
-
-
-
+
+
+
+
@@ -89595,7 +89757,7 @@
-
+
@@ -89821,7 +89983,7 @@
-
+
@@ -89843,7 +90005,7 @@
-
+
@@ -90283,7 +90445,7 @@
-
+
@@ -90370,11 +90532,11 @@
-
-
-
-
-
+
+
+
+
+
@@ -90390,12 +90552,12 @@
-
+
-
+
-
+
@@ -90578,7 +90740,7 @@
-
+
@@ -90917,6 +91079,14 @@
+
+
+
+
+
+
+
+
@@ -90973,7 +91143,7 @@
-
+
@@ -91160,15 +91330,15 @@
-
+
-
+
-
+
-
+
@@ -91265,23 +91435,23 @@
-
+
-
+
-
+
-
+
-
+
-
+
-
+
@@ -91764,7 +91934,7 @@
-
+
@@ -92984,7 +93154,7 @@
-
+
@@ -96007,7 +96177,7 @@
-
+
@@ -96023,7 +96193,7 @@
-
+
@@ -96031,7 +96201,7 @@
-
+
@@ -96039,7 +96209,7 @@
-
+
@@ -96047,7 +96217,7 @@
-
+
@@ -96055,15 +96225,7 @@
-
-
-
-
-
-
-
-
-
+
@@ -96077,7 +96239,18 @@
-
+
+
+
+
+
+
+
+
+
+
+
+
@@ -96085,7 +96258,7 @@
-
+
@@ -96096,7 +96269,7 @@
-
+
@@ -96107,7 +96280,7 @@
-
+
@@ -96115,7 +96288,7 @@
-
+
@@ -96123,52 +96296,52 @@
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
@@ -96759,7 +96932,7 @@
-
+
@@ -96783,7 +96956,7 @@
-
+
@@ -96873,10 +97046,10 @@
-
+
-
+
@@ -96884,7 +97057,7 @@
-
+
@@ -96930,7 +97103,7 @@
-
+
@@ -97494,42 +97667,42 @@
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-
+
@@ -97593,7 +97766,7 @@
-
+
@@ -97640,23 +97813,23 @@
-
+
-
+
-
+
-
+
-
+
-
+
-
+
@@ -97978,7 +98151,7 @@
-
+
@@ -98093,12 +98266,12 @@
-
+
-
+
-
+
@@ -98138,7 +98311,7 @@
-
+
@@ -98325,7 +98498,7 @@
-
+
@@ -98340,7 +98513,7 @@
-
+
@@ -98368,7 +98541,7 @@
-
+
@@ -98385,7 +98558,7 @@
-
+
@@ -98413,22 +98586,22 @@
-
+
-
+
-
+
-
+
@@ -98444,7 +98617,7 @@
-
+
@@ -98452,7 +98625,7 @@
-
+
@@ -98475,7 +98648,7 @@
-
+
@@ -98553,13 +98726,13 @@
-
+
-
+
@@ -98860,7 +99033,7 @@
-
+
@@ -99034,7 +99207,7 @@
-
+
@@ -99913,10 +100086,10 @@
-
+
-
+
@@ -102102,7 +102275,7 @@
-
+
@@ -102804,7 +102977,7 @@
-
+
diff --git a/drivers/android/vendor_hooks.c b/drivers/android/vendor_hooks.c
index 7b4b1abbb537..0d3b4faa7bc7 100644
--- a/drivers/android/vendor_hooks.c
+++ b/drivers/android/vendor_hooks.c
@@ -55,3 +55,4 @@ EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_alter_rwsem_list_add);
EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_alter_futex_plist_add);
EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_is_fpsimd_save);
EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_ipi_stop);
+EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_input_handle_event);
diff --git a/drivers/input/input.c b/drivers/input/input.c
index e2eb9b9b8363..6434e0aec875 100644
--- a/drivers/input/input.c
+++ b/drivers/input/input.c
@@ -26,6 +26,10 @@
#include "input-compat.h"
#include "input-poller.h"
+#ifndef __GENKSYMS__
+#include
+#endif
+
MODULE_AUTHOR("Vojtech Pavlik ");
MODULE_DESCRIPTION("Input core");
MODULE_LICENSE("GPL");
@@ -438,6 +442,7 @@ void input_event(struct input_dev *dev,
if (is_event_supported(type, dev->evbit, EV_MAX)) {
spin_lock_irqsave(&dev->event_lock, flags);
+ trace_android_vh_input_handle_event(dev, type, code, value);
input_handle_event(dev, type, code, value);
spin_unlock_irqrestore(&dev->event_lock, flags);
}
diff --git a/include/linux/netdev_features.h b/include/linux/netdev_features.h
index 4b19c544c59a..34d050bb1ae6 100644
--- a/include/linux/netdev_features.h
+++ b/include/linux/netdev_features.h
@@ -53,8 +53,9 @@ enum {
NETIF_F_GSO_ESP_BIT, /* ... ESP with TSO */
NETIF_F_GSO_UDP_BIT, /* ... UFO, deprecated except tuntap */
NETIF_F_GSO_UDP_L4_BIT, /* ... UDP payload GSO (not UFO) */
+ NETIF_F_GSO_FRAGLIST_BIT, /* ... Fraglist GSO */
/**/NETIF_F_GSO_LAST = /* last bit, see GSO_MASK */
- NETIF_F_GSO_UDP_L4_BIT,
+ NETIF_F_GSO_FRAGLIST_BIT,
NETIF_F_FCOE_CRC_BIT, /* FCoE CRC32 */
NETIF_F_SCTP_CRC_BIT, /* SCTP checksum offload */
@@ -80,6 +81,7 @@ enum {
NETIF_F_GRO_HW_BIT, /* Hardware Generic receive offload */
NETIF_F_HW_TLS_RECORD_BIT, /* Offload TLS record */
+ NETIF_F_GRO_FRAGLIST_BIT, /* Fraglist GRO */
/*
* Add your fresh new feature above and remember to update
@@ -150,6 +152,8 @@ enum {
#define NETIF_F_GSO_UDP_L4 __NETIF_F(GSO_UDP_L4)
#define NETIF_F_HW_TLS_TX __NETIF_F(HW_TLS_TX)
#define NETIF_F_HW_TLS_RX __NETIF_F(HW_TLS_RX)
+#define NETIF_F_GRO_FRAGLIST __NETIF_F(GRO_FRAGLIST)
+#define NETIF_F_GSO_FRAGLIST __NETIF_F(GSO_FRAGLIST)
/* Finds the next feature with the highest number of the range of start till 0.
*/
@@ -226,6 +230,9 @@ static inline int find_next_netdev_feature(u64 feature, unsigned long start)
/* changeable features with no special hardware requirements */
#define NETIF_F_SOFT_FEATURES (NETIF_F_GSO | NETIF_F_GRO)
+/* Changeable features with no special hardware requirements that defaults to off. */
+#define NETIF_F_SOFT_FEATURES_OFF NETIF_F_GRO_FRAGLIST
+
#define NETIF_F_VLAN_FEATURES (NETIF_F_HW_VLAN_CTAG_FILTER | \
NETIF_F_HW_VLAN_CTAG_RX | \
NETIF_F_HW_VLAN_CTAG_TX | \
diff --git a/include/linux/netdevice.h b/include/linux/netdevice.h
index 9c7d93783759..97c1c5d54d0a 100644
--- a/include/linux/netdevice.h
+++ b/include/linux/netdevice.h
@@ -2350,7 +2350,8 @@ struct napi_gro_cb {
/* Number of gro_receive callbacks this packet already went through */
u8 recursion_counter:4;
- /* 1 bit hole */
+ /* GRO is done by frag_list pointer chaining. */
+ u8 is_flist:1;
/* used to support CHECKSUM_COMPLETE for tunneling protocols */
__wsum csum;
@@ -2705,6 +2706,7 @@ struct net_device *dev_get_by_napi_id(unsigned int napi_id);
int netdev_get_name(struct net *net, char *name, int ifindex);
int dev_restart(struct net_device *dev);
int skb_gro_receive(struct sk_buff *p, struct sk_buff *skb);
+int skb_gro_receive_list(struct sk_buff *p, struct sk_buff *skb);
static inline unsigned int skb_gro_offset(const struct sk_buff *skb)
{
@@ -4572,6 +4574,7 @@ static inline bool net_gso_ok(netdev_features_t features, int gso_type)
BUILD_BUG_ON(SKB_GSO_ESP != (NETIF_F_GSO_ESP >> NETIF_F_GSO_SHIFT));
BUILD_BUG_ON(SKB_GSO_UDP != (NETIF_F_GSO_UDP >> NETIF_F_GSO_SHIFT));
BUILD_BUG_ON(SKB_GSO_UDP_L4 != (NETIF_F_GSO_UDP_L4 >> NETIF_F_GSO_SHIFT));
+ BUILD_BUG_ON(SKB_GSO_FRAGLIST != (NETIF_F_GSO_FRAGLIST >> NETIF_F_GSO_SHIFT));
return (features & feature) == feature;
}
diff --git a/include/linux/skbuff.h b/include/linux/skbuff.h
index 542d881978ef..89301656f985 100644
--- a/include/linux/skbuff.h
+++ b/include/linux/skbuff.h
@@ -593,6 +593,8 @@ enum {
SKB_GSO_UDP = 1 << 16,
SKB_GSO_UDP_L4 = 1 << 17,
+
+ SKB_GSO_FRAGLIST = 1 << 18,
};
#if BITS_PER_LONG > 32
@@ -3529,6 +3531,8 @@ void skb_scrub_packet(struct sk_buff *skb, bool xnet);
bool skb_gso_validate_network_len(const struct sk_buff *skb, unsigned int mtu);
bool skb_gso_validate_mac_len(const struct sk_buff *skb, unsigned int len);
struct sk_buff *skb_segment(struct sk_buff *skb, netdev_features_t features);
+struct sk_buff *skb_segment_list(struct sk_buff *skb, netdev_features_t features,
+ unsigned int offset);
struct sk_buff *skb_vlan_untag(struct sk_buff *skb);
int skb_ensure_writable(struct sk_buff *skb, int write_len);
int __skb_vlan_pop(struct sk_buff *skb, u16 *vlan_tci);
diff --git a/include/net/udp.h b/include/net/udp.h
index fabf507bce5d..e55d5f765807 100644
--- a/include/net/udp.h
+++ b/include/net/udp.h
@@ -167,7 +167,7 @@ typedef struct sock *(*udp_lookup_t)(struct sk_buff *skb, __be16 sport,
__be16 dport);
struct sk_buff *udp_gro_receive(struct list_head *head, struct sk_buff *skb,
- struct udphdr *uh, udp_lookup_t lookup);
+ struct udphdr *uh, struct sock *sk);
int udp_gro_complete(struct sk_buff *skb, int nhoff, udp_lookup_t lookup);
struct sk_buff *__udp_gso_segment(struct sk_buff *gso_skb,
diff --git a/include/trace/hooks/debug.h b/include/trace/hooks/debug.h
index d3e310ec4b2d..d3127643ece8 100644
--- a/include/trace/hooks/debug.h
+++ b/include/trace/hooks/debug.h
@@ -12,11 +12,19 @@
#if defined(CONFIG_TRACEPOINTS) && defined(CONFIG_ANDROID_VENDOR_HOOKS)
+struct pt_regs;
DECLARE_HOOK(android_vh_ipi_stop,
TP_PROTO(struct pt_regs *regs),
TP_ARGS(regs))
+
+struct input_dev;
+DECLARE_HOOK(android_vh_input_handle_event,
+ TP_PROTO(struct input_dev *dev, unsigned int type,
+ unsigned int code, int value),
+ TP_ARGS(dev, type, code, value))
#else
#define trace_android_vh_ipi_stop(regs)
+#define trace_android_vh_input_handle_event(dev, type, code, value)
#endif
#endif /* _TRACE_HOOK_DEBUG_H */
diff --git a/net/core/dev.c b/net/core/dev.c
index 782c28482b2a..fb3238c916ab 100644
--- a/net/core/dev.c
+++ b/net/core/dev.c
@@ -3032,7 +3032,7 @@ struct sk_buff *__skb_gso_segment(struct sk_buff *skb,
segs = skb_mac_gso_segment(skb, features);
- if (unlikely(skb_needs_check(skb, tx_path) && !IS_ERR(segs)))
+ if (segs != skb && unlikely(skb_needs_check(skb, tx_path) && !IS_ERR(segs)))
skb_warn_bad_offload(skb);
return segs;
@@ -5609,13 +5609,12 @@ static void napi_skb_free_stolen_head(struct sk_buff *skb)
kmem_cache_free(skbuff_head_cache, skb);
}
-static gro_result_t napi_skb_finish(struct napi_struct *napi,
- struct sk_buff *skb,
- gro_result_t ret)
+static gro_result_t napi_skb_finish(gro_result_t ret, struct sk_buff *skb)
{
switch (ret) {
case GRO_NORMAL:
- gro_normal_one(napi, skb);
+ if (netif_receive_skb_internal(skb))
+ ret = GRO_DROP;
break;
case GRO_DROP:
@@ -5647,7 +5646,7 @@ gro_result_t napi_gro_receive(struct napi_struct *napi, struct sk_buff *skb)
skb_gro_reset_offset(skb);
- ret = napi_skb_finish(napi, skb, dev_gro_receive(napi, skb));
+ ret = napi_skb_finish(dev_gro_receive(napi, skb), skb);
trace_napi_gro_receive_exit(ret);
return ret;
@@ -9042,7 +9041,7 @@ int register_netdevice(struct net_device *dev)
/* Transfer changeable features to wanted_features and enable
* software offloads (GSO and GRO).
*/
- dev->hw_features |= NETIF_F_SOFT_FEATURES;
+ dev->hw_features |= (NETIF_F_SOFT_FEATURES | NETIF_F_SOFT_FEATURES_OFF);
dev->features |= NETIF_F_SOFT_FEATURES;
if (dev->netdev_ops->ndo_udp_tunnel_add) {
diff --git a/net/core/skbuff.c b/net/core/skbuff.c
index 126be82a84f7..e53fe52dd764 100644
--- a/net/core/skbuff.c
+++ b/net/core/skbuff.c
@@ -3640,6 +3640,97 @@ static inline skb_frag_t skb_head_frag_to_page_desc(struct sk_buff *frag_skb)
return head_frag;
}
+struct sk_buff *skb_segment_list(struct sk_buff *skb,
+ netdev_features_t features,
+ unsigned int offset)
+{
+ struct sk_buff *list_skb = skb_shinfo(skb)->frag_list;
+ unsigned int tnl_hlen = skb_tnl_header_len(skb);
+ unsigned int delta_truesize = 0;
+ unsigned int delta_len = 0;
+ struct sk_buff *tail = NULL;
+ struct sk_buff *nskb;
+
+ skb_push(skb, -skb_network_offset(skb) + offset);
+
+ skb_shinfo(skb)->frag_list = NULL;
+
+ do {
+ nskb = list_skb;
+ list_skb = list_skb->next;
+
+ if (!tail)
+ skb->next = nskb;
+ else
+ tail->next = nskb;
+
+ tail = nskb;
+
+ delta_len += nskb->len;
+ delta_truesize += nskb->truesize;
+
+ skb_push(nskb, -skb_network_offset(nskb) + offset);
+
+ __copy_skb_header(nskb, skb);
+
+ skb_headers_offset_update(nskb, skb_headroom(nskb) - skb_headroom(skb));
+ skb_copy_from_linear_data_offset(skb, -tnl_hlen,
+ nskb->data - tnl_hlen,
+ offset + tnl_hlen);
+
+ if (skb_needs_linearize(nskb, features) &&
+ __skb_linearize(nskb))
+ goto err_linearize;
+
+ } while (list_skb);
+
+ skb->truesize = skb->truesize - delta_truesize;
+ skb->data_len = skb->data_len - delta_len;
+ skb->len = skb->len - delta_len;
+
+ skb_gso_reset(skb);
+
+ skb->prev = tail;
+
+ if (skb_needs_linearize(skb, features) &&
+ __skb_linearize(skb))
+ goto err_linearize;
+
+ skb_get(skb);
+
+ return skb;
+
+err_linearize:
+ kfree_skb_list(skb->next);
+ skb->next = NULL;
+ return ERR_PTR(-ENOMEM);
+}
+EXPORT_SYMBOL_GPL(skb_segment_list);
+
+int skb_gro_receive_list(struct sk_buff *p, struct sk_buff *skb)
+{
+ if (unlikely(p->len + skb->len >= 65536))
+ return -E2BIG;
+
+ if (NAPI_GRO_CB(p)->last == p)
+ skb_shinfo(p)->frag_list = skb;
+ else
+ NAPI_GRO_CB(p)->last->next = skb;
+
+ skb_pull(skb, skb_gro_offset(skb));
+
+ NAPI_GRO_CB(p)->last = skb;
+ NAPI_GRO_CB(p)->count++;
+ p->data_len += skb->len;
+ p->truesize += skb->truesize;
+ p->len += skb->len;
+
+ NAPI_GRO_CB(skb)->same_flow = 1;
+
+ return 0;
+}
+EXPORT_SYMBOL_GPL(skb_gro_receive_list);
+
/**
* skb_segment - Perform protocol segmentation on skb.
* @head_skb: buffer to segment
diff --git a/net/ipv4/udp_offload.c b/net/ipv4/udp_offload.c
index a3908e55ed89..9455b0a0776b 100644
--- a/net/ipv4/udp_offload.c
+++ b/net/ipv4/udp_offload.c
@@ -184,6 +184,20 @@ out_unlock:
}
EXPORT_SYMBOL(skb_udp_tunnel_segment);
+static struct sk_buff *__udp_gso_segment_list(struct sk_buff *skb,
+ netdev_features_t features)
+{
+ unsigned int mss = skb_shinfo(skb)->gso_size;
+
+ skb = skb_segment_list(skb, features, skb_mac_header_len(skb));
+ if (IS_ERR(skb))
+ return skb;
+
+ udp_hdr(skb)->len = htons(sizeof(struct udphdr) + mss);
+
+ return skb;
+}
+
struct sk_buff *__udp_gso_segment(struct sk_buff *gso_skb,
netdev_features_t features)
{
@@ -196,6 +210,9 @@ struct sk_buff *__udp_gso_segment(struct sk_buff *gso_skb,
__sum16 check;
__be16 newlen;
+ if (skb_shinfo(gso_skb)->gso_type & SKB_GSO_FRAGLIST)
+ return __udp_gso_segment_list(gso_skb, features);
+
mss = skb_shinfo(gso_skb)->gso_size;
if (gso_skb->len <= sizeof(*uh) + mss)
return ERR_PTR(-EINVAL);
@@ -354,6 +371,7 @@ static struct sk_buff *udp_gro_receive_segment(struct list_head *head,
struct udphdr *uh2;
struct sk_buff *p;
unsigned int ulen;
+ int ret = 0;
/* requires non zero csum, for symmetry with GSO */
if (!uh->check) {
@@ -369,7 +387,6 @@ static struct sk_buff *udp_gro_receive_segment(struct list_head *head,
}
/* pull encapsulating udp header */
skb_gro_pull(skb, sizeof(struct udphdr));
- skb_gro_postpull_rcsum(skb, uh, sizeof(struct udphdr));
list_for_each_entry(p, head, list) {
if (!NAPI_GRO_CB(p)->same_flow)
@@ -383,14 +400,40 @@ static struct sk_buff *udp_gro_receive_segment(struct list_head *head,
continue;
}
+ if (NAPI_GRO_CB(skb)->is_flist != NAPI_GRO_CB(p)->is_flist) {
+ NAPI_GRO_CB(skb)->flush = 1;
+ return p;
+ }
+
/* Terminate the flow on len mismatch or if it grow "too much".
* Under small packet flood GRO count could elsewhere grow a lot
* leading to excessive truesize values.
* On len mismatch merge the first packet shorter than gso_size,
* otherwise complete the GRO packet.
*/
- if (ulen > ntohs(uh2->len) || skb_gro_receive(p, skb) ||
- ulen != ntohs(uh2->len) ||
+ if (ulen > ntohs(uh2->len)) {
+ pp = p;
+ } else {
+ if (NAPI_GRO_CB(skb)->is_flist) {
+ if (!pskb_may_pull(skb, skb_gro_offset(skb))) {
+ NAPI_GRO_CB(skb)->flush = 1;
+ return NULL;
+ }
+ if ((skb->ip_summed != p->ip_summed) ||
+ (skb->csum_level != p->csum_level)) {
+ NAPI_GRO_CB(skb)->flush = 1;
+ return NULL;
+ }
+ ret = skb_gro_receive_list(p, skb);
+ } else {
+ skb_gro_postpull_rcsum(skb, uh,
+ sizeof(struct udphdr));
+
+ ret = skb_gro_receive(p, skb);
+ }
+ }
+
+ if (ret || ulen != ntohs(uh2->len) ||
NAPI_GRO_CB(p)->count >= UDP_GRO_CNT_MAX)
pp = p;
@@ -401,36 +444,30 @@ static struct sk_buff *udp_gro_receive_segment(struct list_head *head,
return NULL;
}
-INDIRECT_CALLABLE_DECLARE(struct sock *udp6_lib_lookup_skb(struct sk_buff *skb,
- __be16 sport, __be16 dport));
struct sk_buff *udp_gro_receive(struct list_head *head, struct sk_buff *skb,
- struct udphdr *uh, udp_lookup_t lookup)
+ struct udphdr *uh, struct sock *sk)
{
struct sk_buff *pp = NULL;
struct sk_buff *p;
struct udphdr *uh2;
unsigned int off = skb_gro_offset(skb);
int flush = 1;
- struct sock *sk;
- rcu_read_lock();
- sk = INDIRECT_CALL_INET(lookup, udp6_lib_lookup_skb,
- udp4_lib_lookup_skb, skb, uh->source, uh->dest);
- if (!sk)
- goto out_unlock;
+ NAPI_GRO_CB(skb)->is_flist = 0;
+ if (skb->dev->features & NETIF_F_GRO_FRAGLIST)
+ NAPI_GRO_CB(skb)->is_flist = sk ? !udp_sk(sk)->gro_enabled: 1;
- if (udp_sk(sk)->gro_enabled) {
+ if ((sk && udp_sk(sk)->gro_enabled) || NAPI_GRO_CB(skb)->is_flist) {
pp = call_gro_receive(udp_gro_receive_segment, head, skb);
- rcu_read_unlock();
return pp;
}
- if (NAPI_GRO_CB(skb)->encap_mark ||
+ if (!sk || NAPI_GRO_CB(skb)->encap_mark ||
(skb->ip_summed != CHECKSUM_PARTIAL &&
NAPI_GRO_CB(skb)->csum_cnt == 0 &&
!NAPI_GRO_CB(skb)->csum_valid) ||
!udp_sk(sk)->gro_receive)
- goto out_unlock;
+ goto out;
/* mark that this skb passed once through the tunnel gro layer */
NAPI_GRO_CB(skb)->encap_mark = 1;
@@ -457,8 +494,7 @@ struct sk_buff *udp_gro_receive(struct list_head *head, struct sk_buff *skb,
skb_gro_postpull_rcsum(skb, uh, sizeof(struct udphdr));
pp = call_gro_receive_sk(udp_sk(sk)->gro_receive, sk, head, skb);
-out_unlock:
- rcu_read_unlock();
+out:
skb_gro_flush_final(skb, pp, flush);
return pp;
}
@@ -468,8 +504,10 @@ INDIRECT_CALLABLE_SCOPE
struct sk_buff *udp4_gro_receive(struct list_head *head, struct sk_buff *skb)
{
struct udphdr *uh = udp_gro_udphdr(skb);
+ struct sk_buff *pp;
+ struct sock *sk;
- if (unlikely(!uh) || !static_branch_unlikely(&udp_encap_needed_key))
+ if (unlikely(!uh))
goto flush;
/* Don't bother verifying checksum if we're going to flush anyway. */
@@ -484,7 +522,11 @@ struct sk_buff *udp4_gro_receive(struct list_head *head, struct sk_buff *skb)
inet_gro_compute_pseudo);
skip:
NAPI_GRO_CB(skb)->is_ipv6 = 0;
- return udp_gro_receive(head, skb, uh, udp4_lib_lookup_skb);
+ rcu_read_lock();
+ sk = static_branch_unlikely(&udp_encap_needed_key) ? udp4_lib_lookup_skb(skb, uh->source, uh->dest) : NULL;
+ pp = udp_gro_receive(head, skb, uh, sk);
+ rcu_read_unlock();
+ return pp;
flush:
NAPI_GRO_CB(skb)->flush = 1;
@@ -517,9 +559,7 @@ int udp_gro_complete(struct sk_buff *skb, int nhoff,
rcu_read_lock();
sk = INDIRECT_CALL_INET(lookup, udp6_lib_lookup_skb,
udp4_lib_lookup_skb, skb, uh->source, uh->dest);
- if (sk && udp_sk(sk)->gro_enabled) {
- err = udp_gro_complete_segment(skb);
- } else if (sk && udp_sk(sk)->gro_complete) {
+ if (sk && udp_sk(sk)->gro_complete) {
skb_shinfo(skb)->gso_type = uh->check ? SKB_GSO_UDP_TUNNEL_CSUM
: SKB_GSO_UDP_TUNNEL;
@@ -529,6 +569,8 @@ int udp_gro_complete(struct sk_buff *skb, int nhoff,
skb->encapsulation = 1;
err = udp_sk(sk)->gro_complete(sk, skb,
nhoff + sizeof(struct udphdr));
+ } else {
+ err = udp_gro_complete_segment(skb);
}
rcu_read_unlock();
@@ -544,6 +586,23 @@ INDIRECT_CALLABLE_SCOPE int udp4_gro_complete(struct sk_buff *skb, int nhoff)
const struct iphdr *iph = ip_hdr(skb);
struct udphdr *uh = (struct udphdr *)(skb->data + nhoff);
+ if (NAPI_GRO_CB(skb)->is_flist) {
+ uh->len = htons(skb->len - nhoff);
+
+ skb_shinfo(skb)->gso_type |= (SKB_GSO_FRAGLIST|SKB_GSO_UDP_L4);
+ skb_shinfo(skb)->gso_segs = NAPI_GRO_CB(skb)->count;
+
+ if (skb->ip_summed == CHECKSUM_UNNECESSARY) {
+ if (skb->csum_level < SKB_MAX_CSUM_LEVEL)
+ skb->csum_level++;
+ } else {
+ skb->ip_summed = CHECKSUM_UNNECESSARY;
+ skb->csum_level = 0;
+ }
+
+ return 0;
+ }
+
if (uh->check)
uh->check = ~udp_v4_check(skb->len - nhoff, iph->saddr,
iph->daddr, 0);
diff --git a/net/ipv6/udp_offload.c b/net/ipv6/udp_offload.c
index 64b8f05d6735..7a46bd33fa2f 100644
--- a/net/ipv6/udp_offload.c
+++ b/net/ipv6/udp_offload.c
@@ -115,8 +115,10 @@ INDIRECT_CALLABLE_SCOPE
struct sk_buff *udp6_gro_receive(struct list_head *head, struct sk_buff *skb)
{
struct udphdr *uh = udp_gro_udphdr(skb);
+ struct sk_buff *pp;
+ struct sock *sk;
- if (unlikely(!uh) || !static_branch_unlikely(&udpv6_encap_needed_key))
+ if (unlikely(!uh))
goto flush;
/* Don't bother verifying checksum if we're going to flush anyway. */
@@ -132,7 +134,11 @@ struct sk_buff *udp6_gro_receive(struct list_head *head, struct sk_buff *skb)
skip:
NAPI_GRO_CB(skb)->is_ipv6 = 1;
- return udp_gro_receive(head, skb, uh, udp6_lib_lookup_skb);
+ rcu_read_lock();
+ sk = static_branch_unlikely(&udpv6_encap_needed_key) ? udp6_lib_lookup_skb(skb, uh->source, uh->dest) : NULL;
+ pp = udp_gro_receive(head, skb, uh, sk);
+ rcu_read_unlock();
+ return pp;
flush:
NAPI_GRO_CB(skb)->flush = 1;
@@ -144,6 +150,23 @@ INDIRECT_CALLABLE_SCOPE int udp6_gro_complete(struct sk_buff *skb, int nhoff)
const struct ipv6hdr *ipv6h = ipv6_hdr(skb);
struct udphdr *uh = (struct udphdr *)(skb->data + nhoff);
+ if (NAPI_GRO_CB(skb)->is_flist) {
+ uh->len = htons(skb->len - nhoff);
+
+ skb_shinfo(skb)->gso_type |= (SKB_GSO_FRAGLIST|SKB_GSO_UDP_L4);
+ skb_shinfo(skb)->gso_segs = NAPI_GRO_CB(skb)->count;
+
+ if (skb->ip_summed == CHECKSUM_UNNECESSARY) {
+ if (skb->csum_level < SKB_MAX_CSUM_LEVEL)
+ skb->csum_level++;
+ } else {
+ skb->ip_summed = CHECKSUM_UNNECESSARY;
+ skb->csum_level = 0;
+ }
+
+ return 0;
+ }
+
if (uh->check)
uh->check = ~udp_v6_check(skb->len - nhoff, &ipv6h->saddr,
&ipv6h->daddr, 0);