net: Allow no-cache copy from user on transmit

This patch uses __copy_from_user_nocache on transmit to bypass data
cache for a performance improvement.  skb_add_data_nocache and
skb_copy_to_page_nocache can be called by sendmsg functions to use
this feature, initial support is in tcp_sendmsg.  This functionality is
configurable per device using ethtool.

Presumably, this feature would only be useful when the driver does
not touch the data.  The feature is turned on by default if a device
indicates that it does some form of checksum offload; it is off by
default for devices that do no checksum offload or indicate no checksum
is necessary.  For the former case copy-checksum is probably done
anyway, in the latter case the device is likely loopback in which case
the no cache copy is probably not beneficial.

This patch was tested using 200 instances of netperf TCP_RR with
1400 byte request and one byte reply.  Platform is 16 core AMD x86.

No-cache copy disabled:
   672703 tps, 97.13% utilization
   50/90/99% latency:244.31 484.205 1028.41

No-cache copy enabled:
   702113 tps, 96.16% utilization,
   50/90/99% latency 238.56 467.56 956.955

Using 14000 byte request and response sizes demonstrate the
effects more dramatically:

No-cache copy disabled:
   79571 tps, 34.34 %utlization
   50/90/95% latency 1584.46 2319.59 5001.76

No-cache copy enabled:
   83856 tps, 34.81% utilization
   50/90/95% latency 2508.42 2622.62 2735.88

Note especially the effect on latency tail (95th percentile).

This seems to provide a nice performance improvement and is
consistent in the tests I ran.  Presumably, this would provide
the greatest benfits in the presence of an application workload
stressing the cache and a lot of transmit data happening.

Signed-off-by: Tom Herbert <therbert@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>

diff --git a/drivers/net/bonding/bond_main.c b/drivers/net/bonding/bond_main.c
index 16d6fe9546959fd679b7db7dde1e994ec8977503..b51e021354b536430ec0260b2a4f5e09d675dbd2 100644 (file)
--- a/drivers/net/bonding/bond_main.c
+++ b/drivers/net/bonding/bond_main.c
@@ -1407,7 +1407,7 @@ static int bond_compute_features(struct bonding *bond)
        int i;
 
        features &= ~(NETIF_F_ALL_CSUM | BOND_VLAN_FEATURES);
-       features |=  NETIF_F_GSO_MASK | NETIF_F_NO_CSUM;
+       features |=  NETIF_F_GSO_MASK | NETIF_F_NO_CSUM | NETIF_F_NOCACHE_COPY;
 
        if (!bond->first_slave)
                goto done;

diff --git a/include/linux/netdevice.h b/include/linux/netdevice.h
index a4664cc68e2bdd4f5e966780a7b55c91adf49430..09d262415769aef45f76baf86b27344ed78e5fb8 100644 (file)
--- a/include/linux/netdevice.h
+++ b/include/linux/netdevice.h
@@ -1066,6 +1066,7 @@ struct net_device {
 #define NETIF_F_NTUPLE         (1 << 27) /* N-tuple filters supported */
 #define NETIF_F_RXHASH         (1 << 28) /* Receive hashing offload */
 #define NETIF_F_RXCSUM         (1 << 29) /* Receive checksumming offload */
+#define NETIF_F_NOCACHE_COPY   (1 << 30) /* Use no-cache copyfromuser */
 
        /* Segmentation offload features */
 #define NETIF_F_GSO_SHIFT      16
@@ -1081,7 +1082,7 @@ struct net_device {
        /* = all defined minus driver/device-class-related */
 #define NETIF_F_NEVER_CHANGE   (NETIF_F_HIGHDMA | NETIF_F_VLAN_CHALLENGED | \
                                  NETIF_F_LLTX | NETIF_F_NETNS_LOCAL)
-#define NETIF_F_ETHTOOL_BITS   (0x3f3fffff & ~NETIF_F_NEVER_CHANGE)
+#define NETIF_F_ETHTOOL_BITS   (0x7f3fffff & ~NETIF_F_NEVER_CHANGE)
 
        /* List of features with software fallbacks. */
 #define NETIF_F_GSO_SOFTWARE   (NETIF_F_TSO | NETIF_F_TSO_ECN | \

diff --git a/include/net/sock.h b/include/net/sock.h
index da0534d3401cdee0ed17ce84150365b73c3c9327..43bd515e92fdea4988876abe7afb65f1c7826bca 100644 (file)
--- a/include/net/sock.h
+++ b/include/net/sock.h
@@ -52,6 +52,7 @@
 #include <linux/mm.h>
 #include <linux/security.h>
 #include <linux/slab.h>
+#include <linux/uaccess.h>
 
 #include <linux/filter.h>
 #include <linux/rculist_nulls.h>
@@ -1389,6 +1390,58 @@ static inline void sk_nocaps_add(struct sock *sk, int flags)
        sk->sk_route_caps &= ~flags;
 }
 
+static inline int skb_do_copy_data_nocache(struct sock *sk, struct sk_buff *skb,
+                                          char __user *from, char *to,
+                                          int copy)
+{
+       if (skb->ip_summed == CHECKSUM_NONE) {
+               int err = 0;
+               __wsum csum = csum_and_copy_from_user(from, to, copy, 0, &err);
+               if (err)
+                       return err;
+               skb->csum = csum_block_add(skb->csum, csum, skb->len);
+       } else if (sk->sk_route_caps & NETIF_F_NOCACHE_COPY) {
+               if (!access_ok(VERIFY_READ, from, copy) ||
+                   __copy_from_user_nocache(to, from, copy))
+                       return -EFAULT;
+       } else if (copy_from_user(to, from, copy))
+               return -EFAULT;
+
+       return 0;
+}
+
+static inline int skb_add_data_nocache(struct sock *sk, struct sk_buff *skb,
+                                      char __user *from, int copy)
+{
+       int err;
+
+       err = skb_do_copy_data_nocache(sk, skb, from, skb_put(skb, copy), copy);
+       if (err)
+               __skb_trim(skb, skb->len);
+
+       return err;
+}
+
+static inline int skb_copy_to_page_nocache(struct sock *sk, char __user *from,
+                                          struct sk_buff *skb,
+                                          struct page *page,
+                                          int off, int copy)
+{
+       int err;
+
+       err = skb_do_copy_data_nocache(sk, skb, from,
+                                      page_address(page) + off, copy);
+       if (err)
+               return err;
+
+       skb->len             += copy;
+       skb->data_len        += copy;
+       skb->truesize        += copy;
+       sk->sk_wmem_queued   += copy;
+       sk_mem_charge(sk, copy);
+       return 0;
+}
+
 static inline int skb_copy_to_page(struct sock *sk, char __user *from,
                                   struct sk_buff *skb, struct page *page,
                                   int off, int copy)

diff --git a/net/core/dev.c b/net/core/dev.c
index 02f56376fe993910b3317e45e73626b399f0f0ba..5d0b4f6f1a72db3d66ce0a2542e80a8d719c8580 100644 (file)
--- a/net/core/dev.c
+++ b/net/core/dev.c
@@ -5425,6 +5425,14 @@ int register_netdevice(struct net_device *dev)
                dev->features &= ~NETIF_F_GSO;
        }
 
+       /* Turn on no cache copy if HW is doing checksum */
+       dev->hw_features |= NETIF_F_NOCACHE_COPY;
+       if ((dev->features & NETIF_F_ALL_CSUM) &&
+           !(dev->features & NETIF_F_NO_CSUM)) {
+               dev->wanted_features |= NETIF_F_NOCACHE_COPY;
+               dev->features |= NETIF_F_NOCACHE_COPY;
+       }
+
        /* Enable GRO and NETIF_F_HIGHDMA for vlans by default,
         * vlan_dev_init() will do the dev->features check, so these features
         * are enabled only if supported by underlying device.
@@ -6182,6 +6190,10 @@ u32 netdev_increment_features(u32 all, u32 one, u32 mask)
                }
        }
 
+       /* If device can't no cache copy, don't do for all */
+       if (!(one & NETIF_F_NOCACHE_COPY))
+               all &= ~NETIF_F_NOCACHE_COPY;
+
        one |= NETIF_F_ALL_CSUM;
 
        one |= all & NETIF_F_ONE_FOR_ALL;

diff --git a/net/core/ethtool.c b/net/core/ethtool.c
index 439e4b0e1312658b402ab33186c2e17a04961c9f..719670ae199c35ec28e5aeb7f1a5aedeba01b02c 100644 (file)
--- a/net/core/ethtool.c
+++ b/net/core/ethtool.c
@@ -359,7 +359,7 @@ static const char netdev_features_strings[ETHTOOL_DEV_FEATURE_WORDS * 32][ETH_GS
        /* NETIF_F_NTUPLE */          "rx-ntuple-filter",
        /* NETIF_F_RXHASH */          "rx-hashing",
        /* NETIF_F_RXCSUM */          "rx-checksum",
-       "",
+       /* NETIF_F_NOCACHE_COPY */    "tx-nocache-copy"
        "",
 };
 

diff --git a/net/ipv4/tcp.c b/net/ipv4/tcp.c
index b22d450105450ae089d42a06486938fcaa2c80dc..054a59d21eb0d4211bf4857f6a15b33d5e80abdc 100644 (file)
--- a/net/ipv4/tcp.c
+++ b/net/ipv4/tcp.c
@@ -999,7 +999,8 @@ new_segment:
                                /* We have some space in skb head. Superb! */
                                if (copy > skb_tailroom(skb))
                                        copy = skb_tailroom(skb);
-                               if ((err = skb_add_data(skb, from, copy)) != 0)
+                               err = skb_add_data_nocache(sk, skb, from, copy);
+                               if (err)
                                        goto do_fault;
                        } else {
                                int merge = 0;
@@ -1042,8 +1043,8 @@ new_segment:
 
                                /* Time to copy data. We are close to
                                 * the end! */
-                               err = skb_copy_to_page(sk, from, skb, page,
-                                                      off, copy);
+                               err = skb_copy_to_page_nocache(sk, from, skb,
+                                                              page, off, copy);
                                if (err) {
                                        /* If this page was new, give it to the
                                         * socket so it does not get leaked.