tcp: refine TSO autosizing

[deliverable/linux.git] / net / ipv4 / tcp_output.c

diff --git a/net/ipv4/tcp_output.c b/net/ipv4/tcp_output.c
index f5bd4bd3f7e669b3fd48a843d55e7313a30a3409..f37ecf53ee8a96827fc08bd203b0ca8857f8fc34 100644 (file)
--- a/net/ipv4/tcp_output.c
+++ b/net/ipv4/tcp_output.c
@@ -1524,6 +1524,27 @@ static bool tcp_nagle_check(bool partial, const struct tcp_sock *tp,
                ((nonagle & TCP_NAGLE_CORK) ||
                 (!nonagle && tp->packets_out && tcp_minshall_check(tp)));
 }
+
+/* Return how many segs we'd like on a TSO packet,
+ * to send one TSO packet per ms
+ */
+static u32 tcp_tso_autosize(const struct sock *sk, unsigned int mss_now)
+{
+       u32 bytes, segs;
+
+       bytes = min(sk->sk_pacing_rate >> 10,
+                   sk->sk_gso_max_size - 1 - MAX_TCP_HEADER);
+
+       /* Goal is to send at least one packet per ms,
+        * not one big TSO packet every 100 ms.
+        * This preserves ACK clocking and is consistent
+        * with tcp_tso_should_defer() heuristic.
+        */
+       segs = max_t(u32, bytes / mss_now, sysctl_tcp_min_tso_segs);
+
+       return min_t(u32, segs, sk->sk_gso_max_segs);
+}
+
 /* Returns the portion of skb which can be sent right away */
 static unsigned int tcp_mss_split_point(const struct sock *sk,
                                        const struct sk_buff *skb,
@@ -1731,7 +1752,7 @@ static int tso_fragment(struct sock *sk, struct sk_buff *skb, unsigned int len,
  * This algorithm is from John Heffner.
  */
 static bool tcp_tso_should_defer(struct sock *sk, struct sk_buff *skb,
-                                bool *is_cwnd_limited)
+                                bool *is_cwnd_limited, u32 max_segs)
 {
        struct tcp_sock *tp = tcp_sk(sk);
        const struct inet_connection_sock *icsk = inet_csk(sk);
@@ -1761,8 +1782,7 @@ static bool tcp_tso_should_defer(struct sock *sk, struct sk_buff *skb,
        limit = min(send_win, cong_win);
 
        /* If a full-sized TSO skb can be sent, do it. */
-       if (limit >= min_t(unsigned int, sk->sk_gso_max_size,
-                          tp->xmit_size_goal_segs * tp->mss_cache))
+       if (limit >= max_segs * tp->mss_cache)
                goto send_now;
 
        /* Middle in queue won't get any more data, full sendable already? */
@@ -1959,6 +1979,7 @@ static bool tcp_write_xmit(struct sock *sk, unsigned int mss_now, int nonagle,
        int cwnd_quota;
        int result;
        bool is_cwnd_limited = false;
+       u32 max_segs;
 
        sent_pkts = 0;
 
@@ -1972,6 +1993,7 @@ static bool tcp_write_xmit(struct sock *sk, unsigned int mss_now, int nonagle,
                }
        }
 
+       max_segs = tcp_tso_autosize(sk, mss_now);
        while ((skb = tcp_send_head(sk))) {
                unsigned int limit;
 
@@ -2004,10 +2026,23 @@ static bool tcp_write_xmit(struct sock *sk, unsigned int mss_now, int nonagle,
                                break;
                } else {
                        if (!push_one &&
-                           tcp_tso_should_defer(sk, skb, &is_cwnd_limited))
+                           tcp_tso_should_defer(sk, skb, &is_cwnd_limited,
+                                                max_segs))
                                break;
                }
 
+               limit = mss_now;
+               if (tso_segs > 1 && !tcp_urg_mode(tp))
+                       limit = tcp_mss_split_point(sk, skb, mss_now,
+                                                   min_t(unsigned int,
+                                                         cwnd_quota,
+                                                         max_segs),
+                                                   nonagle);
+
+               if (skb->len > limit &&
+                   unlikely(tso_fragment(sk, skb, limit, mss_now, gfp)))
+                       break;
+
                /* TCP Small Queues :
                 * Control number of packets in qdisc/devices to two packets / or ~1 ms.
                 * This allows for :
@@ -2018,8 +2053,8 @@ static bool tcp_write_xmit(struct sock *sk, unsigned int mss_now, int nonagle,
                 * of queued bytes to ensure line rate.
                 * One example is wifi aggregation (802.11 AMPDU)
                 */
-               limit = max_t(unsigned int, sysctl_tcp_limit_output_bytes,
-                             sk->sk_pacing_rate >> 10);
+               limit = max(2 * skb->truesize, sk->sk_pacing_rate >> 10);
+               limit = min_t(u32, limit, sysctl_tcp_limit_output_bytes);
 
                if (atomic_read(&sk->sk_wmem_alloc) > limit) {
                        set_bit(TSQ_THROTTLED, &tp->tsq_flags);
@@ -2032,18 +2067,6 @@ static bool tcp_write_xmit(struct sock *sk, unsigned int mss_now, int nonagle,
                                break;
                }
 
-               limit = mss_now;
-               if (tso_segs > 1 && !tcp_urg_mode(tp))
-                       limit = tcp_mss_split_point(sk, skb, mss_now,
-                                                   min_t(unsigned int,
-                                                         cwnd_quota,
-                                                         sk->sk_gso_max_segs),
-                                                   nonagle);
-
-               if (skb->len > limit &&
-                   unlikely(tso_fragment(sk, skb, limit, mss_now, gfp)))
-                       break;
-
                if (unlikely(tcp_transmit_skb(sk, skb, 1, gfp)))
                        break;