Commit | Line | Data |
---|---|---|
06021292 ET |
1 | /* |
2 | * Low Latency Sockets | |
3 | * Copyright(c) 2013 Intel Corporation. | |
4 | * | |
5 | * This program is free software; you can redistribute it and/or modify it | |
6 | * under the terms and conditions of the GNU General Public License, | |
7 | * version 2, as published by the Free Software Foundation. | |
8 | * | |
9 | * This program is distributed in the hope it will be useful, but WITHOUT | |
10 | * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or | |
11 | * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for | |
12 | * more details. | |
13 | * | |
14 | * You should have received a copy of the GNU General Public License along with | |
15 | * this program; if not, write to the Free Software Foundation, Inc., | |
16 | * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA. | |
17 | * | |
18 | * Author: Eliezer Tamir | |
19 | * | |
20 | * Contact Information: | |
21 | * e1000-devel Mailing List <e1000-devel@lists.sourceforge.net> | |
22 | */ | |
23 | ||
24 | /* | |
25 | * For now this depends on CONFIG_X86_TSC | |
26 | */ | |
27 | ||
28 | #ifndef _LINUX_NET_LL_POLL_H | |
29 | #define _LINUX_NET_LL_POLL_H | |
30 | ||
31 | #include <linux/netdevice.h> | |
32 | #include <net/ip.h> | |
33 | ||
34 | #ifdef CONFIG_NET_LL_RX_POLL | |
35 | ||
36 | struct napi_struct; | |
eb6db622 | 37 | extern unsigned int sysctl_net_ll_poll __read_mostly; |
06021292 ET |
38 | |
39 | /* return values from ndo_ll_poll */ | |
40 | #define LL_FLUSH_FAILED -1 | |
41 | #define LL_FLUSH_BUSY -2 | |
42 | ||
43 | /* we don't mind a ~2.5% imprecision */ | |
44 | #define TSC_MHZ (tsc_khz >> 10) | |
45 | ||
46 | static inline cycles_t ll_end_time(void) | |
47 | { | |
eb6db622 ET |
48 | return (cycles_t)TSC_MHZ * ACCESS_ONCE(sysctl_net_ll_poll) |
49 | + get_cycles(); | |
06021292 ET |
50 | } |
51 | ||
52 | static inline bool sk_valid_ll(struct sock *sk) | |
53 | { | |
54 | return sysctl_net_ll_poll && sk->sk_napi_id && | |
55 | !need_resched() && !signal_pending(current); | |
56 | } | |
57 | ||
58 | static inline bool can_poll_ll(cycles_t end_time) | |
59 | { | |
60 | return !time_after((unsigned long)get_cycles(), | |
61 | (unsigned long)end_time); | |
62 | } | |
63 | ||
64 | static inline bool sk_poll_ll(struct sock *sk, int nonblock) | |
65 | { | |
66 | cycles_t end_time = ll_end_time(); | |
67 | const struct net_device_ops *ops; | |
68 | struct napi_struct *napi; | |
69 | int rc = false; | |
70 | ||
71 | /* | |
72 | * rcu read lock for napi hash | |
73 | * bh so we don't race with net_rx_action | |
74 | */ | |
75 | rcu_read_lock_bh(); | |
76 | ||
77 | napi = napi_by_id(sk->sk_napi_id); | |
78 | if (!napi) | |
79 | goto out; | |
80 | ||
81 | ops = napi->dev->netdev_ops; | |
82 | if (!ops->ndo_ll_poll) | |
83 | goto out; | |
84 | ||
85 | do { | |
86 | ||
87 | rc = ops->ndo_ll_poll(napi); | |
88 | ||
89 | if (rc == LL_FLUSH_FAILED) | |
90 | break; /* permanent failure */ | |
91 | ||
92 | if (rc > 0) | |
93 | /* local bh are disabled so it is ok to use _BH */ | |
94 | NET_ADD_STATS_BH(sock_net(sk), | |
95 | LINUX_MIB_LOWLATENCYRXPACKETS, rc); | |
96 | ||
97 | } while (skb_queue_empty(&sk->sk_receive_queue) | |
98 | && can_poll_ll(end_time) && !nonblock); | |
99 | ||
100 | rc = !skb_queue_empty(&sk->sk_receive_queue); | |
101 | out: | |
102 | rcu_read_unlock_bh(); | |
103 | return rc; | |
104 | } | |
105 | ||
106 | /* used in the NIC receive handler to mark the skb */ | |
107 | static inline void skb_mark_ll(struct sk_buff *skb, struct napi_struct *napi) | |
108 | { | |
109 | skb->napi_id = napi->napi_id; | |
110 | } | |
111 | ||
112 | /* used in the protocol hanlder to propagate the napi_id to the socket */ | |
113 | static inline void sk_mark_ll(struct sock *sk, struct sk_buff *skb) | |
114 | { | |
115 | sk->sk_napi_id = skb->napi_id; | |
116 | } | |
117 | ||
118 | #else /* CONFIG_NET_LL_RX_POLL */ | |
119 | ||
120 | static inline cycles_t ll_end_time(void) | |
121 | { | |
122 | return 0; | |
123 | } | |
124 | ||
125 | static inline bool sk_valid_ll(struct sock *sk) | |
126 | { | |
127 | return false; | |
128 | } | |
129 | ||
130 | static inline bool sk_poll_ll(struct sock *sk, int nonblock) | |
131 | { | |
132 | return false; | |
133 | } | |
134 | ||
135 | static inline void skb_mark_ll(struct sk_buff *skb, struct napi_struct *napi) | |
136 | { | |
137 | } | |
138 | ||
139 | static inline void sk_mark_ll(struct sock *sk, struct sk_buff *skb) | |
140 | { | |
141 | } | |
142 | ||
143 | static inline bool can_poll_ll(cycles_t end_time) | |
144 | { | |
145 | return false; | |
146 | } | |
147 | ||
148 | #endif /* CONFIG_NET_LL_RX_POLL */ | |
149 | #endif /* _LINUX_NET_LL_POLL_H */ |