Commit | Line | Data |
---|---|---|
8af3c33f JK |
1 | /******************************************************************************* |
2 | ||
3 | Intel 10 Gigabit PCI Express Linux driver | |
4 | Copyright(c) 1999 - 2012 Intel Corporation. | |
5 | ||
6 | This program is free software; you can redistribute it and/or modify it | |
7 | under the terms and conditions of the GNU General Public License, | |
8 | version 2, as published by the Free Software Foundation. | |
9 | ||
10 | This program is distributed in the hope it will be useful, but WITHOUT | |
11 | ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or | |
12 | FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for | |
13 | more details. | |
14 | ||
15 | You should have received a copy of the GNU General Public License along with | |
16 | this program; if not, write to the Free Software Foundation, Inc., | |
17 | 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA. | |
18 | ||
19 | The full GNU General Public License is included in this distribution in | |
20 | the file called "COPYING". | |
21 | ||
22 | Contact Information: | |
23 | e1000-devel Mailing List <e1000-devel@lists.sourceforge.net> | |
24 | Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497 | |
25 | ||
26 | *******************************************************************************/ | |
27 | ||
28 | #include "ixgbe.h" | |
29 | #include "ixgbe_sriov.h" | |
30 | ||
31 | /** | |
32 | * ixgbe_cache_ring_rss - Descriptor ring to register mapping for RSS | |
33 | * @adapter: board private structure to initialize | |
34 | * | |
35 | * Cache the descriptor ring offsets for RSS to the assigned rings. | |
36 | * | |
37 | **/ | |
38 | static inline bool ixgbe_cache_ring_rss(struct ixgbe_adapter *adapter) | |
39 | { | |
40 | int i; | |
41 | ||
42 | if (!(adapter->flags & IXGBE_FLAG_RSS_ENABLED)) | |
43 | return false; | |
44 | ||
45 | for (i = 0; i < adapter->num_rx_queues; i++) | |
46 | adapter->rx_ring[i]->reg_idx = i; | |
47 | for (i = 0; i < adapter->num_tx_queues; i++) | |
48 | adapter->tx_ring[i]->reg_idx = i; | |
49 | ||
50 | return true; | |
51 | } | |
8af3c33f | 52 | |
800bd607 | 53 | #ifdef CONFIG_IXGBE_DCB |
8af3c33f JK |
54 | /* ixgbe_get_first_reg_idx - Return first register index associated with ring */ |
55 | static void ixgbe_get_first_reg_idx(struct ixgbe_adapter *adapter, u8 tc, | |
56 | unsigned int *tx, unsigned int *rx) | |
57 | { | |
58 | struct net_device *dev = adapter->netdev; | |
59 | struct ixgbe_hw *hw = &adapter->hw; | |
60 | u8 num_tcs = netdev_get_num_tc(dev); | |
61 | ||
62 | *tx = 0; | |
63 | *rx = 0; | |
64 | ||
65 | switch (hw->mac.type) { | |
66 | case ixgbe_mac_82598EB: | |
67 | *tx = tc << 2; | |
68 | *rx = tc << 3; | |
69 | break; | |
70 | case ixgbe_mac_82599EB: | |
71 | case ixgbe_mac_X540: | |
72 | if (num_tcs > 4) { | |
73 | if (tc < 3) { | |
74 | *tx = tc << 5; | |
75 | *rx = tc << 4; | |
76 | } else if (tc < 5) { | |
77 | *tx = ((tc + 2) << 4); | |
78 | *rx = tc << 4; | |
79 | } else if (tc < num_tcs) { | |
80 | *tx = ((tc + 8) << 3); | |
81 | *rx = tc << 4; | |
82 | } | |
83 | } else { | |
84 | *rx = tc << 5; | |
85 | switch (tc) { | |
86 | case 0: | |
87 | *tx = 0; | |
88 | break; | |
89 | case 1: | |
90 | *tx = 64; | |
91 | break; | |
92 | case 2: | |
93 | *tx = 96; | |
94 | break; | |
95 | case 3: | |
96 | *tx = 112; | |
97 | break; | |
98 | default: | |
99 | break; | |
100 | } | |
101 | } | |
102 | break; | |
103 | default: | |
104 | break; | |
105 | } | |
106 | } | |
107 | ||
108 | /** | |
109 | * ixgbe_cache_ring_dcb - Descriptor ring to register mapping for DCB | |
110 | * @adapter: board private structure to initialize | |
111 | * | |
112 | * Cache the descriptor ring offsets for DCB to the assigned rings. | |
113 | * | |
114 | **/ | |
115 | static inline bool ixgbe_cache_ring_dcb(struct ixgbe_adapter *adapter) | |
116 | { | |
117 | struct net_device *dev = adapter->netdev; | |
118 | int i, j, k; | |
119 | u8 num_tcs = netdev_get_num_tc(dev); | |
120 | ||
121 | if (!num_tcs) | |
122 | return false; | |
123 | ||
124 | for (i = 0, k = 0; i < num_tcs; i++) { | |
125 | unsigned int tx_s, rx_s; | |
126 | u16 count = dev->tc_to_txq[i].count; | |
127 | ||
128 | ixgbe_get_first_reg_idx(adapter, i, &tx_s, &rx_s); | |
129 | for (j = 0; j < count; j++, k++) { | |
130 | adapter->tx_ring[k]->reg_idx = tx_s + j; | |
131 | adapter->rx_ring[k]->reg_idx = rx_s + j; | |
132 | adapter->tx_ring[k]->dcb_tc = i; | |
133 | adapter->rx_ring[k]->dcb_tc = i; | |
134 | } | |
135 | } | |
136 | ||
137 | return true; | |
138 | } | |
139 | #endif | |
140 | ||
8af3c33f JK |
141 | #ifdef IXGBE_FCOE |
142 | /** | |
143 | * ixgbe_cache_ring_fcoe - Descriptor ring to register mapping for the FCoE | |
144 | * @adapter: board private structure to initialize | |
145 | * | |
146 | * Cache the descriptor ring offsets for FCoE mode to the assigned rings. | |
147 | * | |
148 | */ | |
149 | static inline bool ixgbe_cache_ring_fcoe(struct ixgbe_adapter *adapter) | |
150 | { | |
151 | struct ixgbe_ring_feature *f = &adapter->ring_feature[RING_F_FCOE]; | |
152 | int i; | |
153 | u8 fcoe_rx_i = 0, fcoe_tx_i = 0; | |
154 | ||
155 | if (!(adapter->flags & IXGBE_FLAG_FCOE_ENABLED)) | |
156 | return false; | |
157 | ||
158 | if (adapter->flags & IXGBE_FLAG_RSS_ENABLED) { | |
0b7f5d0b | 159 | ixgbe_cache_ring_rss(adapter); |
8af3c33f | 160 | |
e4b317e9 AD |
161 | fcoe_rx_i = f->offset; |
162 | fcoe_tx_i = f->offset; | |
8af3c33f JK |
163 | } |
164 | for (i = 0; i < f->indices; i++, fcoe_rx_i++, fcoe_tx_i++) { | |
e4b317e9 AD |
165 | adapter->rx_ring[f->offset + i]->reg_idx = fcoe_rx_i; |
166 | adapter->tx_ring[f->offset + i]->reg_idx = fcoe_tx_i; | |
8af3c33f JK |
167 | } |
168 | return true; | |
169 | } | |
170 | ||
171 | #endif /* IXGBE_FCOE */ | |
172 | /** | |
173 | * ixgbe_cache_ring_sriov - Descriptor ring to register mapping for sriov | |
174 | * @adapter: board private structure to initialize | |
175 | * | |
176 | * SR-IOV doesn't use any descriptor rings but changes the default if | |
177 | * no other mapping is used. | |
178 | * | |
179 | */ | |
180 | static inline bool ixgbe_cache_ring_sriov(struct ixgbe_adapter *adapter) | |
181 | { | |
182 | adapter->rx_ring[0]->reg_idx = adapter->num_vfs * 2; | |
183 | adapter->tx_ring[0]->reg_idx = adapter->num_vfs * 2; | |
184 | if (adapter->num_vfs) | |
185 | return true; | |
186 | else | |
187 | return false; | |
188 | } | |
189 | ||
190 | /** | |
191 | * ixgbe_cache_ring_register - Descriptor ring to register mapping | |
192 | * @adapter: board private structure to initialize | |
193 | * | |
194 | * Once we know the feature-set enabled for the device, we'll cache | |
195 | * the register offset the descriptor ring is assigned to. | |
196 | * | |
197 | * Note, the order the various feature calls is important. It must start with | |
198 | * the "most" features enabled at the same time, then trickle down to the | |
199 | * least amount of features turned on at once. | |
200 | **/ | |
201 | static void ixgbe_cache_ring_register(struct ixgbe_adapter *adapter) | |
202 | { | |
203 | /* start with default case */ | |
204 | adapter->rx_ring[0]->reg_idx = 0; | |
205 | adapter->tx_ring[0]->reg_idx = 0; | |
206 | ||
207 | if (ixgbe_cache_ring_sriov(adapter)) | |
208 | return; | |
209 | ||
210 | #ifdef CONFIG_IXGBE_DCB | |
211 | if (ixgbe_cache_ring_dcb(adapter)) | |
212 | return; | |
213 | #endif | |
214 | ||
215 | #ifdef IXGBE_FCOE | |
216 | if (ixgbe_cache_ring_fcoe(adapter)) | |
217 | return; | |
218 | #endif /* IXGBE_FCOE */ | |
219 | ||
8af3c33f JK |
220 | if (ixgbe_cache_ring_rss(adapter)) |
221 | return; | |
222 | } | |
223 | ||
224 | /** | |
49ce9c2c | 225 | * ixgbe_set_sriov_queues - Allocate queues for IOV use |
8af3c33f JK |
226 | * @adapter: board private structure to initialize |
227 | * | |
228 | * IOV doesn't actually use anything, so just NAK the | |
229 | * request for now and let the other queue routines | |
230 | * figure out what to do. | |
231 | */ | |
232 | static inline bool ixgbe_set_sriov_queues(struct ixgbe_adapter *adapter) | |
233 | { | |
234 | return false; | |
235 | } | |
236 | ||
237 | /** | |
49ce9c2c | 238 | * ixgbe_set_rss_queues - Allocate queues for RSS |
8af3c33f JK |
239 | * @adapter: board private structure to initialize |
240 | * | |
241 | * This is our "base" multiqueue mode. RSS (Receive Side Scaling) will try | |
242 | * to allocate one Rx queue per CPU, and if available, one Tx queue per CPU. | |
243 | * | |
244 | **/ | |
0b7f5d0b | 245 | static bool ixgbe_set_rss_queues(struct ixgbe_adapter *adapter) |
8af3c33f | 246 | { |
0b7f5d0b AD |
247 | struct ixgbe_ring_feature *f; |
248 | u16 rss_i; | |
8af3c33f | 249 | |
0b7f5d0b AD |
250 | if (!(adapter->flags & IXGBE_FLAG_RSS_ENABLED)) { |
251 | adapter->flags &= ~IXGBE_FLAG_FDIR_HASH_CAPABLE; | |
252 | return false; | |
8af3c33f JK |
253 | } |
254 | ||
0b7f5d0b AD |
255 | /* set mask for 16 queue limit of RSS */ |
256 | f = &adapter->ring_feature[RING_F_RSS]; | |
257 | rss_i = f->limit; | |
8af3c33f | 258 | |
0b7f5d0b AD |
259 | f->indices = rss_i; |
260 | f->mask = 0xF; | |
8af3c33f JK |
261 | |
262 | /* | |
0b7f5d0b | 263 | * Use Flow Director in addition to RSS to ensure the best |
8af3c33f JK |
264 | * distribution of flows across cores, even when an FDIR flow |
265 | * isn't matched. | |
266 | */ | |
0b7f5d0b AD |
267 | if (adapter->flags & IXGBE_FLAG_FDIR_HASH_CAPABLE) { |
268 | f = &adapter->ring_feature[RING_F_FDIR]; | |
269 | ||
270 | f->indices = min_t(u16, num_online_cpus(), f->limit); | |
271 | rss_i = max_t(u16, rss_i, f->indices); | |
8af3c33f | 272 | } |
0b7f5d0b AD |
273 | |
274 | adapter->num_rx_queues = rss_i; | |
275 | adapter->num_tx_queues = rss_i; | |
276 | ||
277 | return true; | |
8af3c33f JK |
278 | } |
279 | ||
280 | #ifdef IXGBE_FCOE | |
281 | /** | |
49ce9c2c | 282 | * ixgbe_set_fcoe_queues - Allocate queues for Fiber Channel over Ethernet (FCoE) |
8af3c33f JK |
283 | * @adapter: board private structure to initialize |
284 | * | |
285 | * FCoE RX FCRETA can use up to 8 rx queues for up to 8 different exchanges. | |
e4b317e9 | 286 | * Offset is used as the index of the first rx queue used by FCoE. |
8af3c33f JK |
287 | **/ |
288 | static inline bool ixgbe_set_fcoe_queues(struct ixgbe_adapter *adapter) | |
289 | { | |
290 | struct ixgbe_ring_feature *f = &adapter->ring_feature[RING_F_FCOE]; | |
291 | ||
292 | if (!(adapter->flags & IXGBE_FLAG_FCOE_ENABLED)) | |
293 | return false; | |
294 | ||
c087663e | 295 | f->indices = min_t(int, num_online_cpus(), f->limit); |
8af3c33f JK |
296 | |
297 | adapter->num_rx_queues = 1; | |
298 | adapter->num_tx_queues = 1; | |
299 | ||
300 | if (adapter->flags & IXGBE_FLAG_RSS_ENABLED) { | |
301 | e_info(probe, "FCoE enabled with RSS\n"); | |
0b7f5d0b | 302 | ixgbe_set_rss_queues(adapter); |
8af3c33f JK |
303 | } |
304 | ||
305 | /* adding FCoE rx rings to the end */ | |
e4b317e9 | 306 | f->offset = adapter->num_rx_queues; |
8af3c33f JK |
307 | adapter->num_rx_queues += f->indices; |
308 | adapter->num_tx_queues += f->indices; | |
309 | ||
310 | return true; | |
311 | } | |
312 | #endif /* IXGBE_FCOE */ | |
313 | ||
314 | /* Artificial max queue cap per traffic class in DCB mode */ | |
315 | #define DCB_QUEUE_CAP 8 | |
316 | ||
317 | #ifdef CONFIG_IXGBE_DCB | |
318 | static inline bool ixgbe_set_dcb_queues(struct ixgbe_adapter *adapter) | |
319 | { | |
320 | int per_tc_q, q, i, offset = 0; | |
321 | struct net_device *dev = adapter->netdev; | |
322 | int tcs = netdev_get_num_tc(dev); | |
323 | ||
324 | if (!tcs) | |
325 | return false; | |
326 | ||
327 | /* Map queue offset and counts onto allocated tx queues */ | |
328 | per_tc_q = min_t(unsigned int, dev->num_tx_queues / tcs, DCB_QUEUE_CAP); | |
329 | q = min_t(int, num_online_cpus(), per_tc_q); | |
330 | ||
331 | for (i = 0; i < tcs; i++) { | |
332 | netdev_set_tc_queue(dev, i, q, offset); | |
333 | offset += q; | |
334 | } | |
335 | ||
336 | adapter->num_tx_queues = q * tcs; | |
337 | adapter->num_rx_queues = q * tcs; | |
338 | ||
339 | #ifdef IXGBE_FCOE | |
340 | /* FCoE enabled queues require special configuration indexed | |
e4b317e9 | 341 | * by feature specific indices and offset. Here we map FCoE |
8af3c33f JK |
342 | * indices onto the DCB queue pairs allowing FCoE to own |
343 | * configuration later. | |
344 | */ | |
345 | if (adapter->flags & IXGBE_FLAG_FCOE_ENABLED) { | |
800bd607 | 346 | u8 tc = ixgbe_fcoe_get_tc(adapter); |
8af3c33f JK |
347 | struct ixgbe_ring_feature *f = |
348 | &adapter->ring_feature[RING_F_FCOE]; | |
349 | ||
8af3c33f | 350 | f->indices = dev->tc_to_txq[tc].count; |
e4b317e9 | 351 | f->offset = dev->tc_to_txq[tc].offset; |
8af3c33f JK |
352 | } |
353 | #endif | |
354 | ||
355 | return true; | |
356 | } | |
357 | #endif | |
358 | ||
359 | /** | |
49ce9c2c | 360 | * ixgbe_set_num_queues - Allocate queues for device, feature dependent |
8af3c33f JK |
361 | * @adapter: board private structure to initialize |
362 | * | |
363 | * This is the top level queue allocation routine. The order here is very | |
364 | * important, starting with the "most" number of features turned on at once, | |
365 | * and ending with the smallest set of features. This way large combinations | |
366 | * can be allocated if they're turned on, and smaller combinations are the | |
367 | * fallthrough conditions. | |
368 | * | |
369 | **/ | |
370 | static int ixgbe_set_num_queues(struct ixgbe_adapter *adapter) | |
371 | { | |
372 | /* Start with base case */ | |
373 | adapter->num_rx_queues = 1; | |
374 | adapter->num_tx_queues = 1; | |
375 | adapter->num_rx_pools = adapter->num_rx_queues; | |
376 | adapter->num_rx_queues_per_pool = 1; | |
377 | ||
378 | if (ixgbe_set_sriov_queues(adapter)) | |
379 | goto done; | |
380 | ||
381 | #ifdef CONFIG_IXGBE_DCB | |
382 | if (ixgbe_set_dcb_queues(adapter)) | |
383 | goto done; | |
384 | ||
385 | #endif | |
386 | #ifdef IXGBE_FCOE | |
387 | if (ixgbe_set_fcoe_queues(adapter)) | |
388 | goto done; | |
389 | ||
390 | #endif /* IXGBE_FCOE */ | |
8af3c33f JK |
391 | if (ixgbe_set_rss_queues(adapter)) |
392 | goto done; | |
393 | ||
394 | /* fallback to base case */ | |
395 | adapter->num_rx_queues = 1; | |
396 | adapter->num_tx_queues = 1; | |
397 | ||
398 | done: | |
399 | if ((adapter->netdev->reg_state == NETREG_UNREGISTERED) || | |
400 | (adapter->netdev->reg_state == NETREG_UNREGISTERING)) | |
401 | return 0; | |
402 | ||
403 | /* Notify the stack of the (possibly) reduced queue counts. */ | |
404 | netif_set_real_num_tx_queues(adapter->netdev, adapter->num_tx_queues); | |
405 | return netif_set_real_num_rx_queues(adapter->netdev, | |
406 | adapter->num_rx_queues); | |
407 | } | |
408 | ||
409 | static void ixgbe_acquire_msix_vectors(struct ixgbe_adapter *adapter, | |
410 | int vectors) | |
411 | { | |
412 | int err, vector_threshold; | |
413 | ||
414 | /* We'll want at least 2 (vector_threshold): | |
415 | * 1) TxQ[0] + RxQ[0] handler | |
416 | * 2) Other (Link Status Change, etc.) | |
417 | */ | |
418 | vector_threshold = MIN_MSIX_COUNT; | |
419 | ||
420 | /* | |
421 | * The more we get, the more we will assign to Tx/Rx Cleanup | |
422 | * for the separate queues...where Rx Cleanup >= Tx Cleanup. | |
423 | * Right now, we simply care about how many we'll get; we'll | |
424 | * set them up later while requesting irq's. | |
425 | */ | |
426 | while (vectors >= vector_threshold) { | |
427 | err = pci_enable_msix(adapter->pdev, adapter->msix_entries, | |
428 | vectors); | |
429 | if (!err) /* Success in acquiring all requested vectors. */ | |
430 | break; | |
431 | else if (err < 0) | |
432 | vectors = 0; /* Nasty failure, quit now */ | |
433 | else /* err == number of vectors we should try again with */ | |
434 | vectors = err; | |
435 | } | |
436 | ||
437 | if (vectors < vector_threshold) { | |
438 | /* Can't allocate enough MSI-X interrupts? Oh well. | |
439 | * This just means we'll go with either a single MSI | |
440 | * vector or fall back to legacy interrupts. | |
441 | */ | |
442 | netif_printk(adapter, hw, KERN_DEBUG, adapter->netdev, | |
443 | "Unable to allocate MSI-X interrupts\n"); | |
444 | adapter->flags &= ~IXGBE_FLAG_MSIX_ENABLED; | |
445 | kfree(adapter->msix_entries); | |
446 | adapter->msix_entries = NULL; | |
447 | } else { | |
448 | adapter->flags |= IXGBE_FLAG_MSIX_ENABLED; /* Woot! */ | |
449 | /* | |
450 | * Adjust for only the vectors we'll use, which is minimum | |
451 | * of max_msix_q_vectors + NON_Q_VECTORS, or the number of | |
452 | * vectors we were allocated. | |
453 | */ | |
49c7ffbe AD |
454 | vectors -= NON_Q_VECTORS; |
455 | adapter->num_q_vectors = min(vectors, adapter->max_q_vectors); | |
8af3c33f JK |
456 | } |
457 | } | |
458 | ||
459 | static void ixgbe_add_ring(struct ixgbe_ring *ring, | |
460 | struct ixgbe_ring_container *head) | |
461 | { | |
462 | ring->next = head->ring; | |
463 | head->ring = ring; | |
464 | head->count++; | |
465 | } | |
466 | ||
467 | /** | |
468 | * ixgbe_alloc_q_vector - Allocate memory for a single interrupt vector | |
469 | * @adapter: board private structure to initialize | |
d0bfcdfd | 470 | * @v_count: q_vectors allocated on adapter, used for ring interleaving |
8af3c33f | 471 | * @v_idx: index of vector in adapter struct |
d0bfcdfd AD |
472 | * @txr_count: total number of Tx rings to allocate |
473 | * @txr_idx: index of first Tx ring to allocate | |
474 | * @rxr_count: total number of Rx rings to allocate | |
475 | * @rxr_idx: index of first Rx ring to allocate | |
8af3c33f JK |
476 | * |
477 | * We allocate one q_vector. If allocation fails we return -ENOMEM. | |
478 | **/ | |
d0bfcdfd AD |
479 | static int ixgbe_alloc_q_vector(struct ixgbe_adapter *adapter, |
480 | int v_count, int v_idx, | |
8af3c33f JK |
481 | int txr_count, int txr_idx, |
482 | int rxr_count, int rxr_idx) | |
483 | { | |
484 | struct ixgbe_q_vector *q_vector; | |
485 | struct ixgbe_ring *ring; | |
486 | int node = -1; | |
487 | int cpu = -1; | |
488 | int ring_count, size; | |
489 | ||
490 | ring_count = txr_count + rxr_count; | |
491 | size = sizeof(struct ixgbe_q_vector) + | |
492 | (sizeof(struct ixgbe_ring) * ring_count); | |
493 | ||
494 | /* customize cpu for Flow Director mapping */ | |
495 | if (adapter->flags & IXGBE_FLAG_FDIR_HASH_CAPABLE) { | |
496 | if (cpu_online(v_idx)) { | |
497 | cpu = v_idx; | |
498 | node = cpu_to_node(cpu); | |
499 | } | |
500 | } | |
501 | ||
502 | /* allocate q_vector and rings */ | |
503 | q_vector = kzalloc_node(size, GFP_KERNEL, node); | |
504 | if (!q_vector) | |
505 | q_vector = kzalloc(size, GFP_KERNEL); | |
506 | if (!q_vector) | |
507 | return -ENOMEM; | |
508 | ||
509 | /* setup affinity mask and node */ | |
510 | if (cpu != -1) | |
511 | cpumask_set_cpu(cpu, &q_vector->affinity_mask); | |
512 | else | |
513 | cpumask_copy(&q_vector->affinity_mask, cpu_online_mask); | |
514 | q_vector->numa_node = node; | |
515 | ||
516 | /* initialize NAPI */ | |
517 | netif_napi_add(adapter->netdev, &q_vector->napi, | |
518 | ixgbe_poll, 64); | |
519 | ||
520 | /* tie q_vector and adapter together */ | |
521 | adapter->q_vector[v_idx] = q_vector; | |
522 | q_vector->adapter = adapter; | |
523 | q_vector->v_idx = v_idx; | |
524 | ||
525 | /* initialize work limits */ | |
526 | q_vector->tx.work_limit = adapter->tx_work_limit; | |
527 | ||
528 | /* initialize pointer to rings */ | |
529 | ring = q_vector->ring; | |
530 | ||
531 | while (txr_count) { | |
532 | /* assign generic ring traits */ | |
533 | ring->dev = &adapter->pdev->dev; | |
534 | ring->netdev = adapter->netdev; | |
535 | ||
536 | /* configure backlink on ring */ | |
537 | ring->q_vector = q_vector; | |
538 | ||
539 | /* update q_vector Tx values */ | |
540 | ixgbe_add_ring(ring, &q_vector->tx); | |
541 | ||
542 | /* apply Tx specific ring traits */ | |
543 | ring->count = adapter->tx_ring_count; | |
544 | ring->queue_index = txr_idx; | |
545 | ||
546 | /* assign ring to adapter */ | |
547 | adapter->tx_ring[txr_idx] = ring; | |
548 | ||
549 | /* update count and index */ | |
550 | txr_count--; | |
d0bfcdfd | 551 | txr_idx += v_count; |
8af3c33f JK |
552 | |
553 | /* push pointer to next ring */ | |
554 | ring++; | |
555 | } | |
556 | ||
557 | while (rxr_count) { | |
558 | /* assign generic ring traits */ | |
559 | ring->dev = &adapter->pdev->dev; | |
560 | ring->netdev = adapter->netdev; | |
561 | ||
562 | /* configure backlink on ring */ | |
563 | ring->q_vector = q_vector; | |
564 | ||
565 | /* update q_vector Rx values */ | |
566 | ixgbe_add_ring(ring, &q_vector->rx); | |
567 | ||
568 | /* | |
569 | * 82599 errata, UDP frames with a 0 checksum | |
570 | * can be marked as checksum errors. | |
571 | */ | |
572 | if (adapter->hw.mac.type == ixgbe_mac_82599EB) | |
573 | set_bit(__IXGBE_RX_CSUM_UDP_ZERO_ERR, &ring->state); | |
574 | ||
b2db497e AD |
575 | #ifdef IXGBE_FCOE |
576 | if (adapter->netdev->features & NETIF_F_FCOE_MTU) { | |
577 | struct ixgbe_ring_feature *f; | |
578 | f = &adapter->ring_feature[RING_F_FCOE]; | |
e4b317e9 AD |
579 | if ((rxr_idx >= f->offset) && |
580 | (rxr_idx < f->offset + f->indices)) | |
57efd44c | 581 | set_bit(__IXGBE_RX_FCOE, &ring->state); |
b2db497e AD |
582 | } |
583 | ||
584 | #endif /* IXGBE_FCOE */ | |
8af3c33f JK |
585 | /* apply Rx specific ring traits */ |
586 | ring->count = adapter->rx_ring_count; | |
587 | ring->queue_index = rxr_idx; | |
588 | ||
589 | /* assign ring to adapter */ | |
590 | adapter->rx_ring[rxr_idx] = ring; | |
591 | ||
592 | /* update count and index */ | |
593 | rxr_count--; | |
d0bfcdfd | 594 | rxr_idx += v_count; |
8af3c33f JK |
595 | |
596 | /* push pointer to next ring */ | |
597 | ring++; | |
598 | } | |
599 | ||
600 | return 0; | |
601 | } | |
602 | ||
603 | /** | |
604 | * ixgbe_free_q_vector - Free memory allocated for specific interrupt vector | |
605 | * @adapter: board private structure to initialize | |
606 | * @v_idx: Index of vector to be freed | |
607 | * | |
608 | * This function frees the memory allocated to the q_vector. In addition if | |
609 | * NAPI is enabled it will delete any references to the NAPI struct prior | |
610 | * to freeing the q_vector. | |
611 | **/ | |
612 | static void ixgbe_free_q_vector(struct ixgbe_adapter *adapter, int v_idx) | |
613 | { | |
614 | struct ixgbe_q_vector *q_vector = adapter->q_vector[v_idx]; | |
615 | struct ixgbe_ring *ring; | |
616 | ||
617 | ixgbe_for_each_ring(ring, q_vector->tx) | |
618 | adapter->tx_ring[ring->queue_index] = NULL; | |
619 | ||
620 | ixgbe_for_each_ring(ring, q_vector->rx) | |
621 | adapter->rx_ring[ring->queue_index] = NULL; | |
622 | ||
623 | adapter->q_vector[v_idx] = NULL; | |
624 | netif_napi_del(&q_vector->napi); | |
625 | ||
626 | /* | |
627 | * ixgbe_get_stats64() might access the rings on this vector, | |
628 | * we must wait a grace period before freeing it. | |
629 | */ | |
630 | kfree_rcu(q_vector, rcu); | |
631 | } | |
632 | ||
633 | /** | |
634 | * ixgbe_alloc_q_vectors - Allocate memory for interrupt vectors | |
635 | * @adapter: board private structure to initialize | |
636 | * | |
637 | * We allocate one q_vector per queue interrupt. If allocation fails we | |
638 | * return -ENOMEM. | |
639 | **/ | |
640 | static int ixgbe_alloc_q_vectors(struct ixgbe_adapter *adapter) | |
641 | { | |
49c7ffbe | 642 | int q_vectors = adapter->num_q_vectors; |
8af3c33f JK |
643 | int rxr_remaining = adapter->num_rx_queues; |
644 | int txr_remaining = adapter->num_tx_queues; | |
645 | int rxr_idx = 0, txr_idx = 0, v_idx = 0; | |
646 | int err; | |
647 | ||
648 | /* only one q_vector if MSI-X is disabled. */ | |
649 | if (!(adapter->flags & IXGBE_FLAG_MSIX_ENABLED)) | |
650 | q_vectors = 1; | |
651 | ||
652 | if (q_vectors >= (rxr_remaining + txr_remaining)) { | |
d0bfcdfd AD |
653 | for (; rxr_remaining; v_idx++) { |
654 | err = ixgbe_alloc_q_vector(adapter, q_vectors, v_idx, | |
655 | 0, 0, 1, rxr_idx); | |
8af3c33f JK |
656 | |
657 | if (err) | |
658 | goto err_out; | |
659 | ||
660 | /* update counts and index */ | |
d0bfcdfd AD |
661 | rxr_remaining--; |
662 | rxr_idx++; | |
8af3c33f JK |
663 | } |
664 | } | |
665 | ||
d0bfcdfd AD |
666 | for (; v_idx < q_vectors; v_idx++) { |
667 | int rqpv = DIV_ROUND_UP(rxr_remaining, q_vectors - v_idx); | |
668 | int tqpv = DIV_ROUND_UP(txr_remaining, q_vectors - v_idx); | |
669 | err = ixgbe_alloc_q_vector(adapter, q_vectors, v_idx, | |
8af3c33f JK |
670 | tqpv, txr_idx, |
671 | rqpv, rxr_idx); | |
672 | ||
673 | if (err) | |
674 | goto err_out; | |
675 | ||
676 | /* update counts and index */ | |
677 | rxr_remaining -= rqpv; | |
8af3c33f | 678 | txr_remaining -= tqpv; |
d0bfcdfd AD |
679 | rxr_idx++; |
680 | txr_idx++; | |
8af3c33f JK |
681 | } |
682 | ||
683 | return 0; | |
684 | ||
685 | err_out: | |
49c7ffbe AD |
686 | adapter->num_tx_queues = 0; |
687 | adapter->num_rx_queues = 0; | |
688 | adapter->num_q_vectors = 0; | |
689 | ||
690 | while (v_idx--) | |
8af3c33f | 691 | ixgbe_free_q_vector(adapter, v_idx); |
8af3c33f JK |
692 | |
693 | return -ENOMEM; | |
694 | } | |
695 | ||
696 | /** | |
697 | * ixgbe_free_q_vectors - Free memory allocated for interrupt vectors | |
698 | * @adapter: board private structure to initialize | |
699 | * | |
700 | * This function frees the memory allocated to the q_vectors. In addition if | |
701 | * NAPI is enabled it will delete any references to the NAPI struct prior | |
702 | * to freeing the q_vector. | |
703 | **/ | |
704 | static void ixgbe_free_q_vectors(struct ixgbe_adapter *adapter) | |
705 | { | |
49c7ffbe | 706 | int v_idx = adapter->num_q_vectors; |
8af3c33f | 707 | |
49c7ffbe AD |
708 | adapter->num_tx_queues = 0; |
709 | adapter->num_rx_queues = 0; | |
710 | adapter->num_q_vectors = 0; | |
8af3c33f | 711 | |
49c7ffbe | 712 | while (v_idx--) |
8af3c33f JK |
713 | ixgbe_free_q_vector(adapter, v_idx); |
714 | } | |
715 | ||
716 | static void ixgbe_reset_interrupt_capability(struct ixgbe_adapter *adapter) | |
717 | { | |
718 | if (adapter->flags & IXGBE_FLAG_MSIX_ENABLED) { | |
719 | adapter->flags &= ~IXGBE_FLAG_MSIX_ENABLED; | |
720 | pci_disable_msix(adapter->pdev); | |
721 | kfree(adapter->msix_entries); | |
722 | adapter->msix_entries = NULL; | |
723 | } else if (adapter->flags & IXGBE_FLAG_MSI_ENABLED) { | |
724 | adapter->flags &= ~IXGBE_FLAG_MSI_ENABLED; | |
725 | pci_disable_msi(adapter->pdev); | |
726 | } | |
727 | } | |
728 | ||
729 | /** | |
730 | * ixgbe_set_interrupt_capability - set MSI-X or MSI if supported | |
731 | * @adapter: board private structure to initialize | |
732 | * | |
733 | * Attempt to configure the interrupts using the best available | |
734 | * capabilities of the hardware and the kernel. | |
735 | **/ | |
736 | static int ixgbe_set_interrupt_capability(struct ixgbe_adapter *adapter) | |
737 | { | |
738 | struct ixgbe_hw *hw = &adapter->hw; | |
739 | int err = 0; | |
740 | int vector, v_budget; | |
741 | ||
742 | /* | |
743 | * It's easy to be greedy for MSI-X vectors, but it really | |
744 | * doesn't do us much good if we have a lot more vectors | |
745 | * than CPU's. So let's be conservative and only ask for | |
746 | * (roughly) the same number of vectors as there are CPU's. | |
747 | * The default is to use pairs of vectors. | |
748 | */ | |
749 | v_budget = max(adapter->num_rx_queues, adapter->num_tx_queues); | |
750 | v_budget = min_t(int, v_budget, num_online_cpus()); | |
751 | v_budget += NON_Q_VECTORS; | |
752 | ||
753 | /* | |
754 | * At the same time, hardware can only support a maximum of | |
755 | * hw.mac->max_msix_vectors vectors. With features | |
756 | * such as RSS and VMDq, we can easily surpass the number of Rx and Tx | |
757 | * descriptor queues supported by our device. Thus, we cap it off in | |
758 | * those rare cases where the cpu count also exceeds our vector limit. | |
759 | */ | |
760 | v_budget = min_t(int, v_budget, hw->mac.max_msix_vectors); | |
761 | ||
762 | /* A failure in MSI-X entry allocation isn't fatal, but it does | |
763 | * mean we disable MSI-X capabilities of the adapter. */ | |
764 | adapter->msix_entries = kcalloc(v_budget, | |
765 | sizeof(struct msix_entry), GFP_KERNEL); | |
766 | if (adapter->msix_entries) { | |
767 | for (vector = 0; vector < v_budget; vector++) | |
768 | adapter->msix_entries[vector].entry = vector; | |
769 | ||
770 | ixgbe_acquire_msix_vectors(adapter, v_budget); | |
771 | ||
772 | if (adapter->flags & IXGBE_FLAG_MSIX_ENABLED) | |
773 | goto out; | |
774 | } | |
775 | ||
776 | adapter->flags &= ~IXGBE_FLAG_DCB_ENABLED; | |
777 | adapter->flags &= ~IXGBE_FLAG_RSS_ENABLED; | |
778 | if (adapter->flags & IXGBE_FLAG_FDIR_HASH_CAPABLE) { | |
779 | e_err(probe, | |
780 | "ATR is not supported while multiple " | |
781 | "queues are disabled. Disabling Flow Director\n"); | |
782 | } | |
783 | adapter->flags &= ~IXGBE_FLAG_FDIR_HASH_CAPABLE; | |
784 | adapter->atr_sample_rate = 0; | |
785 | if (adapter->flags & IXGBE_FLAG_SRIOV_ENABLED) | |
786 | ixgbe_disable_sriov(adapter); | |
787 | ||
788 | err = ixgbe_set_num_queues(adapter); | |
789 | if (err) | |
790 | return err; | |
791 | ||
49c7ffbe AD |
792 | adapter->num_q_vectors = 1; |
793 | ||
8af3c33f JK |
794 | err = pci_enable_msi(adapter->pdev); |
795 | if (!err) { | |
796 | adapter->flags |= IXGBE_FLAG_MSI_ENABLED; | |
797 | } else { | |
798 | netif_printk(adapter, hw, KERN_DEBUG, adapter->netdev, | |
799 | "Unable to allocate MSI interrupt, " | |
800 | "falling back to legacy. Error: %d\n", err); | |
801 | /* reset err */ | |
802 | err = 0; | |
803 | } | |
804 | ||
805 | out: | |
806 | return err; | |
807 | } | |
808 | ||
809 | /** | |
810 | * ixgbe_init_interrupt_scheme - Determine proper interrupt scheme | |
811 | * @adapter: board private structure to initialize | |
812 | * | |
813 | * We determine which interrupt scheme to use based on... | |
814 | * - Kernel support (MSI, MSI-X) | |
815 | * - which can be user-defined (via MODULE_PARAM) | |
816 | * - Hardware queue count (num_*_queues) | |
817 | * - defined by miscellaneous hardware support/features (RSS, etc.) | |
818 | **/ | |
819 | int ixgbe_init_interrupt_scheme(struct ixgbe_adapter *adapter) | |
820 | { | |
821 | int err; | |
822 | ||
823 | /* Number of supported queues */ | |
824 | err = ixgbe_set_num_queues(adapter); | |
825 | if (err) | |
826 | return err; | |
827 | ||
828 | err = ixgbe_set_interrupt_capability(adapter); | |
829 | if (err) { | |
830 | e_dev_err("Unable to setup interrupt capabilities\n"); | |
831 | goto err_set_interrupt; | |
832 | } | |
833 | ||
834 | err = ixgbe_alloc_q_vectors(adapter); | |
835 | if (err) { | |
836 | e_dev_err("Unable to allocate memory for queue vectors\n"); | |
837 | goto err_alloc_q_vectors; | |
838 | } | |
839 | ||
840 | ixgbe_cache_ring_register(adapter); | |
841 | ||
842 | e_dev_info("Multiqueue %s: Rx Queue count = %u, Tx Queue count = %u\n", | |
843 | (adapter->num_rx_queues > 1) ? "Enabled" : "Disabled", | |
844 | adapter->num_rx_queues, adapter->num_tx_queues); | |
845 | ||
846 | set_bit(__IXGBE_DOWN, &adapter->state); | |
847 | ||
848 | return 0; | |
849 | ||
850 | err_alloc_q_vectors: | |
851 | ixgbe_reset_interrupt_capability(adapter); | |
852 | err_set_interrupt: | |
853 | return err; | |
854 | } | |
855 | ||
856 | /** | |
857 | * ixgbe_clear_interrupt_scheme - Clear the current interrupt scheme settings | |
858 | * @adapter: board private structure to clear interrupt scheme on | |
859 | * | |
860 | * We go through and clear interrupt specific resources and reset the structure | |
861 | * to pre-load conditions | |
862 | **/ | |
863 | void ixgbe_clear_interrupt_scheme(struct ixgbe_adapter *adapter) | |
864 | { | |
865 | adapter->num_tx_queues = 0; | |
866 | adapter->num_rx_queues = 0; | |
867 | ||
868 | ixgbe_free_q_vectors(adapter); | |
869 | ixgbe_reset_interrupt_capability(adapter); | |
870 | } | |
871 | ||
872 | void ixgbe_tx_ctxtdesc(struct ixgbe_ring *tx_ring, u32 vlan_macip_lens, | |
873 | u32 fcoe_sof_eof, u32 type_tucmd, u32 mss_l4len_idx) | |
874 | { | |
875 | struct ixgbe_adv_tx_context_desc *context_desc; | |
876 | u16 i = tx_ring->next_to_use; | |
877 | ||
878 | context_desc = IXGBE_TX_CTXTDESC(tx_ring, i); | |
879 | ||
880 | i++; | |
881 | tx_ring->next_to_use = (i < tx_ring->count) ? i : 0; | |
882 | ||
883 | /* set bits to identify this as an advanced context descriptor */ | |
884 | type_tucmd |= IXGBE_TXD_CMD_DEXT | IXGBE_ADVTXD_DTYP_CTXT; | |
885 | ||
886 | context_desc->vlan_macip_lens = cpu_to_le32(vlan_macip_lens); | |
887 | context_desc->seqnum_seed = cpu_to_le32(fcoe_sof_eof); | |
888 | context_desc->type_tucmd_mlhl = cpu_to_le32(type_tucmd); | |
889 | context_desc->mss_l4len_idx = cpu_to_le32(mss_l4len_idx); | |
890 | } | |
891 |