projects / deliverable / linux.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
e1000e: cleanup PARENTHESIS_ALIGNMENT checkpatch checks
[deliverable/linux.git] / drivers / net / ethernet / intel / e1000e / netdev.c
1 /*******************************************************************************
2
3 Intel PRO/1000 Linux driver
4 Copyright(c) 1999 - 2013 Intel Corporation.
5
6 This program is free software; you can redistribute it and/or modify it
7 under the terms and conditions of the GNU General Public License,
8 version 2, as published by the Free Software Foundation.
9
10 This program is distributed in the hope it will be useful, but WITHOUT
11 ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
13 more details.
14
15 You should have received a copy of the GNU General Public License along with
16 this program; if not, write to the Free Software Foundation, Inc.,
17 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
18
19 The full GNU General Public License is included in this distribution in
20 the file called "COPYING".
21
22 Contact Information:
23 Linux NICS <linux.nics@intel.com>
24 e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
25 Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
26
27 *******************************************************************************/
28
29 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
30
31 #include <linux/module.h>
32 #include <linux/types.h>
33 #include <linux/init.h>
34 #include <linux/pci.h>
35 #include <linux/vmalloc.h>
36 #include <linux/pagemap.h>
37 #include <linux/delay.h>
38 #include <linux/netdevice.h>
39 #include <linux/interrupt.h>
40 #include <linux/tcp.h>
41 #include <linux/ipv6.h>
42 #include <linux/slab.h>
43 #include <net/checksum.h>
44 #include <net/ip6_checksum.h>
45 #include <linux/ethtool.h>
46 #include <linux/if_vlan.h>
47 #include <linux/cpu.h>
48 #include <linux/smp.h>
49 #include <linux/pm_qos.h>
50 #include <linux/pm_runtime.h>
51 #include <linux/aer.h>
52 #include <linux/prefetch.h>
53
54 #include "e1000.h"
55
56 #define DRV_EXTRAVERSION "-k"
57
58 #define DRV_VERSION "2.2.14" DRV_EXTRAVERSION
59 char e1000e_driver_name[] = "e1000e";
60 const char e1000e_driver_version[] = DRV_VERSION;
61
62 #define DEFAULT_MSG_ENABLE (NETIF_MSG_DRV|NETIF_MSG_PROBE|NETIF_MSG_LINK)
63 static int debug = -1;
64 module_param(debug, int, 0);
65 MODULE_PARM_DESC(debug, "Debug level (0=none,...,16=all)");
66
67 static void e1000e_disable_aspm(struct pci_dev *pdev, u16 state);
68
69 static const struct e1000_info *e1000_info_tbl[] = {
70 [board_82571] = &e1000_82571_info,
71 [board_82572] = &e1000_82572_info,
72 [board_82573] = &e1000_82573_info,
73 [board_82574] = &e1000_82574_info,
74 [board_82583] = &e1000_82583_info,
75 [board_80003es2lan] = &e1000_es2_info,
76 [board_ich8lan] = &e1000_ich8_info,
77 [board_ich9lan] = &e1000_ich9_info,
78 [board_ich10lan] = &e1000_ich10_info,
79 [board_pchlan] = &e1000_pch_info,
80 [board_pch2lan] = &e1000_pch2_info,
81 [board_pch_lpt] = &e1000_pch_lpt_info,
82 };
83
84 struct e1000_reg_info {
85 u32 ofs;
86 char *name;
87 };
88
89 static const struct e1000_reg_info e1000_reg_info_tbl[] = {
90 /* General Registers */
91 {E1000_CTRL, "CTRL"},
92 {E1000_STATUS, "STATUS"},
93 {E1000_CTRL_EXT, "CTRL_EXT"},
94
95 /* Interrupt Registers */
96 {E1000_ICR, "ICR"},
97
98 /* Rx Registers */
99 {E1000_RCTL, "RCTL"},
100 {E1000_RDLEN(0), "RDLEN"},
101 {E1000_RDH(0), "RDH"},
102 {E1000_RDT(0), "RDT"},
103 {E1000_RDTR, "RDTR"},
104 {E1000_RXDCTL(0), "RXDCTL"},
105 {E1000_ERT, "ERT"},
106 {E1000_RDBAL(0), "RDBAL"},
107 {E1000_RDBAH(0), "RDBAH"},
108 {E1000_RDFH, "RDFH"},
109 {E1000_RDFT, "RDFT"},
110 {E1000_RDFHS, "RDFHS"},
111 {E1000_RDFTS, "RDFTS"},
112 {E1000_RDFPC, "RDFPC"},
113
114 /* Tx Registers */
115 {E1000_TCTL, "TCTL"},
116 {E1000_TDBAL(0), "TDBAL"},
117 {E1000_TDBAH(0), "TDBAH"},
118 {E1000_TDLEN(0), "TDLEN"},
119 {E1000_TDH(0), "TDH"},
120 {E1000_TDT(0), "TDT"},
121 {E1000_TIDV, "TIDV"},
122 {E1000_TXDCTL(0), "TXDCTL"},
123 {E1000_TADV, "TADV"},
124 {E1000_TARC(0), "TARC"},
125 {E1000_TDFH, "TDFH"},
126 {E1000_TDFT, "TDFT"},
127 {E1000_TDFHS, "TDFHS"},
128 {E1000_TDFTS, "TDFTS"},
129 {E1000_TDFPC, "TDFPC"},
130
131 /* List Terminator */
132 {0, NULL}
133 };
134
135 /**
136 * e1000_regdump - register printout routine
137 * @hw: pointer to the HW structure
138 * @reginfo: pointer to the register info table
139 **/
140 static void e1000_regdump(struct e1000_hw *hw, struct e1000_reg_info *reginfo)
141 {
142 int n = 0;
143 char rname[16];
144 u32 regs[8];
145
146 switch (reginfo->ofs) {
147 case E1000_RXDCTL(0):
148 for (n = 0; n < 2; n++)
149 regs[n] = __er32(hw, E1000_RXDCTL(n));
150 break;
151 case E1000_TXDCTL(0):
152 for (n = 0; n < 2; n++)
153 regs[n] = __er32(hw, E1000_TXDCTL(n));
154 break;
155 case E1000_TARC(0):
156 for (n = 0; n < 2; n++)
157 regs[n] = __er32(hw, E1000_TARC(n));
158 break;
159 default:
160 pr_info("%-15s %08x\n",
161 reginfo->name, __er32(hw, reginfo->ofs));
162 return;
163 }
164
165 snprintf(rname, 16, "%s%s", reginfo->name, "[0-1]");
166 pr_info("%-15s %08x %08x\n", rname, regs[0], regs[1]);
167 }
168
169 static void e1000e_dump_ps_pages(struct e1000_adapter *adapter,
170 struct e1000_buffer *bi)
171 {
172 int i;
173 struct e1000_ps_page *ps_page;
174
175 for (i = 0; i < adapter->rx_ps_pages; i++) {
176 ps_page = &bi->ps_pages[i];
177
178 if (ps_page->page) {
179 pr_info("packet dump for ps_page %d:\n", i);
180 print_hex_dump(KERN_INFO, "", DUMP_PREFIX_ADDRESS,
181 16, 1, page_address(ps_page->page),
182 PAGE_SIZE, true);
183 }
184 }
185 }
186
187 /**
188 * e1000e_dump - Print registers, Tx-ring and Rx-ring
189 * @adapter: board private structure
190 **/
191 static void e1000e_dump(struct e1000_adapter *adapter)
192 {
193 struct net_device *netdev = adapter->netdev;
194 struct e1000_hw *hw = &adapter->hw;
195 struct e1000_reg_info *reginfo;
196 struct e1000_ring *tx_ring = adapter->tx_ring;
197 struct e1000_tx_desc *tx_desc;
198 struct my_u0 {
199 __le64 a;
200 __le64 b;
201 } *u0;
202 struct e1000_buffer *buffer_info;
203 struct e1000_ring *rx_ring = adapter->rx_ring;
204 union e1000_rx_desc_packet_split *rx_desc_ps;
205 union e1000_rx_desc_extended *rx_desc;
206 struct my_u1 {
207 __le64 a;
208 __le64 b;
209 __le64 c;
210 __le64 d;
211 } *u1;
212 u32 staterr;
213 int i = 0;
214
215 if (!netif_msg_hw(adapter))
216 return;
217
218 /* Print netdevice Info */
219 if (netdev) {
220 dev_info(&adapter->pdev->dev, "Net device Info\n");
221 pr_info("Device Name state trans_start last_rx\n");
222 pr_info("%-15s %016lX %016lX %016lX\n",
223 netdev->name, netdev->state, netdev->trans_start,
224 netdev->last_rx);
225 }
226
227 /* Print Registers */
228 dev_info(&adapter->pdev->dev, "Register Dump\n");
229 pr_info(" Register Name Value\n");
230 for (reginfo = (struct e1000_reg_info *)e1000_reg_info_tbl;
231 reginfo->name; reginfo++) {
232 e1000_regdump(hw, reginfo);
233 }
234
235 /* Print Tx Ring Summary */
236 if (!netdev || !netif_running(netdev))
237 return;
238
239 dev_info(&adapter->pdev->dev, "Tx Ring Summary\n");
240 pr_info("Queue [NTU] [NTC] [bi(ntc)->dma ] leng ntw timestamp\n");
241 buffer_info = &tx_ring->buffer_info[tx_ring->next_to_clean];
242 pr_info(" %5d %5X %5X %016llX %04X %3X %016llX\n",
243 0, tx_ring->next_to_use, tx_ring->next_to_clean,
244 (unsigned long long)buffer_info->dma,
245 buffer_info->length,
246 buffer_info->next_to_watch,
247 (unsigned long long)buffer_info->time_stamp);
248
249 /* Print Tx Ring */
250 if (!netif_msg_tx_done(adapter))
251 goto rx_ring_summary;
252
253 dev_info(&adapter->pdev->dev, "Tx Ring Dump\n");
254
255 /* Transmit Descriptor Formats - DEXT[29] is 0 (Legacy) or 1 (Extended)
256 *
257 * Legacy Transmit Descriptor
258 * +--------------------------------------------------------------+
259 * 0 | Buffer Address [63:0] (Reserved on Write Back) |
260 * +--------------------------------------------------------------+
261 * 8 | Special | CSS | Status | CMD | CSO | Length |
262 * +--------------------------------------------------------------+
263 * 63 48 47 36 35 32 31 24 23 16 15 0
264 *
265 * Extended Context Descriptor (DTYP=0x0) for TSO or checksum offload
266 * 63 48 47 40 39 32 31 16 15 8 7 0
267 * +----------------------------------------------------------------+
268 * 0 | TUCSE | TUCS0 | TUCSS | IPCSE | IPCS0 | IPCSS |
269 * +----------------------------------------------------------------+
270 * 8 | MSS | HDRLEN | RSV | STA | TUCMD | DTYP | PAYLEN |
271 * +----------------------------------------------------------------+
272 * 63 48 47 40 39 36 35 32 31 24 23 20 19 0
273 *
274 * Extended Data Descriptor (DTYP=0x1)
275 * +----------------------------------------------------------------+
276 * 0 | Buffer Address [63:0] |
277 * +----------------------------------------------------------------+
278 * 8 | VLAN tag | POPTS | Rsvd | Status | Command | DTYP | DTALEN |
279 * +----------------------------------------------------------------+
280 * 63 48 47 40 39 36 35 32 31 24 23 20 19 0
281 */
282 pr_info("Tl[desc] [address 63:0 ] [SpeCssSCmCsLen] [bi->dma ] leng ntw timestamp bi->skb <-- Legacy format\n");
283 pr_info("Tc[desc] [Ce CoCsIpceCoS] [MssHlRSCm0Plen] [bi->dma ] leng ntw timestamp bi->skb <-- Ext Context format\n");
284 pr_info("Td[desc] [address 63:0 ] [VlaPoRSCm1Dlen] [bi->dma ] leng ntw timestamp bi->skb <-- Ext Data format\n");
285 for (i = 0; tx_ring->desc && (i < tx_ring->count); i++) {
286 const char *next_desc;
287 tx_desc = E1000_TX_DESC(*tx_ring, i);
288 buffer_info = &tx_ring->buffer_info[i];
289 u0 = (struct my_u0 *)tx_desc;
290 if (i == tx_ring->next_to_use && i == tx_ring->next_to_clean)
291 next_desc = " NTC/U";
292 else if (i == tx_ring->next_to_use)
293 next_desc = " NTU";
294 else if (i == tx_ring->next_to_clean)
295 next_desc = " NTC";
296 else
297 next_desc = "";
298 pr_info("T%c[0x%03X] %016llX %016llX %016llX %04X %3X %016llX %p%s\n",
299 (!(le64_to_cpu(u0->b) & (1 << 29)) ? 'l' :
300 ((le64_to_cpu(u0->b) & (1 << 20)) ? 'd' : 'c')),
301 i,
302 (unsigned long long)le64_to_cpu(u0->a),
303 (unsigned long long)le64_to_cpu(u0->b),
304 (unsigned long long)buffer_info->dma,
305 buffer_info->length, buffer_info->next_to_watch,
306 (unsigned long long)buffer_info->time_stamp,
307 buffer_info->skb, next_desc);
308
309 if (netif_msg_pktdata(adapter) && buffer_info->skb)
310 print_hex_dump(KERN_INFO, "", DUMP_PREFIX_ADDRESS,
311 16, 1, buffer_info->skb->data,
312 buffer_info->skb->len, true);
313 }
314
315 /* Print Rx Ring Summary */
316 rx_ring_summary:
317 dev_info(&adapter->pdev->dev, "Rx Ring Summary\n");
318 pr_info("Queue [NTU] [NTC]\n");
319 pr_info(" %5d %5X %5X\n",
320 0, rx_ring->next_to_use, rx_ring->next_to_clean);
321
322 /* Print Rx Ring */
323 if (!netif_msg_rx_status(adapter))
324 return;
325
326 dev_info(&adapter->pdev->dev, "Rx Ring Dump\n");
327 switch (adapter->rx_ps_pages) {
328 case 1:
329 case 2:
330 case 3:
331 /* [Extended] Packet Split Receive Descriptor Format
332 *
333 * +-----------------------------------------------------+
334 * 0 | Buffer Address 0 [63:0] |
335 * +-----------------------------------------------------+
336 * 8 | Buffer Address 1 [63:0] |
337 * +-----------------------------------------------------+
338 * 16 | Buffer Address 2 [63:0] |
339 * +-----------------------------------------------------+
340 * 24 | Buffer Address 3 [63:0] |
341 * +-----------------------------------------------------+
342 */
343 pr_info("R [desc] [buffer 0 63:0 ] [buffer 1 63:0 ] [buffer 2 63:0 ] [buffer 3 63:0 ] [bi->dma ] [bi->skb] <-- Ext Pkt Split format\n");
344 /* [Extended] Receive Descriptor (Write-Back) Format
345 *
346 * 63 48 47 32 31 13 12 8 7 4 3 0
347 * +------------------------------------------------------+
348 * 0 | Packet | IP | Rsvd | MRQ | Rsvd | MRQ RSS |
349 * | Checksum | Ident | | Queue | | Type |
350 * +------------------------------------------------------+
351 * 8 | VLAN Tag | Length | Extended Error | Extended Status |
352 * +------------------------------------------------------+
353 * 63 48 47 32 31 20 19 0
354 */
355 pr_info("RWB[desc] [ck ipid mrqhsh] [vl l0 ee es] [ l3 l2 l1 hs] [reserved ] ---------------- [bi->skb] <-- Ext Rx Write-Back format\n");
356 for (i = 0; i < rx_ring->count; i++) {
357 const char *next_desc;
358 buffer_info = &rx_ring->buffer_info[i];
359 rx_desc_ps = E1000_RX_DESC_PS(*rx_ring, i);
360 u1 = (struct my_u1 *)rx_desc_ps;
361 staterr =
362 le32_to_cpu(rx_desc_ps->wb.middle.status_error);
363
364 if (i == rx_ring->next_to_use)
365 next_desc = " NTU";
366 else if (i == rx_ring->next_to_clean)
367 next_desc = " NTC";
368 else
369 next_desc = "";
370
371 if (staterr & E1000_RXD_STAT_DD) {
372 /* Descriptor Done */
373 pr_info("%s[0x%03X] %016llX %016llX %016llX %016llX ---------------- %p%s\n",
374 "RWB", i,
375 (unsigned long long)le64_to_cpu(u1->a),
376 (unsigned long long)le64_to_cpu(u1->b),
377 (unsigned long long)le64_to_cpu(u1->c),
378 (unsigned long long)le64_to_cpu(u1->d),
379 buffer_info->skb, next_desc);
380 } else {
381 pr_info("%s[0x%03X] %016llX %016llX %016llX %016llX %016llX %p%s\n",
382 "R ", i,
383 (unsigned long long)le64_to_cpu(u1->a),
384 (unsigned long long)le64_to_cpu(u1->b),
385 (unsigned long long)le64_to_cpu(u1->c),
386 (unsigned long long)le64_to_cpu(u1->d),
387 (unsigned long long)buffer_info->dma,
388 buffer_info->skb, next_desc);
389
390 if (netif_msg_pktdata(adapter))
391 e1000e_dump_ps_pages(adapter,
392 buffer_info);
393 }
394 }
395 break;
396 default:
397 case 0:
398 /* Extended Receive Descriptor (Read) Format
399 *
400 * +-----------------------------------------------------+
401 * 0 | Buffer Address [63:0] |
402 * +-----------------------------------------------------+
403 * 8 | Reserved |
404 * +-----------------------------------------------------+
405 */
406 pr_info("R [desc] [buf addr 63:0 ] [reserved 63:0 ] [bi->dma ] [bi->skb] <-- Ext (Read) format\n");
407 /* Extended Receive Descriptor (Write-Back) Format
408 *
409 * 63 48 47 32 31 24 23 4 3 0
410 * +------------------------------------------------------+
411 * | RSS Hash | | | |
412 * 0 +-------------------+ Rsvd | Reserved | MRQ RSS |
413 * | Packet | IP | | | Type |
414 * | Checksum | Ident | | | |
415 * +------------------------------------------------------+
416 * 8 | VLAN Tag | Length | Extended Error | Extended Status |
417 * +------------------------------------------------------+
418 * 63 48 47 32 31 20 19 0
419 */
420 pr_info("RWB[desc] [cs ipid mrq] [vt ln xe xs] [bi->skb] <-- Ext (Write-Back) format\n");
421
422 for (i = 0; i < rx_ring->count; i++) {
423 const char *next_desc;
424
425 buffer_info = &rx_ring->buffer_info[i];
426 rx_desc = E1000_RX_DESC_EXT(*rx_ring, i);
427 u1 = (struct my_u1 *)rx_desc;
428 staterr = le32_to_cpu(rx_desc->wb.upper.status_error);
429
430 if (i == rx_ring->next_to_use)
431 next_desc = " NTU";
432 else if (i == rx_ring->next_to_clean)
433 next_desc = " NTC";
434 else
435 next_desc = "";
436
437 if (staterr & E1000_RXD_STAT_DD) {
438 /* Descriptor Done */
439 pr_info("%s[0x%03X] %016llX %016llX ---------------- %p%s\n",
440 "RWB", i,
441 (unsigned long long)le64_to_cpu(u1->a),
442 (unsigned long long)le64_to_cpu(u1->b),
443 buffer_info->skb, next_desc);
444 } else {
445 pr_info("%s[0x%03X] %016llX %016llX %016llX %p%s\n",
446 "R ", i,
447 (unsigned long long)le64_to_cpu(u1->a),
448 (unsigned long long)le64_to_cpu(u1->b),
449 (unsigned long long)buffer_info->dma,
450 buffer_info->skb, next_desc);
451
452 if (netif_msg_pktdata(adapter) &&
453 buffer_info->skb)
454 print_hex_dump(KERN_INFO, "",
455 DUMP_PREFIX_ADDRESS, 16,
456 1,
457 buffer_info->skb->data,
458 adapter->rx_buffer_len,
459 true);
460 }
461 }
462 }
463 }
464
465 /**
466 * e1000_desc_unused - calculate if we have unused descriptors
467 **/
468 static int e1000_desc_unused(struct e1000_ring *ring)
469 {
470 if (ring->next_to_clean > ring->next_to_use)
471 return ring->next_to_clean - ring->next_to_use - 1;
472
473 return ring->count + ring->next_to_clean - ring->next_to_use - 1;
474 }
475
476 /**
477 * e1000e_systim_to_hwtstamp - convert system time value to hw time stamp
478 * @adapter: board private structure
479 * @hwtstamps: time stamp structure to update
480 * @systim: unsigned 64bit system time value.
481 *
482 * Convert the system time value stored in the RX/TXSTMP registers into a
483 * hwtstamp which can be used by the upper level time stamping functions.
484 *
485 * The 'systim_lock' spinlock is used to protect the consistency of the
486 * system time value. This is needed because reading the 64 bit time
487 * value involves reading two 32 bit registers. The first read latches the
488 * value.
489 **/
490 static void e1000e_systim_to_hwtstamp(struct e1000_adapter *adapter,
491 struct skb_shared_hwtstamps *hwtstamps,
492 u64 systim)
493 {
494 u64 ns;
495 unsigned long flags;
496
497 spin_lock_irqsave(&adapter->systim_lock, flags);
498 ns = timecounter_cyc2time(&adapter->tc, systim);
499 spin_unlock_irqrestore(&adapter->systim_lock, flags);
500
501 memset(hwtstamps, 0, sizeof(*hwtstamps));
502 hwtstamps->hwtstamp = ns_to_ktime(ns);
503 }
504
505 /**
506 * e1000e_rx_hwtstamp - utility function which checks for Rx time stamp
507 * @adapter: board private structure
508 * @status: descriptor extended error and status field
509 * @skb: particular skb to include time stamp
510 *
511 * If the time stamp is valid, convert it into the timecounter ns value
512 * and store that result into the shhwtstamps structure which is passed
513 * up the network stack.
514 **/
515 static void e1000e_rx_hwtstamp(struct e1000_adapter *adapter, u32 status,
516 struct sk_buff *skb)
517 {
518 struct e1000_hw *hw = &adapter->hw;
519 u64 rxstmp;
520
521 if (!(adapter->flags & FLAG_HAS_HW_TIMESTAMP) ||
522 !(status & E1000_RXDEXT_STATERR_TST) ||
523 !(er32(TSYNCRXCTL) & E1000_TSYNCRXCTL_VALID))
524 return;
525
526 /* The Rx time stamp registers contain the time stamp. No other
527 * received packet will be time stamped until the Rx time stamp
528 * registers are read. Because only one packet can be time stamped
529 * at a time, the register values must belong to this packet and
530 * therefore none of the other additional attributes need to be
531 * compared.
532 */
533 rxstmp = (u64)er32(RXSTMPL);
534 rxstmp |= (u64)er32(RXSTMPH) << 32;
535 e1000e_systim_to_hwtstamp(adapter, skb_hwtstamps(skb), rxstmp);
536
537 adapter->flags2 &= ~FLAG2_CHECK_RX_HWTSTAMP;
538 }
539
540 /**
541 * e1000_receive_skb - helper function to handle Rx indications
542 * @adapter: board private structure
543 * @staterr: descriptor extended error and status field as written by hardware
544 * @vlan: descriptor vlan field as written by hardware (no le/be conversion)
545 * @skb: pointer to sk_buff to be indicated to stack
546 **/
547 static void e1000_receive_skb(struct e1000_adapter *adapter,
548 struct net_device *netdev, struct sk_buff *skb,
549 u32 staterr, __le16 vlan)
550 {
551 u16 tag = le16_to_cpu(vlan);
552
553 e1000e_rx_hwtstamp(adapter, staterr, skb);
554
555 skb->protocol = eth_type_trans(skb, netdev);
556
557 if (staterr & E1000_RXD_STAT_VP)
558 __vlan_hwaccel_put_tag(skb, tag);
559
560 napi_gro_receive(&adapter->napi, skb);
561 }
562
563 /**
564 * e1000_rx_checksum - Receive Checksum Offload
565 * @adapter: board private structure
566 * @status_err: receive descriptor status and error fields
567 * @csum: receive descriptor csum field
568 * @sk_buff: socket buffer with received data
569 **/
570 static void e1000_rx_checksum(struct e1000_adapter *adapter, u32 status_err,
571 struct sk_buff *skb)
572 {
573 u16 status = (u16)status_err;
574 u8 errors = (u8)(status_err >> 24);
575
576 skb_checksum_none_assert(skb);
577
578 /* Rx checksum disabled */
579 if (!(adapter->netdev->features & NETIF_F_RXCSUM))
580 return;
581
582 /* Ignore Checksum bit is set */
583 if (status & E1000_RXD_STAT_IXSM)
584 return;
585
586 /* TCP/UDP checksum error bit or IP checksum error bit is set */
587 if (errors & (E1000_RXD_ERR_TCPE | E1000_RXD_ERR_IPE)) {
588 /* let the stack verify checksum errors */
589 adapter->hw_csum_err++;
590 return;
591 }
592
593 /* TCP/UDP Checksum has not been calculated */
594 if (!(status & (E1000_RXD_STAT_TCPCS | E1000_RXD_STAT_UDPCS)))
595 return;
596
597 /* It must be a TCP or UDP packet with a valid checksum */
598 skb->ip_summed = CHECKSUM_UNNECESSARY;
599 adapter->hw_csum_good++;
600 }
601
602 static void e1000e_update_rdt_wa(struct e1000_ring *rx_ring, unsigned int i)
603 {
604 struct e1000_adapter *adapter = rx_ring->adapter;
605 struct e1000_hw *hw = &adapter->hw;
606 s32 ret_val = __ew32_prepare(hw);
607
608 writel(i, rx_ring->tail);
609
610 if (unlikely(!ret_val && (i != readl(rx_ring->tail)))) {
611 u32 rctl = er32(RCTL);
612 ew32(RCTL, rctl & ~E1000_RCTL_EN);
613 e_err("ME firmware caused invalid RDT - resetting\n");
614 schedule_work(&adapter->reset_task);
615 }
616 }
617
618 static void e1000e_update_tdt_wa(struct e1000_ring *tx_ring, unsigned int i)
619 {
620 struct e1000_adapter *adapter = tx_ring->adapter;
621 struct e1000_hw *hw = &adapter->hw;
622 s32 ret_val = __ew32_prepare(hw);
623
624 writel(i, tx_ring->tail);
625
626 if (unlikely(!ret_val && (i != readl(tx_ring->tail)))) {
627 u32 tctl = er32(TCTL);
628 ew32(TCTL, tctl & ~E1000_TCTL_EN);
629 e_err("ME firmware caused invalid TDT - resetting\n");
630 schedule_work(&adapter->reset_task);
631 }
632 }
633
634 /**
635 * e1000_alloc_rx_buffers - Replace used receive buffers
636 * @rx_ring: Rx descriptor ring
637 **/
638 static void e1000_alloc_rx_buffers(struct e1000_ring *rx_ring,
639 int cleaned_count, gfp_t gfp)
640 {
641 struct e1000_adapter *adapter = rx_ring->adapter;
642 struct net_device *netdev = adapter->netdev;
643 struct pci_dev *pdev = adapter->pdev;
644 union e1000_rx_desc_extended *rx_desc;
645 struct e1000_buffer *buffer_info;
646 struct sk_buff *skb;
647 unsigned int i;
648 unsigned int bufsz = adapter->rx_buffer_len;
649
650 i = rx_ring->next_to_use;
651 buffer_info = &rx_ring->buffer_info[i];
652
653 while (cleaned_count--) {
654 skb = buffer_info->skb;
655 if (skb) {
656 skb_trim(skb, 0);
657 goto map_skb;
658 }
659
660 skb = __netdev_alloc_skb_ip_align(netdev, bufsz, gfp);
661 if (!skb) {
662 /* Better luck next round */
663 adapter->alloc_rx_buff_failed++;
664 break;
665 }
666
667 buffer_info->skb = skb;
668 map_skb:
669 buffer_info->dma = dma_map_single(&pdev->dev, skb->data,
670 adapter->rx_buffer_len,
671 DMA_FROM_DEVICE);
672 if (dma_mapping_error(&pdev->dev, buffer_info->dma)) {
673 dev_err(&pdev->dev, "Rx DMA map failed\n");
674 adapter->rx_dma_failed++;
675 break;
676 }
677
678 rx_desc = E1000_RX_DESC_EXT(*rx_ring, i);
679 rx_desc->read.buffer_addr = cpu_to_le64(buffer_info->dma);
680
681 if (unlikely(!(i & (E1000_RX_BUFFER_WRITE - 1)))) {
682 /* Force memory writes to complete before letting h/w
683 * know there are new descriptors to fetch. (Only
684 * applicable for weak-ordered memory model archs,
685 * such as IA-64).
686 */
687 wmb();
688 if (adapter->flags2 & FLAG2_PCIM2PCI_ARBITER_WA)
689 e1000e_update_rdt_wa(rx_ring, i);
690 else
691 writel(i, rx_ring->tail);
692 }
693 i++;
694 if (i == rx_ring->count)
695 i = 0;
696 buffer_info = &rx_ring->buffer_info[i];
697 }
698
699 rx_ring->next_to_use = i;
700 }
701
702 /**
703 * e1000_alloc_rx_buffers_ps - Replace used receive buffers; packet split
704 * @rx_ring: Rx descriptor ring
705 **/
706 static void e1000_alloc_rx_buffers_ps(struct e1000_ring *rx_ring,
707 int cleaned_count, gfp_t gfp)
708 {
709 struct e1000_adapter *adapter = rx_ring->adapter;
710 struct net_device *netdev = adapter->netdev;
711 struct pci_dev *pdev = adapter->pdev;
712 union e1000_rx_desc_packet_split *rx_desc;
713 struct e1000_buffer *buffer_info;
714 struct e1000_ps_page *ps_page;
715 struct sk_buff *skb;
716 unsigned int i, j;
717
718 i = rx_ring->next_to_use;
719 buffer_info = &rx_ring->buffer_info[i];
720
721 while (cleaned_count--) {
722 rx_desc = E1000_RX_DESC_PS(*rx_ring, i);
723
724 for (j = 0; j < PS_PAGE_BUFFERS; j++) {
725 ps_page = &buffer_info->ps_pages[j];
726 if (j >= adapter->rx_ps_pages) {
727 /* all unused desc entries get hw null ptr */
728 rx_desc->read.buffer_addr[j + 1] =
729 ~cpu_to_le64(0);
730 continue;
731 }
732 if (!ps_page->page) {
733 ps_page->page = alloc_page(gfp);
734 if (!ps_page->page) {
735 adapter->alloc_rx_buff_failed++;
736 goto no_buffers;
737 }
738 ps_page->dma = dma_map_page(&pdev->dev,
739 ps_page->page,
740 0, PAGE_SIZE,
741 DMA_FROM_DEVICE);
742 if (dma_mapping_error(&pdev->dev,
743 ps_page->dma)) {
744 dev_err(&adapter->pdev->dev,
745 "Rx DMA page map failed\n");
746 adapter->rx_dma_failed++;
747 goto no_buffers;
748 }
749 }
750 /* Refresh the desc even if buffer_addrs
751 * didn't change because each write-back
752 * erases this info.
753 */
754 rx_desc->read.buffer_addr[j + 1] =
755 cpu_to_le64(ps_page->dma);
756 }
757
758 skb = __netdev_alloc_skb_ip_align(netdev,
759 adapter->rx_ps_bsize0,
760 gfp);
761
762 if (!skb) {
763 adapter->alloc_rx_buff_failed++;
764 break;
765 }
766
767 buffer_info->skb = skb;
768 buffer_info->dma = dma_map_single(&pdev->dev, skb->data,
769 adapter->rx_ps_bsize0,
770 DMA_FROM_DEVICE);
771 if (dma_mapping_error(&pdev->dev, buffer_info->dma)) {
772 dev_err(&pdev->dev, "Rx DMA map failed\n");
773 adapter->rx_dma_failed++;
774 /* cleanup skb */
775 dev_kfree_skb_any(skb);
776 buffer_info->skb = NULL;
777 break;
778 }
779
780 rx_desc->read.buffer_addr[0] = cpu_to_le64(buffer_info->dma);
781
782 if (unlikely(!(i & (E1000_RX_BUFFER_WRITE - 1)))) {
783 /* Force memory writes to complete before letting h/w
784 * know there are new descriptors to fetch. (Only
785 * applicable for weak-ordered memory model archs,
786 * such as IA-64).
787 */
788 wmb();
789 if (adapter->flags2 & FLAG2_PCIM2PCI_ARBITER_WA)
790 e1000e_update_rdt_wa(rx_ring, i << 1);
791 else
792 writel(i << 1, rx_ring->tail);
793 }
794
795 i++;
796 if (i == rx_ring->count)
797 i = 0;
798 buffer_info = &rx_ring->buffer_info[i];
799 }
800
801 no_buffers:
802 rx_ring->next_to_use = i;
803 }
804
805 /**
806 * e1000_alloc_jumbo_rx_buffers - Replace used jumbo receive buffers
807 * @rx_ring: Rx descriptor ring
808 * @cleaned_count: number of buffers to allocate this pass
809 **/
810
811 static void e1000_alloc_jumbo_rx_buffers(struct e1000_ring *rx_ring,
812 int cleaned_count, gfp_t gfp)
813 {
814 struct e1000_adapter *adapter = rx_ring->adapter;
815 struct net_device *netdev = adapter->netdev;
816 struct pci_dev *pdev = adapter->pdev;
817 union e1000_rx_desc_extended *rx_desc;
818 struct e1000_buffer *buffer_info;
819 struct sk_buff *skb;
820 unsigned int i;
821 unsigned int bufsz = 256 - 16; /* for skb_reserve */
822
823 i = rx_ring->next_to_use;
824 buffer_info = &rx_ring->buffer_info[i];
825
826 while (cleaned_count--) {
827 skb = buffer_info->skb;
828 if (skb) {
829 skb_trim(skb, 0);
830 goto check_page;
831 }
832
833 skb = __netdev_alloc_skb_ip_align(netdev, bufsz, gfp);
834 if (unlikely(!skb)) {
835 /* Better luck next round */
836 adapter->alloc_rx_buff_failed++;
837 break;
838 }
839
840 buffer_info->skb = skb;
841 check_page:
842 /* allocate a new page if necessary */
843 if (!buffer_info->page) {
844 buffer_info->page = alloc_page(gfp);
845 if (unlikely(!buffer_info->page)) {
846 adapter->alloc_rx_buff_failed++;
847 break;
848 }
849 }
850
851 if (!buffer_info->dma)
852 buffer_info->dma = dma_map_page(&pdev->dev,
853 buffer_info->page, 0,
854 PAGE_SIZE,
855 DMA_FROM_DEVICE);
856
857 rx_desc = E1000_RX_DESC_EXT(*rx_ring, i);
858 rx_desc->read.buffer_addr = cpu_to_le64(buffer_info->dma);
859
860 if (unlikely(++i == rx_ring->count))
861 i = 0;
862 buffer_info = &rx_ring->buffer_info[i];
863 }
864
865 if (likely(rx_ring->next_to_use != i)) {
866 rx_ring->next_to_use = i;
867 if (unlikely(i-- == 0))
868 i = (rx_ring->count - 1);
869
870 /* Force memory writes to complete before letting h/w
871 * know there are new descriptors to fetch. (Only
872 * applicable for weak-ordered memory model archs,
873 * such as IA-64).
874 */
875 wmb();
876 if (adapter->flags2 & FLAG2_PCIM2PCI_ARBITER_WA)
877 e1000e_update_rdt_wa(rx_ring, i);
878 else
879 writel(i, rx_ring->tail);
880 }
881 }
882
883 static inline void e1000_rx_hash(struct net_device *netdev, __le32 rss,
884 struct sk_buff *skb)
885 {
886 if (netdev->features & NETIF_F_RXHASH)
887 skb->rxhash = le32_to_cpu(rss);
888 }
889
890 /**
891 * e1000_clean_rx_irq - Send received data up the network stack
892 * @rx_ring: Rx descriptor ring
893 *
894 * the return value indicates whether actual cleaning was done, there
895 * is no guarantee that everything was cleaned
896 **/
897 static bool e1000_clean_rx_irq(struct e1000_ring *rx_ring, int *work_done,
898 int work_to_do)
899 {
900 struct e1000_adapter *adapter = rx_ring->adapter;
901 struct net_device *netdev = adapter->netdev;
902 struct pci_dev *pdev = adapter->pdev;
903 struct e1000_hw *hw = &adapter->hw;
904 union e1000_rx_desc_extended *rx_desc, *next_rxd;
905 struct e1000_buffer *buffer_info, *next_buffer;
906 u32 length, staterr;
907 unsigned int i;
908 int cleaned_count = 0;
909 bool cleaned = false;
910 unsigned int total_rx_bytes = 0, total_rx_packets = 0;
911
912 i = rx_ring->next_to_clean;
913 rx_desc = E1000_RX_DESC_EXT(*rx_ring, i);
914 staterr = le32_to_cpu(rx_desc->wb.upper.status_error);
915 buffer_info = &rx_ring->buffer_info[i];
916
917 while (staterr & E1000_RXD_STAT_DD) {
918 struct sk_buff *skb;
919
920 if (*work_done >= work_to_do)
921 break;
922 (*work_done)++;
923 rmb(); /* read descriptor and rx_buffer_info after status DD */
924
925 skb = buffer_info->skb;
926 buffer_info->skb = NULL;
927
928 prefetch(skb->data - NET_IP_ALIGN);
929
930 i++;
931 if (i == rx_ring->count)
932 i = 0;
933 next_rxd = E1000_RX_DESC_EXT(*rx_ring, i);
934 prefetch(next_rxd);
935
936 next_buffer = &rx_ring->buffer_info[i];
937
938 cleaned = true;
939 cleaned_count++;
940 dma_unmap_single(&pdev->dev,
941 buffer_info->dma,
942 adapter->rx_buffer_len,
943 DMA_FROM_DEVICE);
944 buffer_info->dma = 0;
945
946 length = le16_to_cpu(rx_desc->wb.upper.length);
947
948 /* !EOP means multiple descriptors were used to store a single
949 * packet, if that's the case we need to toss it. In fact, we
950 * need to toss every packet with the EOP bit clear and the
951 * next frame that _does_ have the EOP bit set, as it is by
952 * definition only a frame fragment
953 */
954 if (unlikely(!(staterr & E1000_RXD_STAT_EOP)))
955 adapter->flags2 |= FLAG2_IS_DISCARDING;
956
957 if (adapter->flags2 & FLAG2_IS_DISCARDING) {
958 /* All receives must fit into a single buffer */
959 e_dbg("Receive packet consumed multiple buffers\n");
960 /* recycle */
961 buffer_info->skb = skb;
962 if (staterr & E1000_RXD_STAT_EOP)
963 adapter->flags2 &= ~FLAG2_IS_DISCARDING;
964 goto next_desc;
965 }
966
967 if (unlikely((staterr & E1000_RXDEXT_ERR_FRAME_ERR_MASK) &&
968 !(netdev->features & NETIF_F_RXALL))) {
969 /* recycle */
970 buffer_info->skb = skb;
971 goto next_desc;
972 }
973
974 /* adjust length to remove Ethernet CRC */
975 if (!(adapter->flags2 & FLAG2_CRC_STRIPPING)) {
976 /* If configured to store CRC, don't subtract FCS,
977 * but keep the FCS bytes out of the total_rx_bytes
978 * counter
979 */
980 if (netdev->features & NETIF_F_RXFCS)
981 total_rx_bytes -= 4;
982 else
983 length -= 4;
984 }
985
986 total_rx_bytes += length;
987 total_rx_packets++;
988
989 /* code added for copybreak, this should improve
990 * performance for small packets with large amounts
991 * of reassembly being done in the stack
992 */
993 if (length < copybreak) {
994 struct sk_buff *new_skb =
995 netdev_alloc_skb_ip_align(netdev, length);
996 if (new_skb) {
997 skb_copy_to_linear_data_offset(new_skb,
998 -NET_IP_ALIGN,
999 (skb->data -
1000 NET_IP_ALIGN),
1001 (length +
1002 NET_IP_ALIGN));
1003 /* save the skb in buffer_info as good */
1004 buffer_info->skb = skb;
1005 skb = new_skb;
1006 }
1007 /* else just continue with the old one */
1008 }
1009 /* end copybreak code */
1010 skb_put(skb, length);
1011
1012 /* Receive Checksum Offload */
1013 e1000_rx_checksum(adapter, staterr, skb);
1014
1015 e1000_rx_hash(netdev, rx_desc->wb.lower.hi_dword.rss, skb);
1016
1017 e1000_receive_skb(adapter, netdev, skb, staterr,
1018 rx_desc->wb.upper.vlan);
1019
1020 next_desc:
1021 rx_desc->wb.upper.status_error &= cpu_to_le32(~0xFF);
1022
1023 /* return some buffers to hardware, one at a time is too slow */
1024 if (cleaned_count >= E1000_RX_BUFFER_WRITE) {
1025 adapter->alloc_rx_buf(rx_ring, cleaned_count,
1026 GFP_ATOMIC);
1027 cleaned_count = 0;
1028 }
1029
1030 /* use prefetched values */
1031 rx_desc = next_rxd;
1032 buffer_info = next_buffer;
1033
1034 staterr = le32_to_cpu(rx_desc->wb.upper.status_error);
1035 }
1036 rx_ring->next_to_clean = i;
1037
1038 cleaned_count = e1000_desc_unused(rx_ring);
1039 if (cleaned_count)
1040 adapter->alloc_rx_buf(rx_ring, cleaned_count, GFP_ATOMIC);
1041
1042 adapter->total_rx_bytes += total_rx_bytes;
1043 adapter->total_rx_packets += total_rx_packets;
1044 return cleaned;
1045 }
1046
1047 static void e1000_put_txbuf(struct e1000_ring *tx_ring,
1048 struct e1000_buffer *buffer_info)
1049 {
1050 struct e1000_adapter *adapter = tx_ring->adapter;
1051
1052 if (buffer_info->dma) {
1053 if (buffer_info->mapped_as_page)
1054 dma_unmap_page(&adapter->pdev->dev, buffer_info->dma,
1055 buffer_info->length, DMA_TO_DEVICE);
1056 else
1057 dma_unmap_single(&adapter->pdev->dev, buffer_info->dma,
1058 buffer_info->length, DMA_TO_DEVICE);
1059 buffer_info->dma = 0;
1060 }
1061 if (buffer_info->skb) {
1062 dev_kfree_skb_any(buffer_info->skb);
1063 buffer_info->skb = NULL;
1064 }
1065 buffer_info->time_stamp = 0;
1066 }
1067
1068 static void e1000_print_hw_hang(struct work_struct *work)
1069 {
1070 struct e1000_adapter *adapter = container_of(work,
1071 struct e1000_adapter,
1072 print_hang_task);
1073 struct net_device *netdev = adapter->netdev;
1074 struct e1000_ring *tx_ring = adapter->tx_ring;
1075 unsigned int i = tx_ring->next_to_clean;
1076 unsigned int eop = tx_ring->buffer_info[i].next_to_watch;
1077 struct e1000_tx_desc *eop_desc = E1000_TX_DESC(*tx_ring, eop);
1078 struct e1000_hw *hw = &adapter->hw;
1079 u16 phy_status, phy_1000t_status, phy_ext_status;
1080 u16 pci_status;
1081
1082 if (test_bit(__E1000_DOWN, &adapter->state))
1083 return;
1084
1085 if (!adapter->tx_hang_recheck &&
1086 (adapter->flags2 & FLAG2_DMA_BURST)) {
1087 /* May be block on write-back, flush and detect again
1088 * flush pending descriptor writebacks to memory
1089 */
1090 ew32(TIDV, adapter->tx_int_delay | E1000_TIDV_FPD);
1091 /* execute the writes immediately */
1092 e1e_flush();
1093 /* Due to rare timing issues, write to TIDV again to ensure
1094 * the write is successful
1095 */
1096 ew32(TIDV, adapter->tx_int_delay | E1000_TIDV_FPD);
1097 /* execute the writes immediately */
1098 e1e_flush();
1099 adapter->tx_hang_recheck = true;
1100 return;
1101 }
1102 /* Real hang detected */
1103 adapter->tx_hang_recheck = false;
1104 netif_stop_queue(netdev);
1105
1106 e1e_rphy(hw, MII_BMSR, &phy_status);
1107 e1e_rphy(hw, MII_STAT1000, &phy_1000t_status);
1108 e1e_rphy(hw, MII_ESTATUS, &phy_ext_status);
1109
1110 pci_read_config_word(adapter->pdev, PCI_STATUS, &pci_status);
1111
1112 /* detected Hardware unit hang */
1113 e_err("Detected Hardware Unit Hang:\n"
1114 " TDH <%x>\n"
1115 " TDT <%x>\n"
1116 " next_to_use <%x>\n"
1117 " next_to_clean <%x>\n"
1118 "buffer_info[next_to_clean]:\n"
1119 " time_stamp <%lx>\n"
1120 " next_to_watch <%x>\n"
1121 " jiffies <%lx>\n"
1122 " next_to_watch.status <%x>\n"
1123 "MAC Status <%x>\n"
1124 "PHY Status <%x>\n"
1125 "PHY 1000BASE-T Status <%x>\n"
1126 "PHY Extended Status <%x>\n"
1127 "PCI Status <%x>\n",
1128 readl(tx_ring->head),
1129 readl(tx_ring->tail),
1130 tx_ring->next_to_use,
1131 tx_ring->next_to_clean,
1132 tx_ring->buffer_info[eop].time_stamp,
1133 eop,
1134 jiffies,
1135 eop_desc->upper.fields.status,
1136 er32(STATUS),
1137 phy_status,
1138 phy_1000t_status,
1139 phy_ext_status,
1140 pci_status);
1141
1142 /* Suggest workaround for known h/w issue */
1143 if ((hw->mac.type == e1000_pchlan) && (er32(CTRL) & E1000_CTRL_TFCE))
1144 e_err("Try turning off Tx pause (flow control) via ethtool\n");
1145 }
1146
1147 /**
1148 * e1000e_tx_hwtstamp_work - check for Tx time stamp
1149 * @work: pointer to work struct
1150 *
1151 * This work function polls the TSYNCTXCTL valid bit to determine when a
1152 * timestamp has been taken for the current stored skb. The timestamp must
1153 * be for this skb because only one such packet is allowed in the queue.
1154 */
1155 static void e1000e_tx_hwtstamp_work(struct work_struct *work)
1156 {
1157 struct e1000_adapter *adapter = container_of(work, struct e1000_adapter,
1158 tx_hwtstamp_work);
1159 struct e1000_hw *hw = &adapter->hw;
1160
1161 if (!adapter->tx_hwtstamp_skb)
1162 return;
1163
1164 if (er32(TSYNCTXCTL) & E1000_TSYNCTXCTL_VALID) {
1165 struct skb_shared_hwtstamps shhwtstamps;
1166 u64 txstmp;
1167
1168 txstmp = er32(TXSTMPL);
1169 txstmp |= (u64)er32(TXSTMPH) << 32;
1170
1171 e1000e_systim_to_hwtstamp(adapter, &shhwtstamps, txstmp);
1172
1173 skb_tstamp_tx(adapter->tx_hwtstamp_skb, &shhwtstamps);
1174 dev_kfree_skb_any(adapter->tx_hwtstamp_skb);
1175 adapter->tx_hwtstamp_skb = NULL;
1176 } else {
1177 /* reschedule to check later */
1178 schedule_work(&adapter->tx_hwtstamp_work);
1179 }
1180 }
1181
1182 /**
1183 * e1000_clean_tx_irq - Reclaim resources after transmit completes
1184 * @tx_ring: Tx descriptor ring
1185 *
1186 * the return value indicates whether actual cleaning was done, there
1187 * is no guarantee that everything was cleaned
1188 **/
1189 static bool e1000_clean_tx_irq(struct e1000_ring *tx_ring)
1190 {
1191 struct e1000_adapter *adapter = tx_ring->adapter;
1192 struct net_device *netdev = adapter->netdev;
1193 struct e1000_hw *hw = &adapter->hw;
1194 struct e1000_tx_desc *tx_desc, *eop_desc;
1195 struct e1000_buffer *buffer_info;
1196 unsigned int i, eop;
1197 unsigned int count = 0;
1198 unsigned int total_tx_bytes = 0, total_tx_packets = 0;
1199 unsigned int bytes_compl = 0, pkts_compl = 0;
1200
1201 i = tx_ring->next_to_clean;
1202 eop = tx_ring->buffer_info[i].next_to_watch;
1203 eop_desc = E1000_TX_DESC(*tx_ring, eop);
1204
1205 while ((eop_desc->upper.data & cpu_to_le32(E1000_TXD_STAT_DD)) &&
1206 (count < tx_ring->count)) {
1207 bool cleaned = false;
1208 rmb(); /* read buffer_info after eop_desc */
1209 for (; !cleaned; count++) {
1210 tx_desc = E1000_TX_DESC(*tx_ring, i);
1211 buffer_info = &tx_ring->buffer_info[i];
1212 cleaned = (i == eop);
1213
1214 if (cleaned) {
1215 total_tx_packets += buffer_info->segs;
1216 total_tx_bytes += buffer_info->bytecount;
1217 if (buffer_info->skb) {
1218 bytes_compl += buffer_info->skb->len;
1219 pkts_compl++;
1220 }
1221 }
1222
1223 e1000_put_txbuf(tx_ring, buffer_info);
1224 tx_desc->upper.data = 0;
1225
1226 i++;
1227 if (i == tx_ring->count)
1228 i = 0;
1229 }
1230
1231 if (i == tx_ring->next_to_use)
1232 break;
1233 eop = tx_ring->buffer_info[i].next_to_watch;
1234 eop_desc = E1000_TX_DESC(*tx_ring, eop);
1235 }
1236
1237 tx_ring->next_to_clean = i;
1238
1239 netdev_completed_queue(netdev, pkts_compl, bytes_compl);
1240
1241 #define TX_WAKE_THRESHOLD 32
1242 if (count && netif_carrier_ok(netdev) &&
1243 e1000_desc_unused(tx_ring) >= TX_WAKE_THRESHOLD) {
1244 /* Make sure that anybody stopping the queue after this
1245 * sees the new next_to_clean.
1246 */
1247 smp_mb();
1248
1249 if (netif_queue_stopped(netdev) &&
1250 !(test_bit(__E1000_DOWN, &adapter->state))) {
1251 netif_wake_queue(netdev);
1252 ++adapter->restart_queue;
1253 }
1254 }
1255
1256 if (adapter->detect_tx_hung) {
1257 /* Detect a transmit hang in hardware, this serializes the
1258 * check with the clearing of time_stamp and movement of i
1259 */
1260 adapter->detect_tx_hung = false;
1261 if (tx_ring->buffer_info[i].time_stamp &&
1262 time_after(jiffies, tx_ring->buffer_info[i].time_stamp
1263 + (adapter->tx_timeout_factor * HZ)) &&
1264 !(er32(STATUS) & E1000_STATUS_TXOFF))
1265 schedule_work(&adapter->print_hang_task);
1266 else
1267 adapter->tx_hang_recheck = false;
1268 }
1269 adapter->total_tx_bytes += total_tx_bytes;
1270 adapter->total_tx_packets += total_tx_packets;
1271 return count < tx_ring->count;
1272 }
1273
1274 /**
1275 * e1000_clean_rx_irq_ps - Send received data up the network stack; packet split
1276 * @rx_ring: Rx descriptor ring
1277 *
1278 * the return value indicates whether actual cleaning was done, there
1279 * is no guarantee that everything was cleaned
1280 **/
1281 static bool e1000_clean_rx_irq_ps(struct e1000_ring *rx_ring, int *work_done,
1282 int work_to_do)
1283 {
1284 struct e1000_adapter *adapter = rx_ring->adapter;
1285 struct e1000_hw *hw = &adapter->hw;
1286 union e1000_rx_desc_packet_split *rx_desc, *next_rxd;
1287 struct net_device *netdev = adapter->netdev;
1288 struct pci_dev *pdev = adapter->pdev;
1289 struct e1000_buffer *buffer_info, *next_buffer;
1290 struct e1000_ps_page *ps_page;
1291 struct sk_buff *skb;
1292 unsigned int i, j;
1293 u32 length, staterr;
1294 int cleaned_count = 0;
1295 bool cleaned = false;
1296 unsigned int total_rx_bytes = 0, total_rx_packets = 0;
1297
1298 i = rx_ring->next_to_clean;
1299 rx_desc = E1000_RX_DESC_PS(*rx_ring, i);
1300 staterr = le32_to_cpu(rx_desc->wb.middle.status_error);
1301 buffer_info = &rx_ring->buffer_info[i];
1302
1303 while (staterr & E1000_RXD_STAT_DD) {
1304 if (*work_done >= work_to_do)
1305 break;
1306 (*work_done)++;
1307 skb = buffer_info->skb;
1308 rmb(); /* read descriptor and rx_buffer_info after status DD */
1309
1310 /* in the packet split case this is header only */
1311 prefetch(skb->data - NET_IP_ALIGN);
1312
1313 i++;
1314 if (i == rx_ring->count)
1315 i = 0;
1316 next_rxd = E1000_RX_DESC_PS(*rx_ring, i);
1317 prefetch(next_rxd);
1318
1319 next_buffer = &rx_ring->buffer_info[i];
1320
1321 cleaned = true;
1322 cleaned_count++;
1323 dma_unmap_single(&pdev->dev, buffer_info->dma,
1324 adapter->rx_ps_bsize0, DMA_FROM_DEVICE);
1325 buffer_info->dma = 0;
1326
1327 /* see !EOP comment in other Rx routine */
1328 if (!(staterr & E1000_RXD_STAT_EOP))
1329 adapter->flags2 |= FLAG2_IS_DISCARDING;
1330
1331 if (adapter->flags2 & FLAG2_IS_DISCARDING) {
1332 e_dbg("Packet Split buffers didn't pick up the full packet\n");
1333 dev_kfree_skb_irq(skb);
1334 if (staterr & E1000_RXD_STAT_EOP)
1335 adapter->flags2 &= ~FLAG2_IS_DISCARDING;
1336 goto next_desc;
1337 }
1338
1339 if (unlikely((staterr & E1000_RXDEXT_ERR_FRAME_ERR_MASK) &&
1340 !(netdev->features & NETIF_F_RXALL))) {
1341 dev_kfree_skb_irq(skb);
1342 goto next_desc;
1343 }
1344
1345 length = le16_to_cpu(rx_desc->wb.middle.length0);
1346
1347 if (!length) {
1348 e_dbg("Last part of the packet spanning multiple descriptors\n");
1349 dev_kfree_skb_irq(skb);
1350 goto next_desc;
1351 }
1352
1353 /* Good Receive */
1354 skb_put(skb, length);
1355
1356 {
1357 /* this looks ugly, but it seems compiler issues make
1358 * it more efficient than reusing j
1359 */
1360 int l1 = le16_to_cpu(rx_desc->wb.upper.length[0]);
1361
1362 /* page alloc/put takes too long and effects small
1363 * packet throughput, so unsplit small packets and
1364 * save the alloc/put only valid in softirq (napi)
1365 * context to call kmap_*
1366 */
1367 if (l1 && (l1 <= copybreak) &&
1368 ((length + l1) <= adapter->rx_ps_bsize0)) {
1369 u8 *vaddr;
1370
1371 ps_page = &buffer_info->ps_pages[0];
1372
1373 /* there is no documentation about how to call
1374 * kmap_atomic, so we can't hold the mapping
1375 * very long
1376 */
1377 dma_sync_single_for_cpu(&pdev->dev,
1378 ps_page->dma,
1379 PAGE_SIZE,
1380 DMA_FROM_DEVICE);
1381 vaddr = kmap_atomic(ps_page->page);
1382 memcpy(skb_tail_pointer(skb), vaddr, l1);
1383 kunmap_atomic(vaddr);
1384 dma_sync_single_for_device(&pdev->dev,
1385 ps_page->dma,
1386 PAGE_SIZE,
1387 DMA_FROM_DEVICE);
1388
1389 /* remove the CRC */
1390 if (!(adapter->flags2 & FLAG2_CRC_STRIPPING)) {
1391 if (!(netdev->features & NETIF_F_RXFCS))
1392 l1 -= 4;
1393 }
1394
1395 skb_put(skb, l1);
1396 goto copydone;
1397 } /* if */
1398 }
1399
1400 for (j = 0; j < PS_PAGE_BUFFERS; j++) {
1401 length = le16_to_cpu(rx_desc->wb.upper.length[j]);
1402 if (!length)
1403 break;
1404
1405 ps_page = &buffer_info->ps_pages[j];
1406 dma_unmap_page(&pdev->dev, ps_page->dma, PAGE_SIZE,
1407 DMA_FROM_DEVICE);
1408 ps_page->dma = 0;
1409 skb_fill_page_desc(skb, j, ps_page->page, 0, length);
1410 ps_page->page = NULL;
1411 skb->len += length;
1412 skb->data_len += length;
1413 skb->truesize += PAGE_SIZE;
1414 }
1415
1416 /* strip the ethernet crc, problem is we're using pages now so
1417 * this whole operation can get a little cpu intensive
1418 */
1419 if (!(adapter->flags2 & FLAG2_CRC_STRIPPING)) {
1420 if (!(netdev->features & NETIF_F_RXFCS))
1421 pskb_trim(skb, skb->len - 4);
1422 }
1423
1424 copydone:
1425 total_rx_bytes += skb->len;
1426 total_rx_packets++;
1427
1428 e1000_rx_checksum(adapter, staterr, skb);
1429
1430 e1000_rx_hash(netdev, rx_desc->wb.lower.hi_dword.rss, skb);
1431
1432 if (rx_desc->wb.upper.header_status &
1433 cpu_to_le16(E1000_RXDPS_HDRSTAT_HDRSP))
1434 adapter->rx_hdr_split++;
1435
1436 e1000_receive_skb(adapter, netdev, skb, staterr,
1437 rx_desc->wb.middle.vlan);
1438
1439 next_desc:
1440 rx_desc->wb.middle.status_error &= cpu_to_le32(~0xFF);
1441 buffer_info->skb = NULL;
1442
1443 /* return some buffers to hardware, one at a time is too slow */
1444 if (cleaned_count >= E1000_RX_BUFFER_WRITE) {
1445 adapter->alloc_rx_buf(rx_ring, cleaned_count,
1446 GFP_ATOMIC);
1447 cleaned_count = 0;
1448 }
1449
1450 /* use prefetched values */
1451 rx_desc = next_rxd;
1452 buffer_info = next_buffer;
1453
1454 staterr = le32_to_cpu(rx_desc->wb.middle.status_error);
1455 }
1456 rx_ring->next_to_clean = i;
1457
1458 cleaned_count = e1000_desc_unused(rx_ring);
1459 if (cleaned_count)
1460 adapter->alloc_rx_buf(rx_ring, cleaned_count, GFP_ATOMIC);
1461
1462 adapter->total_rx_bytes += total_rx_bytes;
1463 adapter->total_rx_packets += total_rx_packets;
1464 return cleaned;
1465 }
1466
1467 /**
1468 * e1000_consume_page - helper function
1469 **/
1470 static void e1000_consume_page(struct e1000_buffer *bi, struct sk_buff *skb,
1471 u16 length)
1472 {
1473 bi->page = NULL;
1474 skb->len += length;
1475 skb->data_len += length;
1476 skb->truesize += PAGE_SIZE;
1477 }
1478
1479 /**
1480 * e1000_clean_jumbo_rx_irq - Send received data up the network stack; legacy
1481 * @adapter: board private structure
1482 *
1483 * the return value indicates whether actual cleaning was done, there
1484 * is no guarantee that everything was cleaned
1485 **/
1486 static bool e1000_clean_jumbo_rx_irq(struct e1000_ring *rx_ring, int *work_done,
1487 int work_to_do)
1488 {
1489 struct e1000_adapter *adapter = rx_ring->adapter;
1490 struct net_device *netdev = adapter->netdev;
1491 struct pci_dev *pdev = adapter->pdev;
1492 union e1000_rx_desc_extended *rx_desc, *next_rxd;
1493 struct e1000_buffer *buffer_info, *next_buffer;
1494 u32 length, staterr;
1495 unsigned int i;
1496 int cleaned_count = 0;
1497 bool cleaned = false;
1498 unsigned int total_rx_bytes = 0, total_rx_packets = 0;
1499 struct skb_shared_info *shinfo;
1500
1501 i = rx_ring->next_to_clean;
1502 rx_desc = E1000_RX_DESC_EXT(*rx_ring, i);
1503 staterr = le32_to_cpu(rx_desc->wb.upper.status_error);
1504 buffer_info = &rx_ring->buffer_info[i];
1505
1506 while (staterr & E1000_RXD_STAT_DD) {
1507 struct sk_buff *skb;
1508
1509 if (*work_done >= work_to_do)
1510 break;
1511 (*work_done)++;
1512 rmb(); /* read descriptor and rx_buffer_info after status DD */
1513
1514 skb = buffer_info->skb;
1515 buffer_info->skb = NULL;
1516
1517 ++i;
1518 if (i == rx_ring->count)
1519 i = 0;
1520 next_rxd = E1000_RX_DESC_EXT(*rx_ring, i);
1521 prefetch(next_rxd);
1522
1523 next_buffer = &rx_ring->buffer_info[i];
1524
1525 cleaned = true;
1526 cleaned_count++;
1527 dma_unmap_page(&pdev->dev, buffer_info->dma, PAGE_SIZE,
1528 DMA_FROM_DEVICE);
1529 buffer_info->dma = 0;
1530
1531 length = le16_to_cpu(rx_desc->wb.upper.length);
1532
1533 /* errors is only valid for DD + EOP descriptors */
1534 if (unlikely((staterr & E1000_RXD_STAT_EOP) &&
1535 ((staterr & E1000_RXDEXT_ERR_FRAME_ERR_MASK) &&
1536 !(netdev->features & NETIF_F_RXALL)))) {
1537 /* recycle both page and skb */
1538 buffer_info->skb = skb;
1539 /* an error means any chain goes out the window too */
1540 if (rx_ring->rx_skb_top)
1541 dev_kfree_skb_irq(rx_ring->rx_skb_top);
1542 rx_ring->rx_skb_top = NULL;
1543 goto next_desc;
1544 }
1545
1546 #define rxtop (rx_ring->rx_skb_top)
1547 if (!(staterr & E1000_RXD_STAT_EOP)) {
1548 /* this descriptor is only the beginning (or middle) */
1549 if (!rxtop) {
1550 /* this is the beginning of a chain */
1551 rxtop = skb;
1552 skb_fill_page_desc(rxtop, 0, buffer_info->page,
1553 0, length);
1554 } else {
1555 /* this is the middle of a chain */
1556 shinfo = skb_shinfo(rxtop);
1557 skb_fill_page_desc(rxtop, shinfo->nr_frags,
1558 buffer_info->page, 0,
1559 length);
1560 /* re-use the skb, only consumed the page */
1561 buffer_info->skb = skb;
1562 }
1563 e1000_consume_page(buffer_info, rxtop, length);
1564 goto next_desc;
1565 } else {
1566 if (rxtop) {
1567 /* end of the chain */
1568 shinfo = skb_shinfo(rxtop);
1569 skb_fill_page_desc(rxtop, shinfo->nr_frags,
1570 buffer_info->page, 0,
1571 length);
1572 /* re-use the current skb, we only consumed the
1573 * page
1574 */
1575 buffer_info->skb = skb;
1576 skb = rxtop;
1577 rxtop = NULL;
1578 e1000_consume_page(buffer_info, skb, length);
1579 } else {
1580 /* no chain, got EOP, this buf is the packet
1581 * copybreak to save the put_page/alloc_page
1582 */
1583 if (length <= copybreak &&
1584 skb_tailroom(skb) >= length) {
1585 u8 *vaddr;
1586 vaddr = kmap_atomic(buffer_info->page);
1587 memcpy(skb_tail_pointer(skb), vaddr,
1588 length);
1589 kunmap_atomic(vaddr);
1590 /* re-use the page, so don't erase
1591 * buffer_info->page
1592 */
1593 skb_put(skb, length);
1594 } else {
1595 skb_fill_page_desc(skb, 0,
1596 buffer_info->page, 0,
1597 length);
1598 e1000_consume_page(buffer_info, skb,
1599 length);
1600 }
1601 }
1602 }
1603
1604 /* Receive Checksum Offload */
1605 e1000_rx_checksum(adapter, staterr, skb);
1606
1607 e1000_rx_hash(netdev, rx_desc->wb.lower.hi_dword.rss, skb);
1608
1609 /* probably a little skewed due to removing CRC */
1610 total_rx_bytes += skb->len;
1611 total_rx_packets++;
1612
1613 /* eth type trans needs skb->data to point to something */
1614 if (!pskb_may_pull(skb, ETH_HLEN)) {
1615 e_err("pskb_may_pull failed.\n");
1616 dev_kfree_skb_irq(skb);
1617 goto next_desc;
1618 }
1619
1620 e1000_receive_skb(adapter, netdev, skb, staterr,
1621 rx_desc->wb.upper.vlan);
1622
1623 next_desc:
1624 rx_desc->wb.upper.status_error &= cpu_to_le32(~0xFF);
1625
1626 /* return some buffers to hardware, one at a time is too slow */
1627 if (unlikely(cleaned_count >= E1000_RX_BUFFER_WRITE)) {
1628 adapter->alloc_rx_buf(rx_ring, cleaned_count,
1629 GFP_ATOMIC);
1630 cleaned_count = 0;
1631 }
1632
1633 /* use prefetched values */
1634 rx_desc = next_rxd;
1635 buffer_info = next_buffer;
1636
1637 staterr = le32_to_cpu(rx_desc->wb.upper.status_error);
1638 }
1639 rx_ring->next_to_clean = i;
1640
1641 cleaned_count = e1000_desc_unused(rx_ring);
1642 if (cleaned_count)
1643 adapter->alloc_rx_buf(rx_ring, cleaned_count, GFP_ATOMIC);
1644
1645 adapter->total_rx_bytes += total_rx_bytes;
1646 adapter->total_rx_packets += total_rx_packets;
1647 return cleaned;
1648 }
1649
1650 /**
1651 * e1000_clean_rx_ring - Free Rx Buffers per Queue
1652 * @rx_ring: Rx descriptor ring
1653 **/
1654 static void e1000_clean_rx_ring(struct e1000_ring *rx_ring)
1655 {
1656 struct e1000_adapter *adapter = rx_ring->adapter;
1657 struct e1000_buffer *buffer_info;
1658 struct e1000_ps_page *ps_page;
1659 struct pci_dev *pdev = adapter->pdev;
1660 unsigned int i, j;
1661
1662 /* Free all the Rx ring sk_buffs */
1663 for (i = 0; i < rx_ring->count; i++) {
1664 buffer_info = &rx_ring->buffer_info[i];
1665 if (buffer_info->dma) {
1666 if (adapter->clean_rx == e1000_clean_rx_irq)
1667 dma_unmap_single(&pdev->dev, buffer_info->dma,
1668 adapter->rx_buffer_len,
1669 DMA_FROM_DEVICE);
1670 else if (adapter->clean_rx == e1000_clean_jumbo_rx_irq)
1671 dma_unmap_page(&pdev->dev, buffer_info->dma,
1672 PAGE_SIZE, DMA_FROM_DEVICE);
1673 else if (adapter->clean_rx == e1000_clean_rx_irq_ps)
1674 dma_unmap_single(&pdev->dev, buffer_info->dma,
1675 adapter->rx_ps_bsize0,
1676 DMA_FROM_DEVICE);
1677 buffer_info->dma = 0;
1678 }
1679
1680 if (buffer_info->page) {
1681 put_page(buffer_info->page);
1682 buffer_info->page = NULL;
1683 }
1684
1685 if (buffer_info->skb) {
1686 dev_kfree_skb(buffer_info->skb);
1687 buffer_info->skb = NULL;
1688 }
1689
1690 for (j = 0; j < PS_PAGE_BUFFERS; j++) {
1691 ps_page = &buffer_info->ps_pages[j];
1692 if (!ps_page->page)
1693 break;
1694 dma_unmap_page(&pdev->dev, ps_page->dma, PAGE_SIZE,
1695 DMA_FROM_DEVICE);
1696 ps_page->dma = 0;
1697 put_page(ps_page->page);
1698 ps_page->page = NULL;
1699 }
1700 }
1701
1702 /* there also may be some cached data from a chained receive */
1703 if (rx_ring->rx_skb_top) {
1704 dev_kfree_skb(rx_ring->rx_skb_top);
1705 rx_ring->rx_skb_top = NULL;
1706 }
1707
1708 /* Zero out the descriptor ring */
1709 memset(rx_ring->desc, 0, rx_ring->size);
1710
1711 rx_ring->next_to_clean = 0;
1712 rx_ring->next_to_use = 0;
1713 adapter->flags2 &= ~FLAG2_IS_DISCARDING;
1714
1715 writel(0, rx_ring->head);
1716 if (rx_ring->adapter->flags2 & FLAG2_PCIM2PCI_ARBITER_WA)
1717 e1000e_update_rdt_wa(rx_ring, 0);
1718 else
1719 writel(0, rx_ring->tail);
1720 }
1721
1722 static void e1000e_downshift_workaround(struct work_struct *work)
1723 {
1724 struct e1000_adapter *adapter = container_of(work,
1725 struct e1000_adapter,
1726 downshift_task);
1727
1728 if (test_bit(__E1000_DOWN, &adapter->state))
1729 return;
1730
1731 e1000e_gig_downshift_workaround_ich8lan(&adapter->hw);
1732 }
1733
1734 /**
1735 * e1000_intr_msi - Interrupt Handler
1736 * @irq: interrupt number
1737 * @data: pointer to a network interface device structure
1738 **/
1739 static irqreturn_t e1000_intr_msi(int __always_unused irq, void *data)
1740 {
1741 struct net_device *netdev = data;
1742 struct e1000_adapter *adapter = netdev_priv(netdev);
1743 struct e1000_hw *hw = &adapter->hw;
1744 u32 icr = er32(ICR);
1745
1746 /* read ICR disables interrupts using IAM */
1747 if (icr & E1000_ICR_LSC) {
1748 hw->mac.get_link_status = true;
1749 /* ICH8 workaround-- Call gig speed drop workaround on cable
1750 * disconnect (LSC) before accessing any PHY registers
1751 */
1752 if ((adapter->flags & FLAG_LSC_GIG_SPEED_DROP) &&
1753 (!(er32(STATUS) & E1000_STATUS_LU)))
1754 schedule_work(&adapter->downshift_task);
1755
1756 /* 80003ES2LAN workaround-- For packet buffer work-around on
1757 * link down event; disable receives here in the ISR and reset
1758 * adapter in watchdog
1759 */
1760 if (netif_carrier_ok(netdev) &&
1761 adapter->flags & FLAG_RX_NEEDS_RESTART) {
1762 /* disable receives */
1763 u32 rctl = er32(RCTL);
1764 ew32(RCTL, rctl & ~E1000_RCTL_EN);
1765 adapter->flags |= FLAG_RESTART_NOW;
1766 }
1767 /* guard against interrupt when we're going down */
1768 if (!test_bit(__E1000_DOWN, &adapter->state))
1769 mod_timer(&adapter->watchdog_timer, jiffies + 1);
1770 }
1771
1772 /* Reset on uncorrectable ECC error */
1773 if ((icr & E1000_ICR_ECCER) && (hw->mac.type == e1000_pch_lpt)) {
1774 u32 pbeccsts = er32(PBECCSTS);
1775
1776 adapter->corr_errors +=
1777 pbeccsts & E1000_PBECCSTS_CORR_ERR_CNT_MASK;
1778 adapter->uncorr_errors +=
1779 (pbeccsts & E1000_PBECCSTS_UNCORR_ERR_CNT_MASK) >>
1780 E1000_PBECCSTS_UNCORR_ERR_CNT_SHIFT;
1781
1782 /* Do the reset outside of interrupt context */
1783 schedule_work(&adapter->reset_task);
1784
1785 /* return immediately since reset is imminent */
1786 return IRQ_HANDLED;
1787 }
1788
1789 if (napi_schedule_prep(&adapter->napi)) {
1790 adapter->total_tx_bytes = 0;
1791 adapter->total_tx_packets = 0;
1792 adapter->total_rx_bytes = 0;
1793 adapter->total_rx_packets = 0;
1794 __napi_schedule(&adapter->napi);
1795 }
1796
1797 return IRQ_HANDLED;
1798 }
1799
1800 /**
1801 * e1000_intr - Interrupt Handler
1802 * @irq: interrupt number
1803 * @data: pointer to a network interface device structure
1804 **/
1805 static irqreturn_t e1000_intr(int __always_unused irq, void *data)
1806 {
1807 struct net_device *netdev = data;
1808 struct e1000_adapter *adapter = netdev_priv(netdev);
1809 struct e1000_hw *hw = &adapter->hw;
1810 u32 rctl, icr = er32(ICR);
1811
1812 if (!icr || test_bit(__E1000_DOWN, &adapter->state))
1813 return IRQ_NONE; /* Not our interrupt */
1814
1815 /* IMS will not auto-mask if INT_ASSERTED is not set, and if it is
1816 * not set, then the adapter didn't send an interrupt
1817 */
1818 if (!(icr & E1000_ICR_INT_ASSERTED))
1819 return IRQ_NONE;
1820
1821 /* Interrupt Auto-Mask...upon reading ICR,
1822 * interrupts are masked. No need for the
1823 * IMC write
1824 */
1825
1826 if (icr & E1000_ICR_LSC) {
1827 hw->mac.get_link_status = true;
1828 /* ICH8 workaround-- Call gig speed drop workaround on cable
1829 * disconnect (LSC) before accessing any PHY registers
1830 */
1831 if ((adapter->flags & FLAG_LSC_GIG_SPEED_DROP) &&
1832 (!(er32(STATUS) & E1000_STATUS_LU)))
1833 schedule_work(&adapter->downshift_task);
1834
1835 /* 80003ES2LAN workaround--
1836 * For packet buffer work-around on link down event;
1837 * disable receives here in the ISR and
1838 * reset adapter in watchdog
1839 */
1840 if (netif_carrier_ok(netdev) &&
1841 (adapter->flags & FLAG_RX_NEEDS_RESTART)) {
1842 /* disable receives */
1843 rctl = er32(RCTL);
1844 ew32(RCTL, rctl & ~E1000_RCTL_EN);
1845 adapter->flags |= FLAG_RESTART_NOW;
1846 }
1847 /* guard against interrupt when we're going down */
1848 if (!test_bit(__E1000_DOWN, &adapter->state))
1849 mod_timer(&adapter->watchdog_timer, jiffies + 1);
1850 }
1851
1852 /* Reset on uncorrectable ECC error */
1853 if ((icr & E1000_ICR_ECCER) && (hw->mac.type == e1000_pch_lpt)) {
1854 u32 pbeccsts = er32(PBECCSTS);
1855
1856 adapter->corr_errors +=
1857 pbeccsts & E1000_PBECCSTS_CORR_ERR_CNT_MASK;
1858 adapter->uncorr_errors +=
1859 (pbeccsts & E1000_PBECCSTS_UNCORR_ERR_CNT_MASK) >>
1860 E1000_PBECCSTS_UNCORR_ERR_CNT_SHIFT;
1861
1862 /* Do the reset outside of interrupt context */
1863 schedule_work(&adapter->reset_task);
1864
1865 /* return immediately since reset is imminent */
1866 return IRQ_HANDLED;
1867 }
1868
1869 if (napi_schedule_prep(&adapter->napi)) {
1870 adapter->total_tx_bytes = 0;
1871 adapter->total_tx_packets = 0;
1872 adapter->total_rx_bytes = 0;
1873 adapter->total_rx_packets = 0;
1874 __napi_schedule(&adapter->napi);
1875 }
1876
1877 return IRQ_HANDLED;
1878 }
1879
1880 static irqreturn_t e1000_msix_other(int __always_unused irq, void *data)
1881 {
1882 struct net_device *netdev = data;
1883 struct e1000_adapter *adapter = netdev_priv(netdev);
1884 struct e1000_hw *hw = &adapter->hw;
1885 u32 icr = er32(ICR);
1886
1887 if (!(icr & E1000_ICR_INT_ASSERTED)) {
1888 if (!test_bit(__E1000_DOWN, &adapter->state))
1889 ew32(IMS, E1000_IMS_OTHER);
1890 return IRQ_NONE;
1891 }
1892
1893 if (icr & adapter->eiac_mask)
1894 ew32(ICS, (icr & adapter->eiac_mask));
1895
1896 if (icr & E1000_ICR_OTHER) {
1897 if (!(icr & E1000_ICR_LSC))
1898 goto no_link_interrupt;
1899 hw->mac.get_link_status = true;
1900 /* guard against interrupt when we're going down */
1901 if (!test_bit(__E1000_DOWN, &adapter->state))
1902 mod_timer(&adapter->watchdog_timer, jiffies + 1);
1903 }
1904
1905 no_link_interrupt:
1906 if (!test_bit(__E1000_DOWN, &adapter->state))
1907 ew32(IMS, E1000_IMS_LSC | E1000_IMS_OTHER);
1908
1909 return IRQ_HANDLED;
1910 }
1911
1912 static irqreturn_t e1000_intr_msix_tx(int __always_unused irq, void *data)
1913 {
1914 struct net_device *netdev = data;
1915 struct e1000_adapter *adapter = netdev_priv(netdev);
1916 struct e1000_hw *hw = &adapter->hw;
1917 struct e1000_ring *tx_ring = adapter->tx_ring;
1918
1919
1920 adapter->total_tx_bytes = 0;
1921 adapter->total_tx_packets = 0;
1922
1923 if (!e1000_clean_tx_irq(tx_ring))
1924 /* Ring was not completely cleaned, so fire another interrupt */
1925 ew32(ICS, tx_ring->ims_val);
1926
1927 return IRQ_HANDLED;
1928 }
1929
1930 static irqreturn_t e1000_intr_msix_rx(int __always_unused irq, void *data)
1931 {
1932 struct net_device *netdev = data;
1933 struct e1000_adapter *adapter = netdev_priv(netdev);
1934 struct e1000_ring *rx_ring = adapter->rx_ring;
1935
1936 /* Write the ITR value calculated at the end of the
1937 * previous interrupt.
1938 */
1939 if (rx_ring->set_itr) {
1940 writel(1000000000 / (rx_ring->itr_val * 256),
1941 rx_ring->itr_register);
1942 rx_ring->set_itr = 0;
1943 }
1944
1945 if (napi_schedule_prep(&adapter->napi)) {
1946 adapter->total_rx_bytes = 0;
1947 adapter->total_rx_packets = 0;
1948 __napi_schedule(&adapter->napi);
1949 }
1950 return IRQ_HANDLED;
1951 }
1952
1953 /**
1954 * e1000_configure_msix - Configure MSI-X hardware
1955 *
1956 * e1000_configure_msix sets up the hardware to properly
1957 * generate MSI-X interrupts.
1958 **/
1959 static void e1000_configure_msix(struct e1000_adapter *adapter)
1960 {
1961 struct e1000_hw *hw = &adapter->hw;
1962 struct e1000_ring *rx_ring = adapter->rx_ring;
1963 struct e1000_ring *tx_ring = adapter->tx_ring;
1964 int vector = 0;
1965 u32 ctrl_ext, ivar = 0;
1966
1967 adapter->eiac_mask = 0;
1968
1969 /* Workaround issue with spurious interrupts on 82574 in MSI-X mode */
1970 if (hw->mac.type == e1000_82574) {
1971 u32 rfctl = er32(RFCTL);
1972 rfctl |= E1000_RFCTL_ACK_DIS;
1973 ew32(RFCTL, rfctl);
1974 }
1975
1976 #define E1000_IVAR_INT_ALLOC_VALID 0x8
1977 /* Configure Rx vector */
1978 rx_ring->ims_val = E1000_IMS_RXQ0;
1979 adapter->eiac_mask |= rx_ring->ims_val;
1980 if (rx_ring->itr_val)
1981 writel(1000000000 / (rx_ring->itr_val * 256),
1982 rx_ring->itr_register);
1983 else
1984 writel(1, rx_ring->itr_register);
1985 ivar = E1000_IVAR_INT_ALLOC_VALID | vector;
1986
1987 /* Configure Tx vector */
1988 tx_ring->ims_val = E1000_IMS_TXQ0;
1989 vector++;
1990 if (tx_ring->itr_val)
1991 writel(1000000000 / (tx_ring->itr_val * 256),
1992 tx_ring->itr_register);
1993 else
1994 writel(1, tx_ring->itr_register);
1995 adapter->eiac_mask |= tx_ring->ims_val;
1996 ivar |= ((E1000_IVAR_INT_ALLOC_VALID | vector) << 8);
1997
1998 /* set vector for Other Causes, e.g. link changes */
1999 vector++;
2000 ivar |= ((E1000_IVAR_INT_ALLOC_VALID | vector) << 16);
2001 if (rx_ring->itr_val)
2002 writel(1000000000 / (rx_ring->itr_val * 256),
2003 hw->hw_addr + E1000_EITR_82574(vector));
2004 else
2005 writel(1, hw->hw_addr + E1000_EITR_82574(vector));
2006
2007 /* Cause Tx interrupts on every write back */
2008 ivar |= (1 << 31);
2009
2010 ew32(IVAR, ivar);
2011
2012 /* enable MSI-X PBA support */
2013 ctrl_ext = er32(CTRL_EXT);
2014 ctrl_ext |= E1000_CTRL_EXT_PBA_CLR;
2015
2016 /* Auto-Mask Other interrupts upon ICR read */
2017 ew32(IAM, ~E1000_EIAC_MASK_82574 | E1000_IMS_OTHER);
2018 ctrl_ext |= E1000_CTRL_EXT_EIAME;
2019 ew32(CTRL_EXT, ctrl_ext);
2020 e1e_flush();
2021 }
2022
2023 void e1000e_reset_interrupt_capability(struct e1000_adapter *adapter)
2024 {
2025 if (adapter->msix_entries) {
2026 pci_disable_msix(adapter->pdev);
2027 kfree(adapter->msix_entries);
2028 adapter->msix_entries = NULL;
2029 } else if (adapter->flags & FLAG_MSI_ENABLED) {
2030 pci_disable_msi(adapter->pdev);
2031 adapter->flags &= ~FLAG_MSI_ENABLED;
2032 }
2033 }
2034
2035 /**
2036 * e1000e_set_interrupt_capability - set MSI or MSI-X if supported
2037 *
2038 * Attempt to configure interrupts using the best available
2039 * capabilities of the hardware and kernel.
2040 **/
2041 void e1000e_set_interrupt_capability(struct e1000_adapter *adapter)
2042 {
2043 int err;
2044 int i;
2045
2046 switch (adapter->int_mode) {
2047 case E1000E_INT_MODE_MSIX:
2048 if (adapter->flags & FLAG_HAS_MSIX) {
2049 adapter->num_vectors = 3; /* RxQ0, TxQ0 and other */
2050 adapter->msix_entries = kcalloc(adapter->num_vectors,
2051 sizeof(struct
2052 msix_entry),
2053 GFP_KERNEL);
2054 if (adapter->msix_entries) {
2055 for (i = 0; i < adapter->num_vectors; i++)
2056 adapter->msix_entries[i].entry = i;
2057
2058 err = pci_enable_msix(adapter->pdev,
2059 adapter->msix_entries,
2060 adapter->num_vectors);
2061 if (err == 0)
2062 return;
2063 }
2064 /* MSI-X failed, so fall through and try MSI */
2065 e_err("Failed to initialize MSI-X interrupts. Falling back to MSI interrupts.\n");
2066 e1000e_reset_interrupt_capability(adapter);
2067 }
2068 adapter->int_mode = E1000E_INT_MODE_MSI;
2069 /* Fall through */
2070 case E1000E_INT_MODE_MSI:
2071 if (!pci_enable_msi(adapter->pdev)) {
2072 adapter->flags |= FLAG_MSI_ENABLED;
2073 } else {
2074 adapter->int_mode = E1000E_INT_MODE_LEGACY;
2075 e_err("Failed to initialize MSI interrupts. Falling back to legacy interrupts.\n");
2076 }
2077 /* Fall through */
2078 case E1000E_INT_MODE_LEGACY:
2079 /* Don't do anything; this is the system default */
2080 break;
2081 }
2082
2083 /* store the number of vectors being used */
2084 adapter->num_vectors = 1;
2085 }
2086
2087 /**
2088 * e1000_request_msix - Initialize MSI-X interrupts
2089 *
2090 * e1000_request_msix allocates MSI-X vectors and requests interrupts from the
2091 * kernel.
2092 **/
2093 static int e1000_request_msix(struct e1000_adapter *adapter)
2094 {
2095 struct net_device *netdev = adapter->netdev;
2096 int err = 0, vector = 0;
2097
2098 if (strlen(netdev->name) < (IFNAMSIZ - 5))
2099 snprintf(adapter->rx_ring->name,
2100 sizeof(adapter->rx_ring->name) - 1,
2101 "%s-rx-0", netdev->name);
2102 else
2103 memcpy(adapter->rx_ring->name, netdev->name, IFNAMSIZ);
2104 err = request_irq(adapter->msix_entries[vector].vector,
2105 e1000_intr_msix_rx, 0, adapter->rx_ring->name,
2106 netdev);
2107 if (err)
2108 return err;
2109 adapter->rx_ring->itr_register = adapter->hw.hw_addr +
2110 E1000_EITR_82574(vector);
2111 adapter->rx_ring->itr_val = adapter->itr;
2112 vector++;
2113
2114 if (strlen(netdev->name) < (IFNAMSIZ - 5))
2115 snprintf(adapter->tx_ring->name,
2116 sizeof(adapter->tx_ring->name) - 1,
2117 "%s-tx-0", netdev->name);
2118 else
2119 memcpy(adapter->tx_ring->name, netdev->name, IFNAMSIZ);
2120 err = request_irq(adapter->msix_entries[vector].vector,
2121 e1000_intr_msix_tx, 0, adapter->tx_ring->name,
2122 netdev);
2123 if (err)
2124 return err;
2125 adapter->tx_ring->itr_register = adapter->hw.hw_addr +
2126 E1000_EITR_82574(vector);
2127 adapter->tx_ring->itr_val = adapter->itr;
2128 vector++;
2129
2130 err = request_irq(adapter->msix_entries[vector].vector,
2131 e1000_msix_other, 0, netdev->name, netdev);
2132 if (err)
2133 return err;
2134
2135 e1000_configure_msix(adapter);
2136
2137 return 0;
2138 }
2139
2140 /**
2141 * e1000_request_irq - initialize interrupts
2142 *
2143 * Attempts to configure interrupts using the best available
2144 * capabilities of the hardware and kernel.
2145 **/
2146 static int e1000_request_irq(struct e1000_adapter *adapter)
2147 {
2148 struct net_device *netdev = adapter->netdev;
2149 int err;
2150
2151 if (adapter->msix_entries) {
2152 err = e1000_request_msix(adapter);
2153 if (!err)
2154 return err;
2155 /* fall back to MSI */
2156 e1000e_reset_interrupt_capability(adapter);
2157 adapter->int_mode = E1000E_INT_MODE_MSI;
2158 e1000e_set_interrupt_capability(adapter);
2159 }
2160 if (adapter->flags & FLAG_MSI_ENABLED) {
2161 err = request_irq(adapter->pdev->irq, e1000_intr_msi, 0,
2162 netdev->name, netdev);
2163 if (!err)
2164 return err;
2165
2166 /* fall back to legacy interrupt */
2167 e1000e_reset_interrupt_capability(adapter);
2168 adapter->int_mode = E1000E_INT_MODE_LEGACY;
2169 }
2170
2171 err = request_irq(adapter->pdev->irq, e1000_intr, IRQF_SHARED,
2172 netdev->name, netdev);
2173 if (err)
2174 e_err("Unable to allocate interrupt, Error: %d\n", err);
2175
2176 return err;
2177 }
2178
2179 static void e1000_free_irq(struct e1000_adapter *adapter)
2180 {
2181 struct net_device *netdev = adapter->netdev;
2182
2183 if (adapter->msix_entries) {
2184 int vector = 0;
2185
2186 free_irq(adapter->msix_entries[vector].vector, netdev);
2187 vector++;
2188
2189 free_irq(adapter->msix_entries[vector].vector, netdev);
2190 vector++;
2191
2192 /* Other Causes interrupt vector */
2193 free_irq(adapter->msix_entries[vector].vector, netdev);
2194 return;
2195 }
2196
2197 free_irq(adapter->pdev->irq, netdev);
2198 }
2199
2200 /**
2201 * e1000_irq_disable - Mask off interrupt generation on the NIC
2202 **/
2203 static void e1000_irq_disable(struct e1000_adapter *adapter)
2204 {
2205 struct e1000_hw *hw = &adapter->hw;
2206
2207 ew32(IMC, ~0);
2208 if (adapter->msix_entries)
2209 ew32(EIAC_82574, 0);
2210 e1e_flush();
2211
2212 if (adapter->msix_entries) {
2213 int i;
2214 for (i = 0; i < adapter->num_vectors; i++)
2215 synchronize_irq(adapter->msix_entries[i].vector);
2216 } else {
2217 synchronize_irq(adapter->pdev->irq);
2218 }
2219 }
2220
2221 /**
2222 * e1000_irq_enable - Enable default interrupt generation settings
2223 **/
2224 static void e1000_irq_enable(struct e1000_adapter *adapter)
2225 {
2226 struct e1000_hw *hw = &adapter->hw;
2227
2228 if (adapter->msix_entries) {
2229 ew32(EIAC_82574, adapter->eiac_mask & E1000_EIAC_MASK_82574);
2230 ew32(IMS, adapter->eiac_mask | E1000_IMS_OTHER | E1000_IMS_LSC);
2231 } else if (hw->mac.type == e1000_pch_lpt) {
2232 ew32(IMS, IMS_ENABLE_MASK | E1000_IMS_ECCER);
2233 } else {
2234 ew32(IMS, IMS_ENABLE_MASK);
2235 }
2236 e1e_flush();
2237 }
2238
2239 /**
2240 * e1000e_get_hw_control - get control of the h/w from f/w
2241 * @adapter: address of board private structure
2242 *
2243 * e1000e_get_hw_control sets {CTRL_EXT|SWSM}:DRV_LOAD bit.
2244 * For ASF and Pass Through versions of f/w this means that
2245 * the driver is loaded. For AMT version (only with 82573)
2246 * of the f/w this means that the network i/f is open.
2247 **/
2248 void e1000e_get_hw_control(struct e1000_adapter *adapter)
2249 {
2250 struct e1000_hw *hw = &adapter->hw;
2251 u32 ctrl_ext;
2252 u32 swsm;
2253
2254 /* Let firmware know the driver has taken over */
2255 if (adapter->flags & FLAG_HAS_SWSM_ON_LOAD) {
2256 swsm = er32(SWSM);
2257 ew32(SWSM, swsm | E1000_SWSM_DRV_LOAD);
2258 } else if (adapter->flags & FLAG_HAS_CTRLEXT_ON_LOAD) {
2259 ctrl_ext = er32(CTRL_EXT);
2260 ew32(CTRL_EXT, ctrl_ext | E1000_CTRL_EXT_DRV_LOAD);
2261 }
2262 }
2263
2264 /**
2265 * e1000e_release_hw_control - release control of the h/w to f/w
2266 * @adapter: address of board private structure
2267 *
2268 * e1000e_release_hw_control resets {CTRL_EXT|SWSM}:DRV_LOAD bit.
2269 * For ASF and Pass Through versions of f/w this means that the
2270 * driver is no longer loaded. For AMT version (only with 82573) i
2271 * of the f/w this means that the network i/f is closed.
2272 *
2273 **/
2274 void e1000e_release_hw_control(struct e1000_adapter *adapter)
2275 {
2276 struct e1000_hw *hw = &adapter->hw;
2277 u32 ctrl_ext;
2278 u32 swsm;
2279
2280 /* Let firmware taken over control of h/w */
2281 if (adapter->flags & FLAG_HAS_SWSM_ON_LOAD) {
2282 swsm = er32(SWSM);
2283 ew32(SWSM, swsm & ~E1000_SWSM_DRV_LOAD);
2284 } else if (adapter->flags & FLAG_HAS_CTRLEXT_ON_LOAD) {
2285 ctrl_ext = er32(CTRL_EXT);
2286 ew32(CTRL_EXT, ctrl_ext & ~E1000_CTRL_EXT_DRV_LOAD);
2287 }
2288 }
2289
2290 /**
2291 * e1000_alloc_ring_dma - allocate memory for a ring structure
2292 **/
2293 static int e1000_alloc_ring_dma(struct e1000_adapter *adapter,
2294 struct e1000_ring *ring)
2295 {
2296 struct pci_dev *pdev = adapter->pdev;
2297
2298 ring->desc = dma_alloc_coherent(&pdev->dev, ring->size, &ring->dma,
2299 GFP_KERNEL);
2300 if (!ring->desc)
2301 return -ENOMEM;
2302
2303 return 0;
2304 }
2305
2306 /**
2307 * e1000e_setup_tx_resources - allocate Tx resources (Descriptors)
2308 * @tx_ring: Tx descriptor ring
2309 *
2310 * Return 0 on success, negative on failure
2311 **/
2312 int e1000e_setup_tx_resources(struct e1000_ring *tx_ring)
2313 {
2314 struct e1000_adapter *adapter = tx_ring->adapter;
2315 int err = -ENOMEM, size;
2316
2317 size = sizeof(struct e1000_buffer) * tx_ring->count;
2318 tx_ring->buffer_info = vzalloc(size);
2319 if (!tx_ring->buffer_info)
2320 goto err;
2321
2322 /* round up to nearest 4K */
2323 tx_ring->size = tx_ring->count * sizeof(struct e1000_tx_desc);
2324 tx_ring->size = ALIGN(tx_ring->size, 4096);
2325
2326 err = e1000_alloc_ring_dma(adapter, tx_ring);
2327 if (err)
2328 goto err;
2329
2330 tx_ring->next_to_use = 0;
2331 tx_ring->next_to_clean = 0;
2332
2333 return 0;
2334 err:
2335 vfree(tx_ring->buffer_info);
2336 e_err("Unable to allocate memory for the transmit descriptor ring\n");
2337 return err;
2338 }
2339
2340 /**
2341 * e1000e_setup_rx_resources - allocate Rx resources (Descriptors)
2342 * @rx_ring: Rx descriptor ring
2343 *
2344 * Returns 0 on success, negative on failure
2345 **/
2346 int e1000e_setup_rx_resources(struct e1000_ring *rx_ring)
2347 {
2348 struct e1000_adapter *adapter = rx_ring->adapter;
2349 struct e1000_buffer *buffer_info;
2350 int i, size, desc_len, err = -ENOMEM;
2351
2352 size = sizeof(struct e1000_buffer) * rx_ring->count;
2353 rx_ring->buffer_info = vzalloc(size);
2354 if (!rx_ring->buffer_info)
2355 goto err;
2356
2357 for (i = 0; i < rx_ring->count; i++) {
2358 buffer_info = &rx_ring->buffer_info[i];
2359 buffer_info->ps_pages = kcalloc(PS_PAGE_BUFFERS,
2360 sizeof(struct e1000_ps_page),
2361 GFP_KERNEL);
2362 if (!buffer_info->ps_pages)
2363 goto err_pages;
2364 }
2365
2366 desc_len = sizeof(union e1000_rx_desc_packet_split);
2367
2368 /* Round up to nearest 4K */
2369 rx_ring->size = rx_ring->count * desc_len;
2370 rx_ring->size = ALIGN(rx_ring->size, 4096);
2371
2372 err = e1000_alloc_ring_dma(adapter, rx_ring);
2373 if (err)
2374 goto err_pages;
2375
2376 rx_ring->next_to_clean = 0;
2377 rx_ring->next_to_use = 0;
2378 rx_ring->rx_skb_top = NULL;
2379
2380 return 0;
2381
2382 err_pages:
2383 for (i = 0; i < rx_ring->count; i++) {
2384 buffer_info = &rx_ring->buffer_info[i];
2385 kfree(buffer_info->ps_pages);
2386 }
2387 err:
2388 vfree(rx_ring->buffer_info);
2389 e_err("Unable to allocate memory for the receive descriptor ring\n");
2390 return err;
2391 }
2392
2393 /**
2394 * e1000_clean_tx_ring - Free Tx Buffers
2395 * @tx_ring: Tx descriptor ring
2396 **/
2397 static void e1000_clean_tx_ring(struct e1000_ring *tx_ring)
2398 {
2399 struct e1000_adapter *adapter = tx_ring->adapter;
2400 struct e1000_buffer *buffer_info;
2401 unsigned long size;
2402 unsigned int i;
2403
2404 for (i = 0; i < tx_ring->count; i++) {
2405 buffer_info = &tx_ring->buffer_info[i];
2406 e1000_put_txbuf(tx_ring, buffer_info);
2407 }
2408
2409 netdev_reset_queue(adapter->netdev);
2410 size = sizeof(struct e1000_buffer) * tx_ring->count;
2411 memset(tx_ring->buffer_info, 0, size);
2412
2413 memset(tx_ring->desc, 0, tx_ring->size);
2414
2415 tx_ring->next_to_use = 0;
2416 tx_ring->next_to_clean = 0;
2417
2418 writel(0, tx_ring->head);
2419 if (tx_ring->adapter->flags2 & FLAG2_PCIM2PCI_ARBITER_WA)
2420 e1000e_update_tdt_wa(tx_ring, 0);
2421 else
2422 writel(0, tx_ring->tail);
2423 }
2424
2425 /**
2426 * e1000e_free_tx_resources - Free Tx Resources per Queue
2427 * @tx_ring: Tx descriptor ring
2428 *
2429 * Free all transmit software resources
2430 **/
2431 void e1000e_free_tx_resources(struct e1000_ring *tx_ring)
2432 {
2433 struct e1000_adapter *adapter = tx_ring->adapter;
2434 struct pci_dev *pdev = adapter->pdev;
2435
2436 e1000_clean_tx_ring(tx_ring);
2437
2438 vfree(tx_ring->buffer_info);
2439 tx_ring->buffer_info = NULL;
2440
2441 dma_free_coherent(&pdev->dev, tx_ring->size, tx_ring->desc,
2442 tx_ring->dma);
2443 tx_ring->desc = NULL;
2444 }
2445
2446 /**
2447 * e1000e_free_rx_resources - Free Rx Resources
2448 * @rx_ring: Rx descriptor ring
2449 *
2450 * Free all receive software resources
2451 **/
2452 void e1000e_free_rx_resources(struct e1000_ring *rx_ring)
2453 {
2454 struct e1000_adapter *adapter = rx_ring->adapter;
2455 struct pci_dev *pdev = adapter->pdev;
2456 int i;
2457
2458 e1000_clean_rx_ring(rx_ring);
2459
2460 for (i = 0; i < rx_ring->count; i++)
2461 kfree(rx_ring->buffer_info[i].ps_pages);
2462
2463 vfree(rx_ring->buffer_info);
2464 rx_ring->buffer_info = NULL;
2465
2466 dma_free_coherent(&pdev->dev, rx_ring->size, rx_ring->desc,
2467 rx_ring->dma);
2468 rx_ring->desc = NULL;
2469 }
2470
2471 /**
2472 * e1000_update_itr - update the dynamic ITR value based on statistics
2473 * @adapter: pointer to adapter
2474 * @itr_setting: current adapter->itr
2475 * @packets: the number of packets during this measurement interval
2476 * @bytes: the number of bytes during this measurement interval
2477 *
2478 * Stores a new ITR value based on packets and byte
2479 * counts during the last interrupt. The advantage of per interrupt
2480 * computation is faster updates and more accurate ITR for the current
2481 * traffic pattern. Constants in this function were computed
2482 * based on theoretical maximum wire speed and thresholds were set based
2483 * on testing data as well as attempting to minimize response time
2484 * while increasing bulk throughput. This functionality is controlled
2485 * by the InterruptThrottleRate module parameter.
2486 **/
2487 static unsigned int e1000_update_itr(u16 itr_setting, int packets, int bytes)
2488 {
2489 unsigned int retval = itr_setting;
2490
2491 if (packets == 0)
2492 return itr_setting;
2493
2494 switch (itr_setting) {
2495 case lowest_latency:
2496 /* handle TSO and jumbo frames */
2497 if (bytes / packets > 8000)
2498 retval = bulk_latency;
2499 else if ((packets < 5) && (bytes > 512))
2500 retval = low_latency;
2501 break;
2502 case low_latency: /* 50 usec aka 20000 ints/s */
2503 if (bytes > 10000) {
2504 /* this if handles the TSO accounting */
2505 if (bytes / packets > 8000)
2506 retval = bulk_latency;
2507 else if ((packets < 10) || ((bytes / packets) > 1200))
2508 retval = bulk_latency;
2509 else if ((packets > 35))
2510 retval = lowest_latency;
2511 } else if (bytes / packets > 2000) {
2512 retval = bulk_latency;
2513 } else if (packets <= 2 && bytes < 512) {
2514 retval = lowest_latency;
2515 }
2516 break;
2517 case bulk_latency: /* 250 usec aka 4000 ints/s */
2518 if (bytes > 25000) {
2519 if (packets > 35)
2520 retval = low_latency;
2521 } else if (bytes < 6000) {
2522 retval = low_latency;
2523 }
2524 break;
2525 }
2526
2527 return retval;
2528 }
2529
2530 static void e1000_set_itr(struct e1000_adapter *adapter)
2531 {
2532 u16 current_itr;
2533 u32 new_itr = adapter->itr;
2534
2535 /* for non-gigabit speeds, just fix the interrupt rate at 4000 */
2536 if (adapter->link_speed != SPEED_1000) {
2537 current_itr = 0;
2538 new_itr = 4000;
2539 goto set_itr_now;
2540 }
2541
2542 if (adapter->flags2 & FLAG2_DISABLE_AIM) {
2543 new_itr = 0;
2544 goto set_itr_now;
2545 }
2546
2547 adapter->tx_itr = e1000_update_itr(adapter->tx_itr,
2548 adapter->total_tx_packets,
2549 adapter->total_tx_bytes);
2550 /* conservative mode (itr 3) eliminates the lowest_latency setting */
2551 if (adapter->itr_setting == 3 && adapter->tx_itr == lowest_latency)
2552 adapter->tx_itr = low_latency;
2553
2554 adapter->rx_itr = e1000_update_itr(adapter->rx_itr,
2555 adapter->total_rx_packets,
2556 adapter->total_rx_bytes);
2557 /* conservative mode (itr 3) eliminates the lowest_latency setting */
2558 if (adapter->itr_setting == 3 && adapter->rx_itr == lowest_latency)
2559 adapter->rx_itr = low_latency;
2560
2561 current_itr = max(adapter->rx_itr, adapter->tx_itr);
2562
2563 switch (current_itr) {
2564 /* counts and packets in update_itr are dependent on these numbers */
2565 case lowest_latency:
2566 new_itr = 70000;
2567 break;
2568 case low_latency:
2569 new_itr = 20000; /* aka hwitr = ~200 */
2570 break;
2571 case bulk_latency:
2572 new_itr = 4000;
2573 break;
2574 default:
2575 break;
2576 }
2577
2578 set_itr_now:
2579 if (new_itr != adapter->itr) {
2580 /* this attempts to bias the interrupt rate towards Bulk
2581 * by adding intermediate steps when interrupt rate is
2582 * increasing
2583 */
2584 new_itr = new_itr > adapter->itr ?
2585 min(adapter->itr + (new_itr >> 2), new_itr) : new_itr;
2586 adapter->itr = new_itr;
2587 adapter->rx_ring->itr_val = new_itr;
2588 if (adapter->msix_entries)
2589 adapter->rx_ring->set_itr = 1;
2590 else
2591 e1000e_write_itr(adapter, new_itr);
2592 }
2593 }
2594
2595 /**
2596 * e1000e_write_itr - write the ITR value to the appropriate registers
2597 * @adapter: address of board private structure
2598 * @itr: new ITR value to program
2599 *
2600 * e1000e_write_itr determines if the adapter is in MSI-X mode
2601 * and, if so, writes the EITR registers with the ITR value.
2602 * Otherwise, it writes the ITR value into the ITR register.
2603 **/
2604 void e1000e_write_itr(struct e1000_adapter *adapter, u32 itr)
2605 {
2606 struct e1000_hw *hw = &adapter->hw;
2607 u32 new_itr = itr ? 1000000000 / (itr * 256) : 0;
2608
2609 if (adapter->msix_entries) {
2610 int vector;
2611
2612 for (vector = 0; vector < adapter->num_vectors; vector++)
2613 writel(new_itr, hw->hw_addr + E1000_EITR_82574(vector));
2614 } else {
2615 ew32(ITR, new_itr);
2616 }
2617 }
2618
2619 /**
2620 * e1000_alloc_queues - Allocate memory for all rings
2621 * @adapter: board private structure to initialize
2622 **/
2623 static int e1000_alloc_queues(struct e1000_adapter *adapter)
2624 {
2625 int size = sizeof(struct e1000_ring);
2626
2627 adapter->tx_ring = kzalloc(size, GFP_KERNEL);
2628 if (!adapter->tx_ring)
2629 goto err;
2630 adapter->tx_ring->count = adapter->tx_ring_count;
2631 adapter->tx_ring->adapter = adapter;
2632
2633 adapter->rx_ring = kzalloc(size, GFP_KERNEL);
2634 if (!adapter->rx_ring)
2635 goto err;
2636 adapter->rx_ring->count = adapter->rx_ring_count;
2637 adapter->rx_ring->adapter = adapter;
2638
2639 return 0;
2640 err:
2641 e_err("Unable to allocate memory for queues\n");
2642 kfree(adapter->rx_ring);
2643 kfree(adapter->tx_ring);
2644 return -ENOMEM;
2645 }
2646
2647 /**
2648 * e1000e_poll - NAPI Rx polling callback
2649 * @napi: struct associated with this polling callback
2650 * @weight: number of packets driver is allowed to process this poll
2651 **/
2652 static int e1000e_poll(struct napi_struct *napi, int weight)
2653 {
2654 struct e1000_adapter *adapter = container_of(napi, struct e1000_adapter,
2655 napi);
2656 struct e1000_hw *hw = &adapter->hw;
2657 struct net_device *poll_dev = adapter->netdev;
2658 int tx_cleaned = 1, work_done = 0;
2659
2660 adapter = netdev_priv(poll_dev);
2661
2662 if (!adapter->msix_entries ||
2663 (adapter->rx_ring->ims_val & adapter->tx_ring->ims_val))
2664 tx_cleaned = e1000_clean_tx_irq(adapter->tx_ring);
2665
2666 adapter->clean_rx(adapter->rx_ring, &work_done, weight);
2667
2668 if (!tx_cleaned)
2669 work_done = weight;
2670
2671 /* If weight not fully consumed, exit the polling mode */
2672 if (work_done < weight) {
2673 if (adapter->itr_setting & 3)
2674 e1000_set_itr(adapter);
2675 napi_complete(napi);
2676 if (!test_bit(__E1000_DOWN, &adapter->state)) {
2677 if (adapter->msix_entries)
2678 ew32(IMS, adapter->rx_ring->ims_val);
2679 else
2680 e1000_irq_enable(adapter);
2681 }
2682 }
2683
2684 return work_done;
2685 }
2686
2687 static int e1000_vlan_rx_add_vid(struct net_device *netdev, u16 vid)
2688 {
2689 struct e1000_adapter *adapter = netdev_priv(netdev);
2690 struct e1000_hw *hw = &adapter->hw;
2691 u32 vfta, index;
2692
2693 /* don't update vlan cookie if already programmed */
2694 if ((adapter->hw.mng_cookie.status &
2695 E1000_MNG_DHCP_COOKIE_STATUS_VLAN) &&
2696 (vid == adapter->mng_vlan_id))
2697 return 0;
2698
2699 /* add VID to filter table */
2700 if (adapter->flags & FLAG_HAS_HW_VLAN_FILTER) {
2701 index = (vid >> 5) & 0x7F;
2702 vfta = E1000_READ_REG_ARRAY(hw, E1000_VFTA, index);
2703 vfta |= (1 << (vid & 0x1F));
2704 hw->mac.ops.write_vfta(hw, index, vfta);
2705 }
2706
2707 set_bit(vid, adapter->active_vlans);
2708
2709 return 0;
2710 }
2711
2712 static int e1000_vlan_rx_kill_vid(struct net_device *netdev, u16 vid)
2713 {
2714 struct e1000_adapter *adapter = netdev_priv(netdev);
2715 struct e1000_hw *hw = &adapter->hw;
2716 u32 vfta, index;
2717
2718 if ((adapter->hw.mng_cookie.status &
2719 E1000_MNG_DHCP_COOKIE_STATUS_VLAN) &&
2720 (vid == adapter->mng_vlan_id)) {
2721 /* release control to f/w */
2722 e1000e_release_hw_control(adapter);
2723 return 0;
2724 }
2725
2726 /* remove VID from filter table */
2727 if (adapter->flags & FLAG_HAS_HW_VLAN_FILTER) {
2728 index = (vid >> 5) & 0x7F;
2729 vfta = E1000_READ_REG_ARRAY(hw, E1000_VFTA, index);
2730 vfta &= ~(1 << (vid & 0x1F));
2731 hw->mac.ops.write_vfta(hw, index, vfta);
2732 }
2733
2734 clear_bit(vid, adapter->active_vlans);
2735
2736 return 0;
2737 }
2738
2739 /**
2740 * e1000e_vlan_filter_disable - helper to disable hw VLAN filtering
2741 * @adapter: board private structure to initialize
2742 **/
2743 static void e1000e_vlan_filter_disable(struct e1000_adapter *adapter)
2744 {
2745 struct net_device *netdev = adapter->netdev;
2746 struct e1000_hw *hw = &adapter->hw;
2747 u32 rctl;
2748
2749 if (adapter->flags & FLAG_HAS_HW_VLAN_FILTER) {
2750 /* disable VLAN receive filtering */
2751 rctl = er32(RCTL);
2752 rctl &= ~(E1000_RCTL_VFE | E1000_RCTL_CFIEN);
2753 ew32(RCTL, rctl);
2754
2755 if (adapter->mng_vlan_id != (u16)E1000_MNG_VLAN_NONE) {
2756 e1000_vlan_rx_kill_vid(netdev, adapter->mng_vlan_id);
2757 adapter->mng_vlan_id = E1000_MNG_VLAN_NONE;
2758 }
2759 }
2760 }
2761
2762 /**
2763 * e1000e_vlan_filter_enable - helper to enable HW VLAN filtering
2764 * @adapter: board private structure to initialize
2765 **/
2766 static void e1000e_vlan_filter_enable(struct e1000_adapter *adapter)
2767 {
2768 struct e1000_hw *hw = &adapter->hw;
2769 u32 rctl;
2770
2771 if (adapter->flags & FLAG_HAS_HW_VLAN_FILTER) {
2772 /* enable VLAN receive filtering */
2773 rctl = er32(RCTL);
2774 rctl |= E1000_RCTL_VFE;
2775 rctl &= ~E1000_RCTL_CFIEN;
2776 ew32(RCTL, rctl);
2777 }
2778 }
2779
2780 /**
2781 * e1000e_vlan_strip_enable - helper to disable HW VLAN stripping
2782 * @adapter: board private structure to initialize
2783 **/
2784 static void e1000e_vlan_strip_disable(struct e1000_adapter *adapter)
2785 {
2786 struct e1000_hw *hw = &adapter->hw;
2787 u32 ctrl;
2788
2789 /* disable VLAN tag insert/strip */
2790 ctrl = er32(CTRL);
2791 ctrl &= ~E1000_CTRL_VME;
2792 ew32(CTRL, ctrl);
2793 }
2794
2795 /**
2796 * e1000e_vlan_strip_enable - helper to enable HW VLAN stripping
2797 * @adapter: board private structure to initialize
2798 **/
2799 static void e1000e_vlan_strip_enable(struct e1000_adapter *adapter)
2800 {
2801 struct e1000_hw *hw = &adapter->hw;
2802 u32 ctrl;
2803
2804 /* enable VLAN tag insert/strip */
2805 ctrl = er32(CTRL);
2806 ctrl |= E1000_CTRL_VME;
2807 ew32(CTRL, ctrl);
2808 }
2809
2810 static void e1000_update_mng_vlan(struct e1000_adapter *adapter)
2811 {
2812 struct net_device *netdev = adapter->netdev;
2813 u16 vid = adapter->hw.mng_cookie.vlan_id;
2814 u16 old_vid = adapter->mng_vlan_id;
2815
2816 if (adapter->hw.mng_cookie.status &
2817 E1000_MNG_DHCP_COOKIE_STATUS_VLAN) {
2818 e1000_vlan_rx_add_vid(netdev, vid);
2819 adapter->mng_vlan_id = vid;
2820 }
2821
2822 if ((old_vid != (u16)E1000_MNG_VLAN_NONE) && (vid != old_vid))
2823 e1000_vlan_rx_kill_vid(netdev, old_vid);
2824 }
2825
2826 static void e1000_restore_vlan(struct e1000_adapter *adapter)
2827 {
2828 u16 vid;
2829
2830 e1000_vlan_rx_add_vid(adapter->netdev, 0);
2831
2832 for_each_set_bit(vid, adapter->active_vlans, VLAN_N_VID)
2833 e1000_vlan_rx_add_vid(adapter->netdev, vid);
2834 }
2835
2836 static void e1000_init_manageability_pt(struct e1000_adapter *adapter)
2837 {
2838 struct e1000_hw *hw = &adapter->hw;
2839 u32 manc, manc2h, mdef, i, j;
2840
2841 if (!(adapter->flags & FLAG_MNG_PT_ENABLED))
2842 return;
2843
2844 manc = er32(MANC);
2845
2846 /* enable receiving management packets to the host. this will probably
2847 * generate destination unreachable messages from the host OS, but
2848 * the packets will be handled on SMBUS
2849 */
2850 manc |= E1000_MANC_EN_MNG2HOST;
2851 manc2h = er32(MANC2H);
2852
2853 switch (hw->mac.type) {
2854 default:
2855 manc2h |= (E1000_MANC2H_PORT_623 | E1000_MANC2H_PORT_664);
2856 break;
2857 case e1000_82574:
2858 case e1000_82583:
2859 /* Check if IPMI pass-through decision filter already exists;
2860 * if so, enable it.
2861 */
2862 for (i = 0, j = 0; i < 8; i++) {
2863 mdef = er32(MDEF(i));
2864
2865 /* Ignore filters with anything other than IPMI ports */
2866 if (mdef & ~(E1000_MDEF_PORT_623 | E1000_MDEF_PORT_664))
2867 continue;
2868
2869 /* Enable this decision filter in MANC2H */
2870 if (mdef)
2871 manc2h |= (1 << i);
2872
2873 j |= mdef;
2874 }
2875
2876 if (j == (E1000_MDEF_PORT_623 | E1000_MDEF_PORT_664))
2877 break;
2878
2879 /* Create new decision filter in an empty filter */
2880 for (i = 0, j = 0; i < 8; i++)
2881 if (er32(MDEF(i)) == 0) {
2882 ew32(MDEF(i), (E1000_MDEF_PORT_623 |
2883 E1000_MDEF_PORT_664));
2884 manc2h |= (1 << 1);
2885 j++;
2886 break;
2887 }
2888
2889 if (!j)
2890 e_warn("Unable to create IPMI pass-through filter\n");
2891 break;
2892 }
2893
2894 ew32(MANC2H, manc2h);
2895 ew32(MANC, manc);
2896 }
2897
2898 /**
2899 * e1000_configure_tx - Configure Transmit Unit after Reset
2900 * @adapter: board private structure
2901 *
2902 * Configure the Tx unit of the MAC after a reset.
2903 **/
2904 static void e1000_configure_tx(struct e1000_adapter *adapter)
2905 {
2906 struct e1000_hw *hw = &adapter->hw;
2907 struct e1000_ring *tx_ring = adapter->tx_ring;
2908 u64 tdba;
2909 u32 tdlen, tarc;
2910
2911 /* Setup the HW Tx Head and Tail descriptor pointers */
2912 tdba = tx_ring->dma;
2913 tdlen = tx_ring->count * sizeof(struct e1000_tx_desc);
2914 ew32(TDBAL(0), (tdba & DMA_BIT_MASK(32)));
2915 ew32(TDBAH(0), (tdba >> 32));
2916 ew32(TDLEN(0), tdlen);
2917 ew32(TDH(0), 0);
2918 ew32(TDT(0), 0);
2919 tx_ring->head = adapter->hw.hw_addr + E1000_TDH(0);
2920 tx_ring->tail = adapter->hw.hw_addr + E1000_TDT(0);
2921
2922 /* Set the Tx Interrupt Delay register */
2923 ew32(TIDV, adapter->tx_int_delay);
2924 /* Tx irq moderation */
2925 ew32(TADV, adapter->tx_abs_int_delay);
2926
2927 if (adapter->flags2 & FLAG2_DMA_BURST) {
2928 u32 txdctl = er32(TXDCTL(0));
2929 txdctl &= ~(E1000_TXDCTL_PTHRESH | E1000_TXDCTL_HTHRESH |
2930 E1000_TXDCTL_WTHRESH);
2931 /* set up some performance related parameters to encourage the
2932 * hardware to use the bus more efficiently in bursts, depends
2933 * on the tx_int_delay to be enabled,
2934 * wthresh = 1 ==> burst write is disabled to avoid Tx stalls
2935 * hthresh = 1 ==> prefetch when one or more available
2936 * pthresh = 0x1f ==> prefetch if internal cache 31 or less
2937 * BEWARE: this seems to work but should be considered first if
2938 * there are Tx hangs or other Tx related bugs
2939 */
2940 txdctl |= E1000_TXDCTL_DMA_BURST_ENABLE;
2941 ew32(TXDCTL(0), txdctl);
2942 }
2943 /* erratum work around: set txdctl the same for both queues */
2944 ew32(TXDCTL(1), er32(TXDCTL(0)));
2945
2946 if (adapter->flags & FLAG_TARC_SPEED_MODE_BIT) {
2947 tarc = er32(TARC(0));
2948 /* set the speed mode bit, we'll clear it if we're not at
2949 * gigabit link later
2950 */
2951 #define SPEED_MODE_BIT (1 << 21)
2952 tarc |= SPEED_MODE_BIT;
2953 ew32(TARC(0), tarc);
2954 }
2955
2956 /* errata: program both queues to unweighted RR */
2957 if (adapter->flags & FLAG_TARC_SET_BIT_ZERO) {
2958 tarc = er32(TARC(0));
2959 tarc |= 1;
2960 ew32(TARC(0), tarc);
2961 tarc = er32(TARC(1));
2962 tarc |= 1;
2963 ew32(TARC(1), tarc);
2964 }
2965
2966 /* Setup Transmit Descriptor Settings for eop descriptor */
2967 adapter->txd_cmd = E1000_TXD_CMD_EOP | E1000_TXD_CMD_IFCS;
2968
2969 /* only set IDE if we are delaying interrupts using the timers */
2970 if (adapter->tx_int_delay)
2971 adapter->txd_cmd |= E1000_TXD_CMD_IDE;
2972
2973 /* enable Report Status bit */
2974 adapter->txd_cmd |= E1000_TXD_CMD_RS;
2975
2976 hw->mac.ops.config_collision_dist(hw);
2977 }
2978
2979 /**
2980 * e1000_setup_rctl - configure the receive control registers
2981 * @adapter: Board private structure
2982 **/
2983 #define PAGE_USE_COUNT(S) (((S) >> PAGE_SHIFT) + \
2984 (((S) & (PAGE_SIZE - 1)) ? 1 : 0))
2985 static void e1000_setup_rctl(struct e1000_adapter *adapter)
2986 {
2987 struct e1000_hw *hw = &adapter->hw;
2988 u32 rctl, rfctl;
2989 u32 pages = 0;
2990
2991 /* Workaround Si errata on PCHx - configure jumbo frame flow */
2992 if (hw->mac.type >= e1000_pch2lan) {
2993 s32 ret_val;
2994
2995 if (adapter->netdev->mtu > ETH_DATA_LEN)
2996 ret_val = e1000_lv_jumbo_workaround_ich8lan(hw, true);
2997 else
2998 ret_val = e1000_lv_jumbo_workaround_ich8lan(hw, false);
2999
3000 if (ret_val)
3001 e_dbg("failed to enable jumbo frame workaround mode\n");
3002 }
3003
3004 /* Program MC offset vector base */
3005 rctl = er32(RCTL);
3006 rctl &= ~(3 << E1000_RCTL_MO_SHIFT);
3007 rctl |= E1000_RCTL_EN | E1000_RCTL_BAM |
3008 E1000_RCTL_LBM_NO | E1000_RCTL_RDMTS_HALF |
3009 (adapter->hw.mac.mc_filter_type << E1000_RCTL_MO_SHIFT);
3010
3011 /* Do not Store bad packets */
3012 rctl &= ~E1000_RCTL_SBP;
3013
3014 /* Enable Long Packet receive */
3015 if (adapter->netdev->mtu <= ETH_DATA_LEN)
3016 rctl &= ~E1000_RCTL_LPE;
3017 else
3018 rctl |= E1000_RCTL_LPE;
3019
3020 /* Some systems expect that the CRC is included in SMBUS traffic. The
3021 * hardware strips the CRC before sending to both SMBUS (BMC) and to
3022 * host memory when this is enabled
3023 */
3024 if (adapter->flags2 & FLAG2_CRC_STRIPPING)
3025 rctl |= E1000_RCTL_SECRC;
3026
3027 /* Workaround Si errata on 82577 PHY - configure IPG for jumbos */
3028 if ((hw->phy.type == e1000_phy_82577) && (rctl & E1000_RCTL_LPE)) {
3029 u16 phy_data;
3030
3031 e1e_rphy(hw, PHY_REG(770, 26), &phy_data);
3032 phy_data &= 0xfff8;
3033 phy_data |= (1 << 2);
3034 e1e_wphy(hw, PHY_REG(770, 26), phy_data);
3035
3036 e1e_rphy(hw, 22, &phy_data);
3037 phy_data &= 0x0fff;
3038 phy_data |= (1 << 14);
3039 e1e_wphy(hw, 0x10, 0x2823);
3040 e1e_wphy(hw, 0x11, 0x0003);
3041 e1e_wphy(hw, 22, phy_data);
3042 }
3043
3044 /* Setup buffer sizes */
3045 rctl &= ~E1000_RCTL_SZ_4096;
3046 rctl |= E1000_RCTL_BSEX;
3047 switch (adapter->rx_buffer_len) {
3048 case 2048:
3049 default:
3050 rctl |= E1000_RCTL_SZ_2048;
3051 rctl &= ~E1000_RCTL_BSEX;
3052 break;
3053 case 4096:
3054 rctl |= E1000_RCTL_SZ_4096;
3055 break;
3056 case 8192:
3057 rctl |= E1000_RCTL_SZ_8192;
3058 break;
3059 case 16384:
3060 rctl |= E1000_RCTL_SZ_16384;
3061 break;
3062 }
3063
3064 /* Enable Extended Status in all Receive Descriptors */
3065 rfctl = er32(RFCTL);
3066 rfctl |= E1000_RFCTL_EXTEN;
3067 ew32(RFCTL, rfctl);
3068
3069 /* 82571 and greater support packet-split where the protocol
3070 * header is placed in skb->data and the packet data is
3071 * placed in pages hanging off of skb_shinfo(skb)->nr_frags.
3072 * In the case of a non-split, skb->data is linearly filled,
3073 * followed by the page buffers. Therefore, skb->data is
3074 * sized to hold the largest protocol header.
3075 *
3076 * allocations using alloc_page take too long for regular MTU
3077 * so only enable packet split for jumbo frames
3078 *
3079 * Using pages when the page size is greater than 16k wastes
3080 * a lot of memory, since we allocate 3 pages at all times
3081 * per packet.
3082 */
3083 pages = PAGE_USE_COUNT(adapter->netdev->mtu);
3084 if ((pages <= 3) && (PAGE_SIZE <= 16384) && (rctl & E1000_RCTL_LPE))
3085 adapter->rx_ps_pages = pages;
3086 else
3087 adapter->rx_ps_pages = 0;
3088
3089 if (adapter->rx_ps_pages) {
3090 u32 psrctl = 0;
3091
3092 /* Enable Packet split descriptors */
3093 rctl |= E1000_RCTL_DTYP_PS;
3094
3095 psrctl |= adapter->rx_ps_bsize0 >>
3096 E1000_PSRCTL_BSIZE0_SHIFT;
3097
3098 switch (adapter->rx_ps_pages) {
3099 case 3:
3100 psrctl |= PAGE_SIZE <<
3101 E1000_PSRCTL_BSIZE3_SHIFT;
3102 case 2:
3103 psrctl |= PAGE_SIZE <<
3104 E1000_PSRCTL_BSIZE2_SHIFT;
3105 case 1:
3106 psrctl |= PAGE_SIZE >>
3107 E1000_PSRCTL_BSIZE1_SHIFT;
3108 break;
3109 }
3110
3111 ew32(PSRCTL, psrctl);
3112 }
3113
3114 /* This is useful for sniffing bad packets. */
3115 if (adapter->netdev->features & NETIF_F_RXALL) {
3116 /* UPE and MPE will be handled by normal PROMISC logic
3117 * in e1000e_set_rx_mode
3118 */
3119 rctl |= (E1000_RCTL_SBP | /* Receive bad packets */
3120 E1000_RCTL_BAM | /* RX All Bcast Pkts */
3121 E1000_RCTL_PMCF); /* RX All MAC Ctrl Pkts */
3122
3123 rctl &= ~(E1000_RCTL_VFE | /* Disable VLAN filter */
3124 E1000_RCTL_DPF | /* Allow filtered pause */
3125 E1000_RCTL_CFIEN); /* Dis VLAN CFIEN Filter */
3126 /* Do not mess with E1000_CTRL_VME, it affects transmit as well,
3127 * and that breaks VLANs.
3128 */
3129 }
3130
3131 ew32(RCTL, rctl);
3132 /* just started the receive unit, no need to restart */
3133 adapter->flags &= ~FLAG_RESTART_NOW;
3134 }
3135
3136 /**
3137 * e1000_configure_rx - Configure Receive Unit after Reset
3138 * @adapter: board private structure
3139 *
3140 * Configure the Rx unit of the MAC after a reset.
3141 **/
3142 static void e1000_configure_rx(struct e1000_adapter *adapter)
3143 {
3144 struct e1000_hw *hw = &adapter->hw;
3145 struct e1000_ring *rx_ring = adapter->rx_ring;
3146 u64 rdba;
3147 u32 rdlen, rctl, rxcsum, ctrl_ext;
3148
3149 if (adapter->rx_ps_pages) {
3150 /* this is a 32 byte descriptor */
3151 rdlen = rx_ring->count *
3152 sizeof(union e1000_rx_desc_packet_split);
3153 adapter->clean_rx = e1000_clean_rx_irq_ps;
3154 adapter->alloc_rx_buf = e1000_alloc_rx_buffers_ps;
3155 } else if (adapter->netdev->mtu > ETH_FRAME_LEN + ETH_FCS_LEN) {
3156 rdlen = rx_ring->count * sizeof(union e1000_rx_desc_extended);
3157 adapter->clean_rx = e1000_clean_jumbo_rx_irq;
3158 adapter->alloc_rx_buf = e1000_alloc_jumbo_rx_buffers;
3159 } else {
3160 rdlen = rx_ring->count * sizeof(union e1000_rx_desc_extended);
3161 adapter->clean_rx = e1000_clean_rx_irq;
3162 adapter->alloc_rx_buf = e1000_alloc_rx_buffers;
3163 }
3164
3165 /* disable receives while setting up the descriptors */
3166 rctl = er32(RCTL);
3167 if (!(adapter->flags2 & FLAG2_NO_DISABLE_RX))
3168 ew32(RCTL, rctl & ~E1000_RCTL_EN);
3169 e1e_flush();
3170 usleep_range(10000, 20000);
3171
3172 if (adapter->flags2 & FLAG2_DMA_BURST) {
3173 /* set the writeback threshold (only takes effect if the RDTR
3174 * is set). set GRAN=1 and write back up to 0x4 worth, and
3175 * enable prefetching of 0x20 Rx descriptors
3176 * granularity = 01
3177 * wthresh = 04,
3178 * hthresh = 04,
3179 * pthresh = 0x20
3180 */
3181 ew32(RXDCTL(0), E1000_RXDCTL_DMA_BURST_ENABLE);
3182 ew32(RXDCTL(1), E1000_RXDCTL_DMA_BURST_ENABLE);
3183
3184 /* override the delay timers for enabling bursting, only if
3185 * the value was not set by the user via module options
3186 */
3187 if (adapter->rx_int_delay == DEFAULT_RDTR)
3188 adapter->rx_int_delay = BURST_RDTR;
3189 if (adapter->rx_abs_int_delay == DEFAULT_RADV)
3190 adapter->rx_abs_int_delay = BURST_RADV;
3191 }
3192
3193 /* set the Receive Delay Timer Register */
3194 ew32(RDTR, adapter->rx_int_delay);
3195
3196 /* irq moderation */
3197 ew32(RADV, adapter->rx_abs_int_delay);
3198 if ((adapter->itr_setting != 0) && (adapter->itr != 0))
3199 e1000e_write_itr(adapter, adapter->itr);
3200
3201 ctrl_ext = er32(CTRL_EXT);
3202 /* Auto-Mask interrupts upon ICR access */
3203 ctrl_ext |= E1000_CTRL_EXT_IAME;
3204 ew32(IAM, 0xffffffff);
3205 ew32(CTRL_EXT, ctrl_ext);
3206 e1e_flush();
3207
3208 /* Setup the HW Rx Head and Tail Descriptor Pointers and
3209 * the Base and Length of the Rx Descriptor Ring
3210 */
3211 rdba = rx_ring->dma;
3212 ew32(RDBAL(0), (rdba & DMA_BIT_MASK(32)));
3213 ew32(RDBAH(0), (rdba >> 32));
3214 ew32(RDLEN(0), rdlen);
3215 ew32(RDH(0), 0);
3216 ew32(RDT(0), 0);
3217 rx_ring->head = adapter->hw.hw_addr + E1000_RDH(0);
3218 rx_ring->tail = adapter->hw.hw_addr + E1000_RDT(0);
3219
3220 /* Enable Receive Checksum Offload for TCP and UDP */
3221 rxcsum = er32(RXCSUM);
3222 if (adapter->netdev->features & NETIF_F_RXCSUM)
3223 rxcsum |= E1000_RXCSUM_TUOFL;
3224 else
3225 rxcsum &= ~E1000_RXCSUM_TUOFL;
3226 ew32(RXCSUM, rxcsum);
3227
3228 /* With jumbo frames, excessive C-state transition latencies result
3229 * in dropped transactions.
3230 */
3231 if (adapter->netdev->mtu > ETH_DATA_LEN) {
3232 u32 lat =
3233 ((er32(PBA) & E1000_PBA_RXA_MASK) * 1024 -
3234 adapter->max_frame_size) * 8 / 1000;
3235
3236 if (adapter->flags & FLAG_IS_ICH) {
3237 u32 rxdctl = er32(RXDCTL(0));
3238 ew32(RXDCTL(0), rxdctl | 0x3);
3239 }
3240
3241 pm_qos_update_request(&adapter->netdev->pm_qos_req, lat);
3242 } else {
3243 pm_qos_update_request(&adapter->netdev->pm_qos_req,
3244 PM_QOS_DEFAULT_VALUE);
3245 }
3246
3247 /* Enable Receives */
3248 ew32(RCTL, rctl);
3249 }
3250
3251 /**
3252 * e1000e_write_mc_addr_list - write multicast addresses to MTA
3253 * @netdev: network interface device structure
3254 *
3255 * Writes multicast address list to the MTA hash table.
3256 * Returns: -ENOMEM on failure
3257 * 0 on no addresses written
3258 * X on writing X addresses to MTA
3259 */
3260 static int e1000e_write_mc_addr_list(struct net_device *netdev)
3261 {
3262 struct e1000_adapter *adapter = netdev_priv(netdev);
3263 struct e1000_hw *hw = &adapter->hw;
3264 struct netdev_hw_addr *ha;
3265 u8 *mta_list;
3266 int i;
3267
3268 if (netdev_mc_empty(netdev)) {
3269 /* nothing to program, so clear mc list */
3270 hw->mac.ops.update_mc_addr_list(hw, NULL, 0);
3271 return 0;
3272 }
3273
3274 mta_list = kzalloc(netdev_mc_count(netdev) * ETH_ALEN, GFP_ATOMIC);
3275 if (!mta_list)
3276 return -ENOMEM;
3277
3278 /* update_mc_addr_list expects a packed array of only addresses. */
3279 i = 0;
3280 netdev_for_each_mc_addr(ha, netdev)
3281 memcpy(mta_list + (i++ * ETH_ALEN), ha->addr, ETH_ALEN);
3282
3283 hw->mac.ops.update_mc_addr_list(hw, mta_list, i);
3284 kfree(mta_list);
3285
3286 return netdev_mc_count(netdev);
3287 }
3288
3289 /**
3290 * e1000e_write_uc_addr_list - write unicast addresses to RAR table
3291 * @netdev: network interface device structure
3292 *
3293 * Writes unicast address list to the RAR table.
3294 * Returns: -ENOMEM on failure/insufficient address space
3295 * 0 on no addresses written
3296 * X on writing X addresses to the RAR table
3297 **/
3298 static int e1000e_write_uc_addr_list(struct net_device *netdev)
3299 {
3300 struct e1000_adapter *adapter = netdev_priv(netdev);
3301 struct e1000_hw *hw = &adapter->hw;
3302 unsigned int rar_entries = hw->mac.rar_entry_count;
3303 int count = 0;
3304
3305 /* save a rar entry for our hardware address */
3306 rar_entries--;
3307
3308 /* save a rar entry for the LAA workaround */
3309 if (adapter->flags & FLAG_RESET_OVERWRITES_LAA)
3310 rar_entries--;
3311
3312 /* return ENOMEM indicating insufficient memory for addresses */
3313 if (netdev_uc_count(netdev) > rar_entries)
3314 return -ENOMEM;
3315
3316 if (!netdev_uc_empty(netdev) && rar_entries) {
3317 struct netdev_hw_addr *ha;
3318
3319 /* write the addresses in reverse order to avoid write
3320 * combining
3321 */
3322 netdev_for_each_uc_addr(ha, netdev) {
3323 if (!rar_entries)
3324 break;
3325 hw->mac.ops.rar_set(hw, ha->addr, rar_entries--);
3326 count++;
3327 }
3328 }
3329
3330 /* zero out the remaining RAR entries not used above */
3331 for (; rar_entries > 0; rar_entries--) {
3332 ew32(RAH(rar_entries), 0);
3333 ew32(RAL(rar_entries), 0);
3334 }
3335 e1e_flush();
3336
3337 return count;
3338 }
3339
3340 /**
3341 * e1000e_set_rx_mode - secondary unicast, Multicast and Promiscuous mode set
3342 * @netdev: network interface device structure
3343 *
3344 * The ndo_set_rx_mode entry point is called whenever the unicast or multicast
3345 * address list or the network interface flags are updated. This routine is
3346 * responsible for configuring the hardware for proper unicast, multicast,
3347 * promiscuous mode, and all-multi behavior.
3348 **/
3349 static void e1000e_set_rx_mode(struct net_device *netdev)
3350 {
3351 struct e1000_adapter *adapter = netdev_priv(netdev);
3352 struct e1000_hw *hw = &adapter->hw;
3353 u32 rctl;
3354
3355 /* Check for Promiscuous and All Multicast modes */
3356 rctl = er32(RCTL);
3357
3358 /* clear the affected bits */
3359 rctl &= ~(E1000_RCTL_UPE | E1000_RCTL_MPE);
3360
3361 if (netdev->flags & IFF_PROMISC) {
3362 rctl |= (E1000_RCTL_UPE | E1000_RCTL_MPE);
3363 /* Do not hardware filter VLANs in promisc mode */
3364 e1000e_vlan_filter_disable(adapter);
3365 } else {
3366 int count;
3367
3368 if (netdev->flags & IFF_ALLMULTI) {
3369 rctl |= E1000_RCTL_MPE;
3370 } else {
3371 /* Write addresses to the MTA, if the attempt fails
3372 * then we should just turn on promiscuous mode so
3373 * that we can at least receive multicast traffic
3374 */
3375 count = e1000e_write_mc_addr_list(netdev);
3376 if (count < 0)
3377 rctl |= E1000_RCTL_MPE;
3378 }
3379 e1000e_vlan_filter_enable(adapter);
3380 /* Write addresses to available RAR registers, if there is not
3381 * sufficient space to store all the addresses then enable
3382 * unicast promiscuous mode
3383 */
3384 count = e1000e_write_uc_addr_list(netdev);
3385 if (count < 0)
3386 rctl |= E1000_RCTL_UPE;
3387 }
3388
3389 ew32(RCTL, rctl);
3390
3391 if (netdev->features & NETIF_F_HW_VLAN_RX)
3392 e1000e_vlan_strip_enable(adapter);
3393 else
3394 e1000e_vlan_strip_disable(adapter);
3395 }
3396
3397 static void e1000e_setup_rss_hash(struct e1000_adapter *adapter)
3398 {
3399 struct e1000_hw *hw = &adapter->hw;
3400 u32 mrqc, rxcsum;
3401 int i;
3402 static const u32 rsskey[10] = {
3403 0xda565a6d, 0xc20e5b25, 0x3d256741, 0xb08fa343, 0xcb2bcad0,
3404 0xb4307bae, 0xa32dcb77, 0x0cf23080, 0x3bb7426a, 0xfa01acbe
3405 };
3406
3407 /* Fill out hash function seed */
3408 for (i = 0; i < 10; i++)
3409 ew32(RSSRK(i), rsskey[i]);
3410
3411 /* Direct all traffic to queue 0 */
3412 for (i = 0; i < 32; i++)
3413 ew32(RETA(i), 0);
3414
3415 /* Disable raw packet checksumming so that RSS hash is placed in
3416 * descriptor on writeback.
3417 */
3418 rxcsum = er32(RXCSUM);
3419 rxcsum |= E1000_RXCSUM_PCSD;
3420
3421 ew32(RXCSUM, rxcsum);
3422
3423 mrqc = (E1000_MRQC_RSS_FIELD_IPV4 |
3424 E1000_MRQC_RSS_FIELD_IPV4_TCP |
3425 E1000_MRQC_RSS_FIELD_IPV6 |
3426 E1000_MRQC_RSS_FIELD_IPV6_TCP |
3427 E1000_MRQC_RSS_FIELD_IPV6_TCP_EX);
3428
3429 ew32(MRQC, mrqc);
3430 }
3431
3432 /**
3433 * e1000e_get_base_timinca - get default SYSTIM time increment attributes
3434 * @adapter: board private structure
3435 * @timinca: pointer to returned time increment attributes
3436 *
3437 * Get attributes for incrementing the System Time Register SYSTIML/H at
3438 * the default base frequency, and set the cyclecounter shift value.
3439 **/
3440 s32 e1000e_get_base_timinca(struct e1000_adapter *adapter, u32 *timinca)
3441 {
3442 struct e1000_hw *hw = &adapter->hw;
3443 u32 incvalue, incperiod, shift;
3444
3445 /* Make sure clock is enabled on I217 before checking the frequency */
3446 if ((hw->mac.type == e1000_pch_lpt) &&
3447 !(er32(TSYNCTXCTL) & E1000_TSYNCTXCTL_ENABLED) &&
3448 !(er32(TSYNCRXCTL) & E1000_TSYNCRXCTL_ENABLED)) {
3449 u32 fextnvm7 = er32(FEXTNVM7);
3450
3451 if (!(fextnvm7 & (1 << 0))) {
3452 ew32(FEXTNVM7, fextnvm7 | (1 << 0));
3453 e1e_flush();
3454 }
3455 }
3456
3457 switch (hw->mac.type) {
3458 case e1000_pch2lan:
3459 case e1000_pch_lpt:
3460 /* On I217, the clock frequency is 25MHz or 96MHz as
3461 * indicated by the System Clock Frequency Indication
3462 */
3463 if ((hw->mac.type != e1000_pch_lpt) ||
3464 (er32(TSYNCRXCTL) & E1000_TSYNCRXCTL_SYSCFI)) {
3465 /* Stable 96MHz frequency */
3466 incperiod = INCPERIOD_96MHz;
3467 incvalue = INCVALUE_96MHz;
3468 shift = INCVALUE_SHIFT_96MHz;
3469 adapter->cc.shift = shift + INCPERIOD_SHIFT_96MHz;
3470 break;
3471 }
3472 /* fall-through */
3473 case e1000_82574:
3474 case e1000_82583:
3475 /* Stable 25MHz frequency */
3476 incperiod = INCPERIOD_25MHz;
3477 incvalue = INCVALUE_25MHz;
3478 shift = INCVALUE_SHIFT_25MHz;
3479 adapter->cc.shift = shift;
3480 break;
3481 default:
3482 return -EINVAL;
3483 }
3484
3485 *timinca = ((incperiod << E1000_TIMINCA_INCPERIOD_SHIFT) |
3486 ((incvalue << shift) & E1000_TIMINCA_INCVALUE_MASK));
3487
3488 return 0;
3489 }
3490
3491 /**
3492 * e1000e_config_hwtstamp - configure the hwtstamp registers and enable/disable
3493 * @adapter: board private structure
3494 *
3495 * Outgoing time stamping can be enabled and disabled. Play nice and
3496 * disable it when requested, although it shouldn't cause any overhead
3497 * when no packet needs it. At most one packet in the queue may be
3498 * marked for time stamping, otherwise it would be impossible to tell
3499 * for sure to which packet the hardware time stamp belongs.
3500 *
3501 * Incoming time stamping has to be configured via the hardware filters.
3502 * Not all combinations are supported, in particular event type has to be
3503 * specified. Matching the kind of event packet is not supported, with the
3504 * exception of "all V2 events regardless of level 2 or 4".
3505 **/
3506 static int e1000e_config_hwtstamp(struct e1000_adapter *adapter)
3507 {
3508 struct e1000_hw *hw = &adapter->hw;
3509 struct hwtstamp_config *config = &adapter->hwtstamp_config;
3510 u32 tsync_tx_ctl = E1000_TSYNCTXCTL_ENABLED;
3511 u32 tsync_rx_ctl = E1000_TSYNCRXCTL_ENABLED;
3512 u32 rxmtrl = 0;
3513 u16 rxudp = 0;
3514 bool is_l4 = false;
3515 bool is_l2 = false;
3516 u32 regval;
3517 s32 ret_val;
3518
3519 if (!(adapter->flags & FLAG_HAS_HW_TIMESTAMP))
3520 return -EINVAL;
3521
3522 /* flags reserved for future extensions - must be zero */
3523 if (config->flags)
3524 return -EINVAL;
3525
3526 switch (config->tx_type) {
3527 case HWTSTAMP_TX_OFF:
3528 tsync_tx_ctl = 0;
3529 break;
3530 case HWTSTAMP_TX_ON:
3531 break;
3532 default:
3533 return -ERANGE;
3534 }
3535
3536 switch (config->rx_filter) {
3537 case HWTSTAMP_FILTER_NONE:
3538 tsync_rx_ctl = 0;
3539 break;
3540 case HWTSTAMP_FILTER_PTP_V1_L4_SYNC:
3541 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_L4_V1;
3542 rxmtrl = E1000_RXMTRL_PTP_V1_SYNC_MESSAGE;
3543 is_l4 = true;
3544 break;
3545 case HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ:
3546 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_L4_V1;
3547 rxmtrl = E1000_RXMTRL_PTP_V1_DELAY_REQ_MESSAGE;
3548 is_l4 = true;
3549 break;
3550 case HWTSTAMP_FILTER_PTP_V2_L2_SYNC:
3551 /* Also time stamps V2 L2 Path Delay Request/Response */
3552 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_L2_V2;
3553 rxmtrl = E1000_RXMTRL_PTP_V2_SYNC_MESSAGE;
3554 is_l2 = true;
3555 break;
3556 case HWTSTAMP_FILTER_PTP_V2_L2_DELAY_REQ:
3557 /* Also time stamps V2 L2 Path Delay Request/Response. */
3558 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_L2_V2;
3559 rxmtrl = E1000_RXMTRL_PTP_V2_DELAY_REQ_MESSAGE;
3560 is_l2 = true;
3561 break;
3562 case HWTSTAMP_FILTER_PTP_V2_L4_SYNC:
3563 /* Hardware cannot filter just V2 L4 Sync messages;
3564 * fall-through to V2 (both L2 and L4) Sync.
3565 */
3566 case HWTSTAMP_FILTER_PTP_V2_SYNC:
3567 /* Also time stamps V2 Path Delay Request/Response. */
3568 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_L2_L4_V2;
3569 rxmtrl = E1000_RXMTRL_PTP_V2_SYNC_MESSAGE;
3570 is_l2 = true;
3571 is_l4 = true;
3572 break;
3573 case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ:
3574 /* Hardware cannot filter just V2 L4 Delay Request messages;
3575 * fall-through to V2 (both L2 and L4) Delay Request.
3576 */
3577 case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ:
3578 /* Also time stamps V2 Path Delay Request/Response. */
3579 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_L2_L4_V2;
3580 rxmtrl = E1000_RXMTRL_PTP_V2_DELAY_REQ_MESSAGE;
3581 is_l2 = true;
3582 is_l4 = true;
3583 break;
3584 case HWTSTAMP_FILTER_PTP_V2_L4_EVENT:
3585 case HWTSTAMP_FILTER_PTP_V2_L2_EVENT:
3586 /* Hardware cannot filter just V2 L4 or L2 Event messages;
3587 * fall-through to all V2 (both L2 and L4) Events.
3588 */
3589 case HWTSTAMP_FILTER_PTP_V2_EVENT:
3590 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_EVENT_V2;
3591 config->rx_filter = HWTSTAMP_FILTER_PTP_V2_EVENT;
3592 is_l2 = true;
3593 is_l4 = true;
3594 break;
3595 case HWTSTAMP_FILTER_PTP_V1_L4_EVENT:
3596 /* For V1, the hardware can only filter Sync messages or
3597 * Delay Request messages but not both so fall-through to
3598 * time stamp all packets.
3599 */
3600 case HWTSTAMP_FILTER_ALL:
3601 is_l2 = true;
3602 is_l4 = true;
3603 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_ALL;
3604 config->rx_filter = HWTSTAMP_FILTER_ALL;
3605 break;
3606 default:
3607 return -ERANGE;
3608 }
3609
3610 /* enable/disable Tx h/w time stamping */
3611 regval = er32(TSYNCTXCTL);
3612 regval &= ~E1000_TSYNCTXCTL_ENABLED;
3613 regval |= tsync_tx_ctl;
3614 ew32(TSYNCTXCTL, regval);
3615 if ((er32(TSYNCTXCTL) & E1000_TSYNCTXCTL_ENABLED) !=
3616 (regval & E1000_TSYNCTXCTL_ENABLED)) {
3617 e_err("Timesync Tx Control register not set as expected\n");
3618 return -EAGAIN;
3619 }
3620
3621 /* enable/disable Rx h/w time stamping */
3622 regval = er32(TSYNCRXCTL);
3623 regval &= ~(E1000_TSYNCRXCTL_ENABLED | E1000_TSYNCRXCTL_TYPE_MASK);
3624 regval |= tsync_rx_ctl;
3625 ew32(TSYNCRXCTL, regval);
3626 if ((er32(TSYNCRXCTL) & (E1000_TSYNCRXCTL_ENABLED |
3627 E1000_TSYNCRXCTL_TYPE_MASK)) !=
3628 (regval & (E1000_TSYNCRXCTL_ENABLED |
3629 E1000_TSYNCRXCTL_TYPE_MASK))) {
3630 e_err("Timesync Rx Control register not set as expected\n");
3631 return -EAGAIN;
3632 }
3633
3634 /* L2: define ethertype filter for time stamped packets */
3635 if (is_l2)
3636 rxmtrl |= ETH_P_1588;
3637
3638 /* define which PTP packets get time stamped */
3639 ew32(RXMTRL, rxmtrl);
3640
3641 /* Filter by destination port */
3642 if (is_l4) {
3643 rxudp = PTP_EV_PORT;
3644 cpu_to_be16s(&rxudp);
3645 }
3646 ew32(RXUDP, rxudp);
3647
3648 e1e_flush();
3649
3650 /* Clear TSYNCRXCTL_VALID & TSYNCTXCTL_VALID bit */
3651 er32(RXSTMPH);
3652 er32(TXSTMPH);
3653
3654 /* Get and set the System Time Register SYSTIM base frequency */
3655 ret_val = e1000e_get_base_timinca(adapter, &regval);
3656 if (ret_val)
3657 return ret_val;
3658 ew32(TIMINCA, regval);
3659
3660 /* reset the ns time counter */
3661 timecounter_init(&adapter->tc, &adapter->cc,
3662 ktime_to_ns(ktime_get_real()));
3663
3664 return 0;
3665 }
3666
3667 /**
3668 * e1000_configure - configure the hardware for Rx and Tx
3669 * @adapter: private board structure
3670 **/
3671 static void e1000_configure(struct e1000_adapter *adapter)
3672 {
3673 struct e1000_ring *rx_ring = adapter->rx_ring;
3674
3675 e1000e_set_rx_mode(adapter->netdev);
3676
3677 e1000_restore_vlan(adapter);
3678 e1000_init_manageability_pt(adapter);
3679
3680 e1000_configure_tx(adapter);
3681
3682 if (adapter->netdev->features & NETIF_F_RXHASH)
3683 e1000e_setup_rss_hash(adapter);
3684 e1000_setup_rctl(adapter);
3685 e1000_configure_rx(adapter);
3686 adapter->alloc_rx_buf(rx_ring, e1000_desc_unused(rx_ring), GFP_KERNEL);
3687 }
3688
3689 /**
3690 * e1000e_power_up_phy - restore link in case the phy was powered down
3691 * @adapter: address of board private structure
3692 *
3693 * The phy may be powered down to save power and turn off link when the
3694 * driver is unloaded and wake on lan is not enabled (among others)
3695 * *** this routine MUST be followed by a call to e1000e_reset ***
3696 **/
3697 void e1000e_power_up_phy(struct e1000_adapter *adapter)
3698 {
3699 if (adapter->hw.phy.ops.power_up)
3700 adapter->hw.phy.ops.power_up(&adapter->hw);
3701
3702 adapter->hw.mac.ops.setup_link(&adapter->hw);
3703 }
3704
3705 /**
3706 * e1000_power_down_phy - Power down the PHY
3707 *
3708 * Power down the PHY so no link is implied when interface is down.
3709 * The PHY cannot be powered down if management or WoL is active.
3710 */
3711 static void e1000_power_down_phy(struct e1000_adapter *adapter)
3712 {
3713 /* WoL is enabled */
3714 if (adapter->wol)
3715 return;
3716
3717 if (adapter->hw.phy.ops.power_down)
3718 adapter->hw.phy.ops.power_down(&adapter->hw);
3719 }
3720
3721 /**
3722 * e1000e_reset - bring the hardware into a known good state
3723 *
3724 * This function boots the hardware and enables some settings that
3725 * require a configuration cycle of the hardware - those cannot be
3726 * set/changed during runtime. After reset the device needs to be
3727 * properly configured for Rx, Tx etc.
3728 */
3729 void e1000e_reset(struct e1000_adapter *adapter)
3730 {
3731 struct e1000_mac_info *mac = &adapter->hw.mac;
3732 struct e1000_fc_info *fc = &adapter->hw.fc;
3733 struct e1000_hw *hw = &adapter->hw;
3734 u32 tx_space, min_tx_space, min_rx_space;
3735 u32 pba = adapter->pba;
3736 u16 hwm;
3737
3738 /* reset Packet Buffer Allocation to default */
3739 ew32(PBA, pba);
3740
3741 if (adapter->max_frame_size > ETH_FRAME_LEN + ETH_FCS_LEN) {
3742 /* To maintain wire speed transmits, the Tx FIFO should be
3743 * large enough to accommodate two full transmit packets,
3744 * rounded up to the next 1KB and expressed in KB. Likewise,
3745 * the Rx FIFO should be large enough to accommodate at least
3746 * one full receive packet and is similarly rounded up and
3747 * expressed in KB.
3748 */
3749 pba = er32(PBA);
3750 /* upper 16 bits has Tx packet buffer allocation size in KB */
3751 tx_space = pba >> 16;
3752 /* lower 16 bits has Rx packet buffer allocation size in KB */
3753 pba &= 0xffff;
3754 /* the Tx fifo also stores 16 bytes of information about the Tx
3755 * but don't include ethernet FCS because hardware appends it
3756 */
3757 min_tx_space = (adapter->max_frame_size +
3758 sizeof(struct e1000_tx_desc) -
3759 ETH_FCS_LEN) * 2;
3760 min_tx_space = ALIGN(min_tx_space, 1024);
3761 min_tx_space >>= 10;
3762 /* software strips receive CRC, so leave room for it */
3763 min_rx_space = adapter->max_frame_size;
3764 min_rx_space = ALIGN(min_rx_space, 1024);
3765 min_rx_space >>= 10;
3766
3767 /* If current Tx allocation is less than the min Tx FIFO size,
3768 * and the min Tx FIFO size is less than the current Rx FIFO
3769 * allocation, take space away from current Rx allocation
3770 */
3771 if ((tx_space < min_tx_space) &&
3772 ((min_tx_space - tx_space) < pba)) {
3773 pba -= min_tx_space - tx_space;
3774
3775 /* if short on Rx space, Rx wins and must trump Tx
3776 * adjustment
3777 */
3778 if (pba < min_rx_space)
3779 pba = min_rx_space;
3780 }
3781
3782 ew32(PBA, pba);
3783 }
3784
3785 /* flow control settings
3786 *
3787 * The high water mark must be low enough to fit one full frame
3788 * (or the size used for early receive) above it in the Rx FIFO.
3789 * Set it to the lower of:
3790 * - 90% of the Rx FIFO size, and
3791 * - the full Rx FIFO size minus one full frame
3792 */
3793 if (adapter->flags & FLAG_DISABLE_FC_PAUSE_TIME)
3794 fc->pause_time = 0xFFFF;
3795 else
3796 fc->pause_time = E1000_FC_PAUSE_TIME;
3797 fc->send_xon = true;
3798 fc->current_mode = fc->requested_mode;
3799
3800 switch (hw->mac.type) {
3801 case e1000_ich9lan:
3802 case e1000_ich10lan:
3803 if (adapter->netdev->mtu > ETH_DATA_LEN) {
3804 pba = 14;
3805 ew32(PBA, pba);
3806 fc->high_water = 0x2800;
3807 fc->low_water = fc->high_water - 8;
3808 break;
3809 }
3810 /* fall-through */
3811 default:
3812 hwm = min(((pba << 10) * 9 / 10),
3813 ((pba << 10) - adapter->max_frame_size));
3814
3815 fc->high_water = hwm & E1000_FCRTH_RTH; /* 8-byte granularity */
3816 fc->low_water = fc->high_water - 8;
3817 break;
3818 case e1000_pchlan:
3819 /* Workaround PCH LOM adapter hangs with certain network
3820 * loads. If hangs persist, try disabling Tx flow control.
3821 */
3822 if (adapter->netdev->mtu > ETH_DATA_LEN) {
3823 fc->high_water = 0x3500;
3824 fc->low_water = 0x1500;
3825 } else {
3826 fc->high_water = 0x5000;
3827 fc->low_water = 0x3000;
3828 }
3829 fc->refresh_time = 0x1000;
3830 break;
3831 case e1000_pch2lan:
3832 case e1000_pch_lpt:
3833 fc->refresh_time = 0x0400;
3834
3835 if (adapter->netdev->mtu <= ETH_DATA_LEN) {
3836 fc->high_water = 0x05C20;
3837 fc->low_water = 0x05048;
3838 fc->pause_time = 0x0650;
3839 break;
3840 }
3841
3842 fc->high_water = ((pba << 10) * 9 / 10) & E1000_FCRTH_RTH;
3843 fc->low_water = ((pba << 10) * 8 / 10) & E1000_FCRTL_RTL;
3844 break;
3845 }
3846
3847 /* Alignment of Tx data is on an arbitrary byte boundary with the
3848 * maximum size per Tx descriptor limited only to the transmit
3849 * allocation of the packet buffer minus 96 bytes with an upper
3850 * limit of 24KB due to receive synchronization limitations.
3851 */
3852 adapter->tx_fifo_limit = min_t(u32, ((er32(PBA) >> 16) << 10) - 96,
3853 24 << 10);
3854
3855 /* Disable Adaptive Interrupt Moderation if 2 full packets cannot
3856 * fit in receive buffer.
3857 */
3858 if (adapter->itr_setting & 0x3) {
3859 if ((adapter->max_frame_size * 2) > (pba << 10)) {
3860 if (!(adapter->flags2 & FLAG2_DISABLE_AIM)) {
3861 dev_info(&adapter->pdev->dev,
3862 "Interrupt Throttle Rate off\n");
3863 adapter->flags2 |= FLAG2_DISABLE_AIM;
3864 e1000e_write_itr(adapter, 0);
3865 }
3866 } else if (adapter->flags2 & FLAG2_DISABLE_AIM) {
3867 dev_info(&adapter->pdev->dev,
3868 "Interrupt Throttle Rate on\n");
3869 adapter->flags2 &= ~FLAG2_DISABLE_AIM;
3870 adapter->itr = 20000;
3871 e1000e_write_itr(adapter, adapter->itr);
3872 }
3873 }
3874
3875 /* Allow time for pending master requests to run */
3876 mac->ops.reset_hw(hw);
3877
3878 /* For parts with AMT enabled, let the firmware know
3879 * that the network interface is in control
3880 */
3881 if (adapter->flags & FLAG_HAS_AMT)
3882 e1000e_get_hw_control(adapter);
3883
3884 ew32(WUC, 0);
3885
3886 if (mac->ops.init_hw(hw))
3887 e_err("Hardware Error\n");
3888
3889 e1000_update_mng_vlan(adapter);
3890
3891 /* Enable h/w to recognize an 802.1Q VLAN Ethernet packet */
3892 ew32(VET, ETH_P_8021Q);
3893
3894 e1000e_reset_adaptive(hw);
3895
3896 /* initialize systim and reset the ns time counter */
3897 e1000e_config_hwtstamp(adapter);
3898
3899 if (!netif_running(adapter->netdev) &&
3900 !test_bit(__E1000_TESTING, &adapter->state)) {
3901 e1000_power_down_phy(adapter);
3902 return;
3903 }
3904
3905 e1000_get_phy_info(hw);
3906
3907 if ((adapter->flags & FLAG_HAS_SMART_POWER_DOWN) &&
3908 !(adapter->flags & FLAG_SMART_POWER_DOWN)) {
3909 u16 phy_data = 0;
3910 /* speed up time to link by disabling smart power down, ignore
3911 * the return value of this function because there is nothing
3912 * different we would do if it failed
3913 */
3914 e1e_rphy(hw, IGP02E1000_PHY_POWER_MGMT, &phy_data);
3915 phy_data &= ~IGP02E1000_PM_SPD;
3916 e1e_wphy(hw, IGP02E1000_PHY_POWER_MGMT, phy_data);
3917 }
3918 }
3919
3920 int e1000e_up(struct e1000_adapter *adapter)
3921 {
3922 struct e1000_hw *hw = &adapter->hw;
3923
3924 /* hardware has been reset, we need to reload some things */
3925 e1000_configure(adapter);
3926
3927 clear_bit(__E1000_DOWN, &adapter->state);
3928
3929 if (adapter->msix_entries)
3930 e1000_configure_msix(adapter);
3931 e1000_irq_enable(adapter);
3932
3933 netif_start_queue(adapter->netdev);
3934
3935 /* fire a link change interrupt to start the watchdog */
3936 if (adapter->msix_entries)
3937 ew32(ICS, E1000_ICS_LSC | E1000_ICR_OTHER);
3938 else
3939 ew32(ICS, E1000_ICS_LSC);
3940
3941 return 0;
3942 }
3943
3944 static void e1000e_flush_descriptors(struct e1000_adapter *adapter)
3945 {
3946 struct e1000_hw *hw = &adapter->hw;
3947
3948 if (!(adapter->flags2 & FLAG2_DMA_BURST))
3949 return;
3950
3951 /* flush pending descriptor writebacks to memory */
3952 ew32(TIDV, adapter->tx_int_delay | E1000_TIDV_FPD);
3953 ew32(RDTR, adapter->rx_int_delay | E1000_RDTR_FPD);
3954
3955 /* execute the writes immediately */
3956 e1e_flush();
3957
3958 /* due to rare timing issues, write to TIDV/RDTR again to ensure the
3959 * write is successful
3960 */
3961 ew32(TIDV, adapter->tx_int_delay | E1000_TIDV_FPD);
3962 ew32(RDTR, adapter->rx_int_delay | E1000_RDTR_FPD);
3963
3964 /* execute the writes immediately */
3965 e1e_flush();
3966 }
3967
3968 static void e1000e_update_stats(struct e1000_adapter *adapter);
3969
3970 void e1000e_down(struct e1000_adapter *adapter)
3971 {
3972 struct net_device *netdev = adapter->netdev;
3973 struct e1000_hw *hw = &adapter->hw;
3974 u32 tctl, rctl;
3975
3976 /* signal that we're down so the interrupt handler does not
3977 * reschedule our watchdog timer
3978 */
3979 set_bit(__E1000_DOWN, &adapter->state);
3980
3981 /* disable receives in the hardware */
3982 rctl = er32(RCTL);
3983 if (!(adapter->flags2 & FLAG2_NO_DISABLE_RX))
3984 ew32(RCTL, rctl & ~E1000_RCTL_EN);
3985 /* flush and sleep below */
3986
3987 netif_stop_queue(netdev);
3988
3989 /* disable transmits in the hardware */
3990 tctl = er32(TCTL);
3991 tctl &= ~E1000_TCTL_EN;
3992 ew32(TCTL, tctl);
3993
3994 /* flush both disables and wait for them to finish */
3995 e1e_flush();
3996 usleep_range(10000, 20000);
3997
3998 e1000_irq_disable(adapter);
3999
4000 del_timer_sync(&adapter->watchdog_timer);
4001 del_timer_sync(&adapter->phy_info_timer);
4002
4003 netif_carrier_off(netdev);
4004
4005 spin_lock(&adapter->stats64_lock);
4006 e1000e_update_stats(adapter);
4007 spin_unlock(&adapter->stats64_lock);
4008
4009 e1000e_flush_descriptors(adapter);
4010 e1000_clean_tx_ring(adapter->tx_ring);
4011 e1000_clean_rx_ring(adapter->rx_ring);
4012
4013 adapter->link_speed = 0;
4014 adapter->link_duplex = 0;
4015
4016 if (!pci_channel_offline(adapter->pdev))
4017 e1000e_reset(adapter);
4018
4019 /* TODO: for power management, we could drop the link and
4020 * pci_disable_device here.
4021 */
4022 }
4023
4024 void e1000e_reinit_locked(struct e1000_adapter *adapter)
4025 {
4026 might_sleep();
4027 while (test_and_set_bit(__E1000_RESETTING, &adapter->state))
4028 usleep_range(1000, 2000);
4029 e1000e_down(adapter);
4030 e1000e_up(adapter);
4031 clear_bit(__E1000_RESETTING, &adapter->state);
4032 }
4033
4034 /**
4035 * e1000e_cyclecounter_read - read raw cycle counter (used by time counter)
4036 * @cc: cyclecounter structure
4037 **/
4038 static cycle_t e1000e_cyclecounter_read(const struct cyclecounter *cc)
4039 {
4040 struct e1000_adapter *adapter = container_of(cc, struct e1000_adapter,
4041 cc);
4042 struct e1000_hw *hw = &adapter->hw;
4043 cycle_t systim;
4044
4045 /* latch SYSTIMH on read of SYSTIML */
4046 systim = (cycle_t)er32(SYSTIML);
4047 systim |= (cycle_t)er32(SYSTIMH) << 32;
4048
4049 return systim;
4050 }
4051
4052 /**
4053 * e1000_sw_init - Initialize general software structures (struct e1000_adapter)
4054 * @adapter: board private structure to initialize
4055 *
4056 * e1000_sw_init initializes the Adapter private data structure.
4057 * Fields are initialized based on PCI device information and
4058 * OS network device settings (MTU size).
4059 **/
4060 static int e1000_sw_init(struct e1000_adapter *adapter)
4061 {
4062 struct net_device *netdev = adapter->netdev;
4063
4064 adapter->rx_buffer_len = ETH_FRAME_LEN + VLAN_HLEN + ETH_FCS_LEN;
4065 adapter->rx_ps_bsize0 = 128;
4066 adapter->max_frame_size = netdev->mtu + ETH_HLEN + ETH_FCS_LEN;
4067 adapter->min_frame_size = ETH_ZLEN + ETH_FCS_LEN;
4068 adapter->tx_ring_count = E1000_DEFAULT_TXD;
4069 adapter->rx_ring_count = E1000_DEFAULT_RXD;
4070
4071 spin_lock_init(&adapter->stats64_lock);
4072
4073 e1000e_set_interrupt_capability(adapter);
4074
4075 if (e1000_alloc_queues(adapter))
4076 return -ENOMEM;
4077
4078 /* Setup hardware time stamping cyclecounter */
4079 if (adapter->flags & FLAG_HAS_HW_TIMESTAMP) {
4080 adapter->cc.read = e1000e_cyclecounter_read;
4081 adapter->cc.mask = CLOCKSOURCE_MASK(64);
4082 adapter->cc.mult = 1;
4083 /* cc.shift set in e1000e_get_base_tininca() */
4084
4085 spin_lock_init(&adapter->systim_lock);
4086 INIT_WORK(&adapter->tx_hwtstamp_work, e1000e_tx_hwtstamp_work);
4087 }
4088
4089 /* Explicitly disable IRQ since the NIC can be in any state. */
4090 e1000_irq_disable(adapter);
4091
4092 set_bit(__E1000_DOWN, &adapter->state);
4093 return 0;
4094 }
4095
4096 /**
4097 * e1000_intr_msi_test - Interrupt Handler
4098 * @irq: interrupt number
4099 * @data: pointer to a network interface device structure
4100 **/
4101 static irqreturn_t e1000_intr_msi_test(int __always_unused irq, void *data)
4102 {
4103 struct net_device *netdev = data;
4104 struct e1000_adapter *adapter = netdev_priv(netdev);
4105 struct e1000_hw *hw = &adapter->hw;
4106 u32 icr = er32(ICR);
4107
4108 e_dbg("icr is %08X\n", icr);
4109 if (icr & E1000_ICR_RXSEQ) {
4110 adapter->flags &= ~FLAG_MSI_TEST_FAILED;
4111 /* Force memory writes to complete before acknowledging the
4112 * interrupt is handled.
4113 */
4114 wmb();
4115 }
4116
4117 return IRQ_HANDLED;
4118 }
4119
4120 /**
4121 * e1000_test_msi_interrupt - Returns 0 for successful test
4122 * @adapter: board private struct
4123 *
4124 * code flow taken from tg3.c
4125 **/
4126 static int e1000_test_msi_interrupt(struct e1000_adapter *adapter)
4127 {
4128 struct net_device *netdev = adapter->netdev;
4129 struct e1000_hw *hw = &adapter->hw;
4130 int err;
4131
4132 /* poll_enable hasn't been called yet, so don't need disable */
4133 /* clear any pending events */
4134 er32(ICR);
4135
4136 /* free the real vector and request a test handler */
4137 e1000_free_irq(adapter);
4138 e1000e_reset_interrupt_capability(adapter);
4139
4140 /* Assume that the test fails, if it succeeds then the test
4141 * MSI irq handler will unset this flag
4142 */
4143 adapter->flags |= FLAG_MSI_TEST_FAILED;
4144
4145 err = pci_enable_msi(adapter->pdev);
4146 if (err)
4147 goto msi_test_failed;
4148
4149 err = request_irq(adapter->pdev->irq, e1000_intr_msi_test, 0,
4150 netdev->name, netdev);
4151 if (err) {
4152 pci_disable_msi(adapter->pdev);
4153 goto msi_test_failed;
4154 }
4155
4156 /* Force memory writes to complete before enabling and firing an
4157 * interrupt.
4158 */
4159 wmb();
4160
4161 e1000_irq_enable(adapter);
4162
4163 /* fire an unusual interrupt on the test handler */
4164 ew32(ICS, E1000_ICS_RXSEQ);
4165 e1e_flush();
4166 msleep(100);
4167
4168 e1000_irq_disable(adapter);
4169
4170 rmb(); /* read flags after interrupt has been fired */
4171
4172 if (adapter->flags & FLAG_MSI_TEST_FAILED) {
4173 adapter->int_mode = E1000E_INT_MODE_LEGACY;
4174 e_info("MSI interrupt test failed, using legacy interrupt.\n");
4175 } else {
4176 e_dbg("MSI interrupt test succeeded!\n");
4177 }
4178
4179 free_irq(adapter->pdev->irq, netdev);
4180 pci_disable_msi(adapter->pdev);
4181
4182 msi_test_failed:
4183 e1000e_set_interrupt_capability(adapter);
4184 return e1000_request_irq(adapter);
4185 }
4186
4187 /**
4188 * e1000_test_msi - Returns 0 if MSI test succeeds or INTx mode is restored
4189 * @adapter: board private struct
4190 *
4191 * code flow taken from tg3.c, called with e1000 interrupts disabled.
4192 **/
4193 static int e1000_test_msi(struct e1000_adapter *adapter)
4194 {
4195 int err;
4196 u16 pci_cmd;
4197
4198 if (!(adapter->flags & FLAG_MSI_ENABLED))
4199 return 0;
4200
4201 /* disable SERR in case the MSI write causes a master abort */
4202 pci_read_config_word(adapter->pdev, PCI_COMMAND, &pci_cmd);
4203 if (pci_cmd & PCI_COMMAND_SERR)
4204 pci_write_config_word(adapter->pdev, PCI_COMMAND,
4205 pci_cmd & ~PCI_COMMAND_SERR);
4206
4207 err = e1000_test_msi_interrupt(adapter);
4208
4209 /* re-enable SERR */
4210 if (pci_cmd & PCI_COMMAND_SERR) {
4211 pci_read_config_word(adapter->pdev, PCI_COMMAND, &pci_cmd);
4212 pci_cmd |= PCI_COMMAND_SERR;
4213 pci_write_config_word(adapter->pdev, PCI_COMMAND, pci_cmd);
4214 }
4215
4216 return err;
4217 }
4218
4219 /**
4220 * e1000_open - Called when a network interface is made active
4221 * @netdev: network interface device structure
4222 *
4223 * Returns 0 on success, negative value on failure
4224 *
4225 * The open entry point is called when a network interface is made
4226 * active by the system (IFF_UP). At this point all resources needed
4227 * for transmit and receive operations are allocated, the interrupt
4228 * handler is registered with the OS, the watchdog timer is started,
4229 * and the stack is notified that the interface is ready.
4230 **/
4231 static int e1000_open(struct net_device *netdev)
4232 {
4233 struct e1000_adapter *adapter = netdev_priv(netdev);
4234 struct e1000_hw *hw = &adapter->hw;
4235 struct pci_dev *pdev = adapter->pdev;
4236 int err;
4237
4238 /* disallow open during test */
4239 if (test_bit(__E1000_TESTING, &adapter->state))
4240 return -EBUSY;
4241
4242 pm_runtime_get_sync(&pdev->dev);
4243
4244 netif_carrier_off(netdev);
4245
4246 /* allocate transmit descriptors */
4247 err = e1000e_setup_tx_resources(adapter->tx_ring);
4248 if (err)
4249 goto err_setup_tx;
4250
4251 /* allocate receive descriptors */
4252 err = e1000e_setup_rx_resources(adapter->rx_ring);
4253 if (err)
4254 goto err_setup_rx;
4255
4256 /* If AMT is enabled, let the firmware know that the network
4257 * interface is now open and reset the part to a known state.
4258 */
4259 if (adapter->flags & FLAG_HAS_AMT) {
4260 e1000e_get_hw_control(adapter);
4261 e1000e_reset(adapter);
4262 }
4263
4264 e1000e_power_up_phy(adapter);
4265
4266 adapter->mng_vlan_id = E1000_MNG_VLAN_NONE;
4267 if ((adapter->hw.mng_cookie.status &
4268 E1000_MNG_DHCP_COOKIE_STATUS_VLAN))
4269 e1000_update_mng_vlan(adapter);
4270
4271 /* DMA latency requirement to workaround jumbo issue */
4272 pm_qos_add_request(&adapter->netdev->pm_qos_req, PM_QOS_CPU_DMA_LATENCY,
4273 PM_QOS_DEFAULT_VALUE);
4274
4275 /* before we allocate an interrupt, we must be ready to handle it.
4276 * Setting DEBUG_SHIRQ in the kernel makes it fire an interrupt
4277 * as soon as we call pci_request_irq, so we have to setup our
4278 * clean_rx handler before we do so.
4279 */
4280 e1000_configure(adapter);
4281
4282 err = e1000_request_irq(adapter);
4283 if (err)
4284 goto err_req_irq;
4285
4286 /* Work around PCIe errata with MSI interrupts causing some chipsets to
4287 * ignore e1000e MSI messages, which means we need to test our MSI
4288 * interrupt now
4289 */
4290 if (adapter->int_mode != E1000E_INT_MODE_LEGACY) {
4291 err = e1000_test_msi(adapter);
4292 if (err) {
4293 e_err("Interrupt allocation failed\n");
4294 goto err_req_irq;
4295 }
4296 }
4297
4298 /* From here on the code is the same as e1000e_up() */
4299 clear_bit(__E1000_DOWN, &adapter->state);
4300
4301 napi_enable(&adapter->napi);
4302
4303 e1000_irq_enable(adapter);
4304
4305 adapter->tx_hang_recheck = false;
4306 netif_start_queue(netdev);
4307
4308 adapter->idle_check = true;
4309 pm_runtime_put(&pdev->dev);
4310
4311 /* fire a link status change interrupt to start the watchdog */
4312 if (adapter->msix_entries)
4313 ew32(ICS, E1000_ICS_LSC | E1000_ICR_OTHER);
4314 else
4315 ew32(ICS, E1000_ICS_LSC);
4316
4317 return 0;
4318
4319 err_req_irq:
4320 e1000e_release_hw_control(adapter);
4321 e1000_power_down_phy(adapter);
4322 e1000e_free_rx_resources(adapter->rx_ring);
4323 err_setup_rx:
4324 e1000e_free_tx_resources(adapter->tx_ring);
4325 err_setup_tx:
4326 e1000e_reset(adapter);
4327 pm_runtime_put_sync(&pdev->dev);
4328
4329 return err;
4330 }
4331
4332 /**
4333 * e1000_close - Disables a network interface
4334 * @netdev: network interface device structure
4335 *
4336 * Returns 0, this is not allowed to fail
4337 *
4338 * The close entry point is called when an interface is de-activated
4339 * by the OS. The hardware is still under the drivers control, but
4340 * needs to be disabled. A global MAC reset is issued to stop the
4341 * hardware, and all transmit and receive resources are freed.
4342 **/
4343 static int e1000_close(struct net_device *netdev)
4344 {
4345 struct e1000_adapter *adapter = netdev_priv(netdev);
4346 struct pci_dev *pdev = adapter->pdev;
4347 int count = E1000_CHECK_RESET_COUNT;
4348
4349 while (test_bit(__E1000_RESETTING, &adapter->state) && count--)
4350 usleep_range(10000, 20000);
4351
4352 WARN_ON(test_bit(__E1000_RESETTING, &adapter->state));
4353
4354 pm_runtime_get_sync(&pdev->dev);
4355
4356 napi_disable(&adapter->napi);
4357
4358 if (!test_bit(__E1000_DOWN, &adapter->state)) {
4359 e1000e_down(adapter);
4360 e1000_free_irq(adapter);
4361 }
4362 e1000_power_down_phy(adapter);
4363
4364 e1000e_free_tx_resources(adapter->tx_ring);
4365 e1000e_free_rx_resources(adapter->rx_ring);
4366
4367 /* kill manageability vlan ID if supported, but not if a vlan with
4368 * the same ID is registered on the host OS (let 8021q kill it)
4369 */
4370 if (adapter->hw.mng_cookie.status &
4371 E1000_MNG_DHCP_COOKIE_STATUS_VLAN)
4372 e1000_vlan_rx_kill_vid(netdev, adapter->mng_vlan_id);
4373
4374 /* If AMT is enabled, let the firmware know that the network
4375 * interface is now closed
4376 */
4377 if ((adapter->flags & FLAG_HAS_AMT) &&
4378 !test_bit(__E1000_TESTING, &adapter->state))
4379 e1000e_release_hw_control(adapter);
4380
4381 pm_qos_remove_request(&adapter->netdev->pm_qos_req);
4382
4383 pm_runtime_put_sync(&pdev->dev);
4384
4385 return 0;
4386 }
4387 /**
4388 * e1000_set_mac - Change the Ethernet Address of the NIC
4389 * @netdev: network interface device structure
4390 * @p: pointer to an address structure
4391 *
4392 * Returns 0 on success, negative on failure
4393 **/
4394 static int e1000_set_mac(struct net_device *netdev, void *p)
4395 {
4396 struct e1000_adapter *adapter = netdev_priv(netdev);
4397 struct e1000_hw *hw = &adapter->hw;
4398 struct sockaddr *addr = p;
4399
4400 if (!is_valid_ether_addr(addr->sa_data))
4401 return -EADDRNOTAVAIL;
4402
4403 memcpy(netdev->dev_addr, addr->sa_data, netdev->addr_len);
4404 memcpy(adapter->hw.mac.addr, addr->sa_data, netdev->addr_len);
4405
4406 hw->mac.ops.rar_set(&adapter->hw, adapter->hw.mac.addr, 0);
4407
4408 if (adapter->flags & FLAG_RESET_OVERWRITES_LAA) {
4409 /* activate the work around */
4410 e1000e_set_laa_state_82571(&adapter->hw, 1);
4411
4412 /* Hold a copy of the LAA in RAR[14] This is done so that
4413 * between the time RAR[0] gets clobbered and the time it
4414 * gets fixed (in e1000_watchdog), the actual LAA is in one
4415 * of the RARs and no incoming packets directed to this port
4416 * are dropped. Eventually the LAA will be in RAR[0] and
4417 * RAR[14]
4418 */
4419 hw->mac.ops.rar_set(&adapter->hw, adapter->hw.mac.addr,
4420 adapter->hw.mac.rar_entry_count - 1);
4421 }
4422
4423 return 0;
4424 }
4425
4426 /**
4427 * e1000e_update_phy_task - work thread to update phy
4428 * @work: pointer to our work struct
4429 *
4430 * this worker thread exists because we must acquire a
4431 * semaphore to read the phy, which we could msleep while
4432 * waiting for it, and we can't msleep in a timer.
4433 **/
4434 static void e1000e_update_phy_task(struct work_struct *work)
4435 {
4436 struct e1000_adapter *adapter = container_of(work,
4437 struct e1000_adapter,
4438 update_phy_task);
4439
4440 if (test_bit(__E1000_DOWN, &adapter->state))
4441 return;
4442
4443 e1000_get_phy_info(&adapter->hw);
4444 }
4445
4446 /**
4447 * e1000_update_phy_info - timre call-back to update PHY info
4448 * @data: pointer to adapter cast into an unsigned long
4449 *
4450 * Need to wait a few seconds after link up to get diagnostic information from
4451 * the phy
4452 **/
4453 static void e1000_update_phy_info(unsigned long data)
4454 {
4455 struct e1000_adapter *adapter = (struct e1000_adapter *) data;
4456
4457 if (test_bit(__E1000_DOWN, &adapter->state))
4458 return;
4459
4460 schedule_work(&adapter->update_phy_task);
4461 }
4462
4463 /**
4464 * e1000e_update_phy_stats - Update the PHY statistics counters
4465 * @adapter: board private structure
4466 *
4467 * Read/clear the upper 16-bit PHY registers and read/accumulate lower
4468 **/
4469 static void e1000e_update_phy_stats(struct e1000_adapter *adapter)
4470 {
4471 struct e1000_hw *hw = &adapter->hw;
4472 s32 ret_val;
4473 u16 phy_data;
4474
4475 ret_val = hw->phy.ops.acquire(hw);
4476 if (ret_val)
4477 return;
4478
4479 /* A page set is expensive so check if already on desired page.
4480 * If not, set to the page with the PHY status registers.
4481 */
4482 hw->phy.addr = 1;
4483 ret_val = e1000e_read_phy_reg_mdic(hw, IGP01E1000_PHY_PAGE_SELECT,
4484 &phy_data);
4485 if (ret_val)
4486 goto release;
4487 if (phy_data != (HV_STATS_PAGE << IGP_PAGE_SHIFT)) {
4488 ret_val = hw->phy.ops.set_page(hw,
4489 HV_STATS_PAGE << IGP_PAGE_SHIFT);
4490 if (ret_val)
4491 goto release;
4492 }
4493
4494 /* Single Collision Count */
4495 hw->phy.ops.read_reg_page(hw, HV_SCC_UPPER, &phy_data);
4496 ret_val = hw->phy.ops.read_reg_page(hw, HV_SCC_LOWER, &phy_data);
4497 if (!ret_val)
4498 adapter->stats.scc += phy_data;
4499
4500 /* Excessive Collision Count */
4501 hw->phy.ops.read_reg_page(hw, HV_ECOL_UPPER, &phy_data);
4502 ret_val = hw->phy.ops.read_reg_page(hw, HV_ECOL_LOWER, &phy_data);
4503 if (!ret_val)
4504 adapter->stats.ecol += phy_data;
4505
4506 /* Multiple Collision Count */
4507 hw->phy.ops.read_reg_page(hw, HV_MCC_UPPER, &phy_data);
4508 ret_val = hw->phy.ops.read_reg_page(hw, HV_MCC_LOWER, &phy_data);
4509 if (!ret_val)
4510 adapter->stats.mcc += phy_data;
4511
4512 /* Late Collision Count */
4513 hw->phy.ops.read_reg_page(hw, HV_LATECOL_UPPER, &phy_data);
4514 ret_val = hw->phy.ops.read_reg_page(hw, HV_LATECOL_LOWER, &phy_data);
4515 if (!ret_val)
4516 adapter->stats.latecol += phy_data;
4517
4518 /* Collision Count - also used for adaptive IFS */
4519 hw->phy.ops.read_reg_page(hw, HV_COLC_UPPER, &phy_data);
4520 ret_val = hw->phy.ops.read_reg_page(hw, HV_COLC_LOWER, &phy_data);
4521 if (!ret_val)
4522 hw->mac.collision_delta = phy_data;
4523
4524 /* Defer Count */
4525 hw->phy.ops.read_reg_page(hw, HV_DC_UPPER, &phy_data);
4526 ret_val = hw->phy.ops.read_reg_page(hw, HV_DC_LOWER, &phy_data);
4527 if (!ret_val)
4528 adapter->stats.dc += phy_data;
4529
4530 /* Transmit with no CRS */
4531 hw->phy.ops.read_reg_page(hw, HV_TNCRS_UPPER, &phy_data);
4532 ret_val = hw->phy.ops.read_reg_page(hw, HV_TNCRS_LOWER, &phy_data);
4533 if (!ret_val)
4534 adapter->stats.tncrs += phy_data;
4535
4536 release:
4537 hw->phy.ops.release(hw);
4538 }
4539
4540 /**
4541 * e1000e_update_stats - Update the board statistics counters
4542 * @adapter: board private structure
4543 **/
4544 static void e1000e_update_stats(struct e1000_adapter *adapter)
4545 {
4546 struct net_device *netdev = adapter->netdev;
4547 struct e1000_hw *hw = &adapter->hw;
4548 struct pci_dev *pdev = adapter->pdev;
4549
4550 /* Prevent stats update while adapter is being reset, or if the pci
4551 * connection is down.
4552 */
4553 if (adapter->link_speed == 0)
4554 return;
4555 if (pci_channel_offline(pdev))
4556 return;
4557
4558 adapter->stats.crcerrs += er32(CRCERRS);
4559 adapter->stats.gprc += er32(GPRC);
4560 adapter->stats.gorc += er32(GORCL);
4561 er32(GORCH); /* Clear gorc */
4562 adapter->stats.bprc += er32(BPRC);
4563 adapter->stats.mprc += er32(MPRC);
4564 adapter->stats.roc += er32(ROC);
4565
4566 adapter->stats.mpc += er32(MPC);
4567
4568 /* Half-duplex statistics */
4569 if (adapter->link_duplex == HALF_DUPLEX) {
4570 if (adapter->flags2 & FLAG2_HAS_PHY_STATS) {
4571 e1000e_update_phy_stats(adapter);
4572 } else {
4573 adapter->stats.scc += er32(SCC);
4574 adapter->stats.ecol += er32(ECOL);
4575 adapter->stats.mcc += er32(MCC);
4576 adapter->stats.latecol += er32(LATECOL);
4577 adapter->stats.dc += er32(DC);
4578
4579 hw->mac.collision_delta = er32(COLC);
4580
4581 if ((hw->mac.type != e1000_82574) &&
4582 (hw->mac.type != e1000_82583))
4583 adapter->stats.tncrs += er32(TNCRS);
4584 }
4585 adapter->stats.colc += hw->mac.collision_delta;
4586 }
4587
4588 adapter->stats.xonrxc += er32(XONRXC);
4589 adapter->stats.xontxc += er32(XONTXC);
4590 adapter->stats.xoffrxc += er32(XOFFRXC);
4591 adapter->stats.xofftxc += er32(XOFFTXC);
4592 adapter->stats.gptc += er32(GPTC);
4593 adapter->stats.gotc += er32(GOTCL);
4594 er32(GOTCH); /* Clear gotc */
4595 adapter->stats.rnbc += er32(RNBC);
4596 adapter->stats.ruc += er32(RUC);
4597
4598 adapter->stats.mptc += er32(MPTC);
4599 adapter->stats.bptc += er32(BPTC);
4600
4601 /* used for adaptive IFS */
4602
4603 hw->mac.tx_packet_delta = er32(TPT);
4604 adapter->stats.tpt += hw->mac.tx_packet_delta;
4605
4606 adapter->stats.algnerrc += er32(ALGNERRC);
4607 adapter->stats.rxerrc += er32(RXERRC);
4608 adapter->stats.cexterr += er32(CEXTERR);
4609 adapter->stats.tsctc += er32(TSCTC);
4610 adapter->stats.tsctfc += er32(TSCTFC);
4611
4612 /* Fill out the OS statistics structure */
4613 netdev->stats.multicast = adapter->stats.mprc;
4614 netdev->stats.collisions = adapter->stats.colc;
4615
4616 /* Rx Errors */
4617
4618 /* RLEC on some newer hardware can be incorrect so build
4619 * our own version based on RUC and ROC
4620 */
4621 netdev->stats.rx_errors = adapter->stats.rxerrc +
4622 adapter->stats.crcerrs + adapter->stats.algnerrc +
4623 adapter->stats.ruc + adapter->stats.roc + adapter->stats.cexterr;
4624 netdev->stats.rx_length_errors = adapter->stats.ruc +
4625 adapter->stats.roc;
4626 netdev->stats.rx_crc_errors = adapter->stats.crcerrs;
4627 netdev->stats.rx_frame_errors = adapter->stats.algnerrc;
4628 netdev->stats.rx_missed_errors = adapter->stats.mpc;
4629
4630 /* Tx Errors */
4631 netdev->stats.tx_errors = adapter->stats.ecol + adapter->stats.latecol;
4632 netdev->stats.tx_aborted_errors = adapter->stats.ecol;
4633 netdev->stats.tx_window_errors = adapter->stats.latecol;
4634 netdev->stats.tx_carrier_errors = adapter->stats.tncrs;
4635
4636 /* Tx Dropped needs to be maintained elsewhere */
4637
4638 /* Management Stats */
4639 adapter->stats.mgptc += er32(MGTPTC);
4640 adapter->stats.mgprc += er32(MGTPRC);
4641 adapter->stats.mgpdc += er32(MGTPDC);
4642
4643 /* Correctable ECC Errors */
4644 if (hw->mac.type == e1000_pch_lpt) {
4645 u32 pbeccsts = er32(PBECCSTS);
4646 adapter->corr_errors +=
4647 pbeccsts & E1000_PBECCSTS_CORR_ERR_CNT_MASK;
4648 adapter->uncorr_errors +=
4649 (pbeccsts & E1000_PBECCSTS_UNCORR_ERR_CNT_MASK) >>
4650 E1000_PBECCSTS_UNCORR_ERR_CNT_SHIFT;
4651 }
4652 }
4653
4654 /**
4655 * e1000_phy_read_status - Update the PHY register status snapshot
4656 * @adapter: board private structure
4657 **/
4658 static void e1000_phy_read_status(struct e1000_adapter *adapter)
4659 {
4660 struct e1000_hw *hw = &adapter->hw;
4661 struct e1000_phy_regs *phy = &adapter->phy_regs;
4662
4663 if ((er32(STATUS) & E1000_STATUS_LU) &&
4664 (adapter->hw.phy.media_type == e1000_media_type_copper)) {
4665 int ret_val;
4666
4667 ret_val = e1e_rphy(hw, MII_BMCR, &phy->bmcr);
4668 ret_val |= e1e_rphy(hw, MII_BMSR, &phy->bmsr);
4669 ret_val |= e1e_rphy(hw, MII_ADVERTISE, &phy->advertise);
4670 ret_val |= e1e_rphy(hw, MII_LPA, &phy->lpa);
4671 ret_val |= e1e_rphy(hw, MII_EXPANSION, &phy->expansion);
4672 ret_val |= e1e_rphy(hw, MII_CTRL1000, &phy->ctrl1000);
4673 ret_val |= e1e_rphy(hw, MII_STAT1000, &phy->stat1000);
4674 ret_val |= e1e_rphy(hw, MII_ESTATUS, &phy->estatus);
4675 if (ret_val)
4676 e_warn("Error reading PHY register\n");
4677 } else {
4678 /* Do not read PHY registers if link is not up
4679 * Set values to typical power-on defaults
4680 */
4681 phy->bmcr = (BMCR_SPEED1000 | BMCR_ANENABLE | BMCR_FULLDPLX);
4682 phy->bmsr = (BMSR_100FULL | BMSR_100HALF | BMSR_10FULL |
4683 BMSR_10HALF | BMSR_ESTATEN | BMSR_ANEGCAPABLE |
4684 BMSR_ERCAP);
4685 phy->advertise = (ADVERTISE_PAUSE_ASYM | ADVERTISE_PAUSE_CAP |
4686 ADVERTISE_ALL | ADVERTISE_CSMA);
4687 phy->lpa = 0;
4688 phy->expansion = EXPANSION_ENABLENPAGE;
4689 phy->ctrl1000 = ADVERTISE_1000FULL;
4690 phy->stat1000 = 0;
4691 phy->estatus = (ESTATUS_1000_TFULL | ESTATUS_1000_THALF);
4692 }
4693 }
4694
4695 static void e1000_print_link_info(struct e1000_adapter *adapter)
4696 {
4697 struct e1000_hw *hw = &adapter->hw;
4698 u32 ctrl = er32(CTRL);
4699
4700 /* Link status message must follow this format for user tools */
4701 pr_info("%s NIC Link is Up %d Mbps %s Duplex, Flow Control: %s\n",
4702 adapter->netdev->name, adapter->link_speed,
4703 adapter->link_duplex == FULL_DUPLEX ? "Full" : "Half",
4704 (ctrl & E1000_CTRL_TFCE) && (ctrl & E1000_CTRL_RFCE) ? "Rx/Tx" :
4705 (ctrl & E1000_CTRL_RFCE) ? "Rx" :
4706 (ctrl & E1000_CTRL_TFCE) ? "Tx" : "None");
4707 }
4708
4709 static bool e1000e_has_link(struct e1000_adapter *adapter)
4710 {
4711 struct e1000_hw *hw = &adapter->hw;
4712 bool link_active = false;
4713 s32 ret_val = 0;
4714
4715 /* get_link_status is set on LSC (link status) interrupt or
4716 * Rx sequence error interrupt. get_link_status will stay
4717 * false until the check_for_link establishes link
4718 * for copper adapters ONLY
4719 */
4720 switch (hw->phy.media_type) {
4721 case e1000_media_type_copper:
4722 if (hw->mac.get_link_status) {
4723 ret_val = hw->mac.ops.check_for_link(hw);
4724 link_active = !hw->mac.get_link_status;
4725 } else {
4726 link_active = true;
4727 }
4728 break;
4729 case e1000_media_type_fiber:
4730 ret_val = hw->mac.ops.check_for_link(hw);
4731 link_active = !!(er32(STATUS) & E1000_STATUS_LU);
4732 break;
4733 case e1000_media_type_internal_serdes:
4734 ret_val = hw->mac.ops.check_for_link(hw);
4735 link_active = adapter->hw.mac.serdes_has_link;
4736 break;
4737 default:
4738 case e1000_media_type_unknown:
4739 break;
4740 }
4741
4742 if ((ret_val == E1000_ERR_PHY) && (hw->phy.type == e1000_phy_igp_3) &&
4743 (er32(CTRL) & E1000_PHY_CTRL_GBE_DISABLE)) {
4744 /* See e1000_kmrn_lock_loss_workaround_ich8lan() */
4745 e_info("Gigabit has been disabled, downgrading speed\n");
4746 }
4747
4748 return link_active;
4749 }
4750
4751 static void e1000e_enable_receives(struct e1000_adapter *adapter)
4752 {
4753 /* make sure the receive unit is started */
4754 if ((adapter->flags & FLAG_RX_NEEDS_RESTART) &&
4755 (adapter->flags & FLAG_RESTART_NOW)) {
4756 struct e1000_hw *hw = &adapter->hw;
4757 u32 rctl = er32(RCTL);
4758 ew32(RCTL, rctl | E1000_RCTL_EN);
4759 adapter->flags &= ~FLAG_RESTART_NOW;
4760 }
4761 }
4762
4763 static void e1000e_check_82574_phy_workaround(struct e1000_adapter *adapter)
4764 {
4765 struct e1000_hw *hw = &adapter->hw;
4766
4767 /* With 82574 controllers, PHY needs to be checked periodically
4768 * for hung state and reset, if two calls return true
4769 */
4770 if (e1000_check_phy_82574(hw))
4771 adapter->phy_hang_count++;
4772 else
4773 adapter->phy_hang_count = 0;
4774
4775 if (adapter->phy_hang_count > 1) {
4776 adapter->phy_hang_count = 0;
4777 schedule_work(&adapter->reset_task);
4778 }
4779 }
4780
4781 /**
4782 * e1000_watchdog - Timer Call-back
4783 * @data: pointer to adapter cast into an unsigned long
4784 **/
4785 static void e1000_watchdog(unsigned long data)
4786 {
4787 struct e1000_adapter *adapter = (struct e1000_adapter *) data;
4788
4789 /* Do the rest outside of interrupt context */
4790 schedule_work(&adapter->watchdog_task);
4791
4792 /* TODO: make this use queue_delayed_work() */
4793 }
4794
4795 static void e1000_watchdog_task(struct work_struct *work)
4796 {
4797 struct e1000_adapter *adapter = container_of(work,
4798 struct e1000_adapter,
4799 watchdog_task);
4800 struct net_device *netdev = adapter->netdev;
4801 struct e1000_mac_info *mac = &adapter->hw.mac;
4802 struct e1000_phy_info *phy = &adapter->hw.phy;
4803 struct e1000_ring *tx_ring = adapter->tx_ring;
4804 struct e1000_hw *hw = &adapter->hw;
4805 u32 link, tctl;
4806
4807 if (test_bit(__E1000_DOWN, &adapter->state))
4808 return;
4809
4810 link = e1000e_has_link(adapter);
4811 if ((netif_carrier_ok(netdev)) && link) {
4812 /* Cancel scheduled suspend requests. */
4813 pm_runtime_resume(netdev->dev.parent);
4814
4815 e1000e_enable_receives(adapter);
4816 goto link_up;
4817 }
4818
4819 if ((e1000e_enable_tx_pkt_filtering(hw)) &&
4820 (adapter->mng_vlan_id != adapter->hw.mng_cookie.vlan_id))
4821 e1000_update_mng_vlan(adapter);
4822
4823 if (link) {
4824 if (!netif_carrier_ok(netdev)) {
4825 bool txb2b = true;
4826
4827 /* Cancel scheduled suspend requests. */
4828 pm_runtime_resume(netdev->dev.parent);
4829
4830 /* update snapshot of PHY registers on LSC */
4831 e1000_phy_read_status(adapter);
4832 mac->ops.get_link_up_info(&adapter->hw,
4833 &adapter->link_speed,
4834 &adapter->link_duplex);
4835 e1000_print_link_info(adapter);
4836
4837 /* check if SmartSpeed worked */
4838 e1000e_check_downshift(hw);
4839 if (phy->speed_downgraded)
4840 netdev_warn(netdev,
4841 "Link Speed was downgraded by SmartSpeed\n");
4842
4843 /* On supported PHYs, check for duplex mismatch only
4844 * if link has autonegotiated at 10/100 half
4845 */
4846 if ((hw->phy.type == e1000_phy_igp_3 ||
4847 hw->phy.type == e1000_phy_bm) &&
4848 (hw->mac.autoneg == true) &&
4849 (adapter->link_speed == SPEED_10 ||
4850 adapter->link_speed == SPEED_100) &&
4851 (adapter->link_duplex == HALF_DUPLEX)) {
4852 u16 autoneg_exp;
4853
4854 e1e_rphy(hw, MII_EXPANSION, &autoneg_exp);
4855
4856 if (!(autoneg_exp & EXPANSION_NWAY))
4857 e_info("Autonegotiated half duplex but link partner cannot autoneg. Try forcing full duplex if link gets many collisions.\n");
4858 }
4859
4860 /* adjust timeout factor according to speed/duplex */
4861 adapter->tx_timeout_factor = 1;
4862 switch (adapter->link_speed) {
4863 case SPEED_10:
4864 txb2b = false;
4865 adapter->tx_timeout_factor = 16;
4866 break;
4867 case SPEED_100:
4868 txb2b = false;
4869 adapter->tx_timeout_factor = 10;
4870 break;
4871 }
4872
4873 /* workaround: re-program speed mode bit after
4874 * link-up event
4875 */
4876 if ((adapter->flags & FLAG_TARC_SPEED_MODE_BIT) &&
4877 !txb2b) {
4878 u32 tarc0;
4879 tarc0 = er32(TARC(0));
4880 tarc0 &= ~SPEED_MODE_BIT;
4881 ew32(TARC(0), tarc0);
4882 }
4883
4884 /* disable TSO for pcie and 10/100 speeds, to avoid
4885 * some hardware issues
4886 */
4887 if (!(adapter->flags & FLAG_TSO_FORCE)) {
4888 switch (adapter->link_speed) {
4889 case SPEED_10:
4890 case SPEED_100:
4891 e_info("10/100 speed: disabling TSO\n");
4892 netdev->features &= ~NETIF_F_TSO;
4893 netdev->features &= ~NETIF_F_TSO6;
4894 break;
4895 case SPEED_1000:
4896 netdev->features |= NETIF_F_TSO;
4897 netdev->features |= NETIF_F_TSO6;
4898 break;
4899 default:
4900 /* oops */
4901 break;
4902 }
4903 }
4904
4905 /* enable transmits in the hardware, need to do this
4906 * after setting TARC(0)
4907 */
4908 tctl = er32(TCTL);
4909 tctl |= E1000_TCTL_EN;
4910 ew32(TCTL, tctl);
4911
4912 /* Perform any post-link-up configuration before
4913 * reporting link up.
4914 */
4915 if (phy->ops.cfg_on_link_up)
4916 phy->ops.cfg_on_link_up(hw);
4917
4918 netif_carrier_on(netdev);
4919
4920 if (!test_bit(__E1000_DOWN, &adapter->state))
4921 mod_timer(&adapter->phy_info_timer,
4922 round_jiffies(jiffies + 2 * HZ));
4923 }
4924 } else {
4925 if (netif_carrier_ok(netdev)) {
4926 adapter->link_speed = 0;
4927 adapter->link_duplex = 0;
4928 /* Link status message must follow this format */
4929 pr_info("%s NIC Link is Down\n", adapter->netdev->name);
4930 netif_carrier_off(netdev);
4931 if (!test_bit(__E1000_DOWN, &adapter->state))
4932 mod_timer(&adapter->phy_info_timer,
4933 round_jiffies(jiffies + 2 * HZ));
4934
4935 /* The link is lost so the controller stops DMA.
4936 * If there is queued Tx work that cannot be done
4937 * or if on an 8000ES2LAN which requires a Rx packet
4938 * buffer work-around on link down event, reset the
4939 * controller to flush the Tx/Rx packet buffers.
4940 * (Do the reset outside of interrupt context).
4941 */
4942 if ((adapter->flags & FLAG_RX_NEEDS_RESTART) ||
4943 (e1000_desc_unused(tx_ring) + 1 < tx_ring->count))
4944 adapter->flags |= FLAG_RESTART_NOW;
4945 else
4946 pm_schedule_suspend(netdev->dev.parent,
4947 LINK_TIMEOUT);
4948 }
4949 }
4950
4951 link_up:
4952 spin_lock(&adapter->stats64_lock);
4953 e1000e_update_stats(adapter);
4954
4955 mac->tx_packet_delta = adapter->stats.tpt - adapter->tpt_old;
4956 adapter->tpt_old = adapter->stats.tpt;
4957 mac->collision_delta = adapter->stats.colc - adapter->colc_old;
4958 adapter->colc_old = adapter->stats.colc;
4959
4960 adapter->gorc = adapter->stats.gorc - adapter->gorc_old;
4961 adapter->gorc_old = adapter->stats.gorc;
4962 adapter->gotc = adapter->stats.gotc - adapter->gotc_old;
4963 adapter->gotc_old = adapter->stats.gotc;
4964 spin_unlock(&adapter->stats64_lock);
4965
4966 if (adapter->flags & FLAG_RESTART_NOW) {
4967 schedule_work(&adapter->reset_task);
4968 /* return immediately since reset is imminent */
4969 return;
4970 }
4971
4972 e1000e_update_adaptive(&adapter->hw);
4973
4974 /* Simple mode for Interrupt Throttle Rate (ITR) */
4975 if (adapter->itr_setting == 4) {
4976 /* Symmetric Tx/Rx gets a reduced ITR=2000;
4977 * Total asymmetrical Tx or Rx gets ITR=8000;
4978 * everyone else is between 2000-8000.
4979 */
4980 u32 goc = (adapter->gotc + adapter->gorc) / 10000;
4981 u32 dif = (adapter->gotc > adapter->gorc ?
4982 adapter->gotc - adapter->gorc :
4983 adapter->gorc - adapter->gotc) / 10000;
4984 u32 itr = goc > 0 ? (dif * 6000 / goc + 2000) : 8000;
4985
4986 e1000e_write_itr(adapter, itr);
4987 }
4988
4989 /* Cause software interrupt to ensure Rx ring is cleaned */
4990 if (adapter->msix_entries)
4991 ew32(ICS, adapter->rx_ring->ims_val);
4992 else
4993 ew32(ICS, E1000_ICS_RXDMT0);
4994
4995 /* flush pending descriptors to memory before detecting Tx hang */
4996 e1000e_flush_descriptors(adapter);
4997
4998 /* Force detection of hung controller every watchdog period */
4999 adapter->detect_tx_hung = true;
5000
5001 /* With 82571 controllers, LAA may be overwritten due to controller
5002 * reset from the other port. Set the appropriate LAA in RAR[0]
5003 */
5004 if (e1000e_get_laa_state_82571(hw))
5005 hw->mac.ops.rar_set(hw, adapter->hw.mac.addr, 0);
5006
5007 if (adapter->flags2 & FLAG2_CHECK_PHY_HANG)
5008 e1000e_check_82574_phy_workaround(adapter);
5009
5010 /* Clear valid timestamp stuck in RXSTMPL/H due to a Rx error */
5011 if (adapter->hwtstamp_config.rx_filter != HWTSTAMP_FILTER_NONE) {
5012 if ((adapter->flags2 & FLAG2_CHECK_RX_HWTSTAMP) &&
5013 (er32(TSYNCRXCTL) & E1000_TSYNCRXCTL_VALID)) {
5014 er32(RXSTMPH);
5015 adapter->rx_hwtstamp_cleared++;
5016 } else {
5017 adapter->flags2 |= FLAG2_CHECK_RX_HWTSTAMP;
5018 }
5019 }
5020
5021 /* Reset the timer */
5022 if (!test_bit(__E1000_DOWN, &adapter->state))
5023 mod_timer(&adapter->watchdog_timer,
5024 round_jiffies(jiffies + 2 * HZ));
5025 }
5026
5027 #define E1000_TX_FLAGS_CSUM 0x00000001
5028 #define E1000_TX_FLAGS_VLAN 0x00000002
5029 #define E1000_TX_FLAGS_TSO 0x00000004
5030 #define E1000_TX_FLAGS_IPV4 0x00000008
5031 #define E1000_TX_FLAGS_NO_FCS 0x00000010
5032 #define E1000_TX_FLAGS_HWTSTAMP 0x00000020
5033 #define E1000_TX_FLAGS_VLAN_MASK 0xffff0000
5034 #define E1000_TX_FLAGS_VLAN_SHIFT 16
5035
5036 static int e1000_tso(struct e1000_ring *tx_ring, struct sk_buff *skb)
5037 {
5038 struct e1000_context_desc *context_desc;
5039 struct e1000_buffer *buffer_info;
5040 unsigned int i;
5041 u32 cmd_length = 0;
5042 u16 ipcse = 0, mss;
5043 u8 ipcss, ipcso, tucss, tucso, hdr_len;
5044
5045 if (!skb_is_gso(skb))
5046 return 0;
5047
5048 if (skb_header_cloned(skb)) {
5049 int err = pskb_expand_head(skb, 0, 0, GFP_ATOMIC);
5050
5051 if (err)
5052 return err;
5053 }
5054
5055 hdr_len = skb_transport_offset(skb) + tcp_hdrlen(skb);
5056 mss = skb_shinfo(skb)->gso_size;
5057 if (skb->protocol == htons(ETH_P_IP)) {
5058 struct iphdr *iph = ip_hdr(skb);
5059 iph->tot_len = 0;
5060 iph->check = 0;
5061 tcp_hdr(skb)->check = ~csum_tcpudp_magic(iph->saddr, iph->daddr,
5062 0, IPPROTO_TCP, 0);
5063 cmd_length = E1000_TXD_CMD_IP;
5064 ipcse = skb_transport_offset(skb) - 1;
5065 } else if (skb_is_gso_v6(skb)) {
5066 ipv6_hdr(skb)->payload_len = 0;
5067 tcp_hdr(skb)->check = ~csum_ipv6_magic(&ipv6_hdr(skb)->saddr,
5068 &ipv6_hdr(skb)->daddr,
5069 0, IPPROTO_TCP, 0);
5070 ipcse = 0;
5071 }
5072 ipcss = skb_network_offset(skb);
5073 ipcso = (void *)&(ip_hdr(skb)->check) - (void *)skb->data;
5074 tucss = skb_transport_offset(skb);
5075 tucso = (void *)&(tcp_hdr(skb)->check) - (void *)skb->data;
5076
5077 cmd_length |= (E1000_TXD_CMD_DEXT | E1000_TXD_CMD_TSE |
5078 E1000_TXD_CMD_TCP | (skb->len - (hdr_len)));
5079
5080 i = tx_ring->next_to_use;
5081 context_desc = E1000_CONTEXT_DESC(*tx_ring, i);
5082 buffer_info = &tx_ring->buffer_info[i];
5083
5084 context_desc->lower_setup.ip_fields.ipcss = ipcss;
5085 context_desc->lower_setup.ip_fields.ipcso = ipcso;
5086 context_desc->lower_setup.ip_fields.ipcse = cpu_to_le16(ipcse);
5087 context_desc->upper_setup.tcp_fields.tucss = tucss;
5088 context_desc->upper_setup.tcp_fields.tucso = tucso;
5089 context_desc->upper_setup.tcp_fields.tucse = 0;
5090 context_desc->tcp_seg_setup.fields.mss = cpu_to_le16(mss);
5091 context_desc->tcp_seg_setup.fields.hdr_len = hdr_len;
5092 context_desc->cmd_and_length = cpu_to_le32(cmd_length);
5093
5094 buffer_info->time_stamp = jiffies;
5095 buffer_info->next_to_watch = i;
5096
5097 i++;
5098 if (i == tx_ring->count)
5099 i = 0;
5100 tx_ring->next_to_use = i;
5101
5102 return 1;
5103 }
5104
5105 static bool e1000_tx_csum(struct e1000_ring *tx_ring, struct sk_buff *skb)
5106 {
5107 struct e1000_adapter *adapter = tx_ring->adapter;
5108 struct e1000_context_desc *context_desc;
5109 struct e1000_buffer *buffer_info;
5110 unsigned int i;
5111 u8 css;
5112 u32 cmd_len = E1000_TXD_CMD_DEXT;
5113 __be16 protocol;
5114
5115 if (skb->ip_summed != CHECKSUM_PARTIAL)
5116 return 0;
5117
5118 if (skb->protocol == cpu_to_be16(ETH_P_8021Q))
5119 protocol = vlan_eth_hdr(skb)->h_vlan_encapsulated_proto;
5120 else
5121 protocol = skb->protocol;
5122
5123 switch (protocol) {
5124 case cpu_to_be16(ETH_P_IP):
5125 if (ip_hdr(skb)->protocol == IPPROTO_TCP)
5126 cmd_len |= E1000_TXD_CMD_TCP;
5127 break;
5128 case cpu_to_be16(ETH_P_IPV6):
5129 /* XXX not handling all IPV6 headers */
5130 if (ipv6_hdr(skb)->nexthdr == IPPROTO_TCP)
5131 cmd_len |= E1000_TXD_CMD_TCP;
5132 break;
5133 default:
5134 if (unlikely(net_ratelimit()))
5135 e_warn("checksum_partial proto=%x!\n",
5136 be16_to_cpu(protocol));
5137 break;
5138 }
5139
5140 css = skb_checksum_start_offset(skb);
5141
5142 i = tx_ring->next_to_use;
5143 buffer_info = &tx_ring->buffer_info[i];
5144 context_desc = E1000_CONTEXT_DESC(*tx_ring, i);
5145
5146 context_desc->lower_setup.ip_config = 0;
5147 context_desc->upper_setup.tcp_fields.tucss = css;
5148 context_desc->upper_setup.tcp_fields.tucso = css + skb->csum_offset;
5149 context_desc->upper_setup.tcp_fields.tucse = 0;
5150 context_desc->tcp_seg_setup.data = 0;
5151 context_desc->cmd_and_length = cpu_to_le32(cmd_len);
5152
5153 buffer_info->time_stamp = jiffies;
5154 buffer_info->next_to_watch = i;
5155
5156 i++;
5157 if (i == tx_ring->count)
5158 i = 0;
5159 tx_ring->next_to_use = i;
5160
5161 return 1;
5162 }
5163
5164 static int e1000_tx_map(struct e1000_ring *tx_ring, struct sk_buff *skb,
5165 unsigned int first, unsigned int max_per_txd,
5166 unsigned int nr_frags)
5167 {
5168 struct e1000_adapter *adapter = tx_ring->adapter;
5169 struct pci_dev *pdev = adapter->pdev;
5170 struct e1000_buffer *buffer_info;
5171 unsigned int len = skb_headlen(skb);
5172 unsigned int offset = 0, size, count = 0, i;
5173 unsigned int f, bytecount, segs;
5174
5175 i = tx_ring->next_to_use;
5176
5177 while (len) {
5178 buffer_info = &tx_ring->buffer_info[i];
5179 size = min(len, max_per_txd);
5180
5181 buffer_info->length = size;
5182 buffer_info->time_stamp = jiffies;
5183 buffer_info->next_to_watch = i;
5184 buffer_info->dma = dma_map_single(&pdev->dev,
5185 skb->data + offset,
5186 size, DMA_TO_DEVICE);
5187 buffer_info->mapped_as_page = false;
5188 if (dma_mapping_error(&pdev->dev, buffer_info->dma))
5189 goto dma_error;
5190
5191 len -= size;
5192 offset += size;
5193 count++;
5194
5195 if (len) {
5196 i++;
5197 if (i == tx_ring->count)
5198 i = 0;
5199 }
5200 }
5201
5202 for (f = 0; f < nr_frags; f++) {
5203 const struct skb_frag_struct *frag;
5204
5205 frag = &skb_shinfo(skb)->frags[f];
5206 len = skb_frag_size(frag);
5207 offset = 0;
5208
5209 while (len) {
5210 i++;
5211 if (i == tx_ring->count)
5212 i = 0;
5213
5214 buffer_info = &tx_ring->buffer_info[i];
5215 size = min(len, max_per_txd);
5216
5217 buffer_info->length = size;
5218 buffer_info->time_stamp = jiffies;
5219 buffer_info->next_to_watch = i;
5220 buffer_info->dma = skb_frag_dma_map(&pdev->dev, frag,
5221 offset, size,
5222 DMA_TO_DEVICE);
5223 buffer_info->mapped_as_page = true;
5224 if (dma_mapping_error(&pdev->dev, buffer_info->dma))
5225 goto dma_error;
5226
5227 len -= size;
5228 offset += size;
5229 count++;
5230 }
5231 }
5232
5233 segs = skb_shinfo(skb)->gso_segs ? : 1;
5234 /* multiply data chunks by size of headers */
5235 bytecount = ((segs - 1) * skb_headlen(skb)) + skb->len;
5236
5237 tx_ring->buffer_info[i].skb = skb;
5238 tx_ring->buffer_info[i].segs = segs;
5239 tx_ring->buffer_info[i].bytecount = bytecount;
5240 tx_ring->buffer_info[first].next_to_watch = i;
5241
5242 return count;
5243
5244 dma_error:
5245 dev_err(&pdev->dev, "Tx DMA map failed\n");
5246 buffer_info->dma = 0;
5247 if (count)
5248 count--;
5249
5250 while (count--) {
5251 if (i == 0)
5252 i += tx_ring->count;
5253 i--;
5254 buffer_info = &tx_ring->buffer_info[i];
5255 e1000_put_txbuf(tx_ring, buffer_info);
5256 }
5257
5258 return 0;
5259 }
5260
5261 static void e1000_tx_queue(struct e1000_ring *tx_ring, int tx_flags, int count)
5262 {
5263 struct e1000_adapter *adapter = tx_ring->adapter;
5264 struct e1000_tx_desc *tx_desc = NULL;
5265 struct e1000_buffer *buffer_info;
5266 u32 txd_upper = 0, txd_lower = E1000_TXD_CMD_IFCS;
5267 unsigned int i;
5268
5269 if (tx_flags & E1000_TX_FLAGS_TSO) {
5270 txd_lower |= E1000_TXD_CMD_DEXT | E1000_TXD_DTYP_D |
5271 E1000_TXD_CMD_TSE;
5272 txd_upper |= E1000_TXD_POPTS_TXSM << 8;
5273
5274 if (tx_flags & E1000_TX_FLAGS_IPV4)
5275 txd_upper |= E1000_TXD_POPTS_IXSM << 8;
5276 }
5277
5278 if (tx_flags & E1000_TX_FLAGS_CSUM) {
5279 txd_lower |= E1000_TXD_CMD_DEXT | E1000_TXD_DTYP_D;
5280 txd_upper |= E1000_TXD_POPTS_TXSM << 8;
5281 }
5282
5283 if (tx_flags & E1000_TX_FLAGS_VLAN) {
5284 txd_lower |= E1000_TXD_CMD_VLE;
5285 txd_upper |= (tx_flags & E1000_TX_FLAGS_VLAN_MASK);
5286 }
5287
5288 if (unlikely(tx_flags & E1000_TX_FLAGS_NO_FCS))
5289 txd_lower &= ~(E1000_TXD_CMD_IFCS);
5290
5291 if (unlikely(tx_flags & E1000_TX_FLAGS_HWTSTAMP)) {
5292 txd_lower |= E1000_TXD_CMD_DEXT | E1000_TXD_DTYP_D;
5293 txd_upper |= E1000_TXD_EXTCMD_TSTAMP;
5294 }
5295
5296 i = tx_ring->next_to_use;
5297
5298 do {
5299 buffer_info = &tx_ring->buffer_info[i];
5300 tx_desc = E1000_TX_DESC(*tx_ring, i);
5301 tx_desc->buffer_addr = cpu_to_le64(buffer_info->dma);
5302 tx_desc->lower.data = cpu_to_le32(txd_lower |
5303 buffer_info->length);
5304 tx_desc->upper.data = cpu_to_le32(txd_upper);
5305
5306 i++;
5307 if (i == tx_ring->count)
5308 i = 0;
5309 } while (--count > 0);
5310
5311 tx_desc->lower.data |= cpu_to_le32(adapter->txd_cmd);
5312
5313 /* txd_cmd re-enables FCS, so we'll re-disable it here as desired. */
5314 if (unlikely(tx_flags & E1000_TX_FLAGS_NO_FCS))
5315 tx_desc->lower.data &= ~(cpu_to_le32(E1000_TXD_CMD_IFCS));
5316
5317 /* Force memory writes to complete before letting h/w
5318 * know there are new descriptors to fetch. (Only
5319 * applicable for weak-ordered memory model archs,
5320 * such as IA-64).
5321 */
5322 wmb();
5323
5324 tx_ring->next_to_use = i;
5325
5326 if (adapter->flags2 & FLAG2_PCIM2PCI_ARBITER_WA)
5327 e1000e_update_tdt_wa(tx_ring, i);
5328 else
5329 writel(i, tx_ring->tail);
5330
5331 /* we need this if more than one processor can write to our tail
5332 * at a time, it synchronizes IO on IA64/Altix systems
5333 */
5334 mmiowb();
5335 }
5336
5337 #define MINIMUM_DHCP_PACKET_SIZE 282
5338 static int e1000_transfer_dhcp_info(struct e1000_adapter *adapter,
5339 struct sk_buff *skb)
5340 {
5341 struct e1000_hw *hw = &adapter->hw;
5342 u16 length, offset;
5343
5344 if (vlan_tx_tag_present(skb) &&
5345 !((vlan_tx_tag_get(skb) == adapter->hw.mng_cookie.vlan_id) &&
5346 (adapter->hw.mng_cookie.status &
5347 E1000_MNG_DHCP_COOKIE_STATUS_VLAN)))
5348 return 0;
5349
5350 if (skb->len <= MINIMUM_DHCP_PACKET_SIZE)
5351 return 0;
5352
5353 if (((struct ethhdr *) skb->data)->h_proto != htons(ETH_P_IP))
5354 return 0;
5355
5356 {
5357 const struct iphdr *ip = (struct iphdr *)((u8 *)skb->data + 14);
5358 struct udphdr *udp;
5359
5360 if (ip->protocol != IPPROTO_UDP)
5361 return 0;
5362
5363 udp = (struct udphdr *)((u8 *)ip + (ip->ihl << 2));
5364 if (ntohs(udp->dest) != 67)
5365 return 0;
5366
5367 offset = (u8 *)udp + 8 - skb->data;
5368 length = skb->len - offset;
5369 return e1000e_mng_write_dhcp_info(hw, (u8 *)udp + 8, length);
5370 }
5371
5372 return 0;
5373 }
5374
5375 static int __e1000_maybe_stop_tx(struct e1000_ring *tx_ring, int size)
5376 {
5377 struct e1000_adapter *adapter = tx_ring->adapter;
5378
5379 netif_stop_queue(adapter->netdev);
5380 /* Herbert's original patch had:
5381 * smp_mb__after_netif_stop_queue();
5382 * but since that doesn't exist yet, just open code it.
5383 */
5384 smp_mb();
5385
5386 /* We need to check again in a case another CPU has just
5387 * made room available.
5388 */
5389 if (e1000_desc_unused(tx_ring) < size)
5390 return -EBUSY;
5391
5392 /* A reprieve! */
5393 netif_start_queue(adapter->netdev);
5394 ++adapter->restart_queue;
5395 return 0;
5396 }
5397
5398 static int e1000_maybe_stop_tx(struct e1000_ring *tx_ring, int size)
5399 {
5400 BUG_ON(size > tx_ring->count);
5401
5402 if (e1000_desc_unused(tx_ring) >= size)
5403 return 0;
5404 return __e1000_maybe_stop_tx(tx_ring, size);
5405 }
5406
5407 static netdev_tx_t e1000_xmit_frame(struct sk_buff *skb,
5408 struct net_device *netdev)
5409 {
5410 struct e1000_adapter *adapter = netdev_priv(netdev);
5411 struct e1000_ring *tx_ring = adapter->tx_ring;
5412 unsigned int first;
5413 unsigned int tx_flags = 0;
5414 unsigned int len = skb_headlen(skb);
5415 unsigned int nr_frags;
5416 unsigned int mss;
5417 int count = 0;
5418 int tso;
5419 unsigned int f;
5420
5421 if (test_bit(__E1000_DOWN, &adapter->state)) {
5422 dev_kfree_skb_any(skb);
5423 return NETDEV_TX_OK;
5424 }
5425
5426 if (skb->len <= 0) {
5427 dev_kfree_skb_any(skb);
5428 return NETDEV_TX_OK;
5429 }
5430
5431 /* The minimum packet size with TCTL.PSP set is 17 bytes so
5432 * pad skb in order to meet this minimum size requirement
5433 */
5434 if (unlikely(skb->len < 17)) {
5435 if (skb_pad(skb, 17 - skb->len))
5436 return NETDEV_TX_OK;
5437 skb->len = 17;
5438 skb_set_tail_pointer(skb, 17);
5439 }
5440
5441 mss = skb_shinfo(skb)->gso_size;
5442 if (mss) {
5443 u8 hdr_len;
5444
5445 /* TSO Workaround for 82571/2/3 Controllers -- if skb->data
5446 * points to just header, pull a few bytes of payload from
5447 * frags into skb->data
5448 */
5449 hdr_len = skb_transport_offset(skb) + tcp_hdrlen(skb);
5450 /* we do this workaround for ES2LAN, but it is un-necessary,
5451 * avoiding it could save a lot of cycles
5452 */
5453 if (skb->data_len && (hdr_len == len)) {
5454 unsigned int pull_size;
5455
5456 pull_size = min_t(unsigned int, 4, skb->data_len);
5457 if (!__pskb_pull_tail(skb, pull_size)) {
5458 e_err("__pskb_pull_tail failed.\n");
5459 dev_kfree_skb_any(skb);
5460 return NETDEV_TX_OK;
5461 }
5462 len = skb_headlen(skb);
5463 }
5464 }
5465
5466 /* reserve a descriptor for the offload context */
5467 if ((mss) || (skb->ip_summed == CHECKSUM_PARTIAL))
5468 count++;
5469 count++;
5470
5471 count += DIV_ROUND_UP(len, adapter->tx_fifo_limit);
5472
5473 nr_frags = skb_shinfo(skb)->nr_frags;
5474 for (f = 0; f < nr_frags; f++)
5475 count += DIV_ROUND_UP(skb_frag_size(&skb_shinfo(skb)->frags[f]),
5476 adapter->tx_fifo_limit);
5477
5478 if (adapter->hw.mac.tx_pkt_filtering)
5479 e1000_transfer_dhcp_info(adapter, skb);
5480
5481 /* need: count + 2 desc gap to keep tail from touching
5482 * head, otherwise try next time
5483 */
5484 if (e1000_maybe_stop_tx(tx_ring, count + 2))
5485 return NETDEV_TX_BUSY;
5486
5487 if (vlan_tx_tag_present(skb)) {
5488 tx_flags |= E1000_TX_FLAGS_VLAN;
5489 tx_flags |= (vlan_tx_tag_get(skb) << E1000_TX_FLAGS_VLAN_SHIFT);
5490 }
5491
5492 first = tx_ring->next_to_use;
5493
5494 tso = e1000_tso(tx_ring, skb);
5495 if (tso < 0) {
5496 dev_kfree_skb_any(skb);
5497 return NETDEV_TX_OK;
5498 }
5499
5500 if (tso)
5501 tx_flags |= E1000_TX_FLAGS_TSO;
5502 else if (e1000_tx_csum(tx_ring, skb))
5503 tx_flags |= E1000_TX_FLAGS_CSUM;
5504
5505 /* Old method was to assume IPv4 packet by default if TSO was enabled.
5506 * 82571 hardware supports TSO capabilities for IPv6 as well...
5507 * no longer assume, we must.
5508 */
5509 if (skb->protocol == htons(ETH_P_IP))
5510 tx_flags |= E1000_TX_FLAGS_IPV4;
5511
5512 if (unlikely(skb->no_fcs))
5513 tx_flags |= E1000_TX_FLAGS_NO_FCS;
5514
5515 /* if count is 0 then mapping error has occurred */
5516 count = e1000_tx_map(tx_ring, skb, first, adapter->tx_fifo_limit,
5517 nr_frags);
5518 if (count) {
5519 if (unlikely((skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) &&
5520 !adapter->tx_hwtstamp_skb)) {
5521 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;
5522 tx_flags |= E1000_TX_FLAGS_HWTSTAMP;
5523 adapter->tx_hwtstamp_skb = skb_get(skb);
5524 schedule_work(&adapter->tx_hwtstamp_work);
5525 } else {
5526 skb_tx_timestamp(skb);
5527 }
5528
5529 netdev_sent_queue(netdev, skb->len);
5530 e1000_tx_queue(tx_ring, tx_flags, count);
5531 /* Make sure there is space in the ring for the next send. */
5532 e1000_maybe_stop_tx(tx_ring,
5533 (MAX_SKB_FRAGS *
5534 DIV_ROUND_UP(PAGE_SIZE,
5535 adapter->tx_fifo_limit) + 2));
5536 } else {
5537 dev_kfree_skb_any(skb);
5538 tx_ring->buffer_info[first].time_stamp = 0;
5539 tx_ring->next_to_use = first;
5540 }
5541
5542 return NETDEV_TX_OK;
5543 }
5544
5545 /**
5546 * e1000_tx_timeout - Respond to a Tx Hang
5547 * @netdev: network interface device structure
5548 **/
5549 static void e1000_tx_timeout(struct net_device *netdev)
5550 {
5551 struct e1000_adapter *adapter = netdev_priv(netdev);
5552
5553 /* Do the reset outside of interrupt context */
5554 adapter->tx_timeout_count++;
5555 schedule_work(&adapter->reset_task);
5556 }
5557
5558 static void e1000_reset_task(struct work_struct *work)
5559 {
5560 struct e1000_adapter *adapter;
5561 adapter = container_of(work, struct e1000_adapter, reset_task);
5562
5563 /* don't run the task if already down */
5564 if (test_bit(__E1000_DOWN, &adapter->state))
5565 return;
5566
5567 if (!(adapter->flags & FLAG_RESTART_NOW)) {
5568 e1000e_dump(adapter);
5569 e_err("Reset adapter unexpectedly\n");
5570 }
5571 e1000e_reinit_locked(adapter);
5572 }
5573
5574 /**
5575 * e1000_get_stats64 - Get System Network Statistics
5576 * @netdev: network interface device structure
5577 * @stats: rtnl_link_stats64 pointer
5578 *
5579 * Returns the address of the device statistics structure.
5580 **/
5581 struct rtnl_link_stats64 *e1000e_get_stats64(struct net_device *netdev,
5582 struct rtnl_link_stats64 *stats)
5583 {
5584 struct e1000_adapter *adapter = netdev_priv(netdev);
5585
5586 memset(stats, 0, sizeof(struct rtnl_link_stats64));
5587 spin_lock(&adapter->stats64_lock);
5588 e1000e_update_stats(adapter);
5589 /* Fill out the OS statistics structure */
5590 stats->rx_bytes = adapter->stats.gorc;
5591 stats->rx_packets = adapter->stats.gprc;
5592 stats->tx_bytes = adapter->stats.gotc;
5593 stats->tx_packets = adapter->stats.gptc;
5594 stats->multicast = adapter->stats.mprc;
5595 stats->collisions = adapter->stats.colc;
5596
5597 /* Rx Errors */
5598
5599 /* RLEC on some newer hardware can be incorrect so build
5600 * our own version based on RUC and ROC
5601 */
5602 stats->rx_errors = adapter->stats.rxerrc +
5603 adapter->stats.crcerrs + adapter->stats.algnerrc +
5604 adapter->stats.ruc + adapter->stats.roc + adapter->stats.cexterr;
5605 stats->rx_length_errors = adapter->stats.ruc + adapter->stats.roc;
5606 stats->rx_crc_errors = adapter->stats.crcerrs;
5607 stats->rx_frame_errors = adapter->stats.algnerrc;
5608 stats->rx_missed_errors = adapter->stats.mpc;
5609
5610 /* Tx Errors */
5611 stats->tx_errors = adapter->stats.ecol + adapter->stats.latecol;
5612 stats->tx_aborted_errors = adapter->stats.ecol;
5613 stats->tx_window_errors = adapter->stats.latecol;
5614 stats->tx_carrier_errors = adapter->stats.tncrs;
5615
5616 /* Tx Dropped needs to be maintained elsewhere */
5617
5618 spin_unlock(&adapter->stats64_lock);
5619 return stats;
5620 }
5621
5622 /**
5623 * e1000_change_mtu - Change the Maximum Transfer Unit
5624 * @netdev: network interface device structure
5625 * @new_mtu: new value for maximum frame size
5626 *
5627 * Returns 0 on success, negative on failure
5628 **/
5629 static int e1000_change_mtu(struct net_device *netdev, int new_mtu)
5630 {
5631 struct e1000_adapter *adapter = netdev_priv(netdev);
5632 int max_frame = new_mtu + ETH_HLEN + ETH_FCS_LEN;
5633
5634 /* Jumbo frame support */
5635 if ((max_frame > ETH_FRAME_LEN + ETH_FCS_LEN) &&
5636 !(adapter->flags & FLAG_HAS_JUMBO_FRAMES)) {
5637 e_err("Jumbo Frames not supported.\n");
5638 return -EINVAL;
5639 }
5640
5641 /* Supported frame sizes */
5642 if ((new_mtu < ETH_ZLEN + ETH_FCS_LEN + VLAN_HLEN) ||
5643 (max_frame > adapter->max_hw_frame_size)) {
5644 e_err("Unsupported MTU setting\n");
5645 return -EINVAL;
5646 }
5647
5648 /* Jumbo frame workaround on 82579 and newer requires CRC be stripped */
5649 if ((adapter->hw.mac.type >= e1000_pch2lan) &&
5650 !(adapter->flags2 & FLAG2_CRC_STRIPPING) &&
5651 (new_mtu > ETH_DATA_LEN)) {
5652 e_err("Jumbo Frames not supported on this device when CRC stripping is disabled.\n");
5653 return -EINVAL;
5654 }
5655
5656 while (test_and_set_bit(__E1000_RESETTING, &adapter->state))
5657 usleep_range(1000, 2000);
5658 /* e1000e_down -> e1000e_reset dependent on max_frame_size & mtu */
5659 adapter->max_frame_size = max_frame;
5660 e_info("changing MTU from %d to %d\n", netdev->mtu, new_mtu);
5661 netdev->mtu = new_mtu;
5662 if (netif_running(netdev))
5663 e1000e_down(adapter);
5664
5665 /* NOTE: netdev_alloc_skb reserves 16 bytes, and typically NET_IP_ALIGN
5666 * means we reserve 2 more, this pushes us to allocate from the next
5667 * larger slab size.
5668 * i.e. RXBUFFER_2048 --> size-4096 slab
5669 * However with the new *_jumbo_rx* routines, jumbo receives will use
5670 * fragmented skbs
5671 */
5672
5673 if (max_frame <= 2048)
5674 adapter->rx_buffer_len = 2048;
5675 else
5676 adapter->rx_buffer_len = 4096;
5677
5678 /* adjust allocation if LPE protects us, and we aren't using SBP */
5679 if ((max_frame == ETH_FRAME_LEN + ETH_FCS_LEN) ||
5680 (max_frame == ETH_FRAME_LEN + VLAN_HLEN + ETH_FCS_LEN))
5681 adapter->rx_buffer_len = ETH_FRAME_LEN + VLAN_HLEN
5682 + ETH_FCS_LEN;
5683
5684 if (netif_running(netdev))
5685 e1000e_up(adapter);
5686 else
5687 e1000e_reset(adapter);
5688
5689 clear_bit(__E1000_RESETTING, &adapter->state);
5690
5691 return 0;
5692 }
5693
5694 static int e1000_mii_ioctl(struct net_device *netdev, struct ifreq *ifr,
5695 int cmd)
5696 {
5697 struct e1000_adapter *adapter = netdev_priv(netdev);
5698 struct mii_ioctl_data *data = if_mii(ifr);
5699
5700 if (adapter->hw.phy.media_type != e1000_media_type_copper)
5701 return -EOPNOTSUPP;
5702
5703 switch (cmd) {
5704 case SIOCGMIIPHY:
5705 data->phy_id = adapter->hw.phy.addr;
5706 break;
5707 case SIOCGMIIREG:
5708 e1000_phy_read_status(adapter);
5709
5710 switch (data->reg_num & 0x1F) {
5711 case MII_BMCR:
5712 data->val_out = adapter->phy_regs.bmcr;
5713 break;
5714 case MII_BMSR:
5715 data->val_out = adapter->phy_regs.bmsr;
5716 break;
5717 case MII_PHYSID1:
5718 data->val_out = (adapter->hw.phy.id >> 16);
5719 break;
5720 case MII_PHYSID2:
5721 data->val_out = (adapter->hw.phy.id & 0xFFFF);
5722 break;
5723 case MII_ADVERTISE:
5724 data->val_out = adapter->phy_regs.advertise;
5725 break;
5726 case MII_LPA:
5727 data->val_out = adapter->phy_regs.lpa;
5728 break;
5729 case MII_EXPANSION:
5730 data->val_out = adapter->phy_regs.expansion;
5731 break;
5732 case MII_CTRL1000:
5733 data->val_out = adapter->phy_regs.ctrl1000;
5734 break;
5735 case MII_STAT1000:
5736 data->val_out = adapter->phy_regs.stat1000;
5737 break;
5738 case MII_ESTATUS:
5739 data->val_out = adapter->phy_regs.estatus;
5740 break;
5741 default:
5742 return -EIO;
5743 }
5744 break;
5745 case SIOCSMIIREG:
5746 default:
5747 return -EOPNOTSUPP;
5748 }
5749 return 0;
5750 }
5751
5752 /**
5753 * e1000e_hwtstamp_ioctl - control hardware time stamping
5754 * @netdev: network interface device structure
5755 * @ifreq: interface request
5756 *
5757 * Outgoing time stamping can be enabled and disabled. Play nice and
5758 * disable it when requested, although it shouldn't cause any overhead
5759 * when no packet needs it. At most one packet in the queue may be
5760 * marked for time stamping, otherwise it would be impossible to tell
5761 * for sure to which packet the hardware time stamp belongs.
5762 *
5763 * Incoming time stamping has to be configured via the hardware filters.
5764 * Not all combinations are supported, in particular event type has to be
5765 * specified. Matching the kind of event packet is not supported, with the
5766 * exception of "all V2 events regardless of level 2 or 4".
5767 **/
5768 static int e1000e_hwtstamp_ioctl(struct net_device *netdev, struct ifreq *ifr)
5769 {
5770 struct e1000_adapter *adapter = netdev_priv(netdev);
5771 struct hwtstamp_config config;
5772 int ret_val;
5773
5774 if (copy_from_user(&config, ifr->ifr_data, sizeof(config)))
5775 return -EFAULT;
5776
5777 adapter->hwtstamp_config = config;
5778
5779 ret_val = e1000e_config_hwtstamp(adapter);
5780 if (ret_val)
5781 return ret_val;
5782
5783 config = adapter->hwtstamp_config;
5784
5785 switch (config.rx_filter) {
5786 case HWTSTAMP_FILTER_PTP_V2_L4_SYNC:
5787 case HWTSTAMP_FILTER_PTP_V2_L2_SYNC:
5788 case HWTSTAMP_FILTER_PTP_V2_SYNC:
5789 case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ:
5790 case HWTSTAMP_FILTER_PTP_V2_L2_DELAY_REQ:
5791 case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ:
5792 /* With V2 type filters which specify a Sync or Delay Request,
5793 * Path Delay Request/Response messages are also time stamped
5794 * by hardware so notify the caller the requested packets plus
5795 * some others are time stamped.
5796 */
5797 config.rx_filter = HWTSTAMP_FILTER_SOME;
5798 break;
5799 default:
5800 break;
5801 }
5802
5803 return copy_to_user(ifr->ifr_data, &config,
5804 sizeof(config)) ? -EFAULT : 0;
5805 }
5806
5807 static int e1000_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd)
5808 {
5809 switch (cmd) {
5810 case SIOCGMIIPHY:
5811 case SIOCGMIIREG:
5812 case SIOCSMIIREG:
5813 return e1000_mii_ioctl(netdev, ifr, cmd);
5814 case SIOCSHWTSTAMP:
5815 return e1000e_hwtstamp_ioctl(netdev, ifr);
5816 default:
5817 return -EOPNOTSUPP;
5818 }
5819 }
5820
5821 static int e1000_init_phy_wakeup(struct e1000_adapter *adapter, u32 wufc)
5822 {
5823 struct e1000_hw *hw = &adapter->hw;
5824 u32 i, mac_reg;
5825 u16 phy_reg, wuc_enable;
5826 int retval;
5827
5828 /* copy MAC RARs to PHY RARs */
5829 e1000_copy_rx_addrs_to_phy_ich8lan(hw);
5830
5831 retval = hw->phy.ops.acquire(hw);
5832 if (retval) {
5833 e_err("Could not acquire PHY\n");
5834 return retval;
5835 }
5836
5837 /* Enable access to wakeup registers on and set page to BM_WUC_PAGE */
5838 retval = e1000_enable_phy_wakeup_reg_access_bm(hw, &wuc_enable);
5839 if (retval)
5840 goto release;
5841
5842 /* copy MAC MTA to PHY MTA - only needed for pchlan */
5843 for (i = 0; i < adapter->hw.mac.mta_reg_count; i++) {
5844 mac_reg = E1000_READ_REG_ARRAY(hw, E1000_MTA, i);
5845 hw->phy.ops.write_reg_page(hw, BM_MTA(i),
5846 (u16)(mac_reg & 0xFFFF));
5847 hw->phy.ops.write_reg_page(hw, BM_MTA(i) + 1,
5848 (u16)((mac_reg >> 16) & 0xFFFF));
5849 }
5850
5851 /* configure PHY Rx Control register */
5852 hw->phy.ops.read_reg_page(&adapter->hw, BM_RCTL, &phy_reg);
5853 mac_reg = er32(RCTL);
5854 if (mac_reg & E1000_RCTL_UPE)
5855 phy_reg |= BM_RCTL_UPE;
5856 if (mac_reg & E1000_RCTL_MPE)
5857 phy_reg |= BM_RCTL_MPE;
5858 phy_reg &= ~(BM_RCTL_MO_MASK);
5859 if (mac_reg & E1000_RCTL_MO_3)
5860 phy_reg |= (((mac_reg & E1000_RCTL_MO_3) >> E1000_RCTL_MO_SHIFT)
5861 << BM_RCTL_MO_SHIFT);
5862 if (mac_reg & E1000_RCTL_BAM)
5863 phy_reg |= BM_RCTL_BAM;
5864 if (mac_reg & E1000_RCTL_PMCF)
5865 phy_reg |= BM_RCTL_PMCF;
5866 mac_reg = er32(CTRL);
5867 if (mac_reg & E1000_CTRL_RFCE)
5868 phy_reg |= BM_RCTL_RFCE;
5869 hw->phy.ops.write_reg_page(&adapter->hw, BM_RCTL, phy_reg);
5870
5871 /* enable PHY wakeup in MAC register */
5872 ew32(WUFC, wufc);
5873 ew32(WUC, E1000_WUC_PHY_WAKE | E1000_WUC_PME_EN);
5874
5875 /* configure and enable PHY wakeup in PHY registers */
5876 hw->phy.ops.write_reg_page(&adapter->hw, BM_WUFC, wufc);
5877 hw->phy.ops.write_reg_page(&adapter->hw, BM_WUC, E1000_WUC_PME_EN);
5878
5879 /* activate PHY wakeup */
5880 wuc_enable |= BM_WUC_ENABLE_BIT | BM_WUC_HOST_WU_BIT;
5881 retval = e1000_disable_phy_wakeup_reg_access_bm(hw, &wuc_enable);
5882 if (retval)
5883 e_err("Could not set PHY Host Wakeup bit\n");
5884 release:
5885 hw->phy.ops.release(hw);
5886
5887 return retval;
5888 }
5889
5890 static int __e1000_shutdown(struct pci_dev *pdev, bool *enable_wake,
5891 bool runtime)
5892 {
5893 struct net_device *netdev = pci_get_drvdata(pdev);
5894 struct e1000_adapter *adapter = netdev_priv(netdev);
5895 struct e1000_hw *hw = &adapter->hw;
5896 u32 ctrl, ctrl_ext, rctl, status;
5897 /* Runtime suspend should only enable wakeup for link changes */
5898 u32 wufc = runtime ? E1000_WUFC_LNKC : adapter->wol;
5899 int retval = 0;
5900
5901 netif_device_detach(netdev);
5902
5903 if (netif_running(netdev)) {
5904 int count = E1000_CHECK_RESET_COUNT;
5905
5906 while (test_bit(__E1000_RESETTING, &adapter->state) && count--)
5907 usleep_range(10000, 20000);
5908
5909 WARN_ON(test_bit(__E1000_RESETTING, &adapter->state));
5910 e1000e_down(adapter);
5911 e1000_free_irq(adapter);
5912 }
5913 e1000e_reset_interrupt_capability(adapter);
5914
5915 retval = pci_save_state(pdev);
5916 if (retval)
5917 return retval;
5918
5919 status = er32(STATUS);
5920 if (status & E1000_STATUS_LU)
5921 wufc &= ~E1000_WUFC_LNKC;
5922
5923 if (wufc) {
5924 e1000_setup_rctl(adapter);
5925 e1000e_set_rx_mode(netdev);
5926
5927 /* turn on all-multi mode if wake on multicast is enabled */
5928 if (wufc & E1000_WUFC_MC) {
5929 rctl = er32(RCTL);
5930 rctl |= E1000_RCTL_MPE;
5931 ew32(RCTL, rctl);
5932 }
5933
5934 ctrl = er32(CTRL);
5935 /* advertise wake from D3Cold */
5936 #define E1000_CTRL_ADVD3WUC 0x00100000
5937 /* phy power management enable */
5938 #define E1000_CTRL_EN_PHY_PWR_MGMT 0x00200000
5939 ctrl |= E1000_CTRL_ADVD3WUC;
5940 if (!(adapter->flags2 & FLAG2_HAS_PHY_WAKEUP))
5941 ctrl |= E1000_CTRL_EN_PHY_PWR_MGMT;
5942 ew32(CTRL, ctrl);
5943
5944 if (adapter->hw.phy.media_type == e1000_media_type_fiber ||
5945 adapter->hw.phy.media_type ==
5946 e1000_media_type_internal_serdes) {
5947 /* keep the laser running in D3 */
5948 ctrl_ext = er32(CTRL_EXT);
5949 ctrl_ext |= E1000_CTRL_EXT_SDP3_DATA;
5950 ew32(CTRL_EXT, ctrl_ext);
5951 }
5952
5953 if (adapter->flags & FLAG_IS_ICH)
5954 e1000_suspend_workarounds_ich8lan(&adapter->hw);
5955
5956 /* Allow time for pending master requests to run */
5957 e1000e_disable_pcie_master(&adapter->hw);
5958
5959 if (adapter->flags2 & FLAG2_HAS_PHY_WAKEUP) {
5960 /* enable wakeup by the PHY */
5961 retval = e1000_init_phy_wakeup(adapter, wufc);
5962 if (retval)
5963 return retval;
5964 } else {
5965 /* enable wakeup by the MAC */
5966 ew32(WUFC, wufc);
5967 ew32(WUC, E1000_WUC_PME_EN);
5968 }
5969 } else {
5970 ew32(WUC, 0);
5971 ew32(WUFC, 0);
5972 }
5973
5974 *enable_wake = !!wufc;
5975
5976 /* make sure adapter isn't asleep if manageability is enabled */
5977 if ((adapter->flags & FLAG_MNG_PT_ENABLED) ||
5978 (hw->mac.ops.check_mng_mode(hw)))
5979 *enable_wake = true;
5980
5981 if (adapter->hw.phy.type == e1000_phy_igp_3)
5982 e1000e_igp3_phy_powerdown_workaround_ich8lan(&adapter->hw);
5983
5984 /* Release control of h/w to f/w. If f/w is AMT enabled, this
5985 * would have already happened in close and is redundant.
5986 */
5987 e1000e_release_hw_control(adapter);
5988
5989 pci_disable_device(pdev);
5990
5991 return 0;
5992 }
5993
5994 static void e1000_power_off(struct pci_dev *pdev, bool sleep, bool wake)
5995 {
5996 if (sleep && wake) {
5997 pci_prepare_to_sleep(pdev);
5998 return;
5999 }
6000
6001 pci_wake_from_d3(pdev, wake);
6002 pci_set_power_state(pdev, PCI_D3hot);
6003 }
6004
6005 static void e1000_complete_shutdown(struct pci_dev *pdev, bool sleep, bool wake)
6006 {
6007 struct net_device *netdev = pci_get_drvdata(pdev);
6008 struct e1000_adapter *adapter = netdev_priv(netdev);
6009
6010 /* The pci-e switch on some quad port adapters will report a
6011 * correctable error when the MAC transitions from D0 to D3. To
6012 * prevent this we need to mask off the correctable errors on the
6013 * downstream port of the pci-e switch.
6014 */
6015 if (adapter->flags & FLAG_IS_QUAD_PORT) {
6016 struct pci_dev *us_dev = pdev->bus->self;
6017 u16 devctl;
6018
6019 pcie_capability_read_word(us_dev, PCI_EXP_DEVCTL, &devctl);
6020 pcie_capability_write_word(us_dev, PCI_EXP_DEVCTL,
6021 (devctl & ~PCI_EXP_DEVCTL_CERE));
6022
6023 e1000_power_off(pdev, sleep, wake);
6024
6025 pcie_capability_write_word(us_dev, PCI_EXP_DEVCTL, devctl);
6026 } else {
6027 e1000_power_off(pdev, sleep, wake);
6028 }
6029 }
6030
6031 #ifdef CONFIG_PCIEASPM
6032 static void __e1000e_disable_aspm(struct pci_dev *pdev, u16 state)
6033 {
6034 pci_disable_link_state_locked(pdev, state);
6035 }
6036 #else
6037 static void __e1000e_disable_aspm(struct pci_dev *pdev, u16 state)
6038 {
6039 u16 aspm_ctl = 0;
6040
6041 if (state & PCIE_LINK_STATE_L0S)
6042 aspm_ctl |= PCI_EXP_LNKCTL_ASPM_L0S;
6043 if (state & PCIE_LINK_STATE_L1)
6044 aspm_ctl |= PCI_EXP_LNKCTL_ASPM_L1;
6045
6046 /* Both device and parent should have the same ASPM setting.
6047 * Disable ASPM in downstream component first and then upstream.
6048 */
6049 pcie_capability_clear_word(pdev, PCI_EXP_LNKCTL, aspm_ctl);
6050
6051 if (pdev->bus->self)
6052 pcie_capability_clear_word(pdev->bus->self, PCI_EXP_LNKCTL,
6053 aspm_ctl);
6054 }
6055 #endif
6056 static void e1000e_disable_aspm(struct pci_dev *pdev, u16 state)
6057 {
6058 dev_info(&pdev->dev, "Disabling ASPM %s %s\n",
6059 (state & PCIE_LINK_STATE_L0S) ? "L0s" : "",
6060 (state & PCIE_LINK_STATE_L1) ? "L1" : "");
6061
6062 __e1000e_disable_aspm(pdev, state);
6063 }
6064
6065 #ifdef CONFIG_PM
6066 static bool e1000e_pm_ready(struct e1000_adapter *adapter)
6067 {
6068 return !!adapter->tx_ring->buffer_info;
6069 }
6070
6071 static int __e1000_resume(struct pci_dev *pdev)
6072 {
6073 struct net_device *netdev = pci_get_drvdata(pdev);
6074 struct e1000_adapter *adapter = netdev_priv(netdev);
6075 struct e1000_hw *hw = &adapter->hw;
6076 u16 aspm_disable_flag = 0;
6077 u32 err;
6078
6079 if (adapter->flags2 & FLAG2_DISABLE_ASPM_L0S)
6080 aspm_disable_flag = PCIE_LINK_STATE_L0S;
6081 if (adapter->flags2 & FLAG2_DISABLE_ASPM_L1)
6082 aspm_disable_flag |= PCIE_LINK_STATE_L1;
6083 if (aspm_disable_flag)
6084 e1000e_disable_aspm(pdev, aspm_disable_flag);
6085
6086 pci_set_power_state(pdev, PCI_D0);
6087 pci_restore_state(pdev);
6088 pci_save_state(pdev);
6089
6090 e1000e_set_interrupt_capability(adapter);
6091 if (netif_running(netdev)) {
6092 err = e1000_request_irq(adapter);
6093 if (err)
6094 return err;
6095 }
6096
6097 if (hw->mac.type >= e1000_pch2lan)
6098 e1000_resume_workarounds_pchlan(&adapter->hw);
6099
6100 e1000e_power_up_phy(adapter);
6101
6102 /* report the system wakeup cause from S3/S4 */
6103 if (adapter->flags2 & FLAG2_HAS_PHY_WAKEUP) {
6104 u16 phy_data;
6105
6106 e1e_rphy(&adapter->hw, BM_WUS, &phy_data);
6107 if (phy_data) {
6108 e_info("PHY Wakeup cause - %s\n",
6109 phy_data & E1000_WUS_EX ? "Unicast Packet" :
6110 phy_data & E1000_WUS_MC ? "Multicast Packet" :
6111 phy_data & E1000_WUS_BC ? "Broadcast Packet" :
6112 phy_data & E1000_WUS_MAG ? "Magic Packet" :
6113 phy_data & E1000_WUS_LNKC ?
6114 "Link Status Change" : "other");
6115 }
6116 e1e_wphy(&adapter->hw, BM_WUS, ~0);
6117 } else {
6118 u32 wus = er32(WUS);
6119 if (wus) {
6120 e_info("MAC Wakeup cause - %s\n",
6121 wus & E1000_WUS_EX ? "Unicast Packet" :
6122 wus & E1000_WUS_MC ? "Multicast Packet" :
6123 wus & E1000_WUS_BC ? "Broadcast Packet" :
6124 wus & E1000_WUS_MAG ? "Magic Packet" :
6125 wus & E1000_WUS_LNKC ? "Link Status Change" :
6126 "other");
6127 }
6128 ew32(WUS, ~0);
6129 }
6130
6131 e1000e_reset(adapter);
6132
6133 e1000_init_manageability_pt(adapter);
6134
6135 if (netif_running(netdev))
6136 e1000e_up(adapter);
6137
6138 netif_device_attach(netdev);
6139
6140 /* If the controller has AMT, do not set DRV_LOAD until the interface
6141 * is up. For all other cases, let the f/w know that the h/w is now
6142 * under the control of the driver.
6143 */
6144 if (!(adapter->flags & FLAG_HAS_AMT))
6145 e1000e_get_hw_control(adapter);
6146
6147 return 0;
6148 }
6149
6150 #ifdef CONFIG_PM_SLEEP
6151 static int e1000_suspend(struct device *dev)
6152 {
6153 struct pci_dev *pdev = to_pci_dev(dev);
6154 int retval;
6155 bool wake;
6156
6157 retval = __e1000_shutdown(pdev, &wake, false);
6158 if (!retval)
6159 e1000_complete_shutdown(pdev, true, wake);
6160
6161 return retval;
6162 }
6163
6164 static int e1000_resume(struct device *dev)
6165 {
6166 struct pci_dev *pdev = to_pci_dev(dev);
6167 struct net_device *netdev = pci_get_drvdata(pdev);
6168 struct e1000_adapter *adapter = netdev_priv(netdev);
6169
6170 if (e1000e_pm_ready(adapter))
6171 adapter->idle_check = true;
6172
6173 return __e1000_resume(pdev);
6174 }
6175 #endif /* CONFIG_PM_SLEEP */
6176
6177 #ifdef CONFIG_PM_RUNTIME
6178 static int e1000_runtime_suspend(struct device *dev)
6179 {
6180 struct pci_dev *pdev = to_pci_dev(dev);
6181 struct net_device *netdev = pci_get_drvdata(pdev);
6182 struct e1000_adapter *adapter = netdev_priv(netdev);
6183
6184 if (e1000e_pm_ready(adapter)) {
6185 bool wake;
6186
6187 __e1000_shutdown(pdev, &wake, true);
6188 }
6189
6190 return 0;
6191 }
6192
6193 static int e1000_idle(struct device *dev)
6194 {
6195 struct pci_dev *pdev = to_pci_dev(dev);
6196 struct net_device *netdev = pci_get_drvdata(pdev);
6197 struct e1000_adapter *adapter = netdev_priv(netdev);
6198
6199 if (!e1000e_pm_ready(adapter))
6200 return 0;
6201
6202 if (adapter->idle_check) {
6203 adapter->idle_check = false;
6204 if (!e1000e_has_link(adapter))
6205 pm_schedule_suspend(dev, MSEC_PER_SEC);
6206 }
6207
6208 return -EBUSY;
6209 }
6210
6211 static int e1000_runtime_resume(struct device *dev)
6212 {
6213 struct pci_dev *pdev = to_pci_dev(dev);
6214 struct net_device *netdev = pci_get_drvdata(pdev);
6215 struct e1000_adapter *adapter = netdev_priv(netdev);
6216
6217 if (!e1000e_pm_ready(adapter))
6218 return 0;
6219
6220 adapter->idle_check = !dev->power.runtime_auto;
6221 return __e1000_resume(pdev);
6222 }
6223 #endif /* CONFIG_PM_RUNTIME */
6224 #endif /* CONFIG_PM */
6225
6226 static void e1000_shutdown(struct pci_dev *pdev)
6227 {
6228 bool wake = false;
6229
6230 __e1000_shutdown(pdev, &wake, false);
6231
6232 if (system_state == SYSTEM_POWER_OFF)
6233 e1000_complete_shutdown(pdev, false, wake);
6234 }
6235
6236 #ifdef CONFIG_NET_POLL_CONTROLLER
6237
6238 static irqreturn_t e1000_intr_msix(int __always_unused irq, void *data)
6239 {
6240 struct net_device *netdev = data;
6241 struct e1000_adapter *adapter = netdev_priv(netdev);
6242
6243 if (adapter->msix_entries) {
6244 int vector, msix_irq;
6245
6246 vector = 0;
6247 msix_irq = adapter->msix_entries[vector].vector;
6248 disable_irq(msix_irq);
6249 e1000_intr_msix_rx(msix_irq, netdev);
6250 enable_irq(msix_irq);
6251
6252 vector++;
6253 msix_irq = adapter->msix_entries[vector].vector;
6254 disable_irq(msix_irq);
6255 e1000_intr_msix_tx(msix_irq, netdev);
6256 enable_irq(msix_irq);
6257
6258 vector++;
6259 msix_irq = adapter->msix_entries[vector].vector;
6260 disable_irq(msix_irq);
6261 e1000_msix_other(msix_irq, netdev);
6262 enable_irq(msix_irq);
6263 }
6264
6265 return IRQ_HANDLED;
6266 }
6267
6268 /**
6269 * e1000_netpoll
6270 * @netdev: network interface device structure
6271 *
6272 * Polling 'interrupt' - used by things like netconsole to send skbs
6273 * without having to re-enable interrupts. It's not called while
6274 * the interrupt routine is executing.
6275 */
6276 static void e1000_netpoll(struct net_device *netdev)
6277 {
6278 struct e1000_adapter *adapter = netdev_priv(netdev);
6279
6280 switch (adapter->int_mode) {
6281 case E1000E_INT_MODE_MSIX:
6282 e1000_intr_msix(adapter->pdev->irq, netdev);
6283 break;
6284 case E1000E_INT_MODE_MSI:
6285 disable_irq(adapter->pdev->irq);
6286 e1000_intr_msi(adapter->pdev->irq, netdev);
6287 enable_irq(adapter->pdev->irq);
6288 break;
6289 default: /* E1000E_INT_MODE_LEGACY */
6290 disable_irq(adapter->pdev->irq);
6291 e1000_intr(adapter->pdev->irq, netdev);
6292 enable_irq(adapter->pdev->irq);
6293 break;
6294 }
6295 }
6296 #endif
6297
6298 /**
6299 * e1000_io_error_detected - called when PCI error is detected
6300 * @pdev: Pointer to PCI device
6301 * @state: The current pci connection state
6302 *
6303 * This function is called after a PCI bus error affecting
6304 * this device has been detected.
6305 */
6306 static pci_ers_result_t e1000_io_error_detected(struct pci_dev *pdev,
6307 pci_channel_state_t state)
6308 {
6309 struct net_device *netdev = pci_get_drvdata(pdev);
6310 struct e1000_adapter *adapter = netdev_priv(netdev);
6311
6312 netif_device_detach(netdev);
6313
6314 if (state == pci_channel_io_perm_failure)
6315 return PCI_ERS_RESULT_DISCONNECT;
6316
6317 if (netif_running(netdev))
6318 e1000e_down(adapter);
6319 pci_disable_device(pdev);
6320
6321 /* Request a slot slot reset. */
6322 return PCI_ERS_RESULT_NEED_RESET;
6323 }
6324
6325 /**
6326 * e1000_io_slot_reset - called after the pci bus has been reset.
6327 * @pdev: Pointer to PCI device
6328 *
6329 * Restart the card from scratch, as if from a cold-boot. Implementation
6330 * resembles the first-half of the e1000_resume routine.
6331 */
6332 static pci_ers_result_t e1000_io_slot_reset(struct pci_dev *pdev)
6333 {
6334 struct net_device *netdev = pci_get_drvdata(pdev);
6335 struct e1000_adapter *adapter = netdev_priv(netdev);
6336 struct e1000_hw *hw = &adapter->hw;
6337 u16 aspm_disable_flag = 0;
6338 int err;
6339 pci_ers_result_t result;
6340
6341 if (adapter->flags2 & FLAG2_DISABLE_ASPM_L0S)
6342 aspm_disable_flag = PCIE_LINK_STATE_L0S;
6343 if (adapter->flags2 & FLAG2_DISABLE_ASPM_L1)
6344 aspm_disable_flag |= PCIE_LINK_STATE_L1;
6345 if (aspm_disable_flag)
6346 e1000e_disable_aspm(pdev, aspm_disable_flag);
6347
6348 err = pci_enable_device_mem(pdev);
6349 if (err) {
6350 dev_err(&pdev->dev,
6351 "Cannot re-enable PCI device after reset.\n");
6352 result = PCI_ERS_RESULT_DISCONNECT;
6353 } else {
6354 pci_set_master(pdev);
6355 pdev->state_saved = true;
6356 pci_restore_state(pdev);
6357
6358 pci_enable_wake(pdev, PCI_D3hot, 0);
6359 pci_enable_wake(pdev, PCI_D3cold, 0);
6360
6361 e1000e_reset(adapter);
6362 ew32(WUS, ~0);
6363 result = PCI_ERS_RESULT_RECOVERED;
6364 }
6365
6366 pci_cleanup_aer_uncorrect_error_status(pdev);
6367
6368 return result;
6369 }
6370
6371 /**
6372 * e1000_io_resume - called when traffic can start flowing again.
6373 * @pdev: Pointer to PCI device
6374 *
6375 * This callback is called when the error recovery driver tells us that
6376 * its OK to resume normal operation. Implementation resembles the
6377 * second-half of the e1000_resume routine.
6378 */
6379 static void e1000_io_resume(struct pci_dev *pdev)
6380 {
6381 struct net_device *netdev = pci_get_drvdata(pdev);
6382 struct e1000_adapter *adapter = netdev_priv(netdev);
6383
6384 e1000_init_manageability_pt(adapter);
6385
6386 if (netif_running(netdev)) {
6387 if (e1000e_up(adapter)) {
6388 dev_err(&pdev->dev,
6389 "can't bring device back up after reset\n");
6390 return;
6391 }
6392 }
6393
6394 netif_device_attach(netdev);
6395
6396 /* If the controller has AMT, do not set DRV_LOAD until the interface
6397 * is up. For all other cases, let the f/w know that the h/w is now
6398 * under the control of the driver.
6399 */
6400 if (!(adapter->flags & FLAG_HAS_AMT))
6401 e1000e_get_hw_control(adapter);
6402 }
6403
6404 static void e1000_print_device_info(struct e1000_adapter *adapter)
6405 {
6406 struct e1000_hw *hw = &adapter->hw;
6407 struct net_device *netdev = adapter->netdev;
6408 u32 ret_val;
6409 u8 pba_str[E1000_PBANUM_LENGTH];
6410
6411 /* print bus type/speed/width info */
6412 e_info("(PCI Express:2.5GT/s:%s) %pM\n",
6413 /* bus width */
6414 ((hw->bus.width == e1000_bus_width_pcie_x4) ? "Width x4" :
6415 "Width x1"),
6416 /* MAC address */
6417 netdev->dev_addr);
6418 e_info("Intel(R) PRO/%s Network Connection\n",
6419 (hw->phy.type == e1000_phy_ife) ? "10/100" : "1000");
6420 ret_val = e1000_read_pba_string_generic(hw, pba_str,
6421 E1000_PBANUM_LENGTH);
6422 if (ret_val)
6423 strlcpy((char *)pba_str, "Unknown", sizeof(pba_str));
6424 e_info("MAC: %d, PHY: %d, PBA No: %s\n",
6425 hw->mac.type, hw->phy.type, pba_str);
6426 }
6427
6428 static void e1000_eeprom_checks(struct e1000_adapter *adapter)
6429 {
6430 struct e1000_hw *hw = &adapter->hw;
6431 int ret_val;
6432 u16 buf = 0;
6433
6434 if (hw->mac.type != e1000_82573)
6435 return;
6436
6437 ret_val = e1000_read_nvm(hw, NVM_INIT_CONTROL2_REG, 1, &buf);
6438 le16_to_cpus(&buf);
6439 if (!ret_val && (!(buf & (1 << 0)))) {
6440 /* Deep Smart Power Down (DSPD) */
6441 dev_warn(&adapter->pdev->dev,
6442 "Warning: detected DSPD enabled in EEPROM\n");
6443 }
6444 }
6445
6446 static int e1000_set_features(struct net_device *netdev,
6447 netdev_features_t features)
6448 {
6449 struct e1000_adapter *adapter = netdev_priv(netdev);
6450 netdev_features_t changed = features ^ netdev->features;
6451
6452 if (changed & (NETIF_F_TSO | NETIF_F_TSO6))
6453 adapter->flags |= FLAG_TSO_FORCE;
6454
6455 if (!(changed & (NETIF_F_HW_VLAN_RX | NETIF_F_HW_VLAN_TX |
6456 NETIF_F_RXCSUM | NETIF_F_RXHASH | NETIF_F_RXFCS |
6457 NETIF_F_RXALL)))
6458 return 0;
6459
6460 if (changed & NETIF_F_RXFCS) {
6461 if (features & NETIF_F_RXFCS) {
6462 adapter->flags2 &= ~FLAG2_CRC_STRIPPING;
6463 } else {
6464 /* We need to take it back to defaults, which might mean
6465 * stripping is still disabled at the adapter level.
6466 */
6467 if (adapter->flags2 & FLAG2_DFLT_CRC_STRIPPING)
6468 adapter->flags2 |= FLAG2_CRC_STRIPPING;
6469 else
6470 adapter->flags2 &= ~FLAG2_CRC_STRIPPING;
6471 }
6472 }
6473
6474 netdev->features = features;
6475
6476 if (netif_running(netdev))
6477 e1000e_reinit_locked(adapter);
6478 else
6479 e1000e_reset(adapter);
6480
6481 return 0;
6482 }
6483
6484 static const struct net_device_ops e1000e_netdev_ops = {
6485 .ndo_open = e1000_open,
6486 .ndo_stop = e1000_close,
6487 .ndo_start_xmit = e1000_xmit_frame,
6488 .ndo_get_stats64 = e1000e_get_stats64,
6489 .ndo_set_rx_mode = e1000e_set_rx_mode,
6490 .ndo_set_mac_address = e1000_set_mac,
6491 .ndo_change_mtu = e1000_change_mtu,
6492 .ndo_do_ioctl = e1000_ioctl,
6493 .ndo_tx_timeout = e1000_tx_timeout,
6494 .ndo_validate_addr = eth_validate_addr,
6495
6496 .ndo_vlan_rx_add_vid = e1000_vlan_rx_add_vid,
6497 .ndo_vlan_rx_kill_vid = e1000_vlan_rx_kill_vid,
6498 #ifdef CONFIG_NET_POLL_CONTROLLER
6499 .ndo_poll_controller = e1000_netpoll,
6500 #endif
6501 .ndo_set_features = e1000_set_features,
6502 };
6503
6504 /**
6505 * e1000_probe - Device Initialization Routine
6506 * @pdev: PCI device information struct
6507 * @ent: entry in e1000_pci_tbl
6508 *
6509 * Returns 0 on success, negative on failure
6510 *
6511 * e1000_probe initializes an adapter identified by a pci_dev structure.
6512 * The OS initialization, configuring of the adapter private structure,
6513 * and a hardware reset occur.
6514 **/
6515 static int e1000_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
6516 {
6517 struct net_device *netdev;
6518 struct e1000_adapter *adapter;
6519 struct e1000_hw *hw;
6520 const struct e1000_info *ei = e1000_info_tbl[ent->driver_data];
6521 resource_size_t mmio_start, mmio_len;
6522 resource_size_t flash_start, flash_len;
6523 static int cards_found;
6524 u16 aspm_disable_flag = 0;
6525 int bars, i, err, pci_using_dac;
6526 u16 eeprom_data = 0;
6527 u16 eeprom_apme_mask = E1000_EEPROM_APME;
6528
6529 if (ei->flags2 & FLAG2_DISABLE_ASPM_L0S)
6530 aspm_disable_flag = PCIE_LINK_STATE_L0S;
6531 if (ei->flags2 & FLAG2_DISABLE_ASPM_L1)
6532 aspm_disable_flag |= PCIE_LINK_STATE_L1;
6533 if (aspm_disable_flag)
6534 e1000e_disable_aspm(pdev, aspm_disable_flag);
6535
6536 err = pci_enable_device_mem(pdev);
6537 if (err)
6538 return err;
6539
6540 pci_using_dac = 0;
6541 err = dma_set_mask(&pdev->dev, DMA_BIT_MASK(64));
6542 if (!err) {
6543 err = dma_set_coherent_mask(&pdev->dev, DMA_BIT_MASK(64));
6544 if (!err)
6545 pci_using_dac = 1;
6546 } else {
6547 err = dma_set_mask(&pdev->dev, DMA_BIT_MASK(32));
6548 if (err) {
6549 err = dma_set_coherent_mask(&pdev->dev,
6550 DMA_BIT_MASK(32));
6551 if (err) {
6552 dev_err(&pdev->dev,
6553 "No usable DMA configuration, aborting\n");
6554 goto err_dma;
6555 }
6556 }
6557 }
6558
6559 bars = pci_select_bars(pdev, IORESOURCE_MEM);
6560 err = pci_request_selected_regions_exclusive(pdev, bars,
6561 e1000e_driver_name);
6562 if (err)
6563 goto err_pci_reg;
6564
6565 /* AER (Advanced Error Reporting) hooks */
6566 pci_enable_pcie_error_reporting(pdev);
6567
6568 pci_set_master(pdev);
6569 /* PCI config space info */
6570 err = pci_save_state(pdev);
6571 if (err)
6572 goto err_alloc_etherdev;
6573
6574 err = -ENOMEM;
6575 netdev = alloc_etherdev(sizeof(struct e1000_adapter));
6576 if (!netdev)
6577 goto err_alloc_etherdev;
6578
6579 SET_NETDEV_DEV(netdev, &pdev->dev);
6580
6581 netdev->irq = pdev->irq;
6582
6583 pci_set_drvdata(pdev, netdev);
6584 adapter = netdev_priv(netdev);
6585 hw = &adapter->hw;
6586 adapter->netdev = netdev;
6587 adapter->pdev = pdev;
6588 adapter->ei = ei;
6589 adapter->pba = ei->pba;
6590 adapter->flags = ei->flags;
6591 adapter->flags2 = ei->flags2;
6592 adapter->hw.adapter = adapter;
6593 adapter->hw.mac.type = ei->mac;
6594 adapter->max_hw_frame_size = ei->max_hw_frame_size;
6595 adapter->msg_enable = netif_msg_init(debug, DEFAULT_MSG_ENABLE);
6596
6597 mmio_start = pci_resource_start(pdev, 0);
6598 mmio_len = pci_resource_len(pdev, 0);
6599
6600 err = -EIO;
6601 adapter->hw.hw_addr = ioremap(mmio_start, mmio_len);
6602 if (!adapter->hw.hw_addr)
6603 goto err_ioremap;
6604
6605 if ((adapter->flags & FLAG_HAS_FLASH) &&
6606 (pci_resource_flags(pdev, 1) & IORESOURCE_MEM)) {
6607 flash_start = pci_resource_start(pdev, 1);
6608 flash_len = pci_resource_len(pdev, 1);
6609 adapter->hw.flash_address = ioremap(flash_start, flash_len);
6610 if (!adapter->hw.flash_address)
6611 goto err_flashmap;
6612 }
6613
6614 /* construct the net_device struct */
6615 netdev->netdev_ops = &e1000e_netdev_ops;
6616 e1000e_set_ethtool_ops(netdev);
6617 netdev->watchdog_timeo = 5 * HZ;
6618 netif_napi_add(netdev, &adapter->napi, e1000e_poll, 64);
6619 strlcpy(netdev->name, pci_name(pdev), sizeof(netdev->name));
6620
6621 netdev->mem_start = mmio_start;
6622 netdev->mem_end = mmio_start + mmio_len;
6623
6624 adapter->bd_number = cards_found++;
6625
6626 e1000e_check_options(adapter);
6627
6628 /* setup adapter struct */
6629 err = e1000_sw_init(adapter);
6630 if (err)
6631 goto err_sw_init;
6632
6633 memcpy(&hw->mac.ops, ei->mac_ops, sizeof(hw->mac.ops));
6634 memcpy(&hw->nvm.ops, ei->nvm_ops, sizeof(hw->nvm.ops));
6635 memcpy(&hw->phy.ops, ei->phy_ops, sizeof(hw->phy.ops));
6636
6637 err = ei->get_variants(adapter);
6638 if (err)
6639 goto err_hw_init;
6640
6641 if ((adapter->flags & FLAG_IS_ICH) &&
6642 (adapter->flags & FLAG_READ_ONLY_NVM))
6643 e1000e_write_protect_nvm_ich8lan(&adapter->hw);
6644
6645 hw->mac.ops.get_bus_info(&adapter->hw);
6646
6647 adapter->hw.phy.autoneg_wait_to_complete = 0;
6648
6649 /* Copper options */
6650 if (adapter->hw.phy.media_type == e1000_media_type_copper) {
6651 adapter->hw.phy.mdix = AUTO_ALL_MODES;
6652 adapter->hw.phy.disable_polarity_correction = 0;
6653 adapter->hw.phy.ms_type = e1000_ms_hw_default;
6654 }
6655
6656 if (hw->phy.ops.check_reset_block && hw->phy.ops.check_reset_block(hw))
6657 dev_info(&pdev->dev,
6658 "PHY reset is blocked due to SOL/IDER session.\n");
6659
6660 /* Set initial default active device features */
6661 netdev->features = (NETIF_F_SG |
6662 NETIF_F_HW_VLAN_RX |
6663 NETIF_F_HW_VLAN_TX |
6664 NETIF_F_TSO |
6665 NETIF_F_TSO6 |
6666 NETIF_F_RXHASH |
6667 NETIF_F_RXCSUM |
6668 NETIF_F_HW_CSUM);
6669
6670 /* Set user-changeable features (subset of all device features) */
6671 netdev->hw_features = netdev->features;
6672 netdev->hw_features |= NETIF_F_RXFCS;
6673 netdev->priv_flags |= IFF_SUPP_NOFCS;
6674 netdev->hw_features |= NETIF_F_RXALL;
6675
6676 if (adapter->flags & FLAG_HAS_HW_VLAN_FILTER)
6677 netdev->features |= NETIF_F_HW_VLAN_FILTER;
6678
6679 netdev->vlan_features |= (NETIF_F_SG |
6680 NETIF_F_TSO |
6681 NETIF_F_TSO6 |
6682 NETIF_F_HW_CSUM);
6683
6684 netdev->priv_flags |= IFF_UNICAST_FLT;
6685
6686 if (pci_using_dac) {
6687 netdev->features |= NETIF_F_HIGHDMA;
6688 netdev->vlan_features |= NETIF_F_HIGHDMA;
6689 }
6690
6691 if (e1000e_enable_mng_pass_thru(&adapter->hw))
6692 adapter->flags |= FLAG_MNG_PT_ENABLED;
6693
6694 /* before reading the NVM, reset the controller to
6695 * put the device in a known good starting state
6696 */
6697 adapter->hw.mac.ops.reset_hw(&adapter->hw);
6698
6699 /* systems with ASPM and others may see the checksum fail on the first
6700 * attempt. Let's give it a few tries
6701 */
6702 for (i = 0;; i++) {
6703 if (e1000_validate_nvm_checksum(&adapter->hw) >= 0)
6704 break;
6705 if (i == 2) {
6706 dev_err(&pdev->dev, "The NVM Checksum Is Not Valid\n");
6707 err = -EIO;
6708 goto err_eeprom;
6709 }
6710 }
6711
6712 e1000_eeprom_checks(adapter);
6713
6714 /* copy the MAC address */
6715 if (e1000e_read_mac_addr(&adapter->hw))
6716 dev_err(&pdev->dev,
6717 "NVM Read Error while reading MAC address\n");
6718
6719 memcpy(netdev->dev_addr, adapter->hw.mac.addr, netdev->addr_len);
6720
6721 if (!is_valid_ether_addr(netdev->dev_addr)) {
6722 dev_err(&pdev->dev, "Invalid MAC Address: %pM\n",
6723 netdev->dev_addr);
6724 err = -EIO;
6725 goto err_eeprom;
6726 }
6727
6728 init_timer(&adapter->watchdog_timer);
6729 adapter->watchdog_timer.function = e1000_watchdog;
6730 adapter->watchdog_timer.data = (unsigned long) adapter;
6731
6732 init_timer(&adapter->phy_info_timer);
6733 adapter->phy_info_timer.function = e1000_update_phy_info;
6734 adapter->phy_info_timer.data = (unsigned long) adapter;
6735
6736 INIT_WORK(&adapter->reset_task, e1000_reset_task);
6737 INIT_WORK(&adapter->watchdog_task, e1000_watchdog_task);
6738 INIT_WORK(&adapter->downshift_task, e1000e_downshift_workaround);
6739 INIT_WORK(&adapter->update_phy_task, e1000e_update_phy_task);
6740 INIT_WORK(&adapter->print_hang_task, e1000_print_hw_hang);
6741
6742 /* Initialize link parameters. User can change them with ethtool */
6743 adapter->hw.mac.autoneg = 1;
6744 adapter->fc_autoneg = true;
6745 adapter->hw.fc.requested_mode = e1000_fc_default;
6746 adapter->hw.fc.current_mode = e1000_fc_default;
6747 adapter->hw.phy.autoneg_advertised = 0x2f;
6748
6749 /* ring size defaults */
6750 adapter->rx_ring->count = E1000_DEFAULT_RXD;
6751 adapter->tx_ring->count = E1000_DEFAULT_TXD;
6752
6753 /* Initial Wake on LAN setting - If APM wake is enabled in
6754 * the EEPROM, enable the ACPI Magic Packet filter
6755 */
6756 if (adapter->flags & FLAG_APME_IN_WUC) {
6757 /* APME bit in EEPROM is mapped to WUC.APME */
6758 eeprom_data = er32(WUC);
6759 eeprom_apme_mask = E1000_WUC_APME;
6760 if ((hw->mac.type > e1000_ich10lan) &&
6761 (eeprom_data & E1000_WUC_PHY_WAKE))
6762 adapter->flags2 |= FLAG2_HAS_PHY_WAKEUP;
6763 } else if (adapter->flags & FLAG_APME_IN_CTRL3) {
6764 if (adapter->flags & FLAG_APME_CHECK_PORT_B &&
6765 (adapter->hw.bus.func == 1))
6766 e1000_read_nvm(&adapter->hw, NVM_INIT_CONTROL3_PORT_B,
6767 1, &eeprom_data);
6768 else
6769 e1000_read_nvm(&adapter->hw, NVM_INIT_CONTROL3_PORT_A,
6770 1, &eeprom_data);
6771 }
6772
6773 /* fetch WoL from EEPROM */
6774 if (eeprom_data & eeprom_apme_mask)
6775 adapter->eeprom_wol |= E1000_WUFC_MAG;
6776
6777 /* now that we have the eeprom settings, apply the special cases
6778 * where the eeprom may be wrong or the board simply won't support
6779 * wake on lan on a particular port
6780 */
6781 if (!(adapter->flags & FLAG_HAS_WOL))
6782 adapter->eeprom_wol = 0;
6783
6784 /* initialize the wol settings based on the eeprom settings */
6785 adapter->wol = adapter->eeprom_wol;
6786 device_set_wakeup_enable(&adapter->pdev->dev, adapter->wol);
6787
6788 /* save off EEPROM version number */
6789 e1000_read_nvm(&adapter->hw, 5, 1, &adapter->eeprom_vers);
6790
6791 /* reset the hardware with the new settings */
6792 e1000e_reset(adapter);
6793
6794 /* If the controller has AMT, do not set DRV_LOAD until the interface
6795 * is up. For all other cases, let the f/w know that the h/w is now
6796 * under the control of the driver.
6797 */
6798 if (!(adapter->flags & FLAG_HAS_AMT))
6799 e1000e_get_hw_control(adapter);
6800
6801 strlcpy(netdev->name, "eth%d", sizeof(netdev->name));
6802 err = register_netdev(netdev);
6803 if (err)
6804 goto err_register;
6805
6806 /* carrier off reporting is important to ethtool even BEFORE open */
6807 netif_carrier_off(netdev);
6808
6809 /* init PTP hardware clock */
6810 e1000e_ptp_init(adapter);
6811
6812 e1000_print_device_info(adapter);
6813
6814 if (pci_dev_run_wake(pdev))
6815 pm_runtime_put_noidle(&pdev->dev);
6816
6817 return 0;
6818
6819 err_register:
6820 if (!(adapter->flags & FLAG_HAS_AMT))
6821 e1000e_release_hw_control(adapter);
6822 err_eeprom:
6823 if (hw->phy.ops.check_reset_block && !hw->phy.ops.check_reset_block(hw))
6824 e1000_phy_hw_reset(&adapter->hw);
6825 err_hw_init:
6826 kfree(adapter->tx_ring);
6827 kfree(adapter->rx_ring);
6828 err_sw_init:
6829 if (adapter->hw.flash_address)
6830 iounmap(adapter->hw.flash_address);
6831 e1000e_reset_interrupt_capability(adapter);
6832 err_flashmap:
6833 iounmap(adapter->hw.hw_addr);
6834 err_ioremap:
6835 free_netdev(netdev);
6836 err_alloc_etherdev:
6837 pci_release_selected_regions(pdev,
6838 pci_select_bars(pdev, IORESOURCE_MEM));
6839 err_pci_reg:
6840 err_dma:
6841 pci_disable_device(pdev);
6842 return err;
6843 }
6844
6845 /**
6846 * e1000_remove - Device Removal Routine
6847 * @pdev: PCI device information struct
6848 *
6849 * e1000_remove is called by the PCI subsystem to alert the driver
6850 * that it should release a PCI device. The could be caused by a
6851 * Hot-Plug event, or because the driver is going to be removed from
6852 * memory.
6853 **/
6854 static void e1000_remove(struct pci_dev *pdev)
6855 {
6856 struct net_device *netdev = pci_get_drvdata(pdev);
6857 struct e1000_adapter *adapter = netdev_priv(netdev);
6858 bool down = test_bit(__E1000_DOWN, &adapter->state);
6859
6860 e1000e_ptp_remove(adapter);
6861
6862 /* The timers may be rescheduled, so explicitly disable them
6863 * from being rescheduled.
6864 */
6865 if (!down)
6866 set_bit(__E1000_DOWN, &adapter->state);
6867 del_timer_sync(&adapter->watchdog_timer);
6868 del_timer_sync(&adapter->phy_info_timer);
6869
6870 cancel_work_sync(&adapter->reset_task);
6871 cancel_work_sync(&adapter->watchdog_task);
6872 cancel_work_sync(&adapter->downshift_task);
6873 cancel_work_sync(&adapter->update_phy_task);
6874 cancel_work_sync(&adapter->print_hang_task);
6875
6876 if (adapter->flags & FLAG_HAS_HW_TIMESTAMP) {
6877 cancel_work_sync(&adapter->tx_hwtstamp_work);
6878 if (adapter->tx_hwtstamp_skb) {
6879 dev_kfree_skb_any(adapter->tx_hwtstamp_skb);
6880 adapter->tx_hwtstamp_skb = NULL;
6881 }
6882 }
6883
6884 if (!(netdev->flags & IFF_UP))
6885 e1000_power_down_phy(adapter);
6886
6887 /* Don't lie to e1000_close() down the road. */
6888 if (!down)
6889 clear_bit(__E1000_DOWN, &adapter->state);
6890 unregister_netdev(netdev);
6891
6892 if (pci_dev_run_wake(pdev))
6893 pm_runtime_get_noresume(&pdev->dev);
6894
6895 /* Release control of h/w to f/w. If f/w is AMT enabled, this
6896 * would have already happened in close and is redundant.
6897 */
6898 e1000e_release_hw_control(adapter);
6899
6900 e1000e_reset_interrupt_capability(adapter);
6901 kfree(adapter->tx_ring);
6902 kfree(adapter->rx_ring);
6903
6904 iounmap(adapter->hw.hw_addr);
6905 if (adapter->hw.flash_address)
6906 iounmap(adapter->hw.flash_address);
6907 pci_release_selected_regions(pdev,
6908 pci_select_bars(pdev, IORESOURCE_MEM));
6909
6910 free_netdev(netdev);
6911
6912 /* AER disable */
6913 pci_disable_pcie_error_reporting(pdev);
6914
6915 pci_disable_device(pdev);
6916 }
6917
6918 /* PCI Error Recovery (ERS) */
6919 static const struct pci_error_handlers e1000_err_handler = {
6920 .error_detected = e1000_io_error_detected,
6921 .slot_reset = e1000_io_slot_reset,
6922 .resume = e1000_io_resume,
6923 };
6924
6925 static DEFINE_PCI_DEVICE_TABLE(e1000_pci_tbl) = {
6926 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82571EB_COPPER), board_82571 },
6927 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82571EB_FIBER), board_82571 },
6928 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82571EB_QUAD_COPPER), board_82571 },
6929 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82571EB_QUAD_COPPER_LP),
6930 board_82571 },
6931 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82571EB_QUAD_FIBER), board_82571 },
6932 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82571EB_SERDES), board_82571 },
6933 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82571EB_SERDES_DUAL), board_82571 },
6934 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82571EB_SERDES_QUAD), board_82571 },
6935 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82571PT_QUAD_COPPER), board_82571 },
6936
6937 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82572EI), board_82572 },
6938 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82572EI_COPPER), board_82572 },
6939 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82572EI_FIBER), board_82572 },
6940 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82572EI_SERDES), board_82572 },
6941
6942 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82573E), board_82573 },
6943 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82573E_IAMT), board_82573 },
6944 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82573L), board_82573 },
6945
6946 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82574L), board_82574 },
6947 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82574LA), board_82574 },
6948 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82583V), board_82583 },
6949
6950 { PCI_VDEVICE(INTEL, E1000_DEV_ID_80003ES2LAN_COPPER_DPT),
6951 board_80003es2lan },
6952 { PCI_VDEVICE(INTEL, E1000_DEV_ID_80003ES2LAN_COPPER_SPT),
6953 board_80003es2lan },
6954 { PCI_VDEVICE(INTEL, E1000_DEV_ID_80003ES2LAN_SERDES_DPT),
6955 board_80003es2lan },
6956 { PCI_VDEVICE(INTEL, E1000_DEV_ID_80003ES2LAN_SERDES_SPT),
6957 board_80003es2lan },
6958
6959 { PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH8_IFE), board_ich8lan },
6960 { PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH8_IFE_G), board_ich8lan },
6961 { PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH8_IFE_GT), board_ich8lan },
6962 { PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH8_IGP_AMT), board_ich8lan },
6963 { PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH8_IGP_C), board_ich8lan },
6964 { PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH8_IGP_M), board_ich8lan },
6965 { PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH8_IGP_M_AMT), board_ich8lan },
6966 { PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH8_82567V_3), board_ich8lan },
6967
6968 { PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH9_IFE), board_ich9lan },
6969 { PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH9_IFE_G), board_ich9lan },
6970 { PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH9_IFE_GT), board_ich9lan },
6971 { PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH9_IGP_AMT), board_ich9lan },
6972 { PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH9_IGP_C), board_ich9lan },
6973 { PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH9_BM), board_ich9lan },
6974 { PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH9_IGP_M), board_ich9lan },
6975 { PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH9_IGP_M_AMT), board_ich9lan },
6976 { PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH9_IGP_M_V), board_ich9lan },
6977
6978 { PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH10_R_BM_LM), board_ich9lan },
6979 { PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH10_R_BM_LF), board_ich9lan },
6980 { PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH10_R_BM_V), board_ich9lan },
6981
6982 { PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH10_D_BM_LM), board_ich10lan },
6983 { PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH10_D_BM_LF), board_ich10lan },
6984 { PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH10_D_BM_V), board_ich10lan },
6985
6986 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_M_HV_LM), board_pchlan },
6987 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_M_HV_LC), board_pchlan },
6988 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_D_HV_DM), board_pchlan },
6989 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_D_HV_DC), board_pchlan },
6990
6991 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH2_LV_LM), board_pch2lan },
6992 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH2_LV_V), board_pch2lan },
6993
6994 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_LPT_I217_LM), board_pch_lpt },
6995 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_LPT_I217_V), board_pch_lpt },
6996 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_LPTLP_I218_LM), board_pch_lpt },
6997 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_LPTLP_I218_V), board_pch_lpt },
6998
6999 { 0, 0, 0, 0, 0, 0, 0 } /* terminate list */
7000 };
7001 MODULE_DEVICE_TABLE(pci, e1000_pci_tbl);
7002
7003 #ifdef CONFIG_PM
7004 static const struct dev_pm_ops e1000_pm_ops = {
7005 SET_SYSTEM_SLEEP_PM_OPS(e1000_suspend, e1000_resume)
7006 SET_RUNTIME_PM_OPS(e1000_runtime_suspend, e1000_runtime_resume,
7007 e1000_idle)
7008 };
7009 #endif
7010
7011 /* PCI Device API Driver */
7012 static struct pci_driver e1000_driver = {
7013 .name = e1000e_driver_name,
7014 .id_table = e1000_pci_tbl,
7015 .probe = e1000_probe,
7016 .remove = e1000_remove,
7017 #ifdef CONFIG_PM
7018 .driver = {
7019 .pm = &e1000_pm_ops,
7020 },
7021 #endif
7022 .shutdown = e1000_shutdown,
7023 .err_handler = &e1000_err_handler
7024 };
7025
7026 /**
7027 * e1000_init_module - Driver Registration Routine
7028 *
7029 * e1000_init_module is the first routine called when the driver is
7030 * loaded. All it does is register with the PCI subsystem.
7031 **/
7032 static int __init e1000_init_module(void)
7033 {
7034 int ret;
7035 pr_info("Intel(R) PRO/1000 Network Driver - %s\n",
7036 e1000e_driver_version);
7037 pr_info("Copyright(c) 1999 - 2013 Intel Corporation.\n");
7038 ret = pci_register_driver(&e1000_driver);
7039
7040 return ret;
7041 }
7042 module_init(e1000_init_module);
7043
7044 /**
7045 * e1000_exit_module - Driver Exit Cleanup Routine
7046 *
7047 * e1000_exit_module is called just before the driver is removed
7048 * from memory.
7049 **/
7050 static void __exit e1000_exit_module(void)
7051 {
7052 pci_unregister_driver(&e1000_driver);
7053 }
7054 module_exit(e1000_exit_module);
7055
7056
7057 MODULE_AUTHOR("Intel Corporation, <linux.nics@intel.com>");
7058 MODULE_DESCRIPTION("Intel(R) PRO/1000 Network Driver");
7059 MODULE_LICENSE("GPL");
7060 MODULE_VERSION(DRV_VERSION);
7061
7062 /* netdev.c */
Linux kernel - Deliverables
This page took 0.305649 seconds and 5 git commands to generate.