projects / deliverable / linux.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
e1000e: cleanup - move defines to appropriate header file
[deliverable/linux.git] / drivers / net / ethernet / intel / e1000e / netdev.c
CommitLineData
bc7f75fa
AK		 1/*******************************************************************************
2
3 Intel PRO/1000 Linux driver
bf67044b 4 Copyright(c) 1999 - 2013 Intel Corporation.
bc7f75fa
AK		 5
6 This program is free software; you can redistribute it and/or modify it
7 under the terms and conditions of the GNU General Public License,
8 version 2, as published by the Free Software Foundation.
9
10 This program is distributed in the hope it will be useful, but WITHOUT
11 ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
13 more details.
14
15 You should have received a copy of the GNU General Public License along with
16 this program; if not, write to the Free Software Foundation, Inc.,
17 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
18
19 The full GNU General Public License is included in this distribution in
20 the file called "COPYING".
21
22 Contact Information:
23 Linux NICS <linux.nics@intel.com>
24 e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
25 Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
26
27*******************************************************************************/
28
8544b9f7
BA		 29#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
30
bc7f75fa
AK		 31#include <linux/module.h>
32#include <linux/types.h>
33#include <linux/init.h>
34#include <linux/pci.h>
35#include <linux/vmalloc.h>
36#include <linux/pagemap.h>
37#include <linux/delay.h>
38#include <linux/netdevice.h>
9fb7a5f7 39#include <linux/interrupt.h>
bc7f75fa
AK		 40#include <linux/tcp.h>
41#include <linux/ipv6.h>
5a0e3ad6 42#include <linux/slab.h>
bc7f75fa
AK		 43#include <net/checksum.h>
44#include <net/ip6_checksum.h>
bc7f75fa
AK		 45#include <linux/ethtool.h>
46#include <linux/if_vlan.h>
47#include <linux/cpu.h>
48#include <linux/smp.h>
e8db0be1 49#include <linux/pm_qos.h>
23606cf5 50#include <linux/pm_runtime.h>
111b9dc5 51#include <linux/aer.h>
70c71606 52#include <linux/prefetch.h>
bc7f75fa
AK		 53
54#include "e1000.h"
55
b3ccf267 56#define DRV_EXTRAVERSION "-k"
c14c643b 57
9e019901 58#define DRV_VERSION "2.2.14" DRV_EXTRAVERSION
bc7f75fa
AK		 59char e1000e_driver_name[] = "e1000e";
60const char e1000e_driver_version[] = DRV_VERSION;
61
b3f4d599 62#define DEFAULT_MSG_ENABLE (NETIF_MSG_DRV|NETIF_MSG_PROBE|NETIF_MSG_LINK)
63static int debug = -1;
64module_param(debug, int, 0);
65MODULE_PARM_DESC(debug, "Debug level (0=none,...,16=all)");
66
78cd29d5
BA		 67static void e1000e_disable_aspm(struct pci_dev *pdev, u16 state);
68
bc7f75fa
AK		 69static const struct e1000_info *e1000_info_tbl[] = {
70 [board_82571] = &e1000_82571_info,
71 [board_82572] = &e1000_82572_info,
72 [board_82573] = &e1000_82573_info,
4662e82b 73 [board_82574] = &e1000_82574_info,
8c81c9c3 74 [board_82583] = &e1000_82583_info,
bc7f75fa
AK		 75 [board_80003es2lan] = &e1000_es2_info,
76 [board_ich8lan] = &e1000_ich8_info,
77 [board_ich9lan] = &e1000_ich9_info,
f4187b56 78 [board_ich10lan] = &e1000_ich10_info,
a4f58f54 79 [board_pchlan] = &e1000_pch_info,
d3738bb8 80 [board_pch2lan] = &e1000_pch2_info,
2fbe4526 81 [board_pch_lpt] = &e1000_pch_lpt_info,
bc7f75fa
AK		 82};
83
84f4ee90
TI		 84struct e1000_reg_info {
85 u32 ofs;
86 char *name;
87};
88
84f4ee90 89static const struct e1000_reg_info e1000_reg_info_tbl[] = {
84f4ee90
TI		 90 /* General Registers */
91 {E1000_CTRL, "CTRL"},
92 {E1000_STATUS, "STATUS"},
93 {E1000_CTRL_EXT, "CTRL_EXT"},
94
95 /* Interrupt Registers */
96 {E1000_ICR, "ICR"},
97
af667a29 98 /* Rx Registers */
84f4ee90 99 {E1000_RCTL, "RCTL"},
1e36052e
BA		 100 {E1000_RDLEN(0), "RDLEN"},
101 {E1000_RDH(0), "RDH"},
102 {E1000_RDT(0), "RDT"},
84f4ee90
TI		 103 {E1000_RDTR, "RDTR"},
104 {E1000_RXDCTL(0), "RXDCTL"},
105 {E1000_ERT, "ERT"},
1e36052e
BA		 106 {E1000_RDBAL(0), "RDBAL"},
107 {E1000_RDBAH(0), "RDBAH"},
84f4ee90
TI		 108 {E1000_RDFH, "RDFH"},
109 {E1000_RDFT, "RDFT"},
110 {E1000_RDFHS, "RDFHS"},
111 {E1000_RDFTS, "RDFTS"},
112 {E1000_RDFPC, "RDFPC"},
113
af667a29 114 /* Tx Registers */
84f4ee90 115 {E1000_TCTL, "TCTL"},
1e36052e
BA		 116 {E1000_TDBAL(0), "TDBAL"},
117 {E1000_TDBAH(0), "TDBAH"},
118 {E1000_TDLEN(0), "TDLEN"},
119 {E1000_TDH(0), "TDH"},
120 {E1000_TDT(0), "TDT"},
84f4ee90
TI		 121 {E1000_TIDV, "TIDV"},
122 {E1000_TXDCTL(0), "TXDCTL"},
123 {E1000_TADV, "TADV"},
124 {E1000_TARC(0), "TARC"},
125 {E1000_TDFH, "TDFH"},
126 {E1000_TDFT, "TDFT"},
127 {E1000_TDFHS, "TDFHS"},
128 {E1000_TDFTS, "TDFTS"},
129 {E1000_TDFPC, "TDFPC"},
130
131 /* List Terminator */
f36bb6ca 132 {0, NULL}
84f4ee90
TI		 133};
134
e921eb1a 135/**
84f4ee90 136 * e1000_regdump - register printout routine
e921eb1a
BA		 137 * @hw: pointer to the HW structure
138 * @reginfo: pointer to the register info table
139 **/
84f4ee90
TI		 140static void e1000_regdump(struct e1000_hw *hw, struct e1000_reg_info *reginfo)
141{
142 int n = 0;
143 char rname[16];
144 u32 regs[8];
145
146 switch (reginfo->ofs) {
147 case E1000_RXDCTL(0):
148 for (n = 0; n < 2; n++)
149 regs[n] = __er32(hw, E1000_RXDCTL(n));
150 break;
151 case E1000_TXDCTL(0):
152 for (n = 0; n < 2; n++)
153 regs[n] = __er32(hw, E1000_TXDCTL(n));
154 break;
155 case E1000_TARC(0):
156 for (n = 0; n < 2; n++)
157 regs[n] = __er32(hw, E1000_TARC(n));
158 break;
159 default:
ef456f85
JK		 160 pr_info("%-15s %08x\n",
161 reginfo->name, __er32(hw, reginfo->ofs));
84f4ee90
TI		 162 return;
163 }
164
165 snprintf(rname, 16, "%s%s", reginfo->name, "[0-1]");
ef456f85 166 pr_info("%-15s %08x %08x\n", rname, regs[0], regs[1]);
84f4ee90
TI		 167}
168
f0c5dadf
ET		 169static void e1000e_dump_ps_pages(struct e1000_adapter *adapter,
170 struct e1000_buffer *bi)
171{
172 int i;
173 struct e1000_ps_page *ps_page;
174
175 for (i = 0; i < adapter->rx_ps_pages; i++) {
176 ps_page = &bi->ps_pages[i];
177
178 if (ps_page->page) {
179 pr_info("packet dump for ps_page %d:\n", i);
180 print_hex_dump(KERN_INFO, "", DUMP_PREFIX_ADDRESS,
181 16, 1, page_address(ps_page->page),
182 PAGE_SIZE, true);
183 }
184 }
185}
186
e921eb1a 187/**
af667a29 188 * e1000e_dump - Print registers, Tx-ring and Rx-ring
e921eb1a
BA		 189 * @adapter: board private structure
190 **/
84f4ee90
TI		 191static void e1000e_dump(struct e1000_adapter *adapter)
192{
193 struct net_device *netdev = adapter->netdev;
194 struct e1000_hw *hw = &adapter->hw;
195 struct e1000_reg_info *reginfo;
196 struct e1000_ring *tx_ring = adapter->tx_ring;
197 struct e1000_tx_desc *tx_desc;
af667a29 198 struct my_u0 {
e885d762
BA		 199 __le64 a;
200 __le64 b;
af667a29 201 } *u0;
84f4ee90
TI		 202 struct e1000_buffer *buffer_info;
203 struct e1000_ring *rx_ring = adapter->rx_ring;
204 union e1000_rx_desc_packet_split *rx_desc_ps;
5f450212 205 union e1000_rx_desc_extended *rx_desc;
af667a29 206 struct my_u1 {
e885d762
BA		 207 __le64 a;
208 __le64 b;
209 __le64 c;
210 __le64 d;
af667a29 211 } *u1;
84f4ee90
TI		 212 u32 staterr;
213 int i = 0;
214
215 if (!netif_msg_hw(adapter))
216 return;
217
218 /* Print netdevice Info */
219 if (netdev) {
220 dev_info(&adapter->pdev->dev, "Net device Info\n");
ef456f85 221 pr_info("Device Name state trans_start last_rx\n");
e5fe2541
BA		 222 pr_info("%-15s %016lX %016lX %016lX\n", netdev->name,
223 netdev->state, netdev->trans_start, netdev->last_rx);
84f4ee90
TI		 224 }
225
226 /* Print Registers */
227 dev_info(&adapter->pdev->dev, "Register Dump\n");
ef456f85 228 pr_info(" Register Name Value\n");
84f4ee90
TI		 229 for (reginfo = (struct e1000_reg_info *)e1000_reg_info_tbl;
230 reginfo->name; reginfo++) {
231 e1000_regdump(hw, reginfo);
232 }
233
af667a29 234 /* Print Tx Ring Summary */
84f4ee90 235 if (!netdev || !netif_running(netdev))
fe1e980f 236 return;
84f4ee90 237
af667a29 238 dev_info(&adapter->pdev->dev, "Tx Ring Summary\n");
ef456f85 239 pr_info("Queue [NTU] [NTC] [bi(ntc)->dma ] leng ntw timestamp\n");
84f4ee90 240 buffer_info = &tx_ring->buffer_info[tx_ring->next_to_clean];
ef456f85
JK		 241 pr_info(" %5d %5X %5X %016llX %04X %3X %016llX\n",
242 0, tx_ring->next_to_use, tx_ring->next_to_clean,
243 (unsigned long long)buffer_info->dma,
244 buffer_info->length,
245 buffer_info->next_to_watch,
246 (unsigned long long)buffer_info->time_stamp);
84f4ee90 247
af667a29 248 /* Print Tx Ring */
84f4ee90
TI		 249 if (!netif_msg_tx_done(adapter))
250 goto rx_ring_summary;
251
af667a29 252 dev_info(&adapter->pdev->dev, "Tx Ring Dump\n");
84f4ee90
TI		 253
254 /* Transmit Descriptor Formats - DEXT[29] is 0 (Legacy) or 1 (Extended)
255 *
256 * Legacy Transmit Descriptor
257 * +--------------------------------------------------------------+
258 * 0 | Buffer Address [63:0] (Reserved on Write Back) |
259 * +--------------------------------------------------------------+
260 * 8 | Special | CSS | Status | CMD | CSO | Length |
261 * +--------------------------------------------------------------+
262 * 63 48 47 36 35 32 31 24 23 16 15 0
263 *
264 * Extended Context Descriptor (DTYP=0x0) for TSO or checksum offload
265 * 63 48 47 40 39 32 31 16 15 8 7 0
266 * +----------------------------------------------------------------+
267 * 0 | TUCSE | TUCS0 | TUCSS | IPCSE | IPCS0 | IPCSS |
268 * +----------------------------------------------------------------+
269 * 8 | MSS | HDRLEN | RSV | STA | TUCMD | DTYP | PAYLEN |
270 * +----------------------------------------------------------------+
271 * 63 48 47 40 39 36 35 32 31 24 23 20 19 0
272 *
273 * Extended Data Descriptor (DTYP=0x1)
274 * +----------------------------------------------------------------+
275 * 0 | Buffer Address [63:0] |
276 * +----------------------------------------------------------------+
277 * 8 | VLAN tag | POPTS | Rsvd | Status | Command | DTYP | DTALEN |
278 * +----------------------------------------------------------------+
279 * 63 48 47 40 39 36 35 32 31 24 23 20 19 0
280 */
ef456f85
JK		 281 pr_info("Tl[desc] [address 63:0 ] [SpeCssSCmCsLen] [bi->dma ] leng ntw timestamp bi->skb <-- Legacy format\n");
282 pr_info("Tc[desc] [Ce CoCsIpceCoS] [MssHlRSCm0Plen] [bi->dma ] leng ntw timestamp bi->skb <-- Ext Context format\n");
283 pr_info("Td[desc] [address 63:0 ] [VlaPoRSCm1Dlen] [bi->dma ] leng ntw timestamp bi->skb <-- Ext Data format\n");
84f4ee90 284 for (i = 0; tx_ring->desc && (i < tx_ring->count); i++) {
ef456f85 285 const char *next_desc;
84f4ee90
TI		 286 tx_desc = E1000_TX_DESC(*tx_ring, i);
287 buffer_info = &tx_ring->buffer_info[i];
288 u0 = (struct my_u0 *)tx_desc;
84f4ee90 289 if (i == tx_ring->next_to_use && i == tx_ring->next_to_clean)
ef456f85 290 next_desc = " NTC/U";
84f4ee90 291 else if (i == tx_ring->next_to_use)
ef456f85 292 next_desc = " NTU";
84f4ee90 293 else if (i == tx_ring->next_to_clean)
ef456f85 294 next_desc = " NTC";
84f4ee90 295 else
ef456f85
JK		 296 next_desc = "";
297 pr_info("T%c[0x%03X] %016llX %016llX %016llX %04X %3X %016llX %p%s\n",
298 (!(le64_to_cpu(u0->b) & (1 << 29)) ? 'l' :
299 ((le64_to_cpu(u0->b) & (1 << 20)) ? 'd' : 'c')),
300 i,
301 (unsigned long long)le64_to_cpu(u0->a),
302 (unsigned long long)le64_to_cpu(u0->b),
303 (unsigned long long)buffer_info->dma,
304 buffer_info->length, buffer_info->next_to_watch,
305 (unsigned long long)buffer_info->time_stamp,
306 buffer_info->skb, next_desc);
84f4ee90 307
f0c5dadf 308 if (netif_msg_pktdata(adapter) && buffer_info->skb)
84f4ee90 309 print_hex_dump(KERN_INFO, "", DUMP_PREFIX_ADDRESS,
f0c5dadf
ET		 310 16, 1, buffer_info->skb->data,
311 buffer_info->skb->len, true);
84f4ee90
TI		 312 }
313
af667a29 314 /* Print Rx Ring Summary */
84f4ee90 315rx_ring_summary:
af667a29 316 dev_info(&adapter->pdev->dev, "Rx Ring Summary\n");
ef456f85
JK		 317 pr_info("Queue [NTU] [NTC]\n");
318 pr_info(" %5d %5X %5X\n",
319 0, rx_ring->next_to_use, rx_ring->next_to_clean);
84f4ee90 320
af667a29 321 /* Print Rx Ring */
84f4ee90 322 if (!netif_msg_rx_status(adapter))
fe1e980f 323 return;
84f4ee90 324
af667a29 325 dev_info(&adapter->pdev->dev, "Rx Ring Dump\n");
84f4ee90
TI		 326 switch (adapter->rx_ps_pages) {
327 case 1:
328 case 2:
329 case 3:
330 /* [Extended] Packet Split Receive Descriptor Format
331 *
332 * +-----------------------------------------------------+
333 * 0 | Buffer Address 0 [63:0] |
334 * +-----------------------------------------------------+
335 * 8 | Buffer Address 1 [63:0] |
336 * +-----------------------------------------------------+
337 * 16 | Buffer Address 2 [63:0] |
338 * +-----------------------------------------------------+
339 * 24 | Buffer Address 3 [63:0] |
340 * +-----------------------------------------------------+
341 */
ef456f85 342 pr_info("R [desc] [buffer 0 63:0 ] [buffer 1 63:0 ] [buffer 2 63:0 ] [buffer 3 63:0 ] [bi->dma ] [bi->skb] <-- Ext Pkt Split format\n");
84f4ee90
TI		 343 /* [Extended] Receive Descriptor (Write-Back) Format
344 *
345 * 63 48 47 32 31 13 12 8 7 4 3 0
346 * +------------------------------------------------------+
347 * 0 | Packet | IP | Rsvd | MRQ | Rsvd | MRQ RSS |
348 * | Checksum | Ident | | Queue | | Type |
349 * +------------------------------------------------------+
350 * 8 | VLAN Tag | Length | Extended Error | Extended Status |
351 * +------------------------------------------------------+
352 * 63 48 47 32 31 20 19 0
353 */
ef456f85 354 pr_info("RWB[desc] [ck ipid mrqhsh] [vl l0 ee es] [ l3 l2 l1 hs] [reserved ] ---------------- [bi->skb] <-- Ext Rx Write-Back format\n");
84f4ee90 355 for (i = 0; i < rx_ring->count; i++) {
ef456f85 356 const char *next_desc;
84f4ee90
TI		 357 buffer_info = &rx_ring->buffer_info[i];
358 rx_desc_ps = E1000_RX_DESC_PS(*rx_ring, i);
359 u1 = (struct my_u1 *)rx_desc_ps;
360 staterr =
af667a29 361 le32_to_cpu(rx_desc_ps->wb.middle.status_error);
ef456f85
JK		 362
363 if (i == rx_ring->next_to_use)
364 next_desc = " NTU";
365 else if (i == rx_ring->next_to_clean)
366 next_desc = " NTC";
367 else
368 next_desc = "";
369
84f4ee90
TI		 370 if (staterr & E1000_RXD_STAT_DD) {
371 /* Descriptor Done */
ef456f85
JK		 372 pr_info("%s[0x%03X] %016llX %016llX %016llX %016llX ---------------- %p%s\n",
373 "RWB", i,
374 (unsigned long long)le64_to_cpu(u1->a),
375 (unsigned long long)le64_to_cpu(u1->b),
376 (unsigned long long)le64_to_cpu(u1->c),
377 (unsigned long long)le64_to_cpu(u1->d),
378 buffer_info->skb, next_desc);
84f4ee90 379 } else {
ef456f85
JK		 380 pr_info("%s[0x%03X] %016llX %016llX %016llX %016llX %016llX %p%s\n",
381 "R ", i,
382 (unsigned long long)le64_to_cpu(u1->a),
383 (unsigned long long)le64_to_cpu(u1->b),
384 (unsigned long long)le64_to_cpu(u1->c),
385 (unsigned long long)le64_to_cpu(u1->d),
386 (unsigned long long)buffer_info->dma,
387 buffer_info->skb, next_desc);
84f4ee90
TI		 388
389 if (netif_msg_pktdata(adapter))
f0c5dadf
ET		 390 e1000e_dump_ps_pages(adapter,
391 buffer_info);
84f4ee90 392 }
84f4ee90
TI		 393 }
394 break;
395 default:
396 case 0:
5f450212 397 /* Extended Receive Descriptor (Read) Format
84f4ee90 398 *
5f450212
BA		 399 * +-----------------------------------------------------+
400 * 0 | Buffer Address [63:0] |
401 * +-----------------------------------------------------+
402 * 8 | Reserved |
403 * +-----------------------------------------------------+
84f4ee90 404 */
ef456f85 405 pr_info("R [desc] [buf addr 63:0 ] [reserved 63:0 ] [bi->dma ] [bi->skb] <-- Ext (Read) format\n");
5f450212
BA		 406 /* Extended Receive Descriptor (Write-Back) Format
407 *
408 * 63 48 47 32 31 24 23 4 3 0
409 * +------------------------------------------------------+
410 * | RSS Hash | | | |
411 * 0 +-------------------+ Rsvd | Reserved | MRQ RSS |
412 * | Packet | IP | | | Type |
413 * | Checksum | Ident | | | |
414 * +------------------------------------------------------+
415 * 8 | VLAN Tag | Length | Extended Error | Extended Status |
416 * +------------------------------------------------------+
417 * 63 48 47 32 31 20 19 0
418 */
ef456f85 419 pr_info("RWB[desc] [cs ipid mrq] [vt ln xe xs] [bi->skb] <-- Ext (Write-Back) format\n");
5f450212
BA		 420
421 for (i = 0; i < rx_ring->count; i++) {
ef456f85
JK		 422 const char *next_desc;
423
84f4ee90 424 buffer_info = &rx_ring->buffer_info[i];
5f450212
BA		 425 rx_desc = E1000_RX_DESC_EXT(*rx_ring, i);
426 u1 = (struct my_u1 *)rx_desc;
427 staterr = le32_to_cpu(rx_desc->wb.upper.status_error);
ef456f85
JK		 428
429 if (i == rx_ring->next_to_use)
430 next_desc = " NTU";
431 else if (i == rx_ring->next_to_clean)
432 next_desc = " NTC";
433 else
434 next_desc = "";
435
5f450212
BA		 436 if (staterr & E1000_RXD_STAT_DD) {
437 /* Descriptor Done */
ef456f85
JK		 438 pr_info("%s[0x%03X] %016llX %016llX ---------------- %p%s\n",
439 "RWB", i,
440 (unsigned long long)le64_to_cpu(u1->a),
441 (unsigned long long)le64_to_cpu(u1->b),
442 buffer_info->skb, next_desc);
5f450212 443 } else {
ef456f85
JK		 444 pr_info("%s[0x%03X] %016llX %016llX %016llX %p%s\n",
445 "R ", i,
446 (unsigned long long)le64_to_cpu(u1->a),
447 (unsigned long long)le64_to_cpu(u1->b),
448 (unsigned long long)buffer_info->dma,
449 buffer_info->skb, next_desc);
5f450212 450
f0c5dadf
ET		 451 if (netif_msg_pktdata(adapter) &&
452 buffer_info->skb)
5f450212
BA		 453 print_hex_dump(KERN_INFO, "",
454 DUMP_PREFIX_ADDRESS, 16,
455 1,
f0c5dadf 456 buffer_info->skb->data,
5f450212
BA		 457 adapter->rx_buffer_len,
458 true);
459 }
84f4ee90
TI		 460 }
461 }
84f4ee90
TI		 462}
463
bc7f75fa
AK		 464/**
465 * e1000_desc_unused - calculate if we have unused descriptors
466 **/
467static int e1000_desc_unused(struct e1000_ring *ring)
468{
469 if (ring->next_to_clean > ring->next_to_use)
470 return ring->next_to_clean - ring->next_to_use - 1;
471
472 return ring->count + ring->next_to_clean - ring->next_to_use - 1;
473}
474
b67e1913
BA		 475/**
476 * e1000e_systim_to_hwtstamp - convert system time value to hw time stamp
477 * @adapter: board private structure
478 * @hwtstamps: time stamp structure to update
479 * @systim: unsigned 64bit system time value.
480 *
481 * Convert the system time value stored in the RX/TXSTMP registers into a
482 * hwtstamp which can be used by the upper level time stamping functions.
483 *
484 * The 'systim_lock' spinlock is used to protect the consistency of the
485 * system time value. This is needed because reading the 64 bit time
486 * value involves reading two 32 bit registers. The first read latches the
487 * value.
488 **/
489static void e1000e_systim_to_hwtstamp(struct e1000_adapter *adapter,
490 struct skb_shared_hwtstamps *hwtstamps,
491 u64 systim)
492{
493 u64 ns;
494 unsigned long flags;
495
496 spin_lock_irqsave(&adapter->systim_lock, flags);
497 ns = timecounter_cyc2time(&adapter->tc, systim);
498 spin_unlock_irqrestore(&adapter->systim_lock, flags);
499
500 memset(hwtstamps, 0, sizeof(*hwtstamps));
501 hwtstamps->hwtstamp = ns_to_ktime(ns);
502}
503
504/**
505 * e1000e_rx_hwtstamp - utility function which checks for Rx time stamp
506 * @adapter: board private structure
507 * @status: descriptor extended error and status field
508 * @skb: particular skb to include time stamp
509 *
510 * If the time stamp is valid, convert it into the timecounter ns value
511 * and store that result into the shhwtstamps structure which is passed
512 * up the network stack.
513 **/
514static void e1000e_rx_hwtstamp(struct e1000_adapter *adapter, u32 status,
515 struct sk_buff *skb)
516{
517 struct e1000_hw *hw = &adapter->hw;
518 u64 rxstmp;
519
520 if (!(adapter->flags & FLAG_HAS_HW_TIMESTAMP) ||
521 !(status & E1000_RXDEXT_STATERR_TST) ||
522 !(er32(TSYNCRXCTL) & E1000_TSYNCRXCTL_VALID))
523 return;
524
525 /* The Rx time stamp registers contain the time stamp. No other
526 * received packet will be time stamped until the Rx time stamp
527 * registers are read. Because only one packet can be time stamped
528 * at a time, the register values must belong to this packet and
529 * therefore none of the other additional attributes need to be
530 * compared.
531 */
532 rxstmp = (u64)er32(RXSTMPL);
533 rxstmp |= (u64)er32(RXSTMPH) << 32;
534 e1000e_systim_to_hwtstamp(adapter, skb_hwtstamps(skb), rxstmp);
535
536 adapter->flags2 &= ~FLAG2_CHECK_RX_HWTSTAMP;
537}
538
bc7f75fa 539/**
ad68076e 540 * e1000_receive_skb - helper function to handle Rx indications
bc7f75fa 541 * @adapter: board private structure
b67e1913 542 * @staterr: descriptor extended error and status field as written by hardware
bc7f75fa
AK		 543 * @vlan: descriptor vlan field as written by hardware (no le/be conversion)
544 * @skb: pointer to sk_buff to be indicated to stack
545 **/
546static void e1000_receive_skb(struct e1000_adapter *adapter,
af667a29 547 struct net_device *netdev, struct sk_buff *skb,
b67e1913 548 u32 staterr, __le16 vlan)
bc7f75fa 549{
86d70e53 550 u16 tag = le16_to_cpu(vlan);
b67e1913
BA		 551
552 e1000e_rx_hwtstamp(adapter, staterr, skb);
553
bc7f75fa
AK		 554 skb->protocol = eth_type_trans(skb, netdev);
555
b67e1913 556 if (staterr & E1000_RXD_STAT_VP)
86d70e53
JK		 557 __vlan_hwaccel_put_tag(skb, tag);
558
559 napi_gro_receive(&adapter->napi, skb);
bc7f75fa
AK		 560}
561
562/**
af667a29 563 * e1000_rx_checksum - Receive Checksum Offload
afd12939
BA		 564 * @adapter: board private structure
565 * @status_err: receive descriptor status and error fields
566 * @csum: receive descriptor csum field
567 * @sk_buff: socket buffer with received data
bc7f75fa
AK		 568 **/
569static void e1000_rx_checksum(struct e1000_adapter *adapter, u32 status_err,
2e1706f2 570 struct sk_buff *skb)
bc7f75fa
AK		 571{
572 u16 status = (u16)status_err;
573 u8 errors = (u8)(status_err >> 24);
bc8acf2c
ED		 574
575 skb_checksum_none_assert(skb);
bc7f75fa 576
afd12939
BA		 577 /* Rx checksum disabled */
578 if (!(adapter->netdev->features & NETIF_F_RXCSUM))
579 return;
580
bc7f75fa
AK		 581 /* Ignore Checksum bit is set */
582 if (status & E1000_RXD_STAT_IXSM)
583 return;
afd12939 584
2e1706f2
BA		 585 /* TCP/UDP checksum error bit or IP checksum error bit is set */
586 if (errors & (E1000_RXD_ERR_TCPE | E1000_RXD_ERR_IPE)) {
bc7f75fa
AK		 587 /* let the stack verify checksum errors */
588 adapter->hw_csum_err++;
589 return;
590 }
591
592 /* TCP/UDP Checksum has not been calculated */
593 if (!(status & (E1000_RXD_STAT_TCPCS | E1000_RXD_STAT_UDPCS)))
594 return;
595
596 /* It must be a TCP or UDP packet with a valid checksum */
2e1706f2 597 skb->ip_summed = CHECKSUM_UNNECESSARY;
bc7f75fa
AK		 598 adapter->hw_csum_good++;
599}
600
55aa6985 601static void e1000e_update_rdt_wa(struct e1000_ring *rx_ring, unsigned int i)
c6e7f51e 602{
55aa6985 603 struct e1000_adapter *adapter = rx_ring->adapter;
c6e7f51e 604 struct e1000_hw *hw = &adapter->hw;
bdc125f7
BA		 605 s32 ret_val = __ew32_prepare(hw);
606
607 writel(i, rx_ring->tail);
c6e7f51e 608
bdc125f7 609 if (unlikely(!ret_val && (i != readl(rx_ring->tail)))) {
c6e7f51e
BA		 610 u32 rctl = er32(RCTL);
611 ew32(RCTL, rctl & ~E1000_RCTL_EN);
612 e_err("ME firmware caused invalid RDT - resetting\n");
613 schedule_work(&adapter->reset_task);
614 }
615}
616
55aa6985 617static void e1000e_update_tdt_wa(struct e1000_ring *tx_ring, unsigned int i)
c6e7f51e 618{
55aa6985 619 struct e1000_adapter *adapter = tx_ring->adapter;
c6e7f51e 620 struct e1000_hw *hw = &adapter->hw;
bdc125f7 621 s32 ret_val = __ew32_prepare(hw);
c6e7f51e 622
bdc125f7
BA		 623 writel(i, tx_ring->tail);
624
625 if (unlikely(!ret_val && (i != readl(tx_ring->tail)))) {
c6e7f51e
BA		 626 u32 tctl = er32(TCTL);
627 ew32(TCTL, tctl & ~E1000_TCTL_EN);
628 e_err("ME firmware caused invalid TDT - resetting\n");
629 schedule_work(&adapter->reset_task);
630 }
631}
632
bc7f75fa 633/**
5f450212 634 * e1000_alloc_rx_buffers - Replace used receive buffers
55aa6985 635 * @rx_ring: Rx descriptor ring
bc7f75fa 636 **/
55aa6985 637static void e1000_alloc_rx_buffers(struct e1000_ring *rx_ring,
c2fed996 638 int cleaned_count, gfp_t gfp)
bc7f75fa 639{
55aa6985 640 struct e1000_adapter *adapter = rx_ring->adapter;
bc7f75fa
AK		 641 struct net_device *netdev = adapter->netdev;
642 struct pci_dev *pdev = adapter->pdev;
5f450212 643 union e1000_rx_desc_extended *rx_desc;
bc7f75fa
AK		 644 struct e1000_buffer *buffer_info;
645 struct sk_buff *skb;
646 unsigned int i;
89d71a66 647 unsigned int bufsz = adapter->rx_buffer_len;
bc7f75fa
AK		 648
649 i = rx_ring->next_to_use;
650 buffer_info = &rx_ring->buffer_info[i];
651
652 while (cleaned_count--) {
653 skb = buffer_info->skb;
654 if (skb) {
655 skb_trim(skb, 0);
656 goto map_skb;
657 }
658
c2fed996 659 skb = __netdev_alloc_skb_ip_align(netdev, bufsz, gfp);
bc7f75fa
AK		 660 if (!skb) {
661 /* Better luck next round */
662 adapter->alloc_rx_buff_failed++;
663 break;
664 }
665
bc7f75fa
AK		 666 buffer_info->skb = skb;
667map_skb:
0be3f55f 668 buffer_info->dma = dma_map_single(&pdev->dev, skb->data,
bc7f75fa 669 adapter->rx_buffer_len,
0be3f55f
NN		 670 DMA_FROM_DEVICE);
671 if (dma_mapping_error(&pdev->dev, buffer_info->dma)) {
af667a29 672 dev_err(&pdev->dev, "Rx DMA map failed\n");
bc7f75fa
AK		 673 adapter->rx_dma_failed++;
674 break;
675 }
676
5f450212
BA		 677 rx_desc = E1000_RX_DESC_EXT(*rx_ring, i);
678 rx_desc->read.buffer_addr = cpu_to_le64(buffer_info->dma);
bc7f75fa 679
50849d79 680 if (unlikely(!(i & (E1000_RX_BUFFER_WRITE - 1)))) {
e921eb1a 681 /* Force memory writes to complete before letting h/w
50849d79
TH		 682 * know there are new descriptors to fetch. (Only
683 * applicable for weak-ordered memory model archs,
684 * such as IA-64).
685 */
686 wmb();
c6e7f51e 687 if (adapter->flags2 & FLAG2_PCIM2PCI_ARBITER_WA)
55aa6985 688 e1000e_update_rdt_wa(rx_ring, i);
c6e7f51e 689 else
c5083cf6 690 writel(i, rx_ring->tail);
50849d79 691 }
bc7f75fa
AK		 692 i++;
693 if (i == rx_ring->count)
694 i = 0;
695 buffer_info = &rx_ring->buffer_info[i];
696 }
697
50849d79 698 rx_ring->next_to_use = i;
bc7f75fa
AK		 699}
700
701/**
702 * e1000_alloc_rx_buffers_ps - Replace used receive buffers; packet split
55aa6985 703 * @rx_ring: Rx descriptor ring
bc7f75fa 704 **/
55aa6985 705static void e1000_alloc_rx_buffers_ps(struct e1000_ring *rx_ring,
c2fed996 706 int cleaned_count, gfp_t gfp)
bc7f75fa 707{
55aa6985 708 struct e1000_adapter *adapter = rx_ring->adapter;
bc7f75fa
AK		 709 struct net_device *netdev = adapter->netdev;
710 struct pci_dev *pdev = adapter->pdev;
711 union e1000_rx_desc_packet_split *rx_desc;
bc7f75fa
AK		 712 struct e1000_buffer *buffer_info;
713 struct e1000_ps_page *ps_page;
714 struct sk_buff *skb;
715 unsigned int i, j;
716
717 i = rx_ring->next_to_use;
718 buffer_info = &rx_ring->buffer_info[i];
719
720 while (cleaned_count--) {
721 rx_desc = E1000_RX_DESC_PS(*rx_ring, i);
722
723 for (j = 0; j < PS_PAGE_BUFFERS; j++) {
47f44e40
AK		 724 ps_page = &buffer_info->ps_pages[j];
725 if (j >= adapter->rx_ps_pages) {
726 /* all unused desc entries get hw null ptr */
af667a29
BA		 727 rx_desc->read.buffer_addr[j + 1] =
728 ~cpu_to_le64(0);
47f44e40
AK		 729 continue;
730 }
731 if (!ps_page->page) {
c2fed996 732 ps_page->page = alloc_page(gfp);
bc7f75fa 733 if (!ps_page->page) {
47f44e40
AK		 734 adapter->alloc_rx_buff_failed++;
735 goto no_buffers;
736 }
0be3f55f
NN		 737 ps_page->dma = dma_map_page(&pdev->dev,
738 ps_page->page,
739 0, PAGE_SIZE,
740 DMA_FROM_DEVICE);
741 if (dma_mapping_error(&pdev->dev,
742 ps_page->dma)) {
47f44e40 743 dev_err(&adapter->pdev->dev,
af667a29 744 "Rx DMA page map failed\n");
47f44e40
AK		 745 adapter->rx_dma_failed++;
746 goto no_buffers;
bc7f75fa 747 }
bc7f75fa 748 }
e921eb1a 749 /* Refresh the desc even if buffer_addrs
47f44e40
AK		 750 * didn't change because each write-back
751 * erases this info.
752 */
af667a29
BA		 753 rx_desc->read.buffer_addr[j + 1] =
754 cpu_to_le64(ps_page->dma);
bc7f75fa
AK		 755 }
756
e5fe2541 757 skb = __netdev_alloc_skb_ip_align(netdev, adapter->rx_ps_bsize0,
c2fed996 758 gfp);
bc7f75fa
AK		 759
760 if (!skb) {
761 adapter->alloc_rx_buff_failed++;
762 break;
763 }
764
bc7f75fa 765 buffer_info->skb = skb;
0be3f55f 766 buffer_info->dma = dma_map_single(&pdev->dev, skb->data,
bc7f75fa 767 adapter->rx_ps_bsize0,
0be3f55f
NN		 768 DMA_FROM_DEVICE);
769 if (dma_mapping_error(&pdev->dev, buffer_info->dma)) {
af667a29 770 dev_err(&pdev->dev, "Rx DMA map failed\n");
bc7f75fa
AK		 771 adapter->rx_dma_failed++;
772 /* cleanup skb */
773 dev_kfree_skb_any(skb);
774 buffer_info->skb = NULL;
775 break;
776 }
777
778 rx_desc->read.buffer_addr[0] = cpu_to_le64(buffer_info->dma);
779
50849d79 780 if (unlikely(!(i & (E1000_RX_BUFFER_WRITE - 1)))) {
e921eb1a 781 /* Force memory writes to complete before letting h/w
50849d79
TH		 782 * know there are new descriptors to fetch. (Only
783 * applicable for weak-ordered memory model archs,
784 * such as IA-64).
785 */
786 wmb();
c6e7f51e 787 if (adapter->flags2 & FLAG2_PCIM2PCI_ARBITER_WA)
55aa6985 788 e1000e_update_rdt_wa(rx_ring, i << 1);
c6e7f51e 789 else
c5083cf6 790 writel(i << 1, rx_ring->tail);
50849d79
TH		 791 }
792
bc7f75fa
AK		 793 i++;
794 if (i == rx_ring->count)
795 i = 0;
796 buffer_info = &rx_ring->buffer_info[i];
797 }
798
799no_buffers:
50849d79 800 rx_ring->next_to_use = i;
bc7f75fa
AK		 801}
802
97ac8cae
BA		 803/**
804 * e1000_alloc_jumbo_rx_buffers - Replace used jumbo receive buffers
55aa6985 805 * @rx_ring: Rx descriptor ring
97ac8cae
BA		 806 * @cleaned_count: number of buffers to allocate this pass
807 **/
808
55aa6985 809static void e1000_alloc_jumbo_rx_buffers(struct e1000_ring *rx_ring,
c2fed996 810 int cleaned_count, gfp_t gfp)
97ac8cae 811{
55aa6985 812 struct e1000_adapter *adapter = rx_ring->adapter;
97ac8cae
BA		 813 struct net_device *netdev = adapter->netdev;
814 struct pci_dev *pdev = adapter->pdev;
5f450212 815 union e1000_rx_desc_extended *rx_desc;
97ac8cae
BA		 816 struct e1000_buffer *buffer_info;
817 struct sk_buff *skb;
818 unsigned int i;
2a2293b9 819 unsigned int bufsz = 256 - 16; /* for skb_reserve */
97ac8cae
BA		 820
821 i = rx_ring->next_to_use;
822 buffer_info = &rx_ring->buffer_info[i];
823
824 while (cleaned_count--) {
825 skb = buffer_info->skb;
826 if (skb) {
827 skb_trim(skb, 0);
828 goto check_page;
829 }
830
c2fed996 831 skb = __netdev_alloc_skb_ip_align(netdev, bufsz, gfp);
97ac8cae
BA		 832 if (unlikely(!skb)) {
833 /* Better luck next round */
834 adapter->alloc_rx_buff_failed++;
835 break;
836 }
837
97ac8cae
BA		 838 buffer_info->skb = skb;
839check_page:
840 /* allocate a new page if necessary */
841 if (!buffer_info->page) {
c2fed996 842 buffer_info->page = alloc_page(gfp);
97ac8cae
BA		 843 if (unlikely(!buffer_info->page)) {
844 adapter->alloc_rx_buff_failed++;
845 break;
846 }
847 }
848
849 if (!buffer_info->dma)
0be3f55f 850 buffer_info->dma = dma_map_page(&pdev->dev,
f0ff4398
BA		 851 buffer_info->page, 0,
852 PAGE_SIZE,
0be3f55f 853 DMA_FROM_DEVICE);
97ac8cae 854
5f450212
BA		 855 rx_desc = E1000_RX_DESC_EXT(*rx_ring, i);
856 rx_desc->read.buffer_addr = cpu_to_le64(buffer_info->dma);
97ac8cae
BA		 857
858 if (unlikely(++i == rx_ring->count))
859 i = 0;
860 buffer_info = &rx_ring->buffer_info[i];
861 }
862
863 if (likely(rx_ring->next_to_use != i)) {
864 rx_ring->next_to_use = i;
865 if (unlikely(i-- == 0))
866 i = (rx_ring->count - 1);
867
868 /* Force memory writes to complete before letting h/w
869 * know there are new descriptors to fetch. (Only
870 * applicable for weak-ordered memory model archs,
e921eb1a
BA		 871 * such as IA-64).
872 */
97ac8cae 873 wmb();
c6e7f51e 874 if (adapter->flags2 & FLAG2_PCIM2PCI_ARBITER_WA)
55aa6985 875 e1000e_update_rdt_wa(rx_ring, i);
c6e7f51e 876 else
c5083cf6 877 writel(i, rx_ring->tail);
97ac8cae
BA		 878 }
879}
880
70495a50
BA		 881static inline void e1000_rx_hash(struct net_device *netdev, __le32 rss,
882 struct sk_buff *skb)
883{
884 if (netdev->features & NETIF_F_RXHASH)
885 skb->rxhash = le32_to_cpu(rss);
886}
887
bc7f75fa 888/**
55aa6985
BA		 889 * e1000_clean_rx_irq - Send received data up the network stack
890 * @rx_ring: Rx descriptor ring
bc7f75fa
AK		 891 *
892 * the return value indicates whether actual cleaning was done, there
893 * is no guarantee that everything was cleaned
894 **/
55aa6985
BA		 895static bool e1000_clean_rx_irq(struct e1000_ring *rx_ring, int *work_done,
896 int work_to_do)
bc7f75fa 897{
55aa6985 898 struct e1000_adapter *adapter = rx_ring->adapter;
bc7f75fa
AK		 899 struct net_device *netdev = adapter->netdev;
900 struct pci_dev *pdev = adapter->pdev;
3bb99fe2 901 struct e1000_hw *hw = &adapter->hw;
5f450212 902 union e1000_rx_desc_extended *rx_desc, *next_rxd;
bc7f75fa 903 struct e1000_buffer *buffer_info, *next_buffer;
5f450212 904 u32 length, staterr;
bc7f75fa
AK		 905 unsigned int i;
906 int cleaned_count = 0;
3db1cd5c 907 bool cleaned = false;
bc7f75fa
AK		 908 unsigned int total_rx_bytes = 0, total_rx_packets = 0;
909
910 i = rx_ring->next_to_clean;
5f450212
BA		 911 rx_desc = E1000_RX_DESC_EXT(*rx_ring, i);
912 staterr = le32_to_cpu(rx_desc->wb.upper.status_error);
bc7f75fa
AK		 913 buffer_info = &rx_ring->buffer_info[i];
914
5f450212 915 while (staterr & E1000_RXD_STAT_DD) {
bc7f75fa 916 struct sk_buff *skb;
bc7f75fa
AK		 917
918 if (*work_done >= work_to_do)
919 break;
920 (*work_done)++;
2d0bb1c1 921 rmb(); /* read descriptor and rx_buffer_info after status DD */
bc7f75fa 922
bc7f75fa
AK		 923 skb = buffer_info->skb;
924 buffer_info->skb = NULL;
925
926 prefetch(skb->data - NET_IP_ALIGN);
927
928 i++;
929 if (i == rx_ring->count)
930 i = 0;
5f450212 931 next_rxd = E1000_RX_DESC_EXT(*rx_ring, i);
bc7f75fa
AK		 932 prefetch(next_rxd);
933
934 next_buffer = &rx_ring->buffer_info[i];
935
3db1cd5c 936 cleaned = true;
bc7f75fa 937 cleaned_count++;
e5fe2541
BA		 938 dma_unmap_single(&pdev->dev, buffer_info->dma,
939 adapter->rx_buffer_len, DMA_FROM_DEVICE);
bc7f75fa
AK		 940 buffer_info->dma = 0;
941
5f450212 942 length = le16_to_cpu(rx_desc->wb.upper.length);
bc7f75fa 943
e921eb1a 944 /* !EOP means multiple descriptors were used to store a single
b94b5028
JB		 945 * packet, if that's the case we need to toss it. In fact, we
946 * need to toss every packet with the EOP bit clear and the
947 * next frame that _does_ have the EOP bit set, as it is by
948 * definition only a frame fragment
949 */
5f450212 950 if (unlikely(!(staterr & E1000_RXD_STAT_EOP)))
b94b5028
JB		 951 adapter->flags2 |= FLAG2_IS_DISCARDING;
952
953 if (adapter->flags2 & FLAG2_IS_DISCARDING) {
bc7f75fa 954 /* All receives must fit into a single buffer */
3bb99fe2 955 e_dbg("Receive packet consumed multiple buffers\n");
bc7f75fa
AK		 956 /* recycle */
957 buffer_info->skb = skb;
5f450212 958 if (staterr & E1000_RXD_STAT_EOP)
b94b5028 959 adapter->flags2 &= ~FLAG2_IS_DISCARDING;
bc7f75fa
AK		 960 goto next_desc;
961 }
962
cf955e6c
BG		 963 if (unlikely((staterr & E1000_RXDEXT_ERR_FRAME_ERR_MASK) &&
964 !(netdev->features & NETIF_F_RXALL))) {
bc7f75fa
AK		 965 /* recycle */
966 buffer_info->skb = skb;
967 goto next_desc;
968 }
969
eb7c3adb 970 /* adjust length to remove Ethernet CRC */
0184039a
BG		 971 if (!(adapter->flags2 & FLAG2_CRC_STRIPPING)) {
972 /* If configured to store CRC, don't subtract FCS,
973 * but keep the FCS bytes out of the total_rx_bytes
974 * counter
975 */
976 if (netdev->features & NETIF_F_RXFCS)
977 total_rx_bytes -= 4;
978 else
979 length -= 4;
980 }
eb7c3adb 981
bc7f75fa
AK		 982 total_rx_bytes += length;
983 total_rx_packets++;
984
e921eb1a 985 /* code added for copybreak, this should improve
bc7f75fa 986 * performance for small packets with large amounts
ad68076e
BA		 987 * of reassembly being done in the stack
988 */
bc7f75fa
AK		 989 if (length < copybreak) {
990 struct sk_buff *new_skb =
89d71a66 991 netdev_alloc_skb_ip_align(netdev, length);
bc7f75fa 992 if (new_skb) {
808ff676
BA		 993 skb_copy_to_linear_data_offset(new_skb,
994 -NET_IP_ALIGN,
995 (skb->data -
996 NET_IP_ALIGN),
997 (length +
998 NET_IP_ALIGN));
bc7f75fa
AK		 999 /* save the skb in buffer_info as good */
1000 buffer_info->skb = skb;
1001 skb = new_skb;
1002 }
1003 /* else just continue with the old one */
1004 }
1005 /* end copybreak code */
1006 skb_put(skb, length);
1007
1008 /* Receive Checksum Offload */
2e1706f2 1009 e1000_rx_checksum(adapter, staterr, skb);
bc7f75fa 1010
70495a50
BA		 1011 e1000_rx_hash(netdev, rx_desc->wb.lower.hi_dword.rss, skb);
1012
5f450212
BA		 1013 e1000_receive_skb(adapter, netdev, skb, staterr,
1014 rx_desc->wb.upper.vlan);
bc7f75fa
AK		 1015
1016next_desc:
5f450212 1017 rx_desc->wb.upper.status_error &= cpu_to_le32(~0xFF);
bc7f75fa
AK		 1018
1019 /* return some buffers to hardware, one at a time is too slow */
1020 if (cleaned_count >= E1000_RX_BUFFER_WRITE) {
55aa6985 1021 adapter->alloc_rx_buf(rx_ring, cleaned_count,
c2fed996 1022 GFP_ATOMIC);
bc7f75fa
AK		 1023 cleaned_count = 0;
1024 }
1025
1026 /* use prefetched values */
1027 rx_desc = next_rxd;
1028 buffer_info = next_buffer;
5f450212
BA		 1029
1030 staterr = le32_to_cpu(rx_desc->wb.upper.status_error);
bc7f75fa
AK		 1031 }
1032 rx_ring->next_to_clean = i;
1033
1034 cleaned_count = e1000_desc_unused(rx_ring);
1035 if (cleaned_count)
55aa6985 1036 adapter->alloc_rx_buf(rx_ring, cleaned_count, GFP_ATOMIC);
bc7f75fa 1037
bc7f75fa 1038 adapter->total_rx_bytes += total_rx_bytes;
7c25769f 1039 adapter->total_rx_packets += total_rx_packets;
bc7f75fa
AK		 1040 return cleaned;
1041}
1042
55aa6985
BA		 1043static void e1000_put_txbuf(struct e1000_ring *tx_ring,
1044 struct e1000_buffer *buffer_info)
bc7f75fa 1045{
55aa6985
BA		 1046 struct e1000_adapter *adapter = tx_ring->adapter;
1047
03b1320d
AD		 1048 if (buffer_info->dma) {
1049 if (buffer_info->mapped_as_page)
0be3f55f
NN		 1050 dma_unmap_page(&adapter->pdev->dev, buffer_info->dma,
1051 buffer_info->length, DMA_TO_DEVICE);
03b1320d 1052 else
0be3f55f
NN		 1053 dma_unmap_single(&adapter->pdev->dev, buffer_info->dma,
1054 buffer_info->length, DMA_TO_DEVICE);
03b1320d
AD		 1055 buffer_info->dma = 0;
1056 }
bc7f75fa
AK		 1057 if (buffer_info->skb) {
1058 dev_kfree_skb_any(buffer_info->skb);
1059 buffer_info->skb = NULL;
1060 }
1b7719c4 1061 buffer_info->time_stamp = 0;
bc7f75fa
AK		 1062}
1063
41cec6f1 1064static void e1000_print_hw_hang(struct work_struct *work)
bc7f75fa 1065{
41cec6f1 1066 struct e1000_adapter *adapter = container_of(work,
f0ff4398
BA		 1067 struct e1000_adapter,
1068 print_hang_task);
09357b00 1069 struct net_device *netdev = adapter->netdev;
bc7f75fa
AK		 1070 struct e1000_ring *tx_ring = adapter->tx_ring;
1071 unsigned int i = tx_ring->next_to_clean;
1072 unsigned int eop = tx_ring->buffer_info[i].next_to_watch;
1073 struct e1000_tx_desc *eop_desc = E1000_TX_DESC(*tx_ring, eop);
41cec6f1
BA		 1074 struct e1000_hw *hw = &adapter->hw;
1075 u16 phy_status, phy_1000t_status, phy_ext_status;
1076 u16 pci_status;
1077
615b32af
JB		 1078 if (test_bit(__E1000_DOWN, &adapter->state))
1079 return;
1080
e5fe2541 1081 if (!adapter->tx_hang_recheck && (adapter->flags2 & FLAG2_DMA_BURST)) {
e921eb1a 1082 /* May be block on write-back, flush and detect again
09357b00
JK		 1083 * flush pending descriptor writebacks to memory
1084 */
1085 ew32(TIDV, adapter->tx_int_delay | E1000_TIDV_FPD);
1086 /* execute the writes immediately */
1087 e1e_flush();
e921eb1a 1088 /* Due to rare timing issues, write to TIDV again to ensure
bf03085f
MV		 1089 * the write is successful
1090 */
1091 ew32(TIDV, adapter->tx_int_delay | E1000_TIDV_FPD);
1092 /* execute the writes immediately */
1093 e1e_flush();
09357b00
JK		 1094 adapter->tx_hang_recheck = true;
1095 return;
1096 }
1097 /* Real hang detected */
1098 adapter->tx_hang_recheck = false;
1099 netif_stop_queue(netdev);
1100
c2ade1a4
BA		 1101 e1e_rphy(hw, MII_BMSR, &phy_status);
1102 e1e_rphy(hw, MII_STAT1000, &phy_1000t_status);
1103 e1e_rphy(hw, MII_ESTATUS, &phy_ext_status);
bc7f75fa 1104
41cec6f1
BA		 1105 pci_read_config_word(adapter->pdev, PCI_STATUS, &pci_status);
1106
1107 /* detected Hardware unit hang */
1108 e_err("Detected Hardware Unit Hang:\n"
44defeb3
JK		 1109 " TDH <%x>\n"
1110 " TDT <%x>\n"
1111 " next_to_use <%x>\n"
1112 " next_to_clean <%x>\n"
1113 "buffer_info[next_to_clean]:\n"
1114 " time_stamp <%lx>\n"
1115 " next_to_watch <%x>\n"
1116 " jiffies <%lx>\n"
41cec6f1
BA		 1117 " next_to_watch.status <%x>\n"
1118 "MAC Status <%x>\n"
1119 "PHY Status <%x>\n"
1120 "PHY 1000BASE-T Status <%x>\n"
1121 "PHY Extended Status <%x>\n"
1122 "PCI Status <%x>\n",
e5fe2541
BA		 1123 readl(tx_ring->head), readl(tx_ring->tail), tx_ring->next_to_use,
1124 tx_ring->next_to_clean, tx_ring->buffer_info[eop].time_stamp,
1125 eop, jiffies, eop_desc->upper.fields.status, er32(STATUS),
1126 phy_status, phy_1000t_status, phy_ext_status, pci_status);
7c0427ee
BA		 1127
1128 /* Suggest workaround for known h/w issue */
1129 if ((hw->mac.type == e1000_pchlan) && (er32(CTRL) & E1000_CTRL_TFCE))
1130 e_err("Try turning off Tx pause (flow control) via ethtool\n");
bc7f75fa
AK		 1131}
1132
b67e1913
BA		 1133/**
1134 * e1000e_tx_hwtstamp_work - check for Tx time stamp
1135 * @work: pointer to work struct
1136 *
1137 * This work function polls the TSYNCTXCTL valid bit to determine when a
1138 * timestamp has been taken for the current stored skb. The timestamp must
1139 * be for this skb because only one such packet is allowed in the queue.
1140 */
1141static void e1000e_tx_hwtstamp_work(struct work_struct *work)
1142{
1143 struct e1000_adapter *adapter = container_of(work, struct e1000_adapter,
1144 tx_hwtstamp_work);
1145 struct e1000_hw *hw = &adapter->hw;
1146
1147 if (!adapter->tx_hwtstamp_skb)
1148 return;
1149
1150 if (er32(TSYNCTXCTL) & E1000_TSYNCTXCTL_VALID) {
1151 struct skb_shared_hwtstamps shhwtstamps;
1152 u64 txstmp;
1153
1154 txstmp = er32(TXSTMPL);
1155 txstmp |= (u64)er32(TXSTMPH) << 32;
1156
1157 e1000e_systim_to_hwtstamp(adapter, &shhwtstamps, txstmp);
1158
1159 skb_tstamp_tx(adapter->tx_hwtstamp_skb, &shhwtstamps);
1160 dev_kfree_skb_any(adapter->tx_hwtstamp_skb);
1161 adapter->tx_hwtstamp_skb = NULL;
1162 } else {
1163 /* reschedule to check later */
1164 schedule_work(&adapter->tx_hwtstamp_work);
1165 }
1166}
1167
bc7f75fa
AK		 1168/**
1169 * e1000_clean_tx_irq - Reclaim resources after transmit completes
55aa6985 1170 * @tx_ring: Tx descriptor ring
bc7f75fa
AK		 1171 *
1172 * the return value indicates whether actual cleaning was done, there
1173 * is no guarantee that everything was cleaned
1174 **/
55aa6985 1175static bool e1000_clean_tx_irq(struct e1000_ring *tx_ring)
bc7f75fa 1176{
55aa6985 1177 struct e1000_adapter *adapter = tx_ring->adapter;
bc7f75fa
AK		 1178 struct net_device *netdev = adapter->netdev;
1179 struct e1000_hw *hw = &adapter->hw;
bc7f75fa
AK		 1180 struct e1000_tx_desc *tx_desc, *eop_desc;
1181 struct e1000_buffer *buffer_info;
1182 unsigned int i, eop;
1183 unsigned int count = 0;
bc7f75fa 1184 unsigned int total_tx_bytes = 0, total_tx_packets = 0;
3f0cfa3b 1185 unsigned int bytes_compl = 0, pkts_compl = 0;
bc7f75fa
AK		 1186
1187 i = tx_ring->next_to_clean;
1188 eop = tx_ring->buffer_info[i].next_to_watch;
1189 eop_desc = E1000_TX_DESC(*tx_ring, eop);
1190
12d04a3c
AD		 1191 while ((eop_desc->upper.data & cpu_to_le32(E1000_TXD_STAT_DD)) &&
1192 (count < tx_ring->count)) {
a86043c2 1193 bool cleaned = false;
2d0bb1c1 1194 rmb(); /* read buffer_info after eop_desc */
a86043c2 1195 for (; !cleaned; count++) {
bc7f75fa
AK		 1196 tx_desc = E1000_TX_DESC(*tx_ring, i);
1197 buffer_info = &tx_ring->buffer_info[i];
1198 cleaned = (i == eop);
1199
1200 if (cleaned) {
9ed318d5
TH		 1201 total_tx_packets += buffer_info->segs;
1202 total_tx_bytes += buffer_info->bytecount;
3f0cfa3b
TH		 1203 if (buffer_info->skb) {
1204 bytes_compl += buffer_info->skb->len;
1205 pkts_compl++;
1206 }
bc7f75fa
AK		 1207 }
1208
55aa6985 1209 e1000_put_txbuf(tx_ring, buffer_info);
bc7f75fa
AK		 1210 tx_desc->upper.data = 0;
1211
1212 i++;
1213 if (i == tx_ring->count)
1214 i = 0;
1215 }
1216
dac87619
TL		 1217 if (i == tx_ring->next_to_use)
1218 break;
bc7f75fa
AK		 1219 eop = tx_ring->buffer_info[i].next_to_watch;
1220 eop_desc = E1000_TX_DESC(*tx_ring, eop);
bc7f75fa
AK		 1221 }
1222
1223 tx_ring->next_to_clean = i;
1224
3f0cfa3b
TH		 1225 netdev_completed_queue(netdev, pkts_compl, bytes_compl);
1226
bc7f75fa 1227#define TX_WAKE_THRESHOLD 32
a86043c2
JB		 1228 if (count && netif_carrier_ok(netdev) &&
1229 e1000_desc_unused(tx_ring) >= TX_WAKE_THRESHOLD) {
bc7f75fa
AK		 1230 /* Make sure that anybody stopping the queue after this
1231 * sees the new next_to_clean.
1232 */
1233 smp_mb();
1234
1235 if (netif_queue_stopped(netdev) &&
1236 !(test_bit(__E1000_DOWN, &adapter->state))) {
1237 netif_wake_queue(netdev);
1238 ++adapter->restart_queue;
1239 }
1240 }
1241
1242 if (adapter->detect_tx_hung) {
e921eb1a 1243 /* Detect a transmit hang in hardware, this serializes the
41cec6f1
BA		 1244 * check with the clearing of time_stamp and movement of i
1245 */
3db1cd5c 1246 adapter->detect_tx_hung = false;
12d04a3c
AD		 1247 if (tx_ring->buffer_info[i].time_stamp &&
1248 time_after(jiffies, tx_ring->buffer_info[i].time_stamp
8e95a202 1249 + (adapter->tx_timeout_factor * HZ)) &&
09357b00 1250 !(er32(STATUS) & E1000_STATUS_TXOFF))
41cec6f1 1251 schedule_work(&adapter->print_hang_task);
09357b00
JK		 1252 else
1253 adapter->tx_hang_recheck = false;
bc7f75fa
AK		 1254 }
1255 adapter->total_tx_bytes += total_tx_bytes;
1256 adapter->total_tx_packets += total_tx_packets;
807540ba 1257 return count < tx_ring->count;
bc7f75fa
AK		 1258}
1259
bc7f75fa
AK		 1260/**
1261 * e1000_clean_rx_irq_ps - Send received data up the network stack; packet split
55aa6985 1262 * @rx_ring: Rx descriptor ring
bc7f75fa
AK		 1263 *
1264 * the return value indicates whether actual cleaning was done, there
1265 * is no guarantee that everything was cleaned
1266 **/
55aa6985
BA		 1267static bool e1000_clean_rx_irq_ps(struct e1000_ring *rx_ring, int *work_done,
1268 int work_to_do)
bc7f75fa 1269{
55aa6985 1270 struct e1000_adapter *adapter = rx_ring->adapter;
3bb99fe2 1271 struct e1000_hw *hw = &adapter->hw;
bc7f75fa
AK		 1272 union e1000_rx_desc_packet_split *rx_desc, *next_rxd;
1273 struct net_device *netdev = adapter->netdev;
1274 struct pci_dev *pdev = adapter->pdev;
bc7f75fa
AK		 1275 struct e1000_buffer *buffer_info, *next_buffer;
1276 struct e1000_ps_page *ps_page;
1277 struct sk_buff *skb;
1278 unsigned int i, j;
1279 u32 length, staterr;
1280 int cleaned_count = 0;
3db1cd5c 1281 bool cleaned = false;
bc7f75fa
AK		 1282 unsigned int total_rx_bytes = 0, total_rx_packets = 0;
1283
1284 i = rx_ring->next_to_clean;
1285 rx_desc = E1000_RX_DESC_PS(*rx_ring, i);
1286 staterr = le32_to_cpu(rx_desc->wb.middle.status_error);
1287 buffer_info = &rx_ring->buffer_info[i];
1288
1289 while (staterr & E1000_RXD_STAT_DD) {
1290 if (*work_done >= work_to_do)
1291 break;
1292 (*work_done)++;
1293 skb = buffer_info->skb;
2d0bb1c1 1294 rmb(); /* read descriptor and rx_buffer_info after status DD */
bc7f75fa
AK		 1295
1296 /* in the packet split case this is header only */
1297 prefetch(skb->data - NET_IP_ALIGN);
1298
1299 i++;
1300 if (i == rx_ring->count)
1301 i = 0;
1302 next_rxd = E1000_RX_DESC_PS(*rx_ring, i);
1303 prefetch(next_rxd);
1304
1305 next_buffer = &rx_ring->buffer_info[i];
1306
3db1cd5c 1307 cleaned = true;
bc7f75fa 1308 cleaned_count++;
0be3f55f 1309 dma_unmap_single(&pdev->dev, buffer_info->dma,
af667a29 1310 adapter->rx_ps_bsize0, DMA_FROM_DEVICE);
bc7f75fa
AK		 1311 buffer_info->dma = 0;
1312
af667a29 1313 /* see !EOP comment in other Rx routine */
b94b5028
JB		 1314 if (!(staterr & E1000_RXD_STAT_EOP))
1315 adapter->flags2 |= FLAG2_IS_DISCARDING;
1316
1317 if (adapter->flags2 & FLAG2_IS_DISCARDING) {
ef456f85 1318 e_dbg("Packet Split buffers didn't pick up the full packet\n");
bc7f75fa 1319 dev_kfree_skb_irq(skb);
b94b5028
JB		 1320 if (staterr & E1000_RXD_STAT_EOP)
1321 adapter->flags2 &= ~FLAG2_IS_DISCARDING;
bc7f75fa
AK		 1322 goto next_desc;
1323 }
1324
cf955e6c
BG		 1325 if (unlikely((staterr & E1000_RXDEXT_ERR_FRAME_ERR_MASK) &&
1326 !(netdev->features & NETIF_F_RXALL))) {
bc7f75fa
AK		 1327 dev_kfree_skb_irq(skb);
1328 goto next_desc;
1329 }
1330
1331 length = le16_to_cpu(rx_desc->wb.middle.length0);
1332
1333 if (!length) {
ef456f85 1334 e_dbg("Last part of the packet spanning multiple descriptors\n");
bc7f75fa
AK		 1335 dev_kfree_skb_irq(skb);
1336 goto next_desc;
1337 }
1338
1339 /* Good Receive */
1340 skb_put(skb, length);
1341
1342 {
e921eb1a 1343 /* this looks ugly, but it seems compiler issues make
0e15df49
BA		 1344 * it more efficient than reusing j
1345 */
1346 int l1 = le16_to_cpu(rx_desc->wb.upper.length[0]);
bc7f75fa 1347
e921eb1a 1348 /* page alloc/put takes too long and effects small
0e15df49
BA		 1349 * packet throughput, so unsplit small packets and
1350 * save the alloc/put only valid in softirq (napi)
1351 * context to call kmap_*
ad68076e 1352 */
0e15df49
BA		 1353 if (l1 && (l1 <= copybreak) &&
1354 ((length + l1) <= adapter->rx_ps_bsize0)) {
1355 u8 *vaddr;
1356
1357 ps_page = &buffer_info->ps_pages[0];
1358
e921eb1a 1359 /* there is no documentation about how to call
0e15df49
BA		 1360 * kmap_atomic, so we can't hold the mapping
1361 * very long
1362 */
1363 dma_sync_single_for_cpu(&pdev->dev,
1364 ps_page->dma,
1365 PAGE_SIZE,
1366 DMA_FROM_DEVICE);
9f393834 1367 vaddr = kmap_atomic(ps_page->page);
0e15df49 1368 memcpy(skb_tail_pointer(skb), vaddr, l1);
9f393834 1369 kunmap_atomic(vaddr);
0e15df49
BA		 1370 dma_sync_single_for_device(&pdev->dev,
1371 ps_page->dma,
1372 PAGE_SIZE,
1373 DMA_FROM_DEVICE);
1374
1375 /* remove the CRC */
0184039a
BG		 1376 if (!(adapter->flags2 & FLAG2_CRC_STRIPPING)) {
1377 if (!(netdev->features & NETIF_F_RXFCS))
1378 l1 -= 4;
1379 }
0e15df49
BA		 1380
1381 skb_put(skb, l1);
1382 goto copydone;
1383 } /* if */
bc7f75fa
AK		 1384 }
1385
1386 for (j = 0; j < PS_PAGE_BUFFERS; j++) {
1387 length = le16_to_cpu(rx_desc->wb.upper.length[j]);
1388 if (!length)
1389 break;
1390
47f44e40 1391 ps_page = &buffer_info->ps_pages[j];
0be3f55f
NN		 1392 dma_unmap_page(&pdev->dev, ps_page->dma, PAGE_SIZE,
1393 DMA_FROM_DEVICE);
bc7f75fa
AK		 1394 ps_page->dma = 0;
1395 skb_fill_page_desc(skb, j, ps_page->page, 0, length);
1396 ps_page->page = NULL;
1397 skb->len += length;
1398 skb->data_len += length;
98a045d7 1399 skb->truesize += PAGE_SIZE;
bc7f75fa
AK		 1400 }
1401
eb7c3adb
JK		 1402 /* strip the ethernet crc, problem is we're using pages now so
1403 * this whole operation can get a little cpu intensive
1404 */
0184039a
BG		 1405 if (!(adapter->flags2 & FLAG2_CRC_STRIPPING)) {
1406 if (!(netdev->features & NETIF_F_RXFCS))
1407 pskb_trim(skb, skb->len - 4);
1408 }
eb7c3adb 1409
bc7f75fa
AK		 1410copydone:
1411 total_rx_bytes += skb->len;
1412 total_rx_packets++;
1413
2e1706f2 1414 e1000_rx_checksum(adapter, staterr, skb);
bc7f75fa 1415
70495a50
BA		 1416 e1000_rx_hash(netdev, rx_desc->wb.lower.hi_dword.rss, skb);
1417
bc7f75fa 1418 if (rx_desc->wb.upper.header_status &
17e813ec 1419 cpu_to_le16(E1000_RXDPS_HDRSTAT_HDRSP))
bc7f75fa
AK		 1420 adapter->rx_hdr_split++;
1421
b67e1913
BA		 1422 e1000_receive_skb(adapter, netdev, skb, staterr,
1423 rx_desc->wb.middle.vlan);
bc7f75fa
AK		 1424
1425next_desc:
1426 rx_desc->wb.middle.status_error &= cpu_to_le32(~0xFF);
1427 buffer_info->skb = NULL;
1428
1429 /* return some buffers to hardware, one at a time is too slow */
1430 if (cleaned_count >= E1000_RX_BUFFER_WRITE) {
55aa6985 1431 adapter->alloc_rx_buf(rx_ring, cleaned_count,
c2fed996 1432 GFP_ATOMIC);
bc7f75fa
AK		 1433 cleaned_count = 0;
1434 }
1435
1436 /* use prefetched values */
1437 rx_desc = next_rxd;
1438 buffer_info = next_buffer;
1439
1440 staterr = le32_to_cpu(rx_desc->wb.middle.status_error);
1441 }
1442 rx_ring->next_to_clean = i;
1443
1444 cleaned_count = e1000_desc_unused(rx_ring);
1445 if (cleaned_count)
55aa6985 1446 adapter->alloc_rx_buf(rx_ring, cleaned_count, GFP_ATOMIC);
bc7f75fa 1447
bc7f75fa 1448 adapter->total_rx_bytes += total_rx_bytes;
7c25769f 1449 adapter->total_rx_packets += total_rx_packets;
bc7f75fa
AK		 1450 return cleaned;
1451}
1452
97ac8cae
BA		 1453/**
1454 * e1000_consume_page - helper function
1455 **/
1456static void e1000_consume_page(struct e1000_buffer *bi, struct sk_buff *skb,
66501f56 1457 u16 length)
97ac8cae
BA		 1458{
1459 bi->page = NULL;
1460 skb->len += length;
1461 skb->data_len += length;
98a045d7 1462 skb->truesize += PAGE_SIZE;
97ac8cae
BA		 1463}
1464
1465/**
1466 * e1000_clean_jumbo_rx_irq - Send received data up the network stack; legacy
1467 * @adapter: board private structure
1468 *
1469 * the return value indicates whether actual cleaning was done, there
1470 * is no guarantee that everything was cleaned
1471 **/
55aa6985
BA		 1472static bool e1000_clean_jumbo_rx_irq(struct e1000_ring *rx_ring, int *work_done,
1473 int work_to_do)
97ac8cae 1474{
55aa6985 1475 struct e1000_adapter *adapter = rx_ring->adapter;
97ac8cae
BA		 1476 struct net_device *netdev = adapter->netdev;
1477 struct pci_dev *pdev = adapter->pdev;
5f450212 1478 union e1000_rx_desc_extended *rx_desc, *next_rxd;
97ac8cae 1479 struct e1000_buffer *buffer_info, *next_buffer;
5f450212 1480 u32 length, staterr;
97ac8cae
BA		 1481 unsigned int i;
1482 int cleaned_count = 0;
1483 bool cleaned = false;
362e20ca 1484 unsigned int total_rx_bytes = 0, total_rx_packets = 0;
17e813ec 1485 struct skb_shared_info *shinfo;
97ac8cae
BA		 1486
1487 i = rx_ring->next_to_clean;
5f450212
BA		 1488 rx_desc = E1000_RX_DESC_EXT(*rx_ring, i);
1489 staterr = le32_to_cpu(rx_desc->wb.upper.status_error);
97ac8cae
BA		 1490 buffer_info = &rx_ring->buffer_info[i];
1491
5f450212 1492 while (staterr & E1000_RXD_STAT_DD) {
97ac8cae 1493 struct sk_buff *skb;
97ac8cae
BA		 1494
1495 if (*work_done >= work_to_do)
1496 break;
1497 (*work_done)++;
2d0bb1c1 1498 rmb(); /* read descriptor and rx_buffer_info after status DD */
97ac8cae 1499
97ac8cae
BA		 1500 skb = buffer_info->skb;
1501 buffer_info->skb = NULL;
1502
1503 ++i;
1504 if (i == rx_ring->count)
1505 i = 0;
5f450212 1506 next_rxd = E1000_RX_DESC_EXT(*rx_ring, i);
97ac8cae
BA		 1507 prefetch(next_rxd);
1508
1509 next_buffer = &rx_ring->buffer_info[i];
1510
1511 cleaned = true;
1512 cleaned_count++;
0be3f55f
NN		 1513 dma_unmap_page(&pdev->dev, buffer_info->dma, PAGE_SIZE,
1514 DMA_FROM_DEVICE);
97ac8cae
BA		 1515 buffer_info->dma = 0;
1516
5f450212 1517 length = le16_to_cpu(rx_desc->wb.upper.length);
97ac8cae
BA		 1518
1519 /* errors is only valid for DD + EOP descriptors */
5f450212 1520 if (unlikely((staterr & E1000_RXD_STAT_EOP) &&
cf955e6c
BG		 1521 ((staterr & E1000_RXDEXT_ERR_FRAME_ERR_MASK) &&
1522 !(netdev->features & NETIF_F_RXALL)))) {
5f450212
BA		 1523 /* recycle both page and skb */
1524 buffer_info->skb = skb;
1525 /* an error means any chain goes out the window too */
1526 if (rx_ring->rx_skb_top)
1527 dev_kfree_skb_irq(rx_ring->rx_skb_top);
1528 rx_ring->rx_skb_top = NULL;
1529 goto next_desc;
97ac8cae 1530 }
f0f1a172 1531#define rxtop (rx_ring->rx_skb_top)
5f450212 1532 if (!(staterr & E1000_RXD_STAT_EOP)) {
97ac8cae
BA		 1533 /* this descriptor is only the beginning (or middle) */
1534 if (!rxtop) {
1535 /* this is the beginning of a chain */
1536 rxtop = skb;
1537 skb_fill_page_desc(rxtop, 0, buffer_info->page,
f0ff4398 1538 0, length);
97ac8cae
BA		 1539 } else {
1540 /* this is the middle of a chain */
17e813ec
BA		 1541 shinfo = skb_shinfo(rxtop);
1542 skb_fill_page_desc(rxtop, shinfo->nr_frags,
1543 buffer_info->page, 0,
1544 length);
97ac8cae
BA		 1545 /* re-use the skb, only consumed the page */
1546 buffer_info->skb = skb;
1547 }
1548 e1000_consume_page(buffer_info, rxtop, length);
1549 goto next_desc;
1550 } else {
1551 if (rxtop) {
1552 /* end of the chain */
17e813ec
BA		 1553 shinfo = skb_shinfo(rxtop);
1554 skb_fill_page_desc(rxtop, shinfo->nr_frags,
1555 buffer_info->page, 0,
1556 length);
97ac8cae 1557 /* re-use the current skb, we only consumed the
e921eb1a
BA		 1558 * page
1559 */
97ac8cae
BA		 1560 buffer_info->skb = skb;
1561 skb = rxtop;
1562 rxtop = NULL;
1563 e1000_consume_page(buffer_info, skb, length);
1564 } else {
1565 /* no chain, got EOP, this buf is the packet
e921eb1a
BA		 1566 * copybreak to save the put_page/alloc_page
1567 */
97ac8cae
BA		 1568 if (length <= copybreak &&
1569 skb_tailroom(skb) >= length) {
1570 u8 *vaddr;
4679026d 1571 vaddr = kmap_atomic(buffer_info->page);
97ac8cae
BA		 1572 memcpy(skb_tail_pointer(skb), vaddr,
1573 length);
4679026d 1574 kunmap_atomic(vaddr);
97ac8cae 1575 /* re-use the page, so don't erase
e921eb1a
BA		 1576 * buffer_info->page
1577 */
97ac8cae
BA		 1578 skb_put(skb, length);
1579 } else {
1580 skb_fill_page_desc(skb, 0,
f0ff4398
BA		 1581 buffer_info->page, 0,
1582 length);
97ac8cae 1583 e1000_consume_page(buffer_info, skb,
f0ff4398 1584 length);
97ac8cae
BA		 1585 }
1586 }
1587 }
1588
2e1706f2
BA		 1589 /* Receive Checksum Offload */
1590 e1000_rx_checksum(adapter, staterr, skb);
97ac8cae 1591
70495a50
BA		 1592 e1000_rx_hash(netdev, rx_desc->wb.lower.hi_dword.rss, skb);
1593
97ac8cae
BA		 1594 /* probably a little skewed due to removing CRC */
1595 total_rx_bytes += skb->len;
1596 total_rx_packets++;
1597
1598 /* eth type trans needs skb->data to point to something */
1599 if (!pskb_may_pull(skb, ETH_HLEN)) {
44defeb3 1600 e_err("pskb_may_pull failed.\n");
ef5ab89c 1601 dev_kfree_skb_irq(skb);
97ac8cae
BA		 1602 goto next_desc;
1603 }
1604
5f450212
BA		 1605 e1000_receive_skb(adapter, netdev, skb, staterr,
1606 rx_desc->wb.upper.vlan);
97ac8cae
BA		 1607
1608next_desc:
5f450212 1609 rx_desc->wb.upper.status_error &= cpu_to_le32(~0xFF);
97ac8cae
BA		 1610
1611 /* return some buffers to hardware, one at a time is too slow */
1612 if (unlikely(cleaned_count >= E1000_RX_BUFFER_WRITE)) {
55aa6985 1613 adapter->alloc_rx_buf(rx_ring, cleaned_count,
c2fed996 1614 GFP_ATOMIC);
97ac8cae
BA		 1615 cleaned_count = 0;
1616 }
1617
1618 /* use prefetched values */
1619 rx_desc = next_rxd;
1620 buffer_info = next_buffer;
5f450212
BA		 1621
1622 staterr = le32_to_cpu(rx_desc->wb.upper.status_error);
97ac8cae
BA		 1623 }
1624 rx_ring->next_to_clean = i;
1625
1626 cleaned_count = e1000_desc_unused(rx_ring);
1627 if (cleaned_count)
55aa6985 1628 adapter->alloc_rx_buf(rx_ring, cleaned_count, GFP_ATOMIC);
97ac8cae
BA		 1629
1630 adapter->total_rx_bytes += total_rx_bytes;
1631 adapter->total_rx_packets += total_rx_packets;
97ac8cae
BA		 1632 return cleaned;
1633}
1634
bc7f75fa
AK		 1635/**
1636 * e1000_clean_rx_ring - Free Rx Buffers per Queue
55aa6985 1637 * @rx_ring: Rx descriptor ring
bc7f75fa 1638 **/
55aa6985 1639static void e1000_clean_rx_ring(struct e1000_ring *rx_ring)
bc7f75fa 1640{
55aa6985 1641 struct e1000_adapter *adapter = rx_ring->adapter;
bc7f75fa
AK		 1642 struct e1000_buffer *buffer_info;
1643 struct e1000_ps_page *ps_page;
1644 struct pci_dev *pdev = adapter->pdev;
bc7f75fa
AK		 1645 unsigned int i, j;
1646
1647 /* Free all the Rx ring sk_buffs */
1648 for (i = 0; i < rx_ring->count; i++) {
1649 buffer_info = &rx_ring->buffer_info[i];
1650 if (buffer_info->dma) {
1651 if (adapter->clean_rx == e1000_clean_rx_irq)
0be3f55f 1652 dma_unmap_single(&pdev->dev, buffer_info->dma,
bc7f75fa 1653 adapter->rx_buffer_len,
0be3f55f 1654 DMA_FROM_DEVICE);
97ac8cae 1655 else if (adapter->clean_rx == e1000_clean_jumbo_rx_irq)
0be3f55f 1656 dma_unmap_page(&pdev->dev, buffer_info->dma,
f0ff4398 1657 PAGE_SIZE, DMA_FROM_DEVICE);
bc7f75fa 1658 else if (adapter->clean_rx == e1000_clean_rx_irq_ps)
0be3f55f 1659 dma_unmap_single(&pdev->dev, buffer_info->dma,
bc7f75fa 1660 adapter->rx_ps_bsize0,
0be3f55f 1661 DMA_FROM_DEVICE);
bc7f75fa
AK		 1662 buffer_info->dma = 0;
1663 }
1664
97ac8cae
BA		 1665 if (buffer_info->page) {
1666 put_page(buffer_info->page);
1667 buffer_info->page = NULL;
1668 }
1669
bc7f75fa
AK		 1670 if (buffer_info->skb) {
1671 dev_kfree_skb(buffer_info->skb);
1672 buffer_info->skb = NULL;
1673 }
1674
1675 for (j = 0; j < PS_PAGE_BUFFERS; j++) {
47f44e40 1676 ps_page = &buffer_info->ps_pages[j];
bc7f75fa
AK		 1677 if (!ps_page->page)
1678 break;
0be3f55f
NN		 1679 dma_unmap_page(&pdev->dev, ps_page->dma, PAGE_SIZE,
1680 DMA_FROM_DEVICE);
bc7f75fa
AK		 1681 ps_page->dma = 0;
1682 put_page(ps_page->page);
1683 ps_page->page = NULL;
1684 }
1685 }
1686
1687 /* there also may be some cached data from a chained receive */
1688 if (rx_ring->rx_skb_top) {
1689 dev_kfree_skb(rx_ring->rx_skb_top);
1690 rx_ring->rx_skb_top = NULL;
1691 }
1692
bc7f75fa
AK		 1693 /* Zero out the descriptor ring */
1694 memset(rx_ring->desc, 0, rx_ring->size);
1695
1696 rx_ring->next_to_clean = 0;
1697 rx_ring->next_to_use = 0;
b94b5028 1698 adapter->flags2 &= ~FLAG2_IS_DISCARDING;
bc7f75fa 1699
c5083cf6 1700 writel(0, rx_ring->head);
bdc125f7
BA		 1701 if (rx_ring->adapter->flags2 & FLAG2_PCIM2PCI_ARBITER_WA)
1702 e1000e_update_rdt_wa(rx_ring, 0);
1703 else
1704 writel(0, rx_ring->tail);
bc7f75fa
AK		 1705}
1706
a8f88ff5
JB		 1707static void e1000e_downshift_workaround(struct work_struct *work)
1708{
1709 struct e1000_adapter *adapter = container_of(work,
17e813ec
BA		 1710 struct e1000_adapter,
1711 downshift_task);
a8f88ff5 1712
615b32af
JB		 1713 if (test_bit(__E1000_DOWN, &adapter->state))
1714 return;
1715
a8f88ff5
JB		 1716 e1000e_gig_downshift_workaround_ich8lan(&adapter->hw);
1717}
1718
bc7f75fa
AK		 1719/**
1720 * e1000_intr_msi - Interrupt Handler
1721 * @irq: interrupt number
1722 * @data: pointer to a network interface device structure
1723 **/
8bb62869 1724static irqreturn_t e1000_intr_msi(int __always_unused irq, void *data)
bc7f75fa
AK		 1725{
1726 struct net_device *netdev = data;
1727 struct e1000_adapter *adapter = netdev_priv(netdev);
1728 struct e1000_hw *hw = &adapter->hw;
1729 u32 icr = er32(ICR);
1730
e921eb1a 1731 /* read ICR disables interrupts using IAM */
573cca8c 1732 if (icr & E1000_ICR_LSC) {
f92518dd 1733 hw->mac.get_link_status = true;
e921eb1a 1734 /* ICH8 workaround-- Call gig speed drop workaround on cable
ad68076e
BA		 1735 * disconnect (LSC) before accessing any PHY registers
1736 */
bc7f75fa
AK		 1737 if ((adapter->flags & FLAG_LSC_GIG_SPEED_DROP) &&
1738 (!(er32(STATUS) & E1000_STATUS_LU)))
a8f88ff5 1739 schedule_work(&adapter->downshift_task);
bc7f75fa 1740
e921eb1a 1741 /* 80003ES2LAN workaround-- For packet buffer work-around on
bc7f75fa 1742 * link down event; disable receives here in the ISR and reset
ad68076e
BA		 1743 * adapter in watchdog
1744 */
bc7f75fa
AK		 1745 if (netif_carrier_ok(netdev) &&
1746 adapter->flags & FLAG_RX_NEEDS_RESTART) {
1747 /* disable receives */
1748 u32 rctl = er32(RCTL);
1749 ew32(RCTL, rctl & ~E1000_RCTL_EN);
12d43f7d 1750 adapter->flags |= FLAG_RESTART_NOW;
bc7f75fa
AK		 1751 }
1752 /* guard against interrupt when we're going down */
1753 if (!test_bit(__E1000_DOWN, &adapter->state))
1754 mod_timer(&adapter->watchdog_timer, jiffies + 1);
1755 }
1756
94fb848b
BA		 1757 /* Reset on uncorrectable ECC error */
1758 if ((icr & E1000_ICR_ECCER) && (hw->mac.type == e1000_pch_lpt)) {
1759 u32 pbeccsts = er32(PBECCSTS);
1760
1761 adapter->corr_errors +=
1762 pbeccsts & E1000_PBECCSTS_CORR_ERR_CNT_MASK;
1763 adapter->uncorr_errors +=
1764 (pbeccsts & E1000_PBECCSTS_UNCORR_ERR_CNT_MASK) >>
1765 E1000_PBECCSTS_UNCORR_ERR_CNT_SHIFT;
1766
1767 /* Do the reset outside of interrupt context */
1768 schedule_work(&adapter->reset_task);
1769
1770 /* return immediately since reset is imminent */
1771 return IRQ_HANDLED;
1772 }
1773
288379f0 1774 if (napi_schedule_prep(&adapter->napi)) {
bc7f75fa
AK		 1775 adapter->total_tx_bytes = 0;
1776 adapter->total_tx_packets = 0;
1777 adapter->total_rx_bytes = 0;
1778 adapter->total_rx_packets = 0;
288379f0 1779 __napi_schedule(&adapter->napi);
bc7f75fa
AK		 1780 }
1781
1782 return IRQ_HANDLED;
1783}
1784
1785/**
1786 * e1000_intr - Interrupt Handler
1787 * @irq: interrupt number
1788 * @data: pointer to a network interface device structure
1789 **/
8bb62869 1790static irqreturn_t e1000_intr(int __always_unused irq, void *data)
bc7f75fa
AK		 1791{
1792 struct net_device *netdev = data;
1793 struct e1000_adapter *adapter = netdev_priv(netdev);
1794 struct e1000_hw *hw = &adapter->hw;
bc7f75fa 1795 u32 rctl, icr = er32(ICR);
4662e82b 1796
a68ea775 1797 if (!icr || test_bit(__E1000_DOWN, &adapter->state))
bc7f75fa
AK		 1798 return IRQ_NONE; /* Not our interrupt */
1799
e921eb1a 1800 /* IMS will not auto-mask if INT_ASSERTED is not set, and if it is
ad68076e
BA		 1801 * not set, then the adapter didn't send an interrupt
1802 */
bc7f75fa
AK		 1803 if (!(icr & E1000_ICR_INT_ASSERTED))
1804 return IRQ_NONE;
1805
e921eb1a 1806 /* Interrupt Auto-Mask...upon reading ICR,
ad68076e
BA		 1807 * interrupts are masked. No need for the
1808 * IMC write
1809 */
bc7f75fa 1810
573cca8c 1811 if (icr & E1000_ICR_LSC) {
f92518dd 1812 hw->mac.get_link_status = true;
e921eb1a 1813 /* ICH8 workaround-- Call gig speed drop workaround on cable
ad68076e
BA		 1814 * disconnect (LSC) before accessing any PHY registers
1815 */
bc7f75fa
AK		 1816 if ((adapter->flags & FLAG_LSC_GIG_SPEED_DROP) &&
1817 (!(er32(STATUS) & E1000_STATUS_LU)))
a8f88ff5 1818 schedule_work(&adapter->downshift_task);
bc7f75fa 1819
e921eb1a 1820 /* 80003ES2LAN workaround--
bc7f75fa
AK		 1821 * For packet buffer work-around on link down event;
1822 * disable receives here in the ISR and
1823 * reset adapter in watchdog
1824 */
1825 if (netif_carrier_ok(netdev) &&
1826 (adapter->flags & FLAG_RX_NEEDS_RESTART)) {
1827 /* disable receives */
1828 rctl = er32(RCTL);
1829 ew32(RCTL, rctl & ~E1000_RCTL_EN);
12d43f7d 1830 adapter->flags |= FLAG_RESTART_NOW;
bc7f75fa
AK		 1831 }
1832 /* guard against interrupt when we're going down */
1833 if (!test_bit(__E1000_DOWN, &adapter->state))
1834 mod_timer(&adapter->watchdog_timer, jiffies + 1);
1835 }
1836
94fb848b
BA		 1837 /* Reset on uncorrectable ECC error */
1838 if ((icr & E1000_ICR_ECCER) && (hw->mac.type == e1000_pch_lpt)) {
1839 u32 pbeccsts = er32(PBECCSTS);
1840
1841 adapter->corr_errors +=
1842 pbeccsts & E1000_PBECCSTS_CORR_ERR_CNT_MASK;
1843 adapter->uncorr_errors +=
1844 (pbeccsts & E1000_PBECCSTS_UNCORR_ERR_CNT_MASK) >>
1845 E1000_PBECCSTS_UNCORR_ERR_CNT_SHIFT;
1846
1847 /* Do the reset outside of interrupt context */
1848 schedule_work(&adapter->reset_task);
1849
1850 /* return immediately since reset is imminent */
1851 return IRQ_HANDLED;
1852 }
1853
288379f0 1854 if (napi_schedule_prep(&adapter->napi)) {
bc7f75fa
AK		 1855 adapter->total_tx_bytes = 0;
1856 adapter->total_tx_packets = 0;
1857 adapter->total_rx_bytes = 0;
1858 adapter->total_rx_packets = 0;
288379f0 1859 __napi_schedule(&adapter->napi);
bc7f75fa
AK		 1860 }
1861
1862 return IRQ_HANDLED;
1863}
1864
8bb62869 1865static irqreturn_t e1000_msix_other(int __always_unused irq, void *data)
4662e82b
BA		 1866{
1867 struct net_device *netdev = data;
1868 struct e1000_adapter *adapter = netdev_priv(netdev);
1869 struct e1000_hw *hw = &adapter->hw;
1870 u32 icr = er32(ICR);
1871
1872 if (!(icr & E1000_ICR_INT_ASSERTED)) {
a3c69fef
JB		 1873 if (!test_bit(__E1000_DOWN, &adapter->state))
1874 ew32(IMS, E1000_IMS_OTHER);
4662e82b
BA		 1875 return IRQ_NONE;
1876 }
1877
1878 if (icr & adapter->eiac_mask)
1879 ew32(ICS, (icr & adapter->eiac_mask));
1880
1881 if (icr & E1000_ICR_OTHER) {
1882 if (!(icr & E1000_ICR_LSC))
1883 goto no_link_interrupt;
f92518dd 1884 hw->mac.get_link_status = true;
4662e82b
BA		 1885 /* guard against interrupt when we're going down */
1886 if (!test_bit(__E1000_DOWN, &adapter->state))
1887 mod_timer(&adapter->watchdog_timer, jiffies + 1);
1888 }
1889
1890no_link_interrupt:
a3c69fef
JB		 1891 if (!test_bit(__E1000_DOWN, &adapter->state))
1892 ew32(IMS, E1000_IMS_LSC | E1000_IMS_OTHER);
4662e82b
BA		 1893
1894 return IRQ_HANDLED;
1895}
1896
8bb62869 1897static irqreturn_t e1000_intr_msix_tx(int __always_unused irq, void *data)
4662e82b
BA		 1898{
1899 struct net_device *netdev = data;
1900 struct e1000_adapter *adapter = netdev_priv(netdev);
1901 struct e1000_hw *hw = &adapter->hw;
1902 struct e1000_ring *tx_ring = adapter->tx_ring;
1903
4662e82b
BA		 1904 adapter->total_tx_bytes = 0;
1905 adapter->total_tx_packets = 0;
1906
55aa6985 1907 if (!e1000_clean_tx_irq(tx_ring))
4662e82b
BA		 1908 /* Ring was not completely cleaned, so fire another interrupt */
1909 ew32(ICS, tx_ring->ims_val);
1910
1911 return IRQ_HANDLED;
1912}
1913
8bb62869 1914static irqreturn_t e1000_intr_msix_rx(int __always_unused irq, void *data)
4662e82b
BA		 1915{
1916 struct net_device *netdev = data;
1917 struct e1000_adapter *adapter = netdev_priv(netdev);
55aa6985 1918 struct e1000_ring *rx_ring = adapter->rx_ring;
4662e82b
BA		 1919
1920 /* Write the ITR value calculated at the end of the
1921 * previous interrupt.
1922 */
55aa6985
BA		 1923 if (rx_ring->set_itr) {
1924 writel(1000000000 / (rx_ring->itr_val * 256),
1925 rx_ring->itr_register);
1926 rx_ring->set_itr = 0;
4662e82b
BA		 1927 }
1928
288379f0 1929 if (napi_schedule_prep(&adapter->napi)) {
4662e82b
BA		 1930 adapter->total_rx_bytes = 0;
1931 adapter->total_rx_packets = 0;
288379f0 1932 __napi_schedule(&adapter->napi);
4662e82b
BA		 1933 }
1934 return IRQ_HANDLED;
1935}
1936
1937/**
1938 * e1000_configure_msix - Configure MSI-X hardware
1939 *
1940 * e1000_configure_msix sets up the hardware to properly
1941 * generate MSI-X interrupts.
1942 **/
1943static void e1000_configure_msix(struct e1000_adapter *adapter)
1944{
1945 struct e1000_hw *hw = &adapter->hw;
1946 struct e1000_ring *rx_ring = adapter->rx_ring;
1947 struct e1000_ring *tx_ring = adapter->tx_ring;
1948 int vector = 0;
1949 u32 ctrl_ext, ivar = 0;
1950
1951 adapter->eiac_mask = 0;
1952
1953 /* Workaround issue with spurious interrupts on 82574 in MSI-X mode */
1954 if (hw->mac.type == e1000_82574) {
1955 u32 rfctl = er32(RFCTL);
1956 rfctl |= E1000_RFCTL_ACK_DIS;
1957 ew32(RFCTL, rfctl);
1958 }
1959
4662e82b
BA		 1960 /* Configure Rx vector */
1961 rx_ring->ims_val = E1000_IMS_RXQ0;
1962 adapter->eiac_mask |= rx_ring->ims_val;
1963 if (rx_ring->itr_val)
1964 writel(1000000000 / (rx_ring->itr_val * 256),
c5083cf6 1965 rx_ring->itr_register);
4662e82b 1966 else
c5083cf6 1967 writel(1, rx_ring->itr_register);
4662e82b
BA		 1968 ivar = E1000_IVAR_INT_ALLOC_VALID | vector;
1969
1970 /* Configure Tx vector */
1971 tx_ring->ims_val = E1000_IMS_TXQ0;
1972 vector++;
1973 if (tx_ring->itr_val)
1974 writel(1000000000 / (tx_ring->itr_val * 256),
c5083cf6 1975 tx_ring->itr_register);
4662e82b 1976 else
c5083cf6 1977 writel(1, tx_ring->itr_register);
4662e82b
BA		 1978 adapter->eiac_mask |= tx_ring->ims_val;
1979 ivar |= ((E1000_IVAR_INT_ALLOC_VALID | vector) << 8);
1980
1981 /* set vector for Other Causes, e.g. link changes */
1982 vector++;
1983 ivar |= ((E1000_IVAR_INT_ALLOC_VALID | vector) << 16);
1984 if (rx_ring->itr_val)
1985 writel(1000000000 / (rx_ring->itr_val * 256),
1986 hw->hw_addr + E1000_EITR_82574(vector));
1987 else
1988 writel(1, hw->hw_addr + E1000_EITR_82574(vector));
1989
1990 /* Cause Tx interrupts on every write back */
1991 ivar |= (1 << 31);
1992
1993 ew32(IVAR, ivar);
1994
1995 /* enable MSI-X PBA support */
1996 ctrl_ext = er32(CTRL_EXT);
1997 ctrl_ext |= E1000_CTRL_EXT_PBA_CLR;
1998
1999 /* Auto-Mask Other interrupts upon ICR read */
4662e82b
BA		 2000 ew32(IAM, ~E1000_EIAC_MASK_82574 | E1000_IMS_OTHER);
2001 ctrl_ext |= E1000_CTRL_EXT_EIAME;
2002 ew32(CTRL_EXT, ctrl_ext);
2003 e1e_flush();
2004}
2005
2006void e1000e_reset_interrupt_capability(struct e1000_adapter *adapter)
2007{
2008 if (adapter->msix_entries) {
2009 pci_disable_msix(adapter->pdev);
2010 kfree(adapter->msix_entries);
2011 adapter->msix_entries = NULL;
2012 } else if (adapter->flags & FLAG_MSI_ENABLED) {
2013 pci_disable_msi(adapter->pdev);
2014 adapter->flags &= ~FLAG_MSI_ENABLED;
2015 }
4662e82b
BA		 2016}
2017
2018/**
2019 * e1000e_set_interrupt_capability - set MSI or MSI-X if supported
2020 *
2021 * Attempt to configure interrupts using the best available
2022 * capabilities of the hardware and kernel.
2023 **/
2024void e1000e_set_interrupt_capability(struct e1000_adapter *adapter)
2025{
2026 int err;
8e86acd7 2027 int i;
4662e82b
BA		 2028
2029 switch (adapter->int_mode) {
2030 case E1000E_INT_MODE_MSIX:
2031 if (adapter->flags & FLAG_HAS_MSIX) {
8e86acd7
JK		 2032 adapter->num_vectors = 3; /* RxQ0, TxQ0 and other */
2033 adapter->msix_entries = kcalloc(adapter->num_vectors,
17e813ec
BA		 2034 sizeof(struct
2035 msix_entry),
2036 GFP_KERNEL);
4662e82b 2037 if (adapter->msix_entries) {
8e86acd7 2038 for (i = 0; i < adapter->num_vectors; i++)
4662e82b
BA		 2039 adapter->msix_entries[i].entry = i;
2040
2041 err = pci_enable_msix(adapter->pdev,
2042 adapter->msix_entries,
8e86acd7 2043 adapter->num_vectors);
b1cdfead 2044 if (err == 0)
4662e82b
BA		 2045 return;
2046 }
2047 /* MSI-X failed, so fall through and try MSI */
ef456f85 2048 e_err("Failed to initialize MSI-X interrupts. Falling back to MSI interrupts.\n");
4662e82b
BA		 2049 e1000e_reset_interrupt_capability(adapter);
2050 }
2051 adapter->int_mode = E1000E_INT_MODE_MSI;
2052 /* Fall through */
2053 case E1000E_INT_MODE_MSI:
2054 if (!pci_enable_msi(adapter->pdev)) {
2055 adapter->flags |= FLAG_MSI_ENABLED;
2056 } else {
2057 adapter->int_mode = E1000E_INT_MODE_LEGACY;
ef456f85 2058 e_err("Failed to initialize MSI interrupts. Falling back to legacy interrupts.\n");
4662e82b
BA		 2059 }
2060 /* Fall through */
2061 case E1000E_INT_MODE_LEGACY:
2062 /* Don't do anything; this is the system default */
2063 break;
2064 }
8e86acd7
JK		 2065
2066 /* store the number of vectors being used */
2067 adapter->num_vectors = 1;
4662e82b
BA		 2068}
2069
2070/**
2071 * e1000_request_msix - Initialize MSI-X interrupts
2072 *
2073 * e1000_request_msix allocates MSI-X vectors and requests interrupts from the
2074 * kernel.
2075 **/
2076static int e1000_request_msix(struct e1000_adapter *adapter)
2077{
2078 struct net_device *netdev = adapter->netdev;
2079 int err = 0, vector = 0;
2080
2081 if (strlen(netdev->name) < (IFNAMSIZ - 5))
79f5e840
BA		 2082 snprintf(adapter->rx_ring->name,
2083 sizeof(adapter->rx_ring->name) - 1,
2084 "%s-rx-0", netdev->name);
4662e82b
BA		 2085 else
2086 memcpy(adapter->rx_ring->name, netdev->name, IFNAMSIZ);
2087 err = request_irq(adapter->msix_entries[vector].vector,
a0607fd3 2088 e1000_intr_msix_rx, 0, adapter->rx_ring->name,
4662e82b
BA		 2089 netdev);
2090 if (err)
5015e53a 2091 return err;
c5083cf6
BA		 2092 adapter->rx_ring->itr_register = adapter->hw.hw_addr +
2093 E1000_EITR_82574(vector);
4662e82b
BA		 2094 adapter->rx_ring->itr_val = adapter->itr;
2095 vector++;
2096
2097 if (strlen(netdev->name) < (IFNAMSIZ - 5))
79f5e840
BA		 2098 snprintf(adapter->tx_ring->name,
2099 sizeof(adapter->tx_ring->name) - 1,
2100 "%s-tx-0", netdev->name);
4662e82b
BA		 2101 else
2102 memcpy(adapter->tx_ring->name, netdev->name, IFNAMSIZ);
2103 err = request_irq(adapter->msix_entries[vector].vector,
a0607fd3 2104 e1000_intr_msix_tx, 0, adapter->tx_ring->name,
4662e82b
BA		 2105 netdev);
2106 if (err)
5015e53a 2107 return err;
c5083cf6
BA		 2108 adapter->tx_ring->itr_register = adapter->hw.hw_addr +
2109 E1000_EITR_82574(vector);
4662e82b
BA		 2110 adapter->tx_ring->itr_val = adapter->itr;
2111 vector++;
2112
2113 err = request_irq(adapter->msix_entries[vector].vector,
a0607fd3 2114 e1000_msix_other, 0, netdev->name, netdev);
4662e82b 2115 if (err)
5015e53a 2116 return err;
4662e82b
BA		 2117
2118 e1000_configure_msix(adapter);
5015e53a 2119
4662e82b 2120 return 0;
4662e82b
BA		 2121}
2122
f8d59f78
BA		 2123/**
2124 * e1000_request_irq - initialize interrupts
2125 *
2126 * Attempts to configure interrupts using the best available
2127 * capabilities of the hardware and kernel.
2128 **/
bc7f75fa
AK		 2129static int e1000_request_irq(struct e1000_adapter *adapter)
2130{
2131 struct net_device *netdev = adapter->netdev;
bc7f75fa
AK		 2132 int err;
2133
4662e82b
BA		 2134 if (adapter->msix_entries) {
2135 err = e1000_request_msix(adapter);
2136 if (!err)
2137 return err;
2138 /* fall back to MSI */
2139 e1000e_reset_interrupt_capability(adapter);
2140 adapter->int_mode = E1000E_INT_MODE_MSI;
2141 e1000e_set_interrupt_capability(adapter);
bc7f75fa 2142 }
4662e82b 2143 if (adapter->flags & FLAG_MSI_ENABLED) {
a0607fd3 2144 err = request_irq(adapter->pdev->irq, e1000_intr_msi, 0,
4662e82b
BA		 2145 netdev->name, netdev);
2146 if (!err)
2147 return err;
bc7f75fa 2148
4662e82b
BA		 2149 /* fall back to legacy interrupt */
2150 e1000e_reset_interrupt_capability(adapter);
2151 adapter->int_mode = E1000E_INT_MODE_LEGACY;
bc7f75fa
AK		 2152 }
2153
a0607fd3 2154 err = request_irq(adapter->pdev->irq, e1000_intr, IRQF_SHARED,
4662e82b
BA		 2155 netdev->name, netdev);
2156 if (err)
2157 e_err("Unable to allocate interrupt, Error: %d\n", err);
2158
bc7f75fa
AK		 2159 return err;
2160}
2161
2162static void e1000_free_irq(struct e1000_adapter *adapter)
2163{
2164 struct net_device *netdev = adapter->netdev;
2165
4662e82b
BA		 2166 if (adapter->msix_entries) {
2167 int vector = 0;
2168
2169 free_irq(adapter->msix_entries[vector].vector, netdev);
2170 vector++;
2171
2172 free_irq(adapter->msix_entries[vector].vector, netdev);
2173 vector++;
2174
2175 /* Other Causes interrupt vector */
2176 free_irq(adapter->msix_entries[vector].vector, netdev);
2177 return;
bc7f75fa 2178 }
4662e82b
BA		 2179
2180 free_irq(adapter->pdev->irq, netdev);
bc7f75fa
AK		 2181}
2182
2183/**
2184 * e1000_irq_disable - Mask off interrupt generation on the NIC
2185 **/
2186static void e1000_irq_disable(struct e1000_adapter *adapter)
2187{
2188 struct e1000_hw *hw = &adapter->hw;
2189
bc7f75fa 2190 ew32(IMC, ~0);
4662e82b
BA		 2191 if (adapter->msix_entries)
2192 ew32(EIAC_82574, 0);
bc7f75fa 2193 e1e_flush();
8e86acd7
JK		 2194
2195 if (adapter->msix_entries) {
2196 int i;
2197 for (i = 0; i < adapter->num_vectors; i++)
2198 synchronize_irq(adapter->msix_entries[i].vector);
2199 } else {
2200 synchronize_irq(adapter->pdev->irq);
2201 }
bc7f75fa
AK		 2202}
2203
2204/**
2205 * e1000_irq_enable - Enable default interrupt generation settings
2206 **/
2207static void e1000_irq_enable(struct e1000_adapter *adapter)
2208{
2209 struct e1000_hw *hw = &adapter->hw;
2210
4662e82b
BA		 2211 if (adapter->msix_entries) {
2212 ew32(EIAC_82574, adapter->eiac_mask & E1000_EIAC_MASK_82574);
2213 ew32(IMS, adapter->eiac_mask | E1000_IMS_OTHER | E1000_IMS_LSC);
94fb848b
BA		 2214 } else if (hw->mac.type == e1000_pch_lpt) {
2215 ew32(IMS, IMS_ENABLE_MASK | E1000_IMS_ECCER);
4662e82b
BA		 2216 } else {
2217 ew32(IMS, IMS_ENABLE_MASK);
2218 }
74ef9c39 2219 e1e_flush();
bc7f75fa
AK		 2220}
2221
2222/**
31dbe5b4 2223 * e1000e_get_hw_control - get control of the h/w from f/w
bc7f75fa
AK		 2224 * @adapter: address of board private structure
2225 *
31dbe5b4 2226 * e1000e_get_hw_control sets {CTRL_EXT|SWSM}:DRV_LOAD bit.
bc7f75fa
AK		 2227 * For ASF and Pass Through versions of f/w this means that
2228 * the driver is loaded. For AMT version (only with 82573)
2229 * of the f/w this means that the network i/f is open.
2230 **/
31dbe5b4 2231void e1000e_get_hw_control(struct e1000_adapter *adapter)
bc7f75fa
AK		 2232{
2233 struct e1000_hw *hw = &adapter->hw;
2234 u32 ctrl_ext;
2235 u32 swsm;
2236
2237 /* Let firmware know the driver has taken over */
2238 if (adapter->flags & FLAG_HAS_SWSM_ON_LOAD) {
2239 swsm = er32(SWSM);
2240 ew32(SWSM, swsm | E1000_SWSM_DRV_LOAD);
2241 } else if (adapter->flags & FLAG_HAS_CTRLEXT_ON_LOAD) {
2242 ctrl_ext = er32(CTRL_EXT);
ad68076e 2243 ew32(CTRL_EXT, ctrl_ext | E1000_CTRL_EXT_DRV_LOAD);
bc7f75fa
AK		 2244 }
2245}
2246
2247/**
31dbe5b4 2248 * e1000e_release_hw_control - release control of the h/w to f/w
bc7f75fa
AK		 2249 * @adapter: address of board private structure
2250 *
31dbe5b4 2251 * e1000e_release_hw_control resets {CTRL_EXT|SWSM}:DRV_LOAD bit.
bc7f75fa
AK		 2252 * For ASF and Pass Through versions of f/w this means that the
2253 * driver is no longer loaded. For AMT version (only with 82573) i
2254 * of the f/w this means that the network i/f is closed.
2255 *
2256 **/
31dbe5b4 2257void e1000e_release_hw_control(struct e1000_adapter *adapter)
bc7f75fa
AK		 2258{
2259 struct e1000_hw *hw = &adapter->hw;
2260 u32 ctrl_ext;
2261 u32 swsm;
2262
2263 /* Let firmware taken over control of h/w */
2264 if (adapter->flags & FLAG_HAS_SWSM_ON_LOAD) {
2265 swsm = er32(SWSM);
2266 ew32(SWSM, swsm & ~E1000_SWSM_DRV_LOAD);
2267 } else if (adapter->flags & FLAG_HAS_CTRLEXT_ON_LOAD) {
2268 ctrl_ext = er32(CTRL_EXT);
ad68076e 2269 ew32(CTRL_EXT, ctrl_ext & ~E1000_CTRL_EXT_DRV_LOAD);
bc7f75fa
AK		 2270 }
2271}
2272
bc7f75fa 2273/**
49ce9c2c 2274 * e1000_alloc_ring_dma - allocate memory for a ring structure
bc7f75fa
AK		 2275 **/
2276static int e1000_alloc_ring_dma(struct e1000_adapter *adapter,
2277 struct e1000_ring *ring)
2278{
2279 struct pci_dev *pdev = adapter->pdev;
2280
2281 ring->desc = dma_alloc_coherent(&pdev->dev, ring->size, &ring->dma,
2282 GFP_KERNEL);
2283 if (!ring->desc)
2284 return -ENOMEM;
2285
2286 return 0;
2287}
2288
2289/**
2290 * e1000e_setup_tx_resources - allocate Tx resources (Descriptors)
55aa6985 2291 * @tx_ring: Tx descriptor ring
bc7f75fa
AK		 2292 *
2293 * Return 0 on success, negative on failure
2294 **/
55aa6985 2295int e1000e_setup_tx_resources(struct e1000_ring *tx_ring)
bc7f75fa 2296{
55aa6985 2297 struct e1000_adapter *adapter = tx_ring->adapter;
bc7f75fa
AK		 2298 int err = -ENOMEM, size;
2299
2300 size = sizeof(struct e1000_buffer) * tx_ring->count;
89bf67f1 2301 tx_ring->buffer_info = vzalloc(size);
bc7f75fa
AK		 2302 if (!tx_ring->buffer_info)
2303 goto err;
bc7f75fa
AK		 2304
2305 /* round up to nearest 4K */
2306 tx_ring->size = tx_ring->count * sizeof(struct e1000_tx_desc);
2307 tx_ring->size = ALIGN(tx_ring->size, 4096);
2308
2309 err = e1000_alloc_ring_dma(adapter, tx_ring);
2310 if (err)
2311 goto err;
2312
2313 tx_ring->next_to_use = 0;
2314 tx_ring->next_to_clean = 0;
bc7f75fa
AK		 2315
2316 return 0;
2317err:
2318 vfree(tx_ring->buffer_info);
44defeb3 2319 e_err("Unable to allocate memory for the transmit descriptor ring\n");
bc7f75fa
AK		 2320 return err;
2321}
2322
2323/**
2324 * e1000e_setup_rx_resources - allocate Rx resources (Descriptors)
55aa6985 2325 * @rx_ring: Rx descriptor ring
bc7f75fa
AK		 2326 *
2327 * Returns 0 on success, negative on failure
2328 **/
55aa6985 2329int e1000e_setup_rx_resources(struct e1000_ring *rx_ring)
bc7f75fa 2330{
55aa6985 2331 struct e1000_adapter *adapter = rx_ring->adapter;
47f44e40
AK		 2332 struct e1000_buffer *buffer_info;
2333 int i, size, desc_len, err = -ENOMEM;
bc7f75fa
AK		 2334
2335 size = sizeof(struct e1000_buffer) * rx_ring->count;
89bf67f1 2336 rx_ring->buffer_info = vzalloc(size);
bc7f75fa
AK		 2337 if (!rx_ring->buffer_info)
2338 goto err;
bc7f75fa 2339
47f44e40
AK		 2340 for (i = 0; i < rx_ring->count; i++) {
2341 buffer_info = &rx_ring->buffer_info[i];
2342 buffer_info->ps_pages = kcalloc(PS_PAGE_BUFFERS,
2343 sizeof(struct e1000_ps_page),
2344 GFP_KERNEL);
2345 if (!buffer_info->ps_pages)
2346 goto err_pages;
2347 }
bc7f75fa
AK		 2348
2349 desc_len = sizeof(union e1000_rx_desc_packet_split);
2350
2351 /* Round up to nearest 4K */
2352 rx_ring->size = rx_ring->count * desc_len;
2353 rx_ring->size = ALIGN(rx_ring->size, 4096);
2354
2355 err = e1000_alloc_ring_dma(adapter, rx_ring);
2356 if (err)
47f44e40 2357 goto err_pages;
bc7f75fa
AK		 2358
2359 rx_ring->next_to_clean = 0;
2360 rx_ring->next_to_use = 0;
2361 rx_ring->rx_skb_top = NULL;
2362
2363 return 0;
47f44e40
AK		 2364
2365err_pages:
2366 for (i = 0; i < rx_ring->count; i++) {
2367 buffer_info = &rx_ring->buffer_info[i];
2368 kfree(buffer_info->ps_pages);
2369 }
bc7f75fa
AK		 2370err:
2371 vfree(rx_ring->buffer_info);
e9262447 2372 e_err("Unable to allocate memory for the receive descriptor ring\n");
bc7f75fa
AK		 2373 return err;
2374}
2375
2376/**
2377 * e1000_clean_tx_ring - Free Tx Buffers
55aa6985 2378 * @tx_ring: Tx descriptor ring
bc7f75fa 2379 **/
55aa6985 2380static void e1000_clean_tx_ring(struct e1000_ring *tx_ring)
bc7f75fa 2381{
55aa6985 2382 struct e1000_adapter *adapter = tx_ring->adapter;
bc7f75fa
AK		 2383 struct e1000_buffer *buffer_info;
2384 unsigned long size;
2385 unsigned int i;
2386
2387 for (i = 0; i < tx_ring->count; i++) {
2388 buffer_info = &tx_ring->buffer_info[i];
55aa6985 2389 e1000_put_txbuf(tx_ring, buffer_info);
bc7f75fa
AK		 2390 }
2391
3f0cfa3b 2392 netdev_reset_queue(adapter->netdev);
bc7f75fa
AK		 2393 size = sizeof(struct e1000_buffer) * tx_ring->count;
2394 memset(tx_ring->buffer_info, 0, size);
2395
2396 memset(tx_ring->desc, 0, tx_ring->size);
2397
2398 tx_ring->next_to_use = 0;
2399 tx_ring->next_to_clean = 0;
2400
c5083cf6 2401 writel(0, tx_ring->head);
bdc125f7
BA		 2402 if (tx_ring->adapter->flags2 & FLAG2_PCIM2PCI_ARBITER_WA)
2403 e1000e_update_tdt_wa(tx_ring, 0);
2404 else
2405 writel(0, tx_ring->tail);
bc7f75fa
AK		 2406}
2407
2408/**
2409 * e1000e_free_tx_resources - Free Tx Resources per Queue
55aa6985 2410 * @tx_ring: Tx descriptor ring
bc7f75fa
AK		 2411 *
2412 * Free all transmit software resources
2413 **/
55aa6985 2414void e1000e_free_tx_resources(struct e1000_ring *tx_ring)
bc7f75fa 2415{
55aa6985 2416 struct e1000_adapter *adapter = tx_ring->adapter;
bc7f75fa 2417 struct pci_dev *pdev = adapter->pdev;
bc7f75fa 2418
55aa6985 2419 e1000_clean_tx_ring(tx_ring);
bc7f75fa
AK		 2420
2421 vfree(tx_ring->buffer_info);
2422 tx_ring->buffer_info = NULL;
2423
2424 dma_free_coherent(&pdev->dev, tx_ring->size, tx_ring->desc,
2425 tx_ring->dma);
2426 tx_ring->desc = NULL;
2427}
2428
2429/**
2430 * e1000e_free_rx_resources - Free Rx Resources
55aa6985 2431 * @rx_ring: Rx descriptor ring
bc7f75fa
AK		 2432 *
2433 * Free all receive software resources
2434 **/
55aa6985 2435void e1000e_free_rx_resources(struct e1000_ring *rx_ring)
bc7f75fa 2436{
55aa6985 2437 struct e1000_adapter *adapter = rx_ring->adapter;
bc7f75fa 2438 struct pci_dev *pdev = adapter->pdev;
47f44e40 2439 int i;
bc7f75fa 2440
55aa6985 2441 e1000_clean_rx_ring(rx_ring);
bc7f75fa 2442
b1cdfead 2443 for (i = 0; i < rx_ring->count; i++)
47f44e40 2444 kfree(rx_ring->buffer_info[i].ps_pages);
47f44e40 2445
bc7f75fa
AK		 2446 vfree(rx_ring->buffer_info);
2447 rx_ring->buffer_info = NULL;
2448
bc7f75fa
AK		 2449 dma_free_coherent(&pdev->dev, rx_ring->size, rx_ring->desc,
2450 rx_ring->dma);
2451 rx_ring->desc = NULL;
2452}
2453
2454/**
2455 * e1000_update_itr - update the dynamic ITR value based on statistics
489815ce
AK		 2456 * @adapter: pointer to adapter
2457 * @itr_setting: current adapter->itr
2458 * @packets: the number of packets during this measurement interval
2459 * @bytes: the number of bytes during this measurement interval
2460 *
bc7f75fa
AK		 2461 * Stores a new ITR value based on packets and byte
2462 * counts during the last interrupt. The advantage of per interrupt
2463 * computation is faster updates and more accurate ITR for the current
2464 * traffic pattern. Constants in this function were computed
2465 * based on theoretical maximum wire speed and thresholds were set based
2466 * on testing data as well as attempting to minimize response time
4662e82b
BA		 2467 * while increasing bulk throughput. This functionality is controlled
2468 * by the InterruptThrottleRate module parameter.
bc7f75fa 2469 **/
8bb62869 2470static unsigned int e1000_update_itr(u16 itr_setting, int packets, int bytes)
bc7f75fa
AK		 2471{
2472 unsigned int retval = itr_setting;
2473
2474 if (packets == 0)
5015e53a 2475 return itr_setting;
bc7f75fa
AK		 2476
2477 switch (itr_setting) {
2478 case lowest_latency:
2479 /* handle TSO and jumbo frames */
362e20ca 2480 if (bytes / packets > 8000)
bc7f75fa 2481 retval = bulk_latency;
b1cdfead 2482 else if ((packets < 5) && (bytes > 512))
bc7f75fa 2483 retval = low_latency;
bc7f75fa
AK		 2484 break;
2485 case low_latency: /* 50 usec aka 20000 ints/s */
2486 if (bytes > 10000) {
2487 /* this if handles the TSO accounting */
362e20ca 2488 if (bytes / packets > 8000)
bc7f75fa 2489 retval = bulk_latency;
362e20ca 2490 else if ((packets < 10) || ((bytes / packets) > 1200))
bc7f75fa 2491 retval = bulk_latency;
b1cdfead 2492 else if ((packets > 35))
bc7f75fa 2493 retval = lowest_latency;
362e20ca 2494 } else if (bytes / packets > 2000) {
bc7f75fa
AK		 2495 retval = bulk_latency;
2496 } else if (packets <= 2 && bytes < 512) {
2497 retval = lowest_latency;
2498 }
2499 break;
2500 case bulk_latency: /* 250 usec aka 4000 ints/s */
2501 if (bytes > 25000) {
b1cdfead 2502 if (packets > 35)
bc7f75fa 2503 retval = low_latency;
bc7f75fa
AK		 2504 } else if (bytes < 6000) {
2505 retval = low_latency;
2506 }
2507 break;
2508 }
2509
bc7f75fa
AK		 2510 return retval;
2511}
2512
2513static void e1000_set_itr(struct e1000_adapter *adapter)
2514{
bc7f75fa
AK		 2515 u16 current_itr;
2516 u32 new_itr = adapter->itr;
2517
2518 /* for non-gigabit speeds, just fix the interrupt rate at 4000 */
2519 if (adapter->link_speed != SPEED_1000) {
2520 current_itr = 0;
2521 new_itr = 4000;
2522 goto set_itr_now;
2523 }
2524
828bac87
BA		 2525 if (adapter->flags2 & FLAG2_DISABLE_AIM) {
2526 new_itr = 0;
2527 goto set_itr_now;
2528 }
2529
8bb62869
BA		 2530 adapter->tx_itr = e1000_update_itr(adapter->tx_itr,
2531 adapter->total_tx_packets,
2532 adapter->total_tx_bytes);
bc7f75fa
AK		 2533 /* conservative mode (itr 3) eliminates the lowest_latency setting */
2534 if (adapter->itr_setting == 3 && adapter->tx_itr == lowest_latency)
2535 adapter->tx_itr = low_latency;
2536
8bb62869
BA		 2537 adapter->rx_itr = e1000_update_itr(adapter->rx_itr,
2538 adapter->total_rx_packets,
2539 adapter->total_rx_bytes);
bc7f75fa
AK		 2540 /* conservative mode (itr 3) eliminates the lowest_latency setting */
2541 if (adapter->itr_setting == 3 && adapter->rx_itr == lowest_latency)
2542 adapter->rx_itr = low_latency;
2543
2544 current_itr = max(adapter->rx_itr, adapter->tx_itr);
2545
bc7f75fa 2546 /* counts and packets in update_itr are dependent on these numbers */
33550cec 2547 switch (current_itr) {
bc7f75fa
AK		 2548 case lowest_latency:
2549 new_itr = 70000;
2550 break;
2551 case low_latency:
2552 new_itr = 20000; /* aka hwitr = ~200 */
2553 break;
2554 case bulk_latency:
2555 new_itr = 4000;
2556 break;
2557 default:
2558 break;
2559 }
2560
2561set_itr_now:
2562 if (new_itr != adapter->itr) {
e921eb1a 2563 /* this attempts to bias the interrupt rate towards Bulk
bc7f75fa 2564 * by adding intermediate steps when interrupt rate is
ad68076e
BA		 2565 * increasing
2566 */
bc7f75fa 2567 new_itr = new_itr > adapter->itr ?
f0ff4398 2568 min(adapter->itr + (new_itr >> 2), new_itr) : new_itr;
bc7f75fa 2569 adapter->itr = new_itr;
4662e82b
BA		 2570 adapter->rx_ring->itr_val = new_itr;
2571 if (adapter->msix_entries)
2572 adapter->rx_ring->set_itr = 1;
2573 else
e3d14b08 2574 e1000e_write_itr(adapter, new_itr);
bc7f75fa
AK		 2575 }
2576}
2577
22a4cca2
MV		 2578/**
2579 * e1000e_write_itr - write the ITR value to the appropriate registers
2580 * @adapter: address of board private structure
2581 * @itr: new ITR value to program
2582 *
2583 * e1000e_write_itr determines if the adapter is in MSI-X mode
2584 * and, if so, writes the EITR registers with the ITR value.
2585 * Otherwise, it writes the ITR value into the ITR register.
2586 **/
2587void e1000e_write_itr(struct e1000_adapter *adapter, u32 itr)
2588{
2589 struct e1000_hw *hw = &adapter->hw;
2590 u32 new_itr = itr ? 1000000000 / (itr * 256) : 0;
2591
2592 if (adapter->msix_entries) {
2593 int vector;
2594
2595 for (vector = 0; vector < adapter->num_vectors; vector++)
2596 writel(new_itr, hw->hw_addr + E1000_EITR_82574(vector));
2597 } else {
2598 ew32(ITR, new_itr);
2599 }
2600}
2601
4662e82b
BA		 2602/**
2603 * e1000_alloc_queues - Allocate memory for all rings
2604 * @adapter: board private structure to initialize
2605 **/
9f9a12f8 2606static int e1000_alloc_queues(struct e1000_adapter *adapter)
4662e82b 2607{
55aa6985
BA		 2608 int size = sizeof(struct e1000_ring);
2609
2610 adapter->tx_ring = kzalloc(size, GFP_KERNEL);
4662e82b
BA		 2611 if (!adapter->tx_ring)
2612 goto err;
55aa6985
BA		 2613 adapter->tx_ring->count = adapter->tx_ring_count;
2614 adapter->tx_ring->adapter = adapter;
4662e82b 2615
55aa6985 2616 adapter->rx_ring = kzalloc(size, GFP_KERNEL);
4662e82b
BA		 2617 if (!adapter->rx_ring)
2618 goto err;
55aa6985
BA		 2619 adapter->rx_ring->count = adapter->rx_ring_count;
2620 adapter->rx_ring->adapter = adapter;
4662e82b
BA		 2621
2622 return 0;
2623err:
2624 e_err("Unable to allocate memory for queues\n");
2625 kfree(adapter->rx_ring);
2626 kfree(adapter->tx_ring);
2627 return -ENOMEM;
2628}
2629
bc7f75fa 2630/**
c58c8a78 2631 * e1000e_poll - NAPI Rx polling callback
ad68076e 2632 * @napi: struct associated with this polling callback
c58c8a78 2633 * @weight: number of packets driver is allowed to process this poll
bc7f75fa 2634 **/
c58c8a78 2635static int e1000e_poll(struct napi_struct *napi, int weight)
bc7f75fa 2636{
c58c8a78
BA		 2637 struct e1000_adapter *adapter = container_of(napi, struct e1000_adapter,
2638 napi);
4662e82b 2639 struct e1000_hw *hw = &adapter->hw;
bc7f75fa 2640 struct net_device *poll_dev = adapter->netdev;
679e8a0f 2641 int tx_cleaned = 1, work_done = 0;
bc7f75fa 2642
4cf1653a 2643 adapter = netdev_priv(poll_dev);
bc7f75fa 2644
c58c8a78
BA		 2645 if (!adapter->msix_entries ||
2646 (adapter->rx_ring->ims_val & adapter->tx_ring->ims_val))
2647 tx_cleaned = e1000_clean_tx_irq(adapter->tx_ring);
4662e82b 2648
c58c8a78 2649 adapter->clean_rx(adapter->rx_ring, &work_done, weight);
d2c7ddd6 2650
12d04a3c 2651 if (!tx_cleaned)
c58c8a78 2652 work_done = weight;
bc7f75fa 2653
c58c8a78
BA		 2654 /* If weight not fully consumed, exit the polling mode */
2655 if (work_done < weight) {
bc7f75fa
AK		 2656 if (adapter->itr_setting & 3)
2657 e1000_set_itr(adapter);
288379f0 2658 napi_complete(napi);
a3c69fef
JB		 2659 if (!test_bit(__E1000_DOWN, &adapter->state)) {
2660 if (adapter->msix_entries)
2661 ew32(IMS, adapter->rx_ring->ims_val);
2662 else
2663 e1000_irq_enable(adapter);
2664 }
bc7f75fa
AK		 2665 }
2666
2667 return work_done;
2668}
2669
8e586137 2670static int e1000_vlan_rx_add_vid(struct net_device *netdev, u16 vid)
bc7f75fa
AK		 2671{
2672 struct e1000_adapter *adapter = netdev_priv(netdev);
2673 struct e1000_hw *hw = &adapter->hw;
2674 u32 vfta, index;
2675
2676 /* don't update vlan cookie if already programmed */
2677 if ((adapter->hw.mng_cookie.status &
2678 E1000_MNG_DHCP_COOKIE_STATUS_VLAN) &&
2679 (vid == adapter->mng_vlan_id))
8e586137 2680 return 0;
caaddaf8 2681
bc7f75fa 2682 /* add VID to filter table */
caaddaf8
BA		 2683 if (adapter->flags & FLAG_HAS_HW_VLAN_FILTER) {
2684 index = (vid >> 5) & 0x7F;
2685 vfta = E1000_READ_REG_ARRAY(hw, E1000_VFTA, index);
2686 vfta |= (1 << (vid & 0x1F));
2687 hw->mac.ops.write_vfta(hw, index, vfta);
2688 }
86d70e53
JK		 2689
2690 set_bit(vid, adapter->active_vlans);
8e586137
JP		 2691
2692 return 0;
bc7f75fa
AK		 2693}
2694
8e586137 2695static int e1000_vlan_rx_kill_vid(struct net_device *netdev, u16 vid)
bc7f75fa
AK		 2696{
2697 struct e1000_adapter *adapter = netdev_priv(netdev);
2698 struct e1000_hw *hw = &adapter->hw;
2699 u32 vfta, index;
2700
bc7f75fa
AK		 2701 if ((adapter->hw.mng_cookie.status &
2702 E1000_MNG_DHCP_COOKIE_STATUS_VLAN) &&
2703 (vid == adapter->mng_vlan_id)) {
2704 /* release control to f/w */
31dbe5b4 2705 e1000e_release_hw_control(adapter);
8e586137 2706 return 0;
bc7f75fa
AK		 2707 }
2708
2709 /* remove VID from filter table */
caaddaf8
BA		 2710 if (adapter->flags & FLAG_HAS_HW_VLAN_FILTER) {
2711 index = (vid >> 5) & 0x7F;
2712 vfta = E1000_READ_REG_ARRAY(hw, E1000_VFTA, index);
2713 vfta &= ~(1 << (vid & 0x1F));
2714 hw->mac.ops.write_vfta(hw, index, vfta);
2715 }
86d70e53
JK		 2716
2717 clear_bit(vid, adapter->active_vlans);
8e586137
JP		 2718
2719 return 0;
bc7f75fa
AK		 2720}
2721
86d70e53
JK		 2722/**
2723 * e1000e_vlan_filter_disable - helper to disable hw VLAN filtering
2724 * @adapter: board private structure to initialize
2725 **/
2726static void e1000e_vlan_filter_disable(struct e1000_adapter *adapter)
bc7f75fa
AK		 2727{
2728 struct net_device *netdev = adapter->netdev;
86d70e53
JK		 2729 struct e1000_hw *hw = &adapter->hw;
2730 u32 rctl;
bc7f75fa 2731
86d70e53
JK		 2732 if (adapter->flags & FLAG_HAS_HW_VLAN_FILTER) {
2733 /* disable VLAN receive filtering */
2734 rctl = er32(RCTL);
2735 rctl &= ~(E1000_RCTL_VFE | E1000_RCTL_CFIEN);
2736 ew32(RCTL, rctl);
2737
2738 if (adapter->mng_vlan_id != (u16)E1000_MNG_VLAN_NONE) {
2739 e1000_vlan_rx_kill_vid(netdev, adapter->mng_vlan_id);
2740 adapter->mng_vlan_id = E1000_MNG_VLAN_NONE;
bc7f75fa 2741 }
bc7f75fa
AK		 2742 }
2743}
2744
86d70e53
JK		 2745/**
2746 * e1000e_vlan_filter_enable - helper to enable HW VLAN filtering
2747 * @adapter: board private structure to initialize
2748 **/
2749static void e1000e_vlan_filter_enable(struct e1000_adapter *adapter)
2750{
2751 struct e1000_hw *hw = &adapter->hw;
2752 u32 rctl;
2753
2754 if (adapter->flags & FLAG_HAS_HW_VLAN_FILTER) {
2755 /* enable VLAN receive filtering */
2756 rctl = er32(RCTL);
2757 rctl |= E1000_RCTL_VFE;
2758 rctl &= ~E1000_RCTL_CFIEN;
2759 ew32(RCTL, rctl);
2760 }
2761}
bc7f75fa 2762
86d70e53
JK		 2763/**
2764 * e1000e_vlan_strip_enable - helper to disable HW VLAN stripping
2765 * @adapter: board private structure to initialize
2766 **/
2767static void e1000e_vlan_strip_disable(struct e1000_adapter *adapter)
bc7f75fa 2768{
bc7f75fa 2769 struct e1000_hw *hw = &adapter->hw;
86d70e53 2770 u32 ctrl;
bc7f75fa 2771
86d70e53
JK		 2772 /* disable VLAN tag insert/strip */
2773 ctrl = er32(CTRL);
2774 ctrl &= ~E1000_CTRL_VME;
2775 ew32(CTRL, ctrl);
2776}
bc7f75fa 2777
86d70e53
JK		 2778/**
2779 * e1000e_vlan_strip_enable - helper to enable HW VLAN stripping
2780 * @adapter: board private structure to initialize
2781 **/
2782static void e1000e_vlan_strip_enable(struct e1000_adapter *adapter)
2783{
2784 struct e1000_hw *hw = &adapter->hw;
2785 u32 ctrl;
bc7f75fa 2786
86d70e53
JK		 2787 /* enable VLAN tag insert/strip */
2788 ctrl = er32(CTRL);
2789 ctrl |= E1000_CTRL_VME;
2790 ew32(CTRL, ctrl);
2791}
bc7f75fa 2792
86d70e53
JK		 2793static void e1000_update_mng_vlan(struct e1000_adapter *adapter)
2794{
2795 struct net_device *netdev = adapter->netdev;
2796 u16 vid = adapter->hw.mng_cookie.vlan_id;
2797 u16 old_vid = adapter->mng_vlan_id;
2798
e5fe2541 2799 if (adapter->hw.mng_cookie.status & E1000_MNG_DHCP_COOKIE_STATUS_VLAN) {
86d70e53
JK		 2800 e1000_vlan_rx_add_vid(netdev, vid);
2801 adapter->mng_vlan_id = vid;
bc7f75fa
AK		 2802 }
2803
86d70e53
JK		 2804 if ((old_vid != (u16)E1000_MNG_VLAN_NONE) && (vid != old_vid))
2805 e1000_vlan_rx_kill_vid(netdev, old_vid);
bc7f75fa
AK		 2806}
2807
2808static void e1000_restore_vlan(struct e1000_adapter *adapter)
2809{
2810 u16 vid;
2811
86d70e53 2812 e1000_vlan_rx_add_vid(adapter->netdev, 0);
bc7f75fa 2813
86d70e53 2814 for_each_set_bit(vid, adapter->active_vlans, VLAN_N_VID)
f0ff4398 2815 e1000_vlan_rx_add_vid(adapter->netdev, vid);
bc7f75fa
AK		 2816}
2817
cd791618 2818static void e1000_init_manageability_pt(struct e1000_adapter *adapter)
bc7f75fa
AK		 2819{
2820 struct e1000_hw *hw = &adapter->hw;
cd791618 2821 u32 manc, manc2h, mdef, i, j;
bc7f75fa
AK		 2822
2823 if (!(adapter->flags & FLAG_MNG_PT_ENABLED))
2824 return;
2825
2826 manc = er32(MANC);
2827
e921eb1a 2828 /* enable receiving management packets to the host. this will probably
bc7f75fa 2829 * generate destination unreachable messages from the host OS, but
ad68076e
BA		 2830 * the packets will be handled on SMBUS
2831 */
bc7f75fa
AK		 2832 manc |= E1000_MANC_EN_MNG2HOST;
2833 manc2h = er32(MANC2H);
cd791618
BA		 2834
2835 switch (hw->mac.type) {
2836 default:
2837 manc2h |= (E1000_MANC2H_PORT_623 | E1000_MANC2H_PORT_664);
2838 break;
2839 case e1000_82574:
2840 case e1000_82583:
e921eb1a 2841 /* Check if IPMI pass-through decision filter already exists;
cd791618
BA		 2842 * if so, enable it.
2843 */
2844 for (i = 0, j = 0; i < 8; i++) {
2845 mdef = er32(MDEF(i));
2846
2847 /* Ignore filters with anything other than IPMI ports */
3b21b508 2848 if (mdef & ~(E1000_MDEF_PORT_623 | E1000_MDEF_PORT_664))
cd791618
BA		 2849 continue;
2850
2851 /* Enable this decision filter in MANC2H */
2852 if (mdef)
2853 manc2h |= (1 << i);
2854
2855 j |= mdef;
2856 }
2857
2858 if (j == (E1000_MDEF_PORT_623 | E1000_MDEF_PORT_664))
2859 break;
2860
2861 /* Create new decision filter in an empty filter */
2862 for (i = 0, j = 0; i < 8; i++)
2863 if (er32(MDEF(i)) == 0) {
2864 ew32(MDEF(i), (E1000_MDEF_PORT_623 |
2865 E1000_MDEF_PORT_664));
2866 manc2h |= (1 << 1);
2867 j++;
2868 break;
2869 }
2870
2871 if (!j)
2872 e_warn("Unable to create IPMI pass-through filter\n");
2873 break;
2874 }
2875
bc7f75fa
AK		 2876 ew32(MANC2H, manc2h);
2877 ew32(MANC, manc);
2878}
2879
2880/**
af667a29 2881 * e1000_configure_tx - Configure Transmit Unit after Reset
bc7f75fa
AK		 2882 * @adapter: board private structure
2883 *
2884 * Configure the Tx unit of the MAC after a reset.
2885 **/
2886static void e1000_configure_tx(struct e1000_adapter *adapter)
2887{
2888 struct e1000_hw *hw = &adapter->hw;
2889 struct e1000_ring *tx_ring = adapter->tx_ring;
2890 u64 tdba;
c550b121 2891 u32 tdlen, tarc;
bc7f75fa
AK		 2892
2893 /* Setup the HW Tx Head and Tail descriptor pointers */
2894 tdba = tx_ring->dma;
2895 tdlen = tx_ring->count * sizeof(struct e1000_tx_desc);
1e36052e
BA		 2896 ew32(TDBAL(0), (tdba & DMA_BIT_MASK(32)));
2897 ew32(TDBAH(0), (tdba >> 32));
2898 ew32(TDLEN(0), tdlen);
2899 ew32(TDH(0), 0);
2900 ew32(TDT(0), 0);
2901 tx_ring->head = adapter->hw.hw_addr + E1000_TDH(0);
2902 tx_ring->tail = adapter->hw.hw_addr + E1000_TDT(0);
bc7f75fa 2903
bc7f75fa
AK		 2904 /* Set the Tx Interrupt Delay register */
2905 ew32(TIDV, adapter->tx_int_delay);
ad68076e 2906 /* Tx irq moderation */
bc7f75fa
AK		 2907 ew32(TADV, adapter->tx_abs_int_delay);
2908
3a3b7586
JB		 2909 if (adapter->flags2 & FLAG2_DMA_BURST) {
2910 u32 txdctl = er32(TXDCTL(0));
2911 txdctl &= ~(E1000_TXDCTL_PTHRESH | E1000_TXDCTL_HTHRESH |
2912 E1000_TXDCTL_WTHRESH);
e921eb1a 2913 /* set up some performance related parameters to encourage the
3a3b7586
JB		 2914 * hardware to use the bus more efficiently in bursts, depends
2915 * on the tx_int_delay to be enabled,
8edc0e62 2916 * wthresh = 1 ==> burst write is disabled to avoid Tx stalls
3a3b7586
JB		 2917 * hthresh = 1 ==> prefetch when one or more available
2918 * pthresh = 0x1f ==> prefetch if internal cache 31 or less
2919 * BEWARE: this seems to work but should be considered first if
af667a29 2920 * there are Tx hangs or other Tx related bugs
3a3b7586
JB		 2921 */
2922 txdctl |= E1000_TXDCTL_DMA_BURST_ENABLE;
2923 ew32(TXDCTL(0), txdctl);
3a3b7586 2924 }
56032be7
BA		 2925 /* erratum work around: set txdctl the same for both queues */
2926 ew32(TXDCTL(1), er32(TXDCTL(0)));
3a3b7586 2927
bc7f75fa 2928 if (adapter->flags & FLAG_TARC_SPEED_MODE_BIT) {
e9ec2c0f 2929 tarc = er32(TARC(0));
e921eb1a 2930 /* set the speed mode bit, we'll clear it if we're not at
ad68076e
BA		 2931 * gigabit link later
2932 */
bc7f75fa
AK		 2933#define SPEED_MODE_BIT (1 << 21)
2934 tarc |= SPEED_MODE_BIT;
e9ec2c0f 2935 ew32(TARC(0), tarc);
bc7f75fa
AK		 2936 }
2937
2938 /* errata: program both queues to unweighted RR */
2939 if (adapter->flags & FLAG_TARC_SET_BIT_ZERO) {
e9ec2c0f 2940 tarc = er32(TARC(0));
bc7f75fa 2941 tarc |= 1;
e9ec2c0f
JK		 2942 ew32(TARC(0), tarc);
2943 tarc = er32(TARC(1));
bc7f75fa 2944 tarc |= 1;
e9ec2c0f 2945 ew32(TARC(1), tarc);
bc7f75fa
AK		 2946 }
2947
bc7f75fa
AK		 2948 /* Setup Transmit Descriptor Settings for eop descriptor */
2949 adapter->txd_cmd = E1000_TXD_CMD_EOP | E1000_TXD_CMD_IFCS;
2950
2951 /* only set IDE if we are delaying interrupts using the timers */
2952 if (adapter->tx_int_delay)
2953 adapter->txd_cmd |= E1000_TXD_CMD_IDE;
2954
2955 /* enable Report Status bit */
2956 adapter->txd_cmd |= E1000_TXD_CMD_RS;
2957
57cde763 2958 hw->mac.ops.config_collision_dist(hw);
bc7f75fa
AK		 2959}
2960
2961/**
2962 * e1000_setup_rctl - configure the receive control registers
2963 * @adapter: Board private structure
2964 **/
2965#define PAGE_USE_COUNT(S) (((S) >> PAGE_SHIFT) + \
2966 (((S) & (PAGE_SIZE - 1)) ? 1 : 0))
2967static void e1000_setup_rctl(struct e1000_adapter *adapter)
2968{
2969 struct e1000_hw *hw = &adapter->hw;
2970 u32 rctl, rfctl;
bc7f75fa
AK		 2971 u32 pages = 0;
2972
2fbe4526
BA		 2973 /* Workaround Si errata on PCHx - configure jumbo frame flow */
2974 if (hw->mac.type >= e1000_pch2lan) {
a1ce6473
BA		 2975 s32 ret_val;
2976
2977 if (adapter->netdev->mtu > ETH_DATA_LEN)
2978 ret_val = e1000_lv_jumbo_workaround_ich8lan(hw, true);
2979 else
2980 ret_val = e1000_lv_jumbo_workaround_ich8lan(hw, false);
dd93f95e
BA		 2981
2982 if (ret_val)
2983 e_dbg("failed to enable jumbo frame workaround mode\n");
a1ce6473
BA		 2984 }
2985
bc7f75fa
AK		 2986 /* Program MC offset vector base */
2987 rctl = er32(RCTL);
2988 rctl &= ~(3 << E1000_RCTL_MO_SHIFT);
2989 rctl |= E1000_RCTL_EN | E1000_RCTL_BAM |
f0ff4398
BA		 2990 E1000_RCTL_LBM_NO | E1000_RCTL_RDMTS_HALF |
2991 (adapter->hw.mac.mc_filter_type << E1000_RCTL_MO_SHIFT);
bc7f75fa
AK		 2992
2993 /* Do not Store bad packets */
2994 rctl &= ~E1000_RCTL_SBP;
2995
2996 /* Enable Long Packet receive */
2997 if (adapter->netdev->mtu <= ETH_DATA_LEN)
2998 rctl &= ~E1000_RCTL_LPE;
2999 else
3000 rctl |= E1000_RCTL_LPE;
3001
eb7c3adb
JK		 3002 /* Some systems expect that the CRC is included in SMBUS traffic. The
3003 * hardware strips the CRC before sending to both SMBUS (BMC) and to
3004 * host memory when this is enabled
3005 */
3006 if (adapter->flags2 & FLAG2_CRC_STRIPPING)
3007 rctl |= E1000_RCTL_SECRC;
5918bd88 3008
a4f58f54
BA		 3009 /* Workaround Si errata on 82577 PHY - configure IPG for jumbos */
3010 if ((hw->phy.type == e1000_phy_82577) && (rctl & E1000_RCTL_LPE)) {
3011 u16 phy_data;
3012
3013 e1e_rphy(hw, PHY_REG(770, 26), &phy_data);
3014 phy_data &= 0xfff8;
3015 phy_data |= (1 << 2);
3016 e1e_wphy(hw, PHY_REG(770, 26), phy_data);
3017
3018 e1e_rphy(hw, 22, &phy_data);
3019 phy_data &= 0x0fff;
3020 phy_data |= (1 << 14);
3021 e1e_wphy(hw, 0x10, 0x2823);
3022 e1e_wphy(hw, 0x11, 0x0003);
3023 e1e_wphy(hw, 22, phy_data);
3024 }
3025
bc7f75fa
AK		 3026 /* Setup buffer sizes */
3027 rctl &= ~E1000_RCTL_SZ_4096;
3028 rctl |= E1000_RCTL_BSEX;
3029 switch (adapter->rx_buffer_len) {
bc7f75fa
AK		 3030 case 2048:
3031 default:
3032 rctl |= E1000_RCTL_SZ_2048;
3033 rctl &= ~E1000_RCTL_BSEX;
3034 break;
3035 case 4096:
3036 rctl |= E1000_RCTL_SZ_4096;
3037 break;
3038 case 8192:
3039 rctl |= E1000_RCTL_SZ_8192;
3040 break;
3041 case 16384:
3042 rctl |= E1000_RCTL_SZ_16384;
3043 break;
3044 }
3045
5f450212
BA		 3046 /* Enable Extended Status in all Receive Descriptors */
3047 rfctl = er32(RFCTL);
3048 rfctl |= E1000_RFCTL_EXTEN;
f6bd5577 3049 ew32(RFCTL, rfctl);
5f450212 3050
e921eb1a 3051 /* 82571 and greater support packet-split where the protocol
bc7f75fa
AK		 3052 * header is placed in skb->data and the packet data is
3053 * placed in pages hanging off of skb_shinfo(skb)->nr_frags.
3054 * In the case of a non-split, skb->data is linearly filled,
3055 * followed by the page buffers. Therefore, skb->data is
3056 * sized to hold the largest protocol header.
3057 *
3058 * allocations using alloc_page take too long for regular MTU
3059 * so only enable packet split for jumbo frames
3060 *
3061 * Using pages when the page size is greater than 16k wastes
3062 * a lot of memory, since we allocate 3 pages at all times
3063 * per packet.
3064 */
bc7f75fa 3065 pages = PAGE_USE_COUNT(adapter->netdev->mtu);
79d4e908 3066 if ((pages <= 3) && (PAGE_SIZE <= 16384) && (rctl & E1000_RCTL_LPE))
bc7f75fa 3067 adapter->rx_ps_pages = pages;
97ac8cae
BA		 3068 else
3069 adapter->rx_ps_pages = 0;
bc7f75fa
AK		 3070
3071 if (adapter->rx_ps_pages) {
90da0669
BA		 3072 u32 psrctl = 0;
3073
140a7480
AK		 3074 /* Enable Packet split descriptors */
3075 rctl |= E1000_RCTL_DTYP_PS;
bc7f75fa 3076
e5fe2541 3077 psrctl |= adapter->rx_ps_bsize0 >> E1000_PSRCTL_BSIZE0_SHIFT;
bc7f75fa
AK		 3078
3079 switch (adapter->rx_ps_pages) {
3080 case 3:
e5fe2541
BA		 3081 psrctl |= PAGE_SIZE << E1000_PSRCTL_BSIZE3_SHIFT;
3082 /* fall-through */
bc7f75fa 3083 case 2:
e5fe2541
BA		 3084 psrctl |= PAGE_SIZE << E1000_PSRCTL_BSIZE2_SHIFT;
3085 /* fall-through */
bc7f75fa 3086 case 1:
e5fe2541 3087 psrctl |= PAGE_SIZE >> E1000_PSRCTL_BSIZE1_SHIFT;
bc7f75fa
AK		 3088 break;
3089 }
3090
3091 ew32(PSRCTL, psrctl);
3092 }
3093
cf955e6c
BG		 3094 /* This is useful for sniffing bad packets. */
3095 if (adapter->netdev->features & NETIF_F_RXALL) {
3096 /* UPE and MPE will be handled by normal PROMISC logic
e921eb1a
BA		 3097 * in e1000e_set_rx_mode
3098 */
cf955e6c
BG		 3099 rctl |= (E1000_RCTL_SBP | /* Receive bad packets */
3100 E1000_RCTL_BAM | /* RX All Bcast Pkts */
3101 E1000_RCTL_PMCF); /* RX All MAC Ctrl Pkts */
3102
3103 rctl &= ~(E1000_RCTL_VFE | /* Disable VLAN filter */
3104 E1000_RCTL_DPF | /* Allow filtered pause */
3105 E1000_RCTL_CFIEN); /* Dis VLAN CFIEN Filter */
3106 /* Do not mess with E1000_CTRL_VME, it affects transmit as well,
3107 * and that breaks VLANs.
3108 */
3109 }
3110
bc7f75fa 3111 ew32(RCTL, rctl);
318a94d6 3112 /* just started the receive unit, no need to restart */
12d43f7d 3113 adapter->flags &= ~FLAG_RESTART_NOW;
bc7f75fa
AK		 3114}
3115
3116/**
3117 * e1000_configure_rx - Configure Receive Unit after Reset
3118 * @adapter: board private structure
3119 *
3120 * Configure the Rx unit of the MAC after a reset.
3121 **/
3122static void e1000_configure_rx(struct e1000_adapter *adapter)
3123{
3124 struct e1000_hw *hw = &adapter->hw;
3125 struct e1000_ring *rx_ring = adapter->rx_ring;
3126 u64 rdba;
3127 u32 rdlen, rctl, rxcsum, ctrl_ext;
3128
3129 if (adapter->rx_ps_pages) {
3130 /* this is a 32 byte descriptor */
3131 rdlen = rx_ring->count *
af667a29 3132 sizeof(union e1000_rx_desc_packet_split);
bc7f75fa
AK		 3133 adapter->clean_rx = e1000_clean_rx_irq_ps;
3134 adapter->alloc_rx_buf = e1000_alloc_rx_buffers_ps;
97ac8cae 3135 } else if (adapter->netdev->mtu > ETH_FRAME_LEN + ETH_FCS_LEN) {
5f450212 3136 rdlen = rx_ring->count * sizeof(union e1000_rx_desc_extended);
97ac8cae
BA		 3137 adapter->clean_rx = e1000_clean_jumbo_rx_irq;
3138 adapter->alloc_rx_buf = e1000_alloc_jumbo_rx_buffers;
bc7f75fa 3139 } else {
5f450212 3140 rdlen = rx_ring->count * sizeof(union e1000_rx_desc_extended);
bc7f75fa
AK		 3141 adapter->clean_rx = e1000_clean_rx_irq;
3142 adapter->alloc_rx_buf = e1000_alloc_rx_buffers;
3143 }
3144
3145 /* disable receives while setting up the descriptors */
3146 rctl = er32(RCTL);
7f99ae63
BA		 3147 if (!(adapter->flags2 & FLAG2_NO_DISABLE_RX))
3148 ew32(RCTL, rctl & ~E1000_RCTL_EN);
bc7f75fa 3149 e1e_flush();
1bba4386 3150 usleep_range(10000, 20000);
bc7f75fa 3151
3a3b7586 3152 if (adapter->flags2 & FLAG2_DMA_BURST) {
e921eb1a 3153 /* set the writeback threshold (only takes effect if the RDTR
3a3b7586 3154 * is set). set GRAN=1 and write back up to 0x4 worth, and
af667a29 3155 * enable prefetching of 0x20 Rx descriptors
3a3b7586
JB		 3156 * granularity = 01
3157 * wthresh = 04,
3158 * hthresh = 04,
3159 * pthresh = 0x20
3160 */
3161 ew32(RXDCTL(0), E1000_RXDCTL_DMA_BURST_ENABLE);
3162 ew32(RXDCTL(1), E1000_RXDCTL_DMA_BURST_ENABLE);
3163
e921eb1a 3164 /* override the delay timers for enabling bursting, only if
3a3b7586
JB		 3165 * the value was not set by the user via module options
3166 */
3167 if (adapter->rx_int_delay == DEFAULT_RDTR)
3168 adapter->rx_int_delay = BURST_RDTR;
3169 if (adapter->rx_abs_int_delay == DEFAULT_RADV)
3170 adapter->rx_abs_int_delay = BURST_RADV;
3171 }
3172
bc7f75fa
AK		 3173 /* set the Receive Delay Timer Register */
3174 ew32(RDTR, adapter->rx_int_delay);
3175
3176 /* irq moderation */
3177 ew32(RADV, adapter->rx_abs_int_delay);
828bac87 3178 if ((adapter->itr_setting != 0) && (adapter->itr != 0))
22a4cca2 3179 e1000e_write_itr(adapter, adapter->itr);
bc7f75fa
AK		 3180
3181 ctrl_ext = er32(CTRL_EXT);
bc7f75fa
AK		 3182 /* Auto-Mask interrupts upon ICR access */
3183 ctrl_ext |= E1000_CTRL_EXT_IAME;
3184 ew32(IAM, 0xffffffff);
3185 ew32(CTRL_EXT, ctrl_ext);
3186 e1e_flush();
3187
e921eb1a 3188 /* Setup the HW Rx Head and Tail Descriptor Pointers and
ad68076e
BA		 3189 * the Base and Length of the Rx Descriptor Ring
3190 */
bc7f75fa 3191 rdba = rx_ring->dma;
1e36052e
BA		 3192 ew32(RDBAL(0), (rdba & DMA_BIT_MASK(32)));
3193 ew32(RDBAH(0), (rdba >> 32));
3194 ew32(RDLEN(0), rdlen);
3195 ew32(RDH(0), 0);
3196 ew32(RDT(0), 0);
3197 rx_ring->head = adapter->hw.hw_addr + E1000_RDH(0);
3198 rx_ring->tail = adapter->hw.hw_addr + E1000_RDT(0);
bc7f75fa
AK		 3199
3200 /* Enable Receive Checksum Offload for TCP and UDP */
3201 rxcsum = er32(RXCSUM);
2e1706f2 3202 if (adapter->netdev->features & NETIF_F_RXCSUM)
bc7f75fa 3203 rxcsum |= E1000_RXCSUM_TUOFL;
2e1706f2 3204 else
bc7f75fa 3205 rxcsum &= ~E1000_RXCSUM_TUOFL;
bc7f75fa
AK		 3206 ew32(RXCSUM, rxcsum);
3207
3e35d991
BA		 3208 /* With jumbo frames, excessive C-state transition latencies result
3209 * in dropped transactions.
3210 */
3211 if (adapter->netdev->mtu > ETH_DATA_LEN) {
3212 u32 lat =
3213 ((er32(PBA) & E1000_PBA_RXA_MASK) * 1024 -
3214 adapter->max_frame_size) * 8 / 1000;
3215
3216 if (adapter->flags & FLAG_IS_ICH) {
53ec5498
BA		 3217 u32 rxdctl = er32(RXDCTL(0));
3218 ew32(RXDCTL(0), rxdctl | 0x3);
53ec5498 3219 }
3e35d991
BA		 3220
3221 pm_qos_update_request(&adapter->netdev->pm_qos_req, lat);
3222 } else {
3223 pm_qos_update_request(&adapter->netdev->pm_qos_req,
3224 PM_QOS_DEFAULT_VALUE);
97ac8cae 3225 }
bc7f75fa
AK		 3226
3227 /* Enable Receives */
3228 ew32(RCTL, rctl);
3229}
3230
3231/**
ef9b965a
JB		 3232 * e1000e_write_mc_addr_list - write multicast addresses to MTA
3233 * @netdev: network interface device structure
bc7f75fa 3234 *
ef9b965a
JB		 3235 * Writes multicast address list to the MTA hash table.
3236 * Returns: -ENOMEM on failure
3237 * 0 on no addresses written
3238 * X on writing X addresses to MTA
3239 */
3240static int e1000e_write_mc_addr_list(struct net_device *netdev)
3241{
3242 struct e1000_adapter *adapter = netdev_priv(netdev);
3243 struct e1000_hw *hw = &adapter->hw;
3244 struct netdev_hw_addr *ha;
3245 u8 *mta_list;
3246 int i;
3247
3248 if (netdev_mc_empty(netdev)) {
3249 /* nothing to program, so clear mc list */
3250 hw->mac.ops.update_mc_addr_list(hw, NULL, 0);
3251 return 0;
3252 }
3253
3254 mta_list = kzalloc(netdev_mc_count(netdev) * ETH_ALEN, GFP_ATOMIC);
3255 if (!mta_list)
3256 return -ENOMEM;
3257
3258 /* update_mc_addr_list expects a packed array of only addresses. */
3259 i = 0;
3260 netdev_for_each_mc_addr(ha, netdev)
f0ff4398 3261 memcpy(mta_list + (i++ * ETH_ALEN), ha->addr, ETH_ALEN);
ef9b965a
JB		 3262
3263 hw->mac.ops.update_mc_addr_list(hw, mta_list, i);
3264 kfree(mta_list);
3265
3266 return netdev_mc_count(netdev);
3267}
3268
3269/**
3270 * e1000e_write_uc_addr_list - write unicast addresses to RAR table
3271 * @netdev: network interface device structure
bc7f75fa 3272 *
ef9b965a
JB		 3273 * Writes unicast address list to the RAR table.
3274 * Returns: -ENOMEM on failure/insufficient address space
3275 * 0 on no addresses written
3276 * X on writing X addresses to the RAR table
bc7f75fa 3277 **/
ef9b965a 3278static int e1000e_write_uc_addr_list(struct net_device *netdev)
bc7f75fa 3279{
ef9b965a
JB		 3280 struct e1000_adapter *adapter = netdev_priv(netdev);
3281 struct e1000_hw *hw = &adapter->hw;
3282 unsigned int rar_entries = hw->mac.rar_entry_count;
3283 int count = 0;
3284
3285 /* save a rar entry for our hardware address */
3286 rar_entries--;
3287
3288 /* save a rar entry for the LAA workaround */
3289 if (adapter->flags & FLAG_RESET_OVERWRITES_LAA)
3290 rar_entries--;
3291
3292 /* return ENOMEM indicating insufficient memory for addresses */
3293 if (netdev_uc_count(netdev) > rar_entries)
3294 return -ENOMEM;
3295
3296 if (!netdev_uc_empty(netdev) && rar_entries) {
3297 struct netdev_hw_addr *ha;
3298
e921eb1a 3299 /* write the addresses in reverse order to avoid write
ef9b965a
JB		 3300 * combining
3301 */
3302 netdev_for_each_uc_addr(ha, netdev) {
3303 if (!rar_entries)
3304 break;
69e1e019 3305 hw->mac.ops.rar_set(hw, ha->addr, rar_entries--);
ef9b965a
JB		 3306 count++;
3307 }
3308 }
3309
3310 /* zero out the remaining RAR entries not used above */
3311 for (; rar_entries > 0; rar_entries--) {
3312 ew32(RAH(rar_entries), 0);
3313 ew32(RAL(rar_entries), 0);
3314 }
3315 e1e_flush();
3316
3317 return count;
bc7f75fa
AK		 3318}
3319
3320/**
ef9b965a 3321 * e1000e_set_rx_mode - secondary unicast, Multicast and Promiscuous mode set
bc7f75fa
AK		 3322 * @netdev: network interface device structure
3323 *
ef9b965a
JB		 3324 * The ndo_set_rx_mode entry point is called whenever the unicast or multicast
3325 * address list or the network interface flags are updated. This routine is
3326 * responsible for configuring the hardware for proper unicast, multicast,
bc7f75fa
AK		 3327 * promiscuous mode, and all-multi behavior.
3328 **/
ef9b965a 3329static void e1000e_set_rx_mode(struct net_device *netdev)
bc7f75fa
AK		 3330{
3331 struct e1000_adapter *adapter = netdev_priv(netdev);
3332 struct e1000_hw *hw = &adapter->hw;
bc7f75fa 3333 u32 rctl;
bc7f75fa
AK		 3334
3335 /* Check for Promiscuous and All Multicast modes */
bc7f75fa
AK		 3336 rctl = er32(RCTL);
3337
ef9b965a
JB		 3338 /* clear the affected bits */
3339 rctl &= ~(E1000_RCTL_UPE | E1000_RCTL_MPE);
3340
bc7f75fa
AK		 3341 if (netdev->flags & IFF_PROMISC) {
3342 rctl |= (E1000_RCTL_UPE | E1000_RCTL_MPE);
86d70e53
JK		 3343 /* Do not hardware filter VLANs in promisc mode */
3344 e1000e_vlan_filter_disable(adapter);
bc7f75fa 3345 } else {
ef9b965a 3346 int count;
3d3a1676 3347
746b9f02
PM		 3348 if (netdev->flags & IFF_ALLMULTI) {
3349 rctl |= E1000_RCTL_MPE;
746b9f02 3350 } else {
e921eb1a 3351 /* Write addresses to the MTA, if the attempt fails
ef9b965a
JB		 3352 * then we should just turn on promiscuous mode so
3353 * that we can at least receive multicast traffic
3354 */
3355 count = e1000e_write_mc_addr_list(netdev);
3356 if (count < 0)
3357 rctl |= E1000_RCTL_MPE;
746b9f02 3358 }
86d70e53 3359 e1000e_vlan_filter_enable(adapter);
e921eb1a 3360 /* Write addresses to available RAR registers, if there is not
ef9b965a
JB		 3361 * sufficient space to store all the addresses then enable
3362 * unicast promiscuous mode
bc7f75fa 3363 */
ef9b965a
JB		 3364 count = e1000e_write_uc_addr_list(netdev);
3365 if (count < 0)
3366 rctl |= E1000_RCTL_UPE;
bc7f75fa 3367 }
86d70e53 3368
ef9b965a
JB		 3369 ew32(RCTL, rctl);
3370
86d70e53
JK		 3371 if (netdev->features & NETIF_F_HW_VLAN_RX)
3372 e1000e_vlan_strip_enable(adapter);
3373 else
3374 e1000e_vlan_strip_disable(adapter);
bc7f75fa
AK		 3375}
3376
70495a50
BA		 3377static void e1000e_setup_rss_hash(struct e1000_adapter *adapter)
3378{
3379 struct e1000_hw *hw = &adapter->hw;
3380 u32 mrqc, rxcsum;
3381 int i;
3382 static const u32 rsskey[10] = {
3383 0xda565a6d, 0xc20e5b25, 0x3d256741, 0xb08fa343, 0xcb2bcad0,
3384 0xb4307bae, 0xa32dcb77, 0x0cf23080, 0x3bb7426a, 0xfa01acbe
3385 };
3386
3387 /* Fill out hash function seed */
3388 for (i = 0; i < 10; i++)
3389 ew32(RSSRK(i), rsskey[i]);
3390
3391 /* Direct all traffic to queue 0 */
3392 for (i = 0; i < 32; i++)
3393 ew32(RETA(i), 0);
3394
e921eb1a 3395 /* Disable raw packet checksumming so that RSS hash is placed in
70495a50
BA		 3396 * descriptor on writeback.
3397 */
3398 rxcsum = er32(RXCSUM);
3399 rxcsum |= E1000_RXCSUM_PCSD;
3400
3401 ew32(RXCSUM, rxcsum);
3402
3403 mrqc = (E1000_MRQC_RSS_FIELD_IPV4 |
3404 E1000_MRQC_RSS_FIELD_IPV4_TCP |
3405 E1000_MRQC_RSS_FIELD_IPV6 |
3406 E1000_MRQC_RSS_FIELD_IPV6_TCP |
3407 E1000_MRQC_RSS_FIELD_IPV6_TCP_EX);
3408
3409 ew32(MRQC, mrqc);
3410}
3411
b67e1913
BA		 3412/**
3413 * e1000e_get_base_timinca - get default SYSTIM time increment attributes
3414 * @adapter: board private structure
3415 * @timinca: pointer to returned time increment attributes
3416 *
3417 * Get attributes for incrementing the System Time Register SYSTIML/H at
3418 * the default base frequency, and set the cyclecounter shift value.
3419 **/
d89777bf 3420s32 e1000e_get_base_timinca(struct e1000_adapter *adapter, u32 *timinca)
b67e1913
BA		 3421{
3422 struct e1000_hw *hw = &adapter->hw;
3423 u32 incvalue, incperiod, shift;
3424
3425 /* Make sure clock is enabled on I217 before checking the frequency */
3426 if ((hw->mac.type == e1000_pch_lpt) &&
3427 !(er32(TSYNCTXCTL) & E1000_TSYNCTXCTL_ENABLED) &&
3428 !(er32(TSYNCRXCTL) & E1000_TSYNCRXCTL_ENABLED)) {
3429 u32 fextnvm7 = er32(FEXTNVM7);
3430
3431 if (!(fextnvm7 & (1 << 0))) {
3432 ew32(FEXTNVM7, fextnvm7 | (1 << 0));
3433 e1e_flush();
3434 }
3435 }
3436
3437 switch (hw->mac.type) {
3438 case e1000_pch2lan:
3439 case e1000_pch_lpt:
3440 /* On I217, the clock frequency is 25MHz or 96MHz as
3441 * indicated by the System Clock Frequency Indication
3442 */
3443 if ((hw->mac.type != e1000_pch_lpt) ||
3444 (er32(TSYNCRXCTL) & E1000_TSYNCRXCTL_SYSCFI)) {
3445 /* Stable 96MHz frequency */
3446 incperiod = INCPERIOD_96MHz;
3447 incvalue = INCVALUE_96MHz;
3448 shift = INCVALUE_SHIFT_96MHz;
3449 adapter->cc.shift = shift + INCPERIOD_SHIFT_96MHz;
3450 break;
3451 }
3452 /* fall-through */
3453 case e1000_82574:
3454 case e1000_82583:
3455 /* Stable 25MHz frequency */
3456 incperiod = INCPERIOD_25MHz;
3457 incvalue = INCVALUE_25MHz;
3458 shift = INCVALUE_SHIFT_25MHz;
3459 adapter->cc.shift = shift;
3460 break;
3461 default:
3462 return -EINVAL;
3463 }
3464
3465 *timinca = ((incperiod << E1000_TIMINCA_INCPERIOD_SHIFT) |
3466 ((incvalue << shift) & E1000_TIMINCA_INCVALUE_MASK));
3467
3468 return 0;
3469}
3470
3471/**
3472 * e1000e_config_hwtstamp - configure the hwtstamp registers and enable/disable
3473 * @adapter: board private structure
3474 *
3475 * Outgoing time stamping can be enabled and disabled. Play nice and
3476 * disable it when requested, although it shouldn't cause any overhead
3477 * when no packet needs it. At most one packet in the queue may be
3478 * marked for time stamping, otherwise it would be impossible to tell
3479 * for sure to which packet the hardware time stamp belongs.
3480 *
3481 * Incoming time stamping has to be configured via the hardware filters.
3482 * Not all combinations are supported, in particular event type has to be
3483 * specified. Matching the kind of event packet is not supported, with the
3484 * exception of "all V2 events regardless of level 2 or 4".
3485 **/
3486static int e1000e_config_hwtstamp(struct e1000_adapter *adapter)
3487{
3488 struct e1000_hw *hw = &adapter->hw;
3489 struct hwtstamp_config *config = &adapter->hwtstamp_config;
3490 u32 tsync_tx_ctl = E1000_TSYNCTXCTL_ENABLED;
3491 u32 tsync_rx_ctl = E1000_TSYNCRXCTL_ENABLED;
d89777bf
BA		 3492 u32 rxmtrl = 0;
3493 u16 rxudp = 0;
3494 bool is_l4 = false;
3495 bool is_l2 = false;
b67e1913
BA		 3496 u32 regval;
3497 s32 ret_val;
3498
3499 if (!(adapter->flags & FLAG_HAS_HW_TIMESTAMP))
3500 return -EINVAL;
3501
3502 /* flags reserved for future extensions - must be zero */
3503 if (config->flags)
3504 return -EINVAL;
3505
3506 switch (config->tx_type) {
3507 case HWTSTAMP_TX_OFF:
3508 tsync_tx_ctl = 0;
3509 break;
3510 case HWTSTAMP_TX_ON:
3511 break;
3512 default:
3513 return -ERANGE;
3514 }
3515
3516 switch (config->rx_filter) {
3517 case HWTSTAMP_FILTER_NONE:
3518 tsync_rx_ctl = 0;
3519 break;
d89777bf
BA		 3520 case HWTSTAMP_FILTER_PTP_V1_L4_SYNC:
3521 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_L4_V1;
3522 rxmtrl = E1000_RXMTRL_PTP_V1_SYNC_MESSAGE;
3523 is_l4 = true;
3524 break;
3525 case HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ:
3526 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_L4_V1;
3527 rxmtrl = E1000_RXMTRL_PTP_V1_DELAY_REQ_MESSAGE;
3528 is_l4 = true;
3529 break;
3530 case HWTSTAMP_FILTER_PTP_V2_L2_SYNC:
3531 /* Also time stamps V2 L2 Path Delay Request/Response */
3532 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_L2_V2;
3533 rxmtrl = E1000_RXMTRL_PTP_V2_SYNC_MESSAGE;
3534 is_l2 = true;
3535 break;
3536 case HWTSTAMP_FILTER_PTP_V2_L2_DELAY_REQ:
3537 /* Also time stamps V2 L2 Path Delay Request/Response. */
3538 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_L2_V2;
3539 rxmtrl = E1000_RXMTRL_PTP_V2_DELAY_REQ_MESSAGE;
3540 is_l2 = true;
3541 break;
3542 case HWTSTAMP_FILTER_PTP_V2_L4_SYNC:
3543 /* Hardware cannot filter just V2 L4 Sync messages;
3544 * fall-through to V2 (both L2 and L4) Sync.
3545 */
3546 case HWTSTAMP_FILTER_PTP_V2_SYNC:
3547 /* Also time stamps V2 Path Delay Request/Response. */
3548 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_L2_L4_V2;
3549 rxmtrl = E1000_RXMTRL_PTP_V2_SYNC_MESSAGE;
3550 is_l2 = true;
3551 is_l4 = true;
3552 break;
3553 case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ:
3554 /* Hardware cannot filter just V2 L4 Delay Request messages;
3555 * fall-through to V2 (both L2 and L4) Delay Request.
3556 */
3557 case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ:
3558 /* Also time stamps V2 Path Delay Request/Response. */
3559 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_L2_L4_V2;
3560 rxmtrl = E1000_RXMTRL_PTP_V2_DELAY_REQ_MESSAGE;
3561 is_l2 = true;
3562 is_l4 = true;
3563 break;
3564 case HWTSTAMP_FILTER_PTP_V2_L4_EVENT:
3565 case HWTSTAMP_FILTER_PTP_V2_L2_EVENT:
3566 /* Hardware cannot filter just V2 L4 or L2 Event messages;
3567 * fall-through to all V2 (both L2 and L4) Events.
3568 */
3569 case HWTSTAMP_FILTER_PTP_V2_EVENT:
3570 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_EVENT_V2;
3571 config->rx_filter = HWTSTAMP_FILTER_PTP_V2_EVENT;
3572 is_l2 = true;
3573 is_l4 = true;
3574 break;
3575 case HWTSTAMP_FILTER_PTP_V1_L4_EVENT:
3576 /* For V1, the hardware can only filter Sync messages or
3577 * Delay Request messages but not both so fall-through to
3578 * time stamp all packets.
3579 */
b67e1913 3580 case HWTSTAMP_FILTER_ALL:
d89777bf
BA		 3581 is_l2 = true;
3582 is_l4 = true;
b67e1913
BA		 3583 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_ALL;
3584 config->rx_filter = HWTSTAMP_FILTER_ALL;
3585 break;
3586 default:
3587 return -ERANGE;
3588 }
3589
3590 /* enable/disable Tx h/w time stamping */
3591 regval = er32(TSYNCTXCTL);
3592 regval &= ~E1000_TSYNCTXCTL_ENABLED;
3593 regval |= tsync_tx_ctl;
3594 ew32(TSYNCTXCTL, regval);
3595 if ((er32(TSYNCTXCTL) & E1000_TSYNCTXCTL_ENABLED) !=
3596 (regval & E1000_TSYNCTXCTL_ENABLED)) {
3597 e_err("Timesync Tx Control register not set as expected\n");
3598 return -EAGAIN;
3599 }
3600
3601 /* enable/disable Rx h/w time stamping */
3602 regval = er32(TSYNCRXCTL);
3603 regval &= ~(E1000_TSYNCRXCTL_ENABLED | E1000_TSYNCRXCTL_TYPE_MASK);
3604 regval |= tsync_rx_ctl;
3605 ew32(TSYNCRXCTL, regval);
3606 if ((er32(TSYNCRXCTL) & (E1000_TSYNCRXCTL_ENABLED |
3607 E1000_TSYNCRXCTL_TYPE_MASK)) !=
3608 (regval & (E1000_TSYNCRXCTL_ENABLED |
3609 E1000_TSYNCRXCTL_TYPE_MASK))) {
3610 e_err("Timesync Rx Control register not set as expected\n");
3611 return -EAGAIN;
3612 }
3613
d89777bf
BA		 3614 /* L2: define ethertype filter for time stamped packets */
3615 if (is_l2)
3616 rxmtrl |= ETH_P_1588;
3617
3618 /* define which PTP packets get time stamped */
3619 ew32(RXMTRL, rxmtrl);
3620
3621 /* Filter by destination port */
3622 if (is_l4) {
3623 rxudp = PTP_EV_PORT;
3624 cpu_to_be16s(&rxudp);
3625 }
3626 ew32(RXUDP, rxudp);
3627
3628 e1e_flush();
3629
b67e1913 3630 /* Clear TSYNCRXCTL_VALID & TSYNCTXCTL_VALID bit */
70806a7f
BA		 3631 er32(RXSTMPH);
3632 er32(TXSTMPH);
b67e1913
BA		 3633
3634 /* Get and set the System Time Register SYSTIM base frequency */
3635 ret_val = e1000e_get_base_timinca(adapter, &regval);
3636 if (ret_val)
3637 return ret_val;
3638 ew32(TIMINCA, regval);
3639
3640 /* reset the ns time counter */
3641 timecounter_init(&adapter->tc, &adapter->cc,
3642 ktime_to_ns(ktime_get_real()));
3643
3644 return 0;
3645}
3646
bc7f75fa 3647/**
ad68076e 3648 * e1000_configure - configure the hardware for Rx and Tx
bc7f75fa
AK		 3649 * @adapter: private board structure
3650 **/
3651static void e1000_configure(struct e1000_adapter *adapter)
3652{
55aa6985
BA		 3653 struct e1000_ring *rx_ring = adapter->rx_ring;
3654
ef9b965a 3655 e1000e_set_rx_mode(adapter->netdev);
bc7f75fa
AK		 3656
3657 e1000_restore_vlan(adapter);
cd791618 3658 e1000_init_manageability_pt(adapter);
bc7f75fa
AK		 3659
3660 e1000_configure_tx(adapter);
70495a50
BA		 3661
3662 if (adapter->netdev->features & NETIF_F_RXHASH)
3663 e1000e_setup_rss_hash(adapter);
bc7f75fa
AK		 3664 e1000_setup_rctl(adapter);
3665 e1000_configure_rx(adapter);
55aa6985 3666 adapter->alloc_rx_buf(rx_ring, e1000_desc_unused(rx_ring), GFP_KERNEL);
bc7f75fa
AK		 3667}
3668
3669/**
3670 * e1000e_power_up_phy - restore link in case the phy was powered down
3671 * @adapter: address of board private structure
3672 *
3673 * The phy may be powered down to save power and turn off link when the
3674 * driver is unloaded and wake on lan is not enabled (among others)
3675 * *** this routine MUST be followed by a call to e1000e_reset ***
3676 **/
3677void e1000e_power_up_phy(struct e1000_adapter *adapter)
3678{
17f208de
BA		 3679 if (adapter->hw.phy.ops.power_up)
3680 adapter->hw.phy.ops.power_up(&adapter->hw);
bc7f75fa
AK		 3681
3682 adapter->hw.mac.ops.setup_link(&adapter->hw);
3683}
3684
3685/**
3686 * e1000_power_down_phy - Power down the PHY
3687 *
17f208de
BA		 3688 * Power down the PHY so no link is implied when interface is down.
3689 * The PHY cannot be powered down if management or WoL is active.
bc7f75fa
AK		 3690 */
3691static void e1000_power_down_phy(struct e1000_adapter *adapter)
3692{
bc7f75fa 3693 /* WoL is enabled */
23b66e2b 3694 if (adapter->wol)
bc7f75fa
AK		 3695 return;
3696
17f208de
BA		 3697 if (adapter->hw.phy.ops.power_down)
3698 adapter->hw.phy.ops.power_down(&adapter->hw);
bc7f75fa
AK		 3699}
3700
3701/**
3702 * e1000e_reset - bring the hardware into a known good state
3703 *
3704 * This function boots the hardware and enables some settings that
3705 * require a configuration cycle of the hardware - those cannot be
3706 * set/changed during runtime. After reset the device needs to be
ad68076e 3707 * properly configured for Rx, Tx etc.
bc7f75fa
AK		 3708 */
3709void e1000e_reset(struct e1000_adapter *adapter)
3710{
3711 struct e1000_mac_info *mac = &adapter->hw.mac;
318a94d6 3712 struct e1000_fc_info *fc = &adapter->hw.fc;
bc7f75fa
AK		 3713 struct e1000_hw *hw = &adapter->hw;
3714 u32 tx_space, min_tx_space, min_rx_space;
318a94d6 3715 u32 pba = adapter->pba;
bc7f75fa
AK		 3716 u16 hwm;
3717
ad68076e 3718 /* reset Packet Buffer Allocation to default */
318a94d6 3719 ew32(PBA, pba);
df762464 3720
318a94d6 3721 if (adapter->max_frame_size > ETH_FRAME_LEN + ETH_FCS_LEN) {
e921eb1a 3722 /* To maintain wire speed transmits, the Tx FIFO should be
bc7f75fa
AK		 3723 * large enough to accommodate two full transmit packets,
3724 * rounded up to the next 1KB and expressed in KB. Likewise,
3725 * the Rx FIFO should be large enough to accommodate at least
3726 * one full receive packet and is similarly rounded up and
ad68076e
BA		 3727 * expressed in KB.
3728 */
df762464 3729 pba = er32(PBA);
bc7f75fa 3730 /* upper 16 bits has Tx packet buffer allocation size in KB */
df762464 3731 tx_space = pba >> 16;
bc7f75fa 3732 /* lower 16 bits has Rx packet buffer allocation size in KB */
df762464 3733 pba &= 0xffff;
e921eb1a 3734 /* the Tx fifo also stores 16 bytes of information about the Tx
ad68076e 3735 * but don't include ethernet FCS because hardware appends it
318a94d6
JK		 3736 */
3737 min_tx_space = (adapter->max_frame_size +
e5fe2541 3738 sizeof(struct e1000_tx_desc) - ETH_FCS_LEN) * 2;
bc7f75fa
AK		 3739 min_tx_space = ALIGN(min_tx_space, 1024);
3740 min_tx_space >>= 10;
3741 /* software strips receive CRC, so leave room for it */
318a94d6 3742 min_rx_space = adapter->max_frame_size;
bc7f75fa
AK		 3743 min_rx_space = ALIGN(min_rx_space, 1024);
3744 min_rx_space >>= 10;
3745
e921eb1a 3746 /* If current Tx allocation is less than the min Tx FIFO size,
bc7f75fa 3747 * and the min Tx FIFO size is less than the current Rx FIFO
ad68076e
BA		 3748 * allocation, take space away from current Rx allocation
3749 */
df762464
AK		 3750 if ((tx_space < min_tx_space) &&
3751 ((min_tx_space - tx_space) < pba)) {
3752 pba -= min_tx_space - tx_space;
bc7f75fa 3753
e921eb1a 3754 /* if short on Rx space, Rx wins and must trump Tx
419e551c 3755 * adjustment
ad68076e 3756 */
79d4e908 3757 if (pba < min_rx_space)
df762464 3758 pba = min_rx_space;
bc7f75fa 3759 }
df762464
AK		 3760
3761 ew32(PBA, pba);
bc7f75fa
AK		 3762 }
3763
e921eb1a 3764 /* flow control settings
ad68076e 3765 *
38eb394e 3766 * The high water mark must be low enough to fit one full frame
bc7f75fa
AK		 3767 * (or the size used for early receive) above it in the Rx FIFO.
3768 * Set it to the lower of:
3769 * - 90% of the Rx FIFO size, and
38eb394e 3770 * - the full Rx FIFO size minus one full frame
ad68076e 3771 */
d3738bb8
BA		 3772 if (adapter->flags & FLAG_DISABLE_FC_PAUSE_TIME)
3773 fc->pause_time = 0xFFFF;
3774 else
3775 fc->pause_time = E1000_FC_PAUSE_TIME;
b20caa80 3776 fc->send_xon = true;
d3738bb8
BA		 3777 fc->current_mode = fc->requested_mode;
3778
3779 switch (hw->mac.type) {
79d4e908
BA		 3780 case e1000_ich9lan:
3781 case e1000_ich10lan:
3782 if (adapter->netdev->mtu > ETH_DATA_LEN) {
3783 pba = 14;
3784 ew32(PBA, pba);
3785 fc->high_water = 0x2800;
3786 fc->low_water = fc->high_water - 8;
3787 break;
3788 }
3789 /* fall-through */
d3738bb8 3790 default:
79d4e908
BA		 3791 hwm = min(((pba << 10) * 9 / 10),
3792 ((pba << 10) - adapter->max_frame_size));
d3738bb8
BA		 3793
3794 fc->high_water = hwm & E1000_FCRTH_RTH; /* 8-byte granularity */
3795 fc->low_water = fc->high_water - 8;
3796 break;
3797 case e1000_pchlan:
e921eb1a 3798 /* Workaround PCH LOM adapter hangs with certain network
38eb394e
BA		 3799 * loads. If hangs persist, try disabling Tx flow control.
3800 */
3801 if (adapter->netdev->mtu > ETH_DATA_LEN) {
3802 fc->high_water = 0x3500;
3803 fc->low_water = 0x1500;
3804 } else {
3805 fc->high_water = 0x5000;
3806 fc->low_water = 0x3000;
3807 }
a305595b 3808 fc->refresh_time = 0x1000;
d3738bb8
BA		 3809 break;
3810 case e1000_pch2lan:
2fbe4526 3811 case e1000_pch_lpt:
d3738bb8 3812 fc->refresh_time = 0x0400;
347b5201
BA		 3813
3814 if (adapter->netdev->mtu <= ETH_DATA_LEN) {
3815 fc->high_water = 0x05C20;
3816 fc->low_water = 0x05048;
3817 fc->pause_time = 0x0650;
3818 break;
828bac87 3819 }
347b5201
BA		 3820
3821 fc->high_water = ((pba << 10) * 9 / 10) & E1000_FCRTH_RTH;
3822 fc->low_water = ((pba << 10) * 8 / 10) & E1000_FCRTL_RTL;
d3738bb8 3823 break;
38eb394e 3824 }
bc7f75fa 3825
e921eb1a 3826 /* Alignment of Tx data is on an arbitrary byte boundary with the
d821a4c4
BA		 3827 * maximum size per Tx descriptor limited only to the transmit
3828 * allocation of the packet buffer minus 96 bytes with an upper
3829 * limit of 24KB due to receive synchronization limitations.
3830 */
3831 adapter->tx_fifo_limit = min_t(u32, ((er32(PBA) >> 16) << 10) - 96,
3832 24 << 10);
3833
e921eb1a 3834 /* Disable Adaptive Interrupt Moderation if 2 full packets cannot
79d4e908 3835 * fit in receive buffer.
828bac87
BA		 3836 */
3837 if (adapter->itr_setting & 0x3) {
79d4e908 3838 if ((adapter->max_frame_size * 2) > (pba << 10)) {
828bac87
BA		 3839 if (!(adapter->flags2 & FLAG2_DISABLE_AIM)) {
3840 dev_info(&adapter->pdev->dev,
17e813ec 3841 "Interrupt Throttle Rate off\n");
828bac87 3842 adapter->flags2 |= FLAG2_DISABLE_AIM;
22a4cca2 3843 e1000e_write_itr(adapter, 0);
828bac87
BA		 3844 }
3845 } else if (adapter->flags2 & FLAG2_DISABLE_AIM) {
3846 dev_info(&adapter->pdev->dev,
17e813ec 3847 "Interrupt Throttle Rate on\n");
828bac87
BA		 3848 adapter->flags2 &= ~FLAG2_DISABLE_AIM;
3849 adapter->itr = 20000;
22a4cca2 3850 e1000e_write_itr(adapter, adapter->itr);
828bac87
BA		 3851 }
3852 }
3853
bc7f75fa
AK		 3854 /* Allow time for pending master requests to run */
3855 mac->ops.reset_hw(hw);
97ac8cae 3856
e921eb1a 3857 /* For parts with AMT enabled, let the firmware know
97ac8cae
BA		 3858 * that the network interface is in control
3859 */
c43bc57e 3860 if (adapter->flags & FLAG_HAS_AMT)
31dbe5b4 3861 e1000e_get_hw_control(adapter);
97ac8cae 3862
bc7f75fa
AK		 3863 ew32(WUC, 0);
3864
3865 if (mac->ops.init_hw(hw))
44defeb3 3866 e_err("Hardware Error\n");
bc7f75fa
AK		 3867
3868 e1000_update_mng_vlan(adapter);
3869
3870 /* Enable h/w to recognize an 802.1Q VLAN Ethernet packet */
3871 ew32(VET, ETH_P_8021Q);
3872
3873 e1000e_reset_adaptive(hw);
31dbe5b4 3874
b67e1913
BA		 3875 /* initialize systim and reset the ns time counter */
3876 e1000e_config_hwtstamp(adapter);
3877
31dbe5b4
BA		 3878 if (!netif_running(adapter->netdev) &&
3879 !test_bit(__E1000_TESTING, &adapter->state)) {
3880 e1000_power_down_phy(adapter);
3881 return;
3882 }
3883
bc7f75fa
AK		 3884 e1000_get_phy_info(hw);
3885
918d7197
BA		 3886 if ((adapter->flags & FLAG_HAS_SMART_POWER_DOWN) &&
3887 !(adapter->flags & FLAG_SMART_POWER_DOWN)) {
bc7f75fa 3888 u16 phy_data = 0;
e921eb1a 3889 /* speed up time to link by disabling smart power down, ignore
bc7f75fa 3890 * the return value of this function because there is nothing
ad68076e
BA		 3891 * different we would do if it failed
3892 */
bc7f75fa
AK		 3893 e1e_rphy(hw, IGP02E1000_PHY_POWER_MGMT, &phy_data);
3894 phy_data &= ~IGP02E1000_PM_SPD;
3895 e1e_wphy(hw, IGP02E1000_PHY_POWER_MGMT, phy_data);
3896 }
bc7f75fa
AK		 3897}
3898
3899int e1000e_up(struct e1000_adapter *adapter)
3900{
3901 struct e1000_hw *hw = &adapter->hw;
3902
3903 /* hardware has been reset, we need to reload some things */
3904 e1000_configure(adapter);
3905
3906 clear_bit(__E1000_DOWN, &adapter->state);
3907
4662e82b
BA		 3908 if (adapter->msix_entries)
3909 e1000_configure_msix(adapter);
bc7f75fa
AK		 3910 e1000_irq_enable(adapter);
3911
400484fa 3912 netif_start_queue(adapter->netdev);
4cb9be7a 3913
bc7f75fa 3914 /* fire a link change interrupt to start the watchdog */
52a9b231
BA		 3915 if (adapter->msix_entries)
3916 ew32(ICS, E1000_ICS_LSC | E1000_ICR_OTHER);
3917 else
3918 ew32(ICS, E1000_ICS_LSC);
3919
bc7f75fa
AK		 3920 return 0;
3921}
3922
713b3c9e
JB		 3923static void e1000e_flush_descriptors(struct e1000_adapter *adapter)
3924{
3925 struct e1000_hw *hw = &adapter->hw;
3926
3927 if (!(adapter->flags2 & FLAG2_DMA_BURST))
3928 return;
3929
3930 /* flush pending descriptor writebacks to memory */
3931 ew32(TIDV, adapter->tx_int_delay | E1000_TIDV_FPD);
3932 ew32(RDTR, adapter->rx_int_delay | E1000_RDTR_FPD);
3933
3934 /* execute the writes immediately */
3935 e1e_flush();
bf03085f 3936
e921eb1a 3937 /* due to rare timing issues, write to TIDV/RDTR again to ensure the
bf03085f
MV		 3938 * write is successful
3939 */
3940 ew32(TIDV, adapter->tx_int_delay | E1000_TIDV_FPD);
3941 ew32(RDTR, adapter->rx_int_delay | E1000_RDTR_FPD);
713b3c9e
JB		 3942
3943 /* execute the writes immediately */
3944 e1e_flush();
3945}
3946
67fd4fcb
JK		 3947static void e1000e_update_stats(struct e1000_adapter *adapter);
3948
bc7f75fa
AK		 3949void e1000e_down(struct e1000_adapter *adapter)
3950{
3951 struct net_device *netdev = adapter->netdev;
3952 struct e1000_hw *hw = &adapter->hw;
3953 u32 tctl, rctl;
3954
e921eb1a 3955 /* signal that we're down so the interrupt handler does not
ad68076e
BA		 3956 * reschedule our watchdog timer
3957 */
bc7f75fa
AK		 3958 set_bit(__E1000_DOWN, &adapter->state);
3959
3960 /* disable receives in the hardware */
3961 rctl = er32(RCTL);
7f99ae63
BA		 3962 if (!(adapter->flags2 & FLAG2_NO_DISABLE_RX))
3963 ew32(RCTL, rctl & ~E1000_RCTL_EN);
bc7f75fa
AK		 3964 /* flush and sleep below */
3965
4cb9be7a 3966 netif_stop_queue(netdev);
bc7f75fa
AK		 3967
3968 /* disable transmits in the hardware */
3969 tctl = er32(TCTL);
3970 tctl &= ~E1000_TCTL_EN;
3971 ew32(TCTL, tctl);
7f99ae63 3972
bc7f75fa
AK		 3973 /* flush both disables and wait for them to finish */
3974 e1e_flush();
1bba4386 3975 usleep_range(10000, 20000);
bc7f75fa 3976
bc7f75fa
AK		 3977 e1000_irq_disable(adapter);
3978
3979 del_timer_sync(&adapter->watchdog_timer);
3980 del_timer_sync(&adapter->phy_info_timer);
3981
bc7f75fa 3982 netif_carrier_off(netdev);
67fd4fcb
JK		 3983
3984 spin_lock(&adapter->stats64_lock);
3985 e1000e_update_stats(adapter);
3986 spin_unlock(&adapter->stats64_lock);
3987
400484fa 3988 e1000e_flush_descriptors(adapter);
55aa6985
BA		 3989 e1000_clean_tx_ring(adapter->tx_ring);
3990 e1000_clean_rx_ring(adapter->rx_ring);
400484fa 3991
bc7f75fa
AK		 3992 adapter->link_speed = 0;
3993 adapter->link_duplex = 0;
3994
52cc3086
JK		 3995 if (!pci_channel_offline(adapter->pdev))
3996 e1000e_reset(adapter);
713b3c9e 3997
e921eb1a 3998 /* TODO: for power management, we could drop the link and
bc7f75fa
AK		 3999 * pci_disable_device here.
4000 */
4001}
4002
4003void e1000e_reinit_locked(struct e1000_adapter *adapter)
4004{
4005 might_sleep();
4006 while (test_and_set_bit(__E1000_RESETTING, &adapter->state))
1bba4386 4007 usleep_range(1000, 2000);
bc7f75fa
AK		 4008 e1000e_down(adapter);
4009 e1000e_up(adapter);
4010 clear_bit(__E1000_RESETTING, &adapter->state);
4011}
4012
b67e1913
BA		 4013/**
4014 * e1000e_cyclecounter_read - read raw cycle counter (used by time counter)
4015 * @cc: cyclecounter structure
4016 **/
4017static cycle_t e1000e_cyclecounter_read(const struct cyclecounter *cc)
4018{
4019 struct e1000_adapter *adapter = container_of(cc, struct e1000_adapter,
4020 cc);
4021 struct e1000_hw *hw = &adapter->hw;
4022 cycle_t systim;
4023
4024 /* latch SYSTIMH on read of SYSTIML */
4025 systim = (cycle_t)er32(SYSTIML);
4026 systim |= (cycle_t)er32(SYSTIMH) << 32;
4027
4028 return systim;
4029}
4030
bc7f75fa
AK		 4031/**
4032 * e1000_sw_init - Initialize general software structures (struct e1000_adapter)
4033 * @adapter: board private structure to initialize
4034 *
4035 * e1000_sw_init initializes the Adapter private data structure.
4036 * Fields are initialized based on PCI device information and
4037 * OS network device settings (MTU size).
4038 **/
9f9a12f8 4039static int e1000_sw_init(struct e1000_adapter *adapter)
bc7f75fa 4040{
bc7f75fa
AK		 4041 struct net_device *netdev = adapter->netdev;
4042
4043 adapter->rx_buffer_len = ETH_FRAME_LEN + VLAN_HLEN + ETH_FCS_LEN;
4044 adapter->rx_ps_bsize0 = 128;
318a94d6
JK		 4045 adapter->max_frame_size = netdev->mtu + ETH_HLEN + ETH_FCS_LEN;
4046 adapter->min_frame_size = ETH_ZLEN + ETH_FCS_LEN;
55aa6985
BA		 4047 adapter->tx_ring_count = E1000_DEFAULT_TXD;
4048 adapter->rx_ring_count = E1000_DEFAULT_RXD;
bc7f75fa 4049
67fd4fcb
JK		 4050 spin_lock_init(&adapter->stats64_lock);
4051
4662e82b 4052 e1000e_set_interrupt_capability(adapter);
bc7f75fa 4053
4662e82b
BA		 4054 if (e1000_alloc_queues(adapter))
4055 return -ENOMEM;
bc7f75fa 4056
b67e1913
BA		 4057 /* Setup hardware time stamping cyclecounter */
4058 if (adapter->flags & FLAG_HAS_HW_TIMESTAMP) {
4059 adapter->cc.read = e1000e_cyclecounter_read;
4060 adapter->cc.mask = CLOCKSOURCE_MASK(64);
4061 adapter->cc.mult = 1;
4062 /* cc.shift set in e1000e_get_base_tininca() */
4063
4064 spin_lock_init(&adapter->systim_lock);
4065 INIT_WORK(&adapter->tx_hwtstamp_work, e1000e_tx_hwtstamp_work);
4066 }
4067
bc7f75fa 4068 /* Explicitly disable IRQ since the NIC can be in any state. */
bc7f75fa
AK		 4069 e1000_irq_disable(adapter);
4070
bc7f75fa
AK		 4071 set_bit(__E1000_DOWN, &adapter->state);
4072 return 0;
bc7f75fa
AK		 4073}
4074
f8d59f78
BA		 4075/**
4076 * e1000_intr_msi_test - Interrupt Handler
4077 * @irq: interrupt number
4078 * @data: pointer to a network interface device structure
4079 **/
8bb62869 4080static irqreturn_t e1000_intr_msi_test(int __always_unused irq, void *data)
f8d59f78
BA		 4081{
4082 struct net_device *netdev = data;
4083 struct e1000_adapter *adapter = netdev_priv(netdev);
4084 struct e1000_hw *hw = &adapter->hw;
4085 u32 icr = er32(ICR);
4086
3bb99fe2 4087 e_dbg("icr is %08X\n", icr);
f8d59f78
BA		 4088 if (icr & E1000_ICR_RXSEQ) {
4089 adapter->flags &= ~FLAG_MSI_TEST_FAILED;
e921eb1a 4090 /* Force memory writes to complete before acknowledging the
bc76329d
BA		 4091 * interrupt is handled.
4092 */
f8d59f78
BA		 4093 wmb();
4094 }
4095
4096 return IRQ_HANDLED;
4097}
4098
4099/**
4100 * e1000_test_msi_interrupt - Returns 0 for successful test
4101 * @adapter: board private struct
4102 *
4103 * code flow taken from tg3.c
4104 **/
4105static int e1000_test_msi_interrupt(struct e1000_adapter *adapter)
4106{
4107 struct net_device *netdev = adapter->netdev;
4108 struct e1000_hw *hw = &adapter->hw;
4109 int err;
4110
4111 /* poll_enable hasn't been called yet, so don't need disable */
4112 /* clear any pending events */
4113 er32(ICR);
4114
4115 /* free the real vector and request a test handler */
4116 e1000_free_irq(adapter);
4662e82b 4117 e1000e_reset_interrupt_capability(adapter);
f8d59f78
BA		 4118
4119 /* Assume that the test fails, if it succeeds then the test
e921eb1a
BA		 4120 * MSI irq handler will unset this flag
4121 */
f8d59f78
BA		 4122 adapter->flags |= FLAG_MSI_TEST_FAILED;
4123
4124 err = pci_enable_msi(adapter->pdev);
4125 if (err)
4126 goto msi_test_failed;
4127
a0607fd3 4128 err = request_irq(adapter->pdev->irq, e1000_intr_msi_test, 0,
f8d59f78
BA		 4129 netdev->name, netdev);
4130 if (err) {
4131 pci_disable_msi(adapter->pdev);
4132 goto msi_test_failed;
4133 }
4134
e921eb1a 4135 /* Force memory writes to complete before enabling and firing an
bc76329d
BA		 4136 * interrupt.
4137 */
f8d59f78
BA		 4138 wmb();
4139
4140 e1000_irq_enable(adapter);
4141
4142 /* fire an unusual interrupt on the test handler */
4143 ew32(ICS, E1000_ICS_RXSEQ);
4144 e1e_flush();
569a3aff 4145 msleep(100);
f8d59f78
BA		 4146
4147 e1000_irq_disable(adapter);
4148
bc76329d 4149 rmb(); /* read flags after interrupt has been fired */
f8d59f78
BA		 4150
4151 if (adapter->flags & FLAG_MSI_TEST_FAILED) {
4662e82b 4152 adapter->int_mode = E1000E_INT_MODE_LEGACY;
068e8a30 4153 e_info("MSI interrupt test failed, using legacy interrupt.\n");
24b706b2 4154 } else {
068e8a30 4155 e_dbg("MSI interrupt test succeeded!\n");
24b706b2 4156 }
f8d59f78
BA		 4157
4158 free_irq(adapter->pdev->irq, netdev);
4159 pci_disable_msi(adapter->pdev);
4160
f8d59f78 4161msi_test_failed:
4662e82b 4162 e1000e_set_interrupt_capability(adapter);
068e8a30 4163 return e1000_request_irq(adapter);
f8d59f78
BA		 4164}
4165
4166/**
4167 * e1000_test_msi - Returns 0 if MSI test succeeds or INTx mode is restored
4168 * @adapter: board private struct
4169 *
4170 * code flow taken from tg3.c, called with e1000 interrupts disabled.
4171 **/
4172static int e1000_test_msi(struct e1000_adapter *adapter)
4173{
4174 int err;
4175 u16 pci_cmd;
4176
4177 if (!(adapter->flags & FLAG_MSI_ENABLED))
4178 return 0;
4179
4180 /* disable SERR in case the MSI write causes a master abort */
4181 pci_read_config_word(adapter->pdev, PCI_COMMAND, &pci_cmd);
36f2407f
DN		 4182 if (pci_cmd & PCI_COMMAND_SERR)
4183 pci_write_config_word(adapter->pdev, PCI_COMMAND,
4184 pci_cmd & ~PCI_COMMAND_SERR);
f8d59f78
BA		 4185
4186 err = e1000_test_msi_interrupt(adapter);
4187
36f2407f
DN		 4188 /* re-enable SERR */
4189 if (pci_cmd & PCI_COMMAND_SERR) {
4190 pci_read_config_word(adapter->pdev, PCI_COMMAND, &pci_cmd);
4191 pci_cmd |= PCI_COMMAND_SERR;
4192 pci_write_config_word(adapter->pdev, PCI_COMMAND, pci_cmd);
4193 }
f8d59f78 4194
f8d59f78
BA		 4195 return err;
4196}
4197
bc7f75fa
AK		 4198/**
4199 * e1000_open - Called when a network interface is made active
4200 * @netdev: network interface device structure
4201 *
4202 * Returns 0 on success, negative value on failure
4203 *
4204 * The open entry point is called when a network interface is made
4205 * active by the system (IFF_UP). At this point all resources needed
4206 * for transmit and receive operations are allocated, the interrupt
4207 * handler is registered with the OS, the watchdog timer is started,
4208 * and the stack is notified that the interface is ready.
4209 **/
4210static int e1000_open(struct net_device *netdev)
4211{
4212 struct e1000_adapter *adapter = netdev_priv(netdev);
4213 struct e1000_hw *hw = &adapter->hw;
23606cf5 4214 struct pci_dev *pdev = adapter->pdev;
bc7f75fa
AK		 4215 int err;
4216
4217 /* disallow open during test */
4218 if (test_bit(__E1000_TESTING, &adapter->state))
4219 return -EBUSY;
4220
23606cf5
RW		 4221 pm_runtime_get_sync(&pdev->dev);
4222
9c563d20
JB		 4223 netif_carrier_off(netdev);
4224
bc7f75fa 4225 /* allocate transmit descriptors */
55aa6985 4226 err = e1000e_setup_tx_resources(adapter->tx_ring);
bc7f75fa
AK		 4227 if (err)
4228 goto err_setup_tx;
4229
4230 /* allocate receive descriptors */
55aa6985 4231 err = e1000e_setup_rx_resources(adapter->rx_ring);
bc7f75fa
AK		 4232 if (err)
4233 goto err_setup_rx;
4234
e921eb1a 4235 /* If AMT is enabled, let the firmware know that the network
11b08be8
BA		 4236 * interface is now open and reset the part to a known state.
4237 */
4238 if (adapter->flags & FLAG_HAS_AMT) {
31dbe5b4 4239 e1000e_get_hw_control(adapter);
11b08be8
BA		 4240 e1000e_reset(adapter);
4241 }
4242
bc7f75fa
AK		 4243 e1000e_power_up_phy(adapter);
4244
4245 adapter->mng_vlan_id = E1000_MNG_VLAN_NONE;
e5fe2541 4246 if ((adapter->hw.mng_cookie.status & E1000_MNG_DHCP_COOKIE_STATUS_VLAN))
bc7f75fa
AK		 4247 e1000_update_mng_vlan(adapter);
4248
79d4e908 4249 /* DMA latency requirement to workaround jumbo issue */
3e35d991
BA		 4250 pm_qos_add_request(&adapter->netdev->pm_qos_req, PM_QOS_CPU_DMA_LATENCY,
4251 PM_QOS_DEFAULT_VALUE);
c128ec29 4252
e921eb1a 4253 /* before we allocate an interrupt, we must be ready to handle it.
bc7f75fa
AK		 4254 * Setting DEBUG_SHIRQ in the kernel makes it fire an interrupt
4255 * as soon as we call pci_request_irq, so we have to setup our
ad68076e
BA		 4256 * clean_rx handler before we do so.
4257 */
bc7f75fa
AK		 4258 e1000_configure(adapter);
4259
4260 err = e1000_request_irq(adapter);
4261 if (err)
4262 goto err_req_irq;
4263
e921eb1a 4264 /* Work around PCIe errata with MSI interrupts causing some chipsets to
f8d59f78
BA		 4265 * ignore e1000e MSI messages, which means we need to test our MSI
4266 * interrupt now
4267 */
4662e82b 4268 if (adapter->int_mode != E1000E_INT_MODE_LEGACY) {
f8d59f78
BA		 4269 err = e1000_test_msi(adapter);
4270 if (err) {
4271 e_err("Interrupt allocation failed\n");
4272 goto err_req_irq;
4273 }
4274 }
4275
bc7f75fa
AK		 4276 /* From here on the code is the same as e1000e_up() */
4277 clear_bit(__E1000_DOWN, &adapter->state);
4278
4279 napi_enable(&adapter->napi);
4280
4281 e1000_irq_enable(adapter);
4282
09357b00 4283 adapter->tx_hang_recheck = false;
4cb9be7a 4284 netif_start_queue(netdev);
d55b53ff 4285
23606cf5
RW		 4286 adapter->idle_check = true;
4287 pm_runtime_put(&pdev->dev);
4288
bc7f75fa 4289 /* fire a link status change interrupt to start the watchdog */
52a9b231
BA		 4290 if (adapter->msix_entries)
4291 ew32(ICS, E1000_ICS_LSC | E1000_ICR_OTHER);
4292 else
4293 ew32(ICS, E1000_ICS_LSC);
bc7f75fa
AK		 4294
4295 return 0;
4296
4297err_req_irq:
31dbe5b4 4298 e1000e_release_hw_control(adapter);
bc7f75fa 4299 e1000_power_down_phy(adapter);
55aa6985 4300 e1000e_free_rx_resources(adapter->rx_ring);
bc7f75fa 4301err_setup_rx:
55aa6985 4302 e1000e_free_tx_resources(adapter->tx_ring);
bc7f75fa
AK		 4303err_setup_tx:
4304 e1000e_reset(adapter);
23606cf5 4305 pm_runtime_put_sync(&pdev->dev);
bc7f75fa
AK		 4306
4307 return err;
4308}
4309
4310/**
4311 * e1000_close - Disables a network interface
4312 * @netdev: network interface device structure
4313 *
4314 * Returns 0, this is not allowed to fail
4315 *
4316 * The close entry point is called when an interface is de-activated
4317 * by the OS. The hardware is still under the drivers control, but
4318 * needs to be disabled. A global MAC reset is issued to stop the
4319 * hardware, and all transmit and receive resources are freed.
4320 **/
4321static int e1000_close(struct net_device *netdev)
4322{
4323 struct e1000_adapter *adapter = netdev_priv(netdev);
23606cf5 4324 struct pci_dev *pdev = adapter->pdev;
bb9e44d0
BA		 4325 int count = E1000_CHECK_RESET_COUNT;
4326
4327 while (test_bit(__E1000_RESETTING, &adapter->state) && count--)
4328 usleep_range(10000, 20000);
bc7f75fa
AK		 4329
4330 WARN_ON(test_bit(__E1000_RESETTING, &adapter->state));
23606cf5
RW		 4331
4332 pm_runtime_get_sync(&pdev->dev);
4333
5f4a780d
BA		 4334 napi_disable(&adapter->napi);
4335
23606cf5
RW		 4336 if (!test_bit(__E1000_DOWN, &adapter->state)) {
4337 e1000e_down(adapter);
4338 e1000_free_irq(adapter);
4339 }
bc7f75fa 4340 e1000_power_down_phy(adapter);
bc7f75fa 4341
55aa6985
BA		 4342 e1000e_free_tx_resources(adapter->tx_ring);
4343 e1000e_free_rx_resources(adapter->rx_ring);
bc7f75fa 4344
e921eb1a 4345 /* kill manageability vlan ID if supported, but not if a vlan with
ad68076e
BA		 4346 * the same ID is registered on the host OS (let 8021q kill it)
4347 */
e5fe2541 4348 if (adapter->hw.mng_cookie.status & E1000_MNG_DHCP_COOKIE_STATUS_VLAN)
bc7f75fa
AK		 4349 e1000_vlan_rx_kill_vid(netdev, adapter->mng_vlan_id);
4350
e921eb1a 4351 /* If AMT is enabled, let the firmware know that the network
ad68076e
BA		 4352 * interface is now closed
4353 */
31dbe5b4
BA		 4354 if ((adapter->flags & FLAG_HAS_AMT) &&
4355 !test_bit(__E1000_TESTING, &adapter->state))
4356 e1000e_release_hw_control(adapter);
bc7f75fa 4357
3e35d991 4358 pm_qos_remove_request(&adapter->netdev->pm_qos_req);
c128ec29 4359
23606cf5
RW		 4360 pm_runtime_put_sync(&pdev->dev);
4361
bc7f75fa
AK		 4362 return 0;
4363}
fc830b78 4364
bc7f75fa
AK		 4365/**
4366 * e1000_set_mac - Change the Ethernet Address of the NIC
4367 * @netdev: network interface device structure
4368 * @p: pointer to an address structure
4369 *
4370 * Returns 0 on success, negative on failure
4371 **/
4372static int e1000_set_mac(struct net_device *netdev, void *p)
4373{
4374 struct e1000_adapter *adapter = netdev_priv(netdev);
69e1e019 4375 struct e1000_hw *hw = &adapter->hw;
bc7f75fa
AK		 4376 struct sockaddr *addr = p;
4377
4378 if (!is_valid_ether_addr(addr->sa_data))
4379 return -EADDRNOTAVAIL;
4380
4381 memcpy(netdev->dev_addr, addr->sa_data, netdev->addr_len);
4382 memcpy(adapter->hw.mac.addr, addr->sa_data, netdev->addr_len);
4383
69e1e019 4384 hw->mac.ops.rar_set(&adapter->hw, adapter->hw.mac.addr, 0);
bc7f75fa
AK		 4385
4386 if (adapter->flags & FLAG_RESET_OVERWRITES_LAA) {
4387 /* activate the work around */
4388 e1000e_set_laa_state_82571(&adapter->hw, 1);
4389
e921eb1a 4390 /* Hold a copy of the LAA in RAR[14] This is done so that
bc7f75fa
AK		 4391 * between the time RAR[0] gets clobbered and the time it
4392 * gets fixed (in e1000_watchdog), the actual LAA is in one
4393 * of the RARs and no incoming packets directed to this port
4394 * are dropped. Eventually the LAA will be in RAR[0] and
ad68076e
BA		 4395 * RAR[14]
4396 */
69e1e019
BA		 4397 hw->mac.ops.rar_set(&adapter->hw, adapter->hw.mac.addr,
4398 adapter->hw.mac.rar_entry_count - 1);
bc7f75fa
AK		 4399 }
4400
4401 return 0;
4402}
4403
a8f88ff5
JB		 4404/**
4405 * e1000e_update_phy_task - work thread to update phy
4406 * @work: pointer to our work struct
4407 *
4408 * this worker thread exists because we must acquire a
4409 * semaphore to read the phy, which we could msleep while
4410 * waiting for it, and we can't msleep in a timer.
4411 **/
4412static void e1000e_update_phy_task(struct work_struct *work)
4413{
4414 struct e1000_adapter *adapter = container_of(work,
17e813ec
BA		 4415 struct e1000_adapter,
4416 update_phy_task);
615b32af
JB		 4417
4418 if (test_bit(__E1000_DOWN, &adapter->state))
4419 return;
4420
a8f88ff5
JB		 4421 e1000_get_phy_info(&adapter->hw);
4422}
4423
e921eb1a
BA		 4424/**
4425 * e1000_update_phy_info - timre call-back to update PHY info
4426 * @data: pointer to adapter cast into an unsigned long
4427 *
ad68076e
BA		 4428 * Need to wait a few seconds after link up to get diagnostic information from
4429 * the phy
e921eb1a 4430 **/
bc7f75fa
AK		 4431static void e1000_update_phy_info(unsigned long data)
4432{
53aa82da 4433 struct e1000_adapter *adapter = (struct e1000_adapter *)data;
615b32af
JB		 4434
4435 if (test_bit(__E1000_DOWN, &adapter->state))
4436 return;
4437
a8f88ff5 4438 schedule_work(&adapter->update_phy_task);
bc7f75fa
AK		 4439}
4440
8c7bbb92
BA		 4441/**
4442 * e1000e_update_phy_stats - Update the PHY statistics counters
4443 * @adapter: board private structure
2b6b168d
BA		 4444 *
4445 * Read/clear the upper 16-bit PHY registers and read/accumulate lower
8c7bbb92
BA		 4446 **/
4447static void e1000e_update_phy_stats(struct e1000_adapter *adapter)
4448{
4449 struct e1000_hw *hw = &adapter->hw;
4450 s32 ret_val;
4451 u16 phy_data;
4452
4453 ret_val = hw->phy.ops.acquire(hw);
4454 if (ret_val)
4455 return;
4456
e921eb1a 4457 /* A page set is expensive so check if already on desired page.
8c7bbb92
BA		 4458 * If not, set to the page with the PHY status registers.
4459 */
2b6b168d 4460 hw->phy.addr = 1;
8c7bbb92
BA		 4461 ret_val = e1000e_read_phy_reg_mdic(hw, IGP01E1000_PHY_PAGE_SELECT,
4462 &phy_data);
4463 if (ret_val)
4464 goto release;
2b6b168d
BA		 4465 if (phy_data != (HV_STATS_PAGE << IGP_PAGE_SHIFT)) {
4466 ret_val = hw->phy.ops.set_page(hw,
4467 HV_STATS_PAGE << IGP_PAGE_SHIFT);
8c7bbb92
BA		 4468 if (ret_val)
4469 goto release;
4470 }
4471
8c7bbb92 4472 /* Single Collision Count */
2b6b168d
BA		 4473 hw->phy.ops.read_reg_page(hw, HV_SCC_UPPER, &phy_data);
4474 ret_val = hw->phy.ops.read_reg_page(hw, HV_SCC_LOWER, &phy_data);
8c7bbb92
BA		 4475 if (!ret_val)
4476 adapter->stats.scc += phy_data;
4477
4478 /* Excessive Collision Count */
2b6b168d
BA		 4479 hw->phy.ops.read_reg_page(hw, HV_ECOL_UPPER, &phy_data);
4480 ret_val = hw->phy.ops.read_reg_page(hw, HV_ECOL_LOWER, &phy_data);
8c7bbb92
BA		 4481 if (!ret_val)
4482 adapter->stats.ecol += phy_data;
4483
4484 /* Multiple Collision Count */
2b6b168d
BA		 4485 hw->phy.ops.read_reg_page(hw, HV_MCC_UPPER, &phy_data);
4486 ret_val = hw->phy.ops.read_reg_page(hw, HV_MCC_LOWER, &phy_data);
8c7bbb92
BA		 4487 if (!ret_val)
4488 adapter->stats.mcc += phy_data;
4489
4490 /* Late Collision Count */
2b6b168d
BA		 4491 hw->phy.ops.read_reg_page(hw, HV_LATECOL_UPPER, &phy_data);
4492 ret_val = hw->phy.ops.read_reg_page(hw, HV_LATECOL_LOWER, &phy_data);
8c7bbb92
BA		 4493 if (!ret_val)
4494 adapter->stats.latecol += phy_data;
4495
4496 /* Collision Count - also used for adaptive IFS */
2b6b168d
BA		 4497 hw->phy.ops.read_reg_page(hw, HV_COLC_UPPER, &phy_data);
4498 ret_val = hw->phy.ops.read_reg_page(hw, HV_COLC_LOWER, &phy_data);
8c7bbb92
BA		 4499 if (!ret_val)
4500 hw->mac.collision_delta = phy_data;
4501
4502 /* Defer Count */
2b6b168d
BA		 4503 hw->phy.ops.read_reg_page(hw, HV_DC_UPPER, &phy_data);
4504 ret_val = hw->phy.ops.read_reg_page(hw, HV_DC_LOWER, &phy_data);
8c7bbb92
BA		 4505 if (!ret_val)
4506 adapter->stats.dc += phy_data;
4507
4508 /* Transmit with no CRS */
2b6b168d
BA		 4509 hw->phy.ops.read_reg_page(hw, HV_TNCRS_UPPER, &phy_data);
4510 ret_val = hw->phy.ops.read_reg_page(hw, HV_TNCRS_LOWER, &phy_data);
8c7bbb92
BA		 4511 if (!ret_val)
4512 adapter->stats.tncrs += phy_data;
4513
4514release:
4515 hw->phy.ops.release(hw);
4516}
4517
bc7f75fa
AK		 4518/**
4519 * e1000e_update_stats - Update the board statistics counters
4520 * @adapter: board private structure
4521 **/
67fd4fcb 4522static void e1000e_update_stats(struct e1000_adapter *adapter)
bc7f75fa 4523{
7274c20f 4524 struct net_device *netdev = adapter->netdev;
bc7f75fa
AK		 4525 struct e1000_hw *hw = &adapter->hw;
4526 struct pci_dev *pdev = adapter->pdev;
bc7f75fa 4527
e921eb1a 4528 /* Prevent stats update while adapter is being reset, or if the pci
bc7f75fa
AK		 4529 * connection is down.
4530 */
4531 if (adapter->link_speed == 0)
4532 return;
4533 if (pci_channel_offline(pdev))
4534 return;
4535
bc7f75fa
AK		 4536 adapter->stats.crcerrs += er32(CRCERRS);
4537 adapter->stats.gprc += er32(GPRC);
7c25769f
BA		 4538 adapter->stats.gorc += er32(GORCL);
4539 er32(GORCH); /* Clear gorc */
bc7f75fa
AK		 4540 adapter->stats.bprc += er32(BPRC);
4541 adapter->stats.mprc += er32(MPRC);
4542 adapter->stats.roc += er32(ROC);
4543
bc7f75fa 4544 adapter->stats.mpc += er32(MPC);
8c7bbb92
BA		 4545
4546 /* Half-duplex statistics */
4547 if (adapter->link_duplex == HALF_DUPLEX) {
4548 if (adapter->flags2 & FLAG2_HAS_PHY_STATS) {
4549 e1000e_update_phy_stats(adapter);
4550 } else {
4551 adapter->stats.scc += er32(SCC);
4552 adapter->stats.ecol += er32(ECOL);
4553 adapter->stats.mcc += er32(MCC);
4554 adapter->stats.latecol += er32(LATECOL);
4555 adapter->stats.dc += er32(DC);
4556
4557 hw->mac.collision_delta = er32(COLC);
4558
4559 if ((hw->mac.type != e1000_82574) &&
4560 (hw->mac.type != e1000_82583))
4561 adapter->stats.tncrs += er32(TNCRS);
4562 }
4563 adapter->stats.colc += hw->mac.collision_delta;
a4f58f54 4564 }
8c7bbb92 4565
bc7f75fa
AK		 4566 adapter->stats.xonrxc += er32(XONRXC);
4567 adapter->stats.xontxc += er32(XONTXC);
4568 adapter->stats.xoffrxc += er32(XOFFRXC);
4569 adapter->stats.xofftxc += er32(XOFFTXC);
bc7f75fa 4570 adapter->stats.gptc += er32(GPTC);
7c25769f
BA		 4571 adapter->stats.gotc += er32(GOTCL);
4572 er32(GOTCH); /* Clear gotc */
bc7f75fa
AK		 4573 adapter->stats.rnbc += er32(RNBC);
4574 adapter->stats.ruc += er32(RUC);
bc7f75fa
AK		 4575
4576 adapter->stats.mptc += er32(MPTC);
4577 adapter->stats.bptc += er32(BPTC);
4578
4579 /* used for adaptive IFS */
4580
4581 hw->mac.tx_packet_delta = er32(TPT);
4582 adapter->stats.tpt += hw->mac.tx_packet_delta;
bc7f75fa
AK		 4583
4584 adapter->stats.algnerrc += er32(ALGNERRC);
4585 adapter->stats.rxerrc += er32(RXERRC);
bc7f75fa
AK		 4586 adapter->stats.cexterr += er32(CEXTERR);
4587 adapter->stats.tsctc += er32(TSCTC);
4588 adapter->stats.tsctfc += er32(TSCTFC);
4589
bc7f75fa 4590 /* Fill out the OS statistics structure */
7274c20f
AK		 4591 netdev->stats.multicast = adapter->stats.mprc;
4592 netdev->stats.collisions = adapter->stats.colc;
bc7f75fa
AK		 4593
4594 /* Rx Errors */
4595
e921eb1a 4596 /* RLEC on some newer hardware can be incorrect so build
ad68076e
BA		 4597 * our own version based on RUC and ROC
4598 */
7274c20f 4599 netdev->stats.rx_errors = adapter->stats.rxerrc +
f0ff4398
BA		 4600 adapter->stats.crcerrs + adapter->stats.algnerrc +
4601 adapter->stats.ruc + adapter->stats.roc + adapter->stats.cexterr;
7274c20f 4602 netdev->stats.rx_length_errors = adapter->stats.ruc +
f0ff4398 4603 adapter->stats.roc;
7274c20f
AK		 4604 netdev->stats.rx_crc_errors = adapter->stats.crcerrs;
4605 netdev->stats.rx_frame_errors = adapter->stats.algnerrc;
4606 netdev->stats.rx_missed_errors = adapter->stats.mpc;
bc7f75fa
AK		 4607
4608 /* Tx Errors */
f0ff4398 4609 netdev->stats.tx_errors = adapter->stats.ecol + adapter->stats.latecol;
7274c20f
AK		 4610 netdev->stats.tx_aborted_errors = adapter->stats.ecol;
4611 netdev->stats.tx_window_errors = adapter->stats.latecol;
4612 netdev->stats.tx_carrier_errors = adapter->stats.tncrs;
bc7f75fa
AK		 4613
4614 /* Tx Dropped needs to be maintained elsewhere */
4615
bc7f75fa
AK		 4616 /* Management Stats */
4617 adapter->stats.mgptc += er32(MGTPTC);
4618 adapter->stats.mgprc += er32(MGTPRC);
4619 adapter->stats.mgpdc += er32(MGTPDC);
94fb848b
BA		 4620
4621 /* Correctable ECC Errors */
4622 if (hw->mac.type == e1000_pch_lpt) {
4623 u32 pbeccsts = er32(PBECCSTS);
4624 adapter->corr_errors +=
4625 pbeccsts & E1000_PBECCSTS_CORR_ERR_CNT_MASK;
4626 adapter->uncorr_errors +=
4627 (pbeccsts & E1000_PBECCSTS_UNCORR_ERR_CNT_MASK) >>
4628 E1000_PBECCSTS_UNCORR_ERR_CNT_SHIFT;
4629 }
bc7f75fa
AK		 4630}
4631
7c25769f
BA		 4632/**
4633 * e1000_phy_read_status - Update the PHY register status snapshot
4634 * @adapter: board private structure
4635 **/
4636static void e1000_phy_read_status(struct e1000_adapter *adapter)
4637{
4638 struct e1000_hw *hw = &adapter->hw;
4639 struct e1000_phy_regs *phy = &adapter->phy_regs;
7c25769f
BA		 4640
4641 if ((er32(STATUS) & E1000_STATUS_LU) &&
4642 (adapter->hw.phy.media_type == e1000_media_type_copper)) {
90da0669
BA		 4643 int ret_val;
4644
c2ade1a4
BA		 4645 ret_val = e1e_rphy(hw, MII_BMCR, &phy->bmcr);
4646 ret_val |= e1e_rphy(hw, MII_BMSR, &phy->bmsr);
4647 ret_val |= e1e_rphy(hw, MII_ADVERTISE, &phy->advertise);
4648 ret_val |= e1e_rphy(hw, MII_LPA, &phy->lpa);
4649 ret_val |= e1e_rphy(hw, MII_EXPANSION, &phy->expansion);
4650 ret_val |= e1e_rphy(hw, MII_CTRL1000, &phy->ctrl1000);
4651 ret_val |= e1e_rphy(hw, MII_STAT1000, &phy->stat1000);
4652 ret_val |= e1e_rphy(hw, MII_ESTATUS, &phy->estatus);
7c25769f 4653 if (ret_val)
44defeb3 4654 e_warn("Error reading PHY register\n");
7c25769f 4655 } else {
e921eb1a 4656 /* Do not read PHY registers if link is not up
7c25769f
BA		 4657 * Set values to typical power-on defaults
4658 */
4659 phy->bmcr = (BMCR_SPEED1000 | BMCR_ANENABLE | BMCR_FULLDPLX);
4660 phy->bmsr = (BMSR_100FULL | BMSR_100HALF | BMSR_10FULL |
4661 BMSR_10HALF | BMSR_ESTATEN | BMSR_ANEGCAPABLE |
4662 BMSR_ERCAP);
4663 phy->advertise = (ADVERTISE_PAUSE_ASYM | ADVERTISE_PAUSE_CAP |
4664 ADVERTISE_ALL | ADVERTISE_CSMA);
4665 phy->lpa = 0;
4666 phy->expansion = EXPANSION_ENABLENPAGE;
4667 phy->ctrl1000 = ADVERTISE_1000FULL;
4668 phy->stat1000 = 0;
4669 phy->estatus = (ESTATUS_1000_TFULL | ESTATUS_1000_THALF);
4670 }
7c25769f
BA		 4671}
4672
bc7f75fa
AK		 4673static void e1000_print_link_info(struct e1000_adapter *adapter)
4674{
bc7f75fa
AK		 4675 struct e1000_hw *hw = &adapter->hw;
4676 u32 ctrl = er32(CTRL);
4677
8f12fe86 4678 /* Link status message must follow this format for user tools */
7dbc1672
BA		 4679 pr_info("%s NIC Link is Up %d Mbps %s Duplex, Flow Control: %s\n",
4680 adapter->netdev->name, adapter->link_speed,
ef456f85
JK		 4681 adapter->link_duplex == FULL_DUPLEX ? "Full" : "Half",
4682 (ctrl & E1000_CTRL_TFCE) && (ctrl & E1000_CTRL_RFCE) ? "Rx/Tx" :
4683 (ctrl & E1000_CTRL_RFCE) ? "Rx" :
4684 (ctrl & E1000_CTRL_TFCE) ? "Tx" : "None");
bc7f75fa
AK		 4685}
4686
0c6bdb30 4687static bool e1000e_has_link(struct e1000_adapter *adapter)
318a94d6
JK		 4688{
4689 struct e1000_hw *hw = &adapter->hw;
3db1cd5c 4690 bool link_active = false;
318a94d6
JK		 4691 s32 ret_val = 0;
4692
e921eb1a 4693 /* get_link_status is set on LSC (link status) interrupt or
318a94d6
JK		 4694 * Rx sequence error interrupt. get_link_status will stay
4695 * false until the check_for_link establishes link
4696 * for copper adapters ONLY
4697 */
4698 switch (hw->phy.media_type) {
4699 case e1000_media_type_copper:
4700 if (hw->mac.get_link_status) {
4701 ret_val = hw->mac.ops.check_for_link(hw);
4702 link_active = !hw->mac.get_link_status;
4703 } else {
3db1cd5c 4704 link_active = true;
318a94d6
JK		 4705 }
4706 break;
4707 case e1000_media_type_fiber:
4708 ret_val = hw->mac.ops.check_for_link(hw);
4709 link_active = !!(er32(STATUS) & E1000_STATUS_LU);
4710 break;
4711 case e1000_media_type_internal_serdes:
4712 ret_val = hw->mac.ops.check_for_link(hw);
4713 link_active = adapter->hw.mac.serdes_has_link;
4714 break;
4715 default:
4716 case e1000_media_type_unknown:
4717 break;
4718 }
4719
4720 if ((ret_val == E1000_ERR_PHY) && (hw->phy.type == e1000_phy_igp_3) &&
4721 (er32(CTRL) & E1000_PHY_CTRL_GBE_DISABLE)) {
4722 /* See e1000_kmrn_lock_loss_workaround_ich8lan() */
44defeb3 4723 e_info("Gigabit has been disabled, downgrading speed\n");
318a94d6
JK		 4724 }
4725
4726 return link_active;
4727}
4728
4729static void e1000e_enable_receives(struct e1000_adapter *adapter)
4730{
4731 /* make sure the receive unit is started */
4732 if ((adapter->flags & FLAG_RX_NEEDS_RESTART) &&
12d43f7d 4733 (adapter->flags & FLAG_RESTART_NOW)) {
318a94d6
JK		 4734 struct e1000_hw *hw = &adapter->hw;
4735 u32 rctl = er32(RCTL);
4736 ew32(RCTL, rctl | E1000_RCTL_EN);
12d43f7d 4737 adapter->flags &= ~FLAG_RESTART_NOW;
318a94d6
JK		 4738 }
4739}
4740
ff10e13c
CW		 4741static void e1000e_check_82574_phy_workaround(struct e1000_adapter *adapter)
4742{
4743 struct e1000_hw *hw = &adapter->hw;
4744
e921eb1a 4745 /* With 82574 controllers, PHY needs to be checked periodically
ff10e13c
CW		 4746 * for hung state and reset, if two calls return true
4747 */
4748 if (e1000_check_phy_82574(hw))
4749 adapter->phy_hang_count++;
4750 else
4751 adapter->phy_hang_count = 0;
4752
4753 if (adapter->phy_hang_count > 1) {
4754 adapter->phy_hang_count = 0;
4755 schedule_work(&adapter->reset_task);
4756 }
4757}
4758
bc7f75fa
AK		 4759/**
4760 * e1000_watchdog - Timer Call-back
4761 * @data: pointer to adapter cast into an unsigned long
4762 **/
4763static void e1000_watchdog(unsigned long data)
4764{
53aa82da 4765 struct e1000_adapter *adapter = (struct e1000_adapter *)data;
bc7f75fa
AK		 4766
4767 /* Do the rest outside of interrupt context */
4768 schedule_work(&adapter->watchdog_task);
4769
4770 /* TODO: make this use queue_delayed_work() */
4771}
4772
4773static void e1000_watchdog_task(struct work_struct *work)
4774{
4775 struct e1000_adapter *adapter = container_of(work,
17e813ec
BA		 4776 struct e1000_adapter,
4777 watchdog_task);
bc7f75fa
AK		 4778 struct net_device *netdev = adapter->netdev;
4779 struct e1000_mac_info *mac = &adapter->hw.mac;
75eb0fad 4780 struct e1000_phy_info *phy = &adapter->hw.phy;
bc7f75fa
AK		 4781 struct e1000_ring *tx_ring = adapter->tx_ring;
4782 struct e1000_hw *hw = &adapter->hw;
4783 u32 link, tctl;
bc7f75fa 4784
615b32af
JB		 4785 if (test_bit(__E1000_DOWN, &adapter->state))
4786 return;
4787
b405e8df 4788 link = e1000e_has_link(adapter);
318a94d6 4789 if ((netif_carrier_ok(netdev)) && link) {
23606cf5
RW		 4790 /* Cancel scheduled suspend requests. */
4791 pm_runtime_resume(netdev->dev.parent);
4792
318a94d6 4793 e1000e_enable_receives(adapter);
bc7f75fa 4794 goto link_up;
bc7f75fa
AK		 4795 }
4796
4797 if ((e1000e_enable_tx_pkt_filtering(hw)) &&
4798 (adapter->mng_vlan_id != adapter->hw.mng_cookie.vlan_id))
4799 e1000_update_mng_vlan(adapter);
4800
bc7f75fa
AK		 4801 if (link) {
4802 if (!netif_carrier_ok(netdev)) {
3db1cd5c 4803 bool txb2b = true;
23606cf5
RW		 4804
4805 /* Cancel scheduled suspend requests. */
4806 pm_runtime_resume(netdev->dev.parent);
4807
318a94d6 4808 /* update snapshot of PHY registers on LSC */
7c25769f 4809 e1000_phy_read_status(adapter);
bc7f75fa 4810 mac->ops.get_link_up_info(&adapter->hw,
17e813ec
BA		 4811 &adapter->link_speed,
4812 &adapter->link_duplex);
bc7f75fa 4813 e1000_print_link_info(adapter);
e792cd91
KS		 4814
4815 /* check if SmartSpeed worked */
4816 e1000e_check_downshift(hw);
4817 if (phy->speed_downgraded)
4818 netdev_warn(netdev,
4819 "Link Speed was downgraded by SmartSpeed\n");
4820
e921eb1a 4821 /* On supported PHYs, check for duplex mismatch only
f4187b56
BA		 4822 * if link has autonegotiated at 10/100 half
4823 */
4824 if ((hw->phy.type == e1000_phy_igp_3 ||
4825 hw->phy.type == e1000_phy_bm) &&
4826 (hw->mac.autoneg == true) &&
4827 (adapter->link_speed == SPEED_10 ||
4828 adapter->link_speed == SPEED_100) &&
4829 (adapter->link_duplex == HALF_DUPLEX)) {
4830 u16 autoneg_exp;
4831
c2ade1a4 4832 e1e_rphy(hw, MII_EXPANSION, &autoneg_exp);
f4187b56 4833
c2ade1a4 4834 if (!(autoneg_exp & EXPANSION_NWAY))
ef456f85 4835 e_info("Autonegotiated half duplex but link partner cannot autoneg. Try forcing full duplex if link gets many collisions.\n");
f4187b56
BA		 4836 }
4837
f49c57e1 4838 /* adjust timeout factor according to speed/duplex */
bc7f75fa
AK		 4839 adapter->tx_timeout_factor = 1;
4840 switch (adapter->link_speed) {
4841 case SPEED_10:
3db1cd5c 4842 txb2b = false;
10f1b492 4843 adapter->tx_timeout_factor = 16;
bc7f75fa
AK		 4844 break;
4845 case SPEED_100:
3db1cd5c 4846 txb2b = false;
4c86e0b9 4847 adapter->tx_timeout_factor = 10;
bc7f75fa
AK		 4848 break;
4849 }
4850
e921eb1a 4851 /* workaround: re-program speed mode bit after
ad68076e
BA		 4852 * link-up event
4853 */
bc7f75fa
AK		 4854 if ((adapter->flags & FLAG_TARC_SPEED_MODE_BIT) &&
4855 !txb2b) {
4856 u32 tarc0;
e9ec2c0f 4857 tarc0 = er32(TARC(0));
bc7f75fa 4858 tarc0 &= ~SPEED_MODE_BIT;
e9ec2c0f 4859 ew32(TARC(0), tarc0);
bc7f75fa
AK		 4860 }
4861
e921eb1a 4862 /* disable TSO for pcie and 10/100 speeds, to avoid
ad68076e
BA		 4863 * some hardware issues
4864 */
bc7f75fa
AK		 4865 if (!(adapter->flags & FLAG_TSO_FORCE)) {
4866 switch (adapter->link_speed) {
4867 case SPEED_10:
4868 case SPEED_100:
44defeb3 4869 e_info("10/100 speed: disabling TSO\n");
bc7f75fa
AK		 4870 netdev->features &= ~NETIF_F_TSO;
4871 netdev->features &= ~NETIF_F_TSO6;
4872 break;
4873 case SPEED_1000:
4874 netdev->features |= NETIF_F_TSO;
4875 netdev->features |= NETIF_F_TSO6;
4876 break;
4877 default:
4878 /* oops */
4879 break;
4880 }
4881 }
4882
e921eb1a 4883 /* enable transmits in the hardware, need to do this
ad68076e
BA		 4884 * after setting TARC(0)
4885 */
bc7f75fa
AK		 4886 tctl = er32(TCTL);
4887 tctl |= E1000_TCTL_EN;
4888 ew32(TCTL, tctl);
4889
e921eb1a 4890 /* Perform any post-link-up configuration before
75eb0fad
BA		 4891 * reporting link up.
4892 */
4893 if (phy->ops.cfg_on_link_up)
4894 phy->ops.cfg_on_link_up(hw);
4895
bc7f75fa 4896 netif_carrier_on(netdev);
bc7f75fa
AK		 4897
4898 if (!test_bit(__E1000_DOWN, &adapter->state))
4899 mod_timer(&adapter->phy_info_timer,
4900 round_jiffies(jiffies + 2 * HZ));
bc7f75fa
AK		 4901 }
4902 } else {
4903 if (netif_carrier_ok(netdev)) {
4904 adapter->link_speed = 0;
4905 adapter->link_duplex = 0;
8f12fe86 4906 /* Link status message must follow this format */
7dbc1672 4907 pr_info("%s NIC Link is Down\n", adapter->netdev->name);
bc7f75fa 4908 netif_carrier_off(netdev);
bc7f75fa
AK		 4909 if (!test_bit(__E1000_DOWN, &adapter->state))
4910 mod_timer(&adapter->phy_info_timer,
4911 round_jiffies(jiffies + 2 * HZ));
4912
12d43f7d
BA		 4913 /* The link is lost so the controller stops DMA.
4914 * If there is queued Tx work that cannot be done
4915 * or if on an 8000ES2LAN which requires a Rx packet
4916 * buffer work-around on link down event, reset the
4917 * controller to flush the Tx/Rx packet buffers.
4918 * (Do the reset outside of interrupt context).
4919 */
4920 if ((adapter->flags & FLAG_RX_NEEDS_RESTART) ||
4921 (e1000_desc_unused(tx_ring) + 1 < tx_ring->count))
4922 adapter->flags |= FLAG_RESTART_NOW;
23606cf5
RW		 4923 else
4924 pm_schedule_suspend(netdev->dev.parent,
17e813ec 4925 LINK_TIMEOUT);
bc7f75fa
AK		 4926 }
4927 }
4928
4929link_up:
67fd4fcb 4930 spin_lock(&adapter->stats64_lock);
bc7f75fa
AK		 4931 e1000e_update_stats(adapter);
4932
4933 mac->tx_packet_delta = adapter->stats.tpt - adapter->tpt_old;
4934 adapter->tpt_old = adapter->stats.tpt;
4935 mac->collision_delta = adapter->stats.colc - adapter->colc_old;
4936 adapter->colc_old = adapter->stats.colc;
4937
7c25769f
BA		 4938 adapter->gorc = adapter->stats.gorc - adapter->gorc_old;
4939 adapter->gorc_old = adapter->stats.gorc;
4940 adapter->gotc = adapter->stats.gotc - adapter->gotc_old;
4941 adapter->gotc_old = adapter->stats.gotc;
2084b114 4942 spin_unlock(&adapter->stats64_lock);
bc7f75fa 4943
12d43f7d 4944 if (adapter->flags & FLAG_RESTART_NOW) {
90da0669
BA		 4945 schedule_work(&adapter->reset_task);
4946 /* return immediately since reset is imminent */
4947 return;
bc7f75fa
AK		 4948 }
4949
12d43f7d
BA		 4950 e1000e_update_adaptive(&adapter->hw);
4951
eab2abf5
JB		 4952 /* Simple mode for Interrupt Throttle Rate (ITR) */
4953 if (adapter->itr_setting == 4) {
e921eb1a 4954 /* Symmetric Tx/Rx gets a reduced ITR=2000;
eab2abf5
JB		 4955 * Total asymmetrical Tx or Rx gets ITR=8000;
4956 * everyone else is between 2000-8000.
4957 */
4958 u32 goc = (adapter->gotc + adapter->gorc) / 10000;
4959 u32 dif = (adapter->gotc > adapter->gorc ?
17e813ec
BA		 4960 adapter->gotc - adapter->gorc :
4961 adapter->gorc - adapter->gotc) / 10000;
eab2abf5
JB		 4962 u32 itr = goc > 0 ? (dif * 6000 / goc + 2000) : 8000;
4963
22a4cca2 4964 e1000e_write_itr(adapter, itr);
eab2abf5
JB		 4965 }
4966
ad68076e 4967 /* Cause software interrupt to ensure Rx ring is cleaned */
4662e82b
BA		 4968 if (adapter->msix_entries)
4969 ew32(ICS, adapter->rx_ring->ims_val);
4970 else
4971 ew32(ICS, E1000_ICS_RXDMT0);
bc7f75fa 4972
713b3c9e
JB		 4973 /* flush pending descriptors to memory before detecting Tx hang */
4974 e1000e_flush_descriptors(adapter);
4975
bc7f75fa 4976 /* Force detection of hung controller every watchdog period */
3db1cd5c 4977 adapter->detect_tx_hung = true;
bc7f75fa 4978
e921eb1a 4979 /* With 82571 controllers, LAA may be overwritten due to controller
ad68076e
BA		 4980 * reset from the other port. Set the appropriate LAA in RAR[0]
4981 */
bc7f75fa 4982 if (e1000e_get_laa_state_82571(hw))
69e1e019 4983 hw->mac.ops.rar_set(hw, adapter->hw.mac.addr, 0);
bc7f75fa 4984
ff10e13c
CW		 4985 if (adapter->flags2 & FLAG2_CHECK_PHY_HANG)
4986 e1000e_check_82574_phy_workaround(adapter);
4987
b67e1913
BA		 4988 /* Clear valid timestamp stuck in RXSTMPL/H due to a Rx error */
4989 if (adapter->hwtstamp_config.rx_filter != HWTSTAMP_FILTER_NONE) {
4990 if ((adapter->flags2 & FLAG2_CHECK_RX_HWTSTAMP) &&
4991 (er32(TSYNCRXCTL) & E1000_TSYNCRXCTL_VALID)) {
4992 er32(RXSTMPH);
4993 adapter->rx_hwtstamp_cleared++;
4994 } else {
4995 adapter->flags2 |= FLAG2_CHECK_RX_HWTSTAMP;
4996 }
4997 }
4998
bc7f75fa
AK		 4999 /* Reset the timer */
5000 if (!test_bit(__E1000_DOWN, &adapter->state))
5001 mod_timer(&adapter->watchdog_timer,
5002 round_jiffies(jiffies + 2 * HZ));
5003}
5004
5005#define E1000_TX_FLAGS_CSUM 0x00000001
5006#define E1000_TX_FLAGS_VLAN 0x00000002
5007#define E1000_TX_FLAGS_TSO 0x00000004
5008#define E1000_TX_FLAGS_IPV4 0x00000008
943146de 5009#define E1000_TX_FLAGS_NO_FCS 0x00000010
b67e1913 5010#define E1000_TX_FLAGS_HWTSTAMP 0x00000020
bc7f75fa
AK		 5011#define E1000_TX_FLAGS_VLAN_MASK 0xffff0000
5012#define E1000_TX_FLAGS_VLAN_SHIFT 16
5013
55aa6985 5014static int e1000_tso(struct e1000_ring *tx_ring, struct sk_buff *skb)
bc7f75fa 5015{
bc7f75fa
AK		 5016 struct e1000_context_desc *context_desc;
5017 struct e1000_buffer *buffer_info;
5018 unsigned int i;
5019 u32 cmd_length = 0;
70443ae9 5020 u16 ipcse = 0, mss;
bc7f75fa 5021 u8 ipcss, ipcso, tucss, tucso, hdr_len;
bc7f75fa 5022
3d5e33c9
BA		 5023 if (!skb_is_gso(skb))
5024 return 0;
bc7f75fa 5025
3d5e33c9 5026 if (skb_header_cloned(skb)) {
90da0669
BA		 5027 int err = pskb_expand_head(skb, 0, 0, GFP_ATOMIC);
5028
3d5e33c9
BA		 5029 if (err)
5030 return err;
bc7f75fa
AK		 5031 }
5032
3d5e33c9
BA		 5033 hdr_len = skb_transport_offset(skb) + tcp_hdrlen(skb);
5034 mss = skb_shinfo(skb)->gso_size;
5035 if (skb->protocol == htons(ETH_P_IP)) {
5036 struct iphdr *iph = ip_hdr(skb);
5037 iph->tot_len = 0;
5038 iph->check = 0;
5039 tcp_hdr(skb)->check = ~csum_tcpudp_magic(iph->saddr, iph->daddr,
f0ff4398 5040 0, IPPROTO_TCP, 0);
3d5e33c9
BA		 5041 cmd_length = E1000_TXD_CMD_IP;
5042 ipcse = skb_transport_offset(skb) - 1;
8e1e8a47 5043 } else if (skb_is_gso_v6(skb)) {
3d5e33c9
BA		 5044 ipv6_hdr(skb)->payload_len = 0;
5045 tcp_hdr(skb)->check = ~csum_ipv6_magic(&ipv6_hdr(skb)->saddr,
f0ff4398
BA		 5046 &ipv6_hdr(skb)->daddr,
5047 0, IPPROTO_TCP, 0);
3d5e33c9
BA		 5048 ipcse = 0;
5049 }
5050 ipcss = skb_network_offset(skb);
5051 ipcso = (void *)&(ip_hdr(skb)->check) - (void *)skb->data;
5052 tucss = skb_transport_offset(skb);
5053 tucso = (void *)&(tcp_hdr(skb)->check) - (void *)skb->data;
3d5e33c9
BA		 5054
5055 cmd_length |= (E1000_TXD_CMD_DEXT | E1000_TXD_CMD_TSE |
f0ff4398 5056 E1000_TXD_CMD_TCP | (skb->len - (hdr_len)));
3d5e33c9
BA		 5057
5058 i = tx_ring->next_to_use;
5059 context_desc = E1000_CONTEXT_DESC(*tx_ring, i);
5060 buffer_info = &tx_ring->buffer_info[i];
5061
5062 context_desc->lower_setup.ip_fields.ipcss = ipcss;
5063 context_desc->lower_setup.ip_fields.ipcso = ipcso;
5064 context_desc->lower_setup.ip_fields.ipcse = cpu_to_le16(ipcse);
5065 context_desc->upper_setup.tcp_fields.tucss = tucss;
5066 context_desc->upper_setup.tcp_fields.tucso = tucso;
70443ae9 5067 context_desc->upper_setup.tcp_fields.tucse = 0;
3d5e33c9
BA		 5068 context_desc->tcp_seg_setup.fields.mss = cpu_to_le16(mss);
5069 context_desc->tcp_seg_setup.fields.hdr_len = hdr_len;
5070 context_desc->cmd_and_length = cpu_to_le32(cmd_length);
5071
5072 buffer_info->time_stamp = jiffies;
5073 buffer_info->next_to_watch = i;
5074
5075 i++;
5076 if (i == tx_ring->count)
5077 i = 0;
5078 tx_ring->next_to_use = i;
5079
5080 return 1;
bc7f75fa
AK		 5081}
5082
55aa6985 5083static bool e1000_tx_csum(struct e1000_ring *tx_ring, struct sk_buff *skb)
bc7f75fa 5084{
55aa6985 5085 struct e1000_adapter *adapter = tx_ring->adapter;
bc7f75fa
AK		 5086 struct e1000_context_desc *context_desc;
5087 struct e1000_buffer *buffer_info;
5088 unsigned int i;
5089 u8 css;
af807c82 5090 u32 cmd_len = E1000_TXD_CMD_DEXT;
5f66f208 5091 __be16 protocol;
bc7f75fa 5092
af807c82
DG		 5093 if (skb->ip_summed != CHECKSUM_PARTIAL)
5094 return 0;
bc7f75fa 5095
5f66f208
AJ		 5096 if (skb->protocol == cpu_to_be16(ETH_P_8021Q))
5097 protocol = vlan_eth_hdr(skb)->h_vlan_encapsulated_proto;
5098 else
5099 protocol = skb->protocol;
5100
3f518390 5101 switch (protocol) {
09640e63 5102 case cpu_to_be16(ETH_P_IP):
af807c82
DG		 5103 if (ip_hdr(skb)->protocol == IPPROTO_TCP)
5104 cmd_len |= E1000_TXD_CMD_TCP;
5105 break;
09640e63 5106 case cpu_to_be16(ETH_P_IPV6):
af807c82
DG		 5107 /* XXX not handling all IPV6 headers */
5108 if (ipv6_hdr(skb)->nexthdr == IPPROTO_TCP)
5109 cmd_len |= E1000_TXD_CMD_TCP;
5110 break;
5111 default:
5112 if (unlikely(net_ratelimit()))
5f66f208
AJ		 5113 e_warn("checksum_partial proto=%x!\n",
5114 be16_to_cpu(protocol));
af807c82 5115 break;
bc7f75fa
AK		 5116 }
5117
0d0b1672 5118 css = skb_checksum_start_offset(skb);
af807c82
DG		 5119
5120 i = tx_ring->next_to_use;
5121 buffer_info = &tx_ring->buffer_info[i];
5122 context_desc = E1000_CONTEXT_DESC(*tx_ring, i);
5123
5124 context_desc->lower_setup.ip_config = 0;
5125 context_desc->upper_setup.tcp_fields.tucss = css;
f0ff4398 5126 context_desc->upper_setup.tcp_fields.tucso = css + skb->csum_offset;
af807c82
DG		 5127 context_desc->upper_setup.tcp_fields.tucse = 0;
5128 context_desc->tcp_seg_setup.data = 0;
5129 context_desc->cmd_and_length = cpu_to_le32(cmd_len);
5130
5131 buffer_info->time_stamp = jiffies;
5132 buffer_info->next_to_watch = i;
5133
5134 i++;
5135 if (i == tx_ring->count)
5136 i = 0;
5137 tx_ring->next_to_use = i;
5138
5139 return 1;
bc7f75fa
AK		 5140}
5141
55aa6985
BA		 5142static int e1000_tx_map(struct e1000_ring *tx_ring, struct sk_buff *skb,
5143 unsigned int first, unsigned int max_per_txd,
d821a4c4 5144 unsigned int nr_frags)
bc7f75fa 5145{
55aa6985 5146 struct e1000_adapter *adapter = tx_ring->adapter;
03b1320d 5147 struct pci_dev *pdev = adapter->pdev;
1b7719c4 5148 struct e1000_buffer *buffer_info;
8ddc951c 5149 unsigned int len = skb_headlen(skb);
03b1320d 5150 unsigned int offset = 0, size, count = 0, i;
9ed318d5 5151 unsigned int f, bytecount, segs;
bc7f75fa
AK		 5152
5153 i = tx_ring->next_to_use;
5154
5155 while (len) {
1b7719c4 5156 buffer_info = &tx_ring->buffer_info[i];
bc7f75fa
AK		 5157 size = min(len, max_per_txd);
5158
bc7f75fa 5159 buffer_info->length = size;
bc7f75fa 5160 buffer_info->time_stamp = jiffies;
bc7f75fa 5161 buffer_info->next_to_watch = i;
0be3f55f
NN		 5162 buffer_info->dma = dma_map_single(&pdev->dev,
5163 skb->data + offset,
af667a29 5164 size, DMA_TO_DEVICE);
03b1320d 5165 buffer_info->mapped_as_page = false;
0be3f55f 5166 if (dma_mapping_error(&pdev->dev, buffer_info->dma))
03b1320d 5167 goto dma_error;
bc7f75fa
AK		 5168
5169 len -= size;
5170 offset += size;
03b1320d 5171 count++;
1b7719c4
AD		 5172
5173 if (len) {
5174 i++;
5175 if (i == tx_ring->count)
5176 i = 0;
5177 }
bc7f75fa
AK		 5178 }
5179
5180 for (f = 0; f < nr_frags; f++) {
9e903e08 5181 const struct skb_frag_struct *frag;
bc7f75fa
AK		 5182
5183 frag = &skb_shinfo(skb)->frags[f];
9e903e08 5184 len = skb_frag_size(frag);
877749bf 5185 offset = 0;
bc7f75fa
AK		 5186
5187 while (len) {
1b7719c4
AD		 5188 i++;
5189 if (i == tx_ring->count)
5190 i = 0;
5191
bc7f75fa
AK		 5192 buffer_info = &tx_ring->buffer_info[i];
5193 size = min(len, max_per_txd);
bc7f75fa
AK		 5194
5195 buffer_info->length = size;
5196 buffer_info->time_stamp = jiffies;
bc7f75fa 5197 buffer_info->next_to_watch = i;
877749bf 5198 buffer_info->dma = skb_frag_dma_map(&pdev->dev, frag,
17e813ec
BA		 5199 offset, size,
5200 DMA_TO_DEVICE);
03b1320d 5201 buffer_info->mapped_as_page = true;
0be3f55f 5202 if (dma_mapping_error(&pdev->dev, buffer_info->dma))
03b1320d 5203 goto dma_error;
bc7f75fa
AK		 5204
5205 len -= size;
5206 offset += size;
5207 count++;
bc7f75fa
AK		 5208 }
5209 }
5210
af667a29 5211 segs = skb_shinfo(skb)->gso_segs ? : 1;
9ed318d5
TH		 5212 /* multiply data chunks by size of headers */
5213 bytecount = ((segs - 1) * skb_headlen(skb)) + skb->len;
5214
bc7f75fa 5215 tx_ring->buffer_info[i].skb = skb;
9ed318d5
TH		 5216 tx_ring->buffer_info[i].segs = segs;
5217 tx_ring->buffer_info[i].bytecount = bytecount;
bc7f75fa
AK		 5218 tx_ring->buffer_info[first].next_to_watch = i;
5219
5220 return count;
03b1320d
AD		 5221
5222dma_error:
af667a29 5223 dev_err(&pdev->dev, "Tx DMA map failed\n");
03b1320d 5224 buffer_info->dma = 0;
c1fa347f 5225 if (count)
03b1320d 5226 count--;
c1fa347f
RK		 5227
5228 while (count--) {
af667a29 5229 if (i == 0)
03b1320d 5230 i += tx_ring->count;
c1fa347f 5231 i--;
03b1320d 5232 buffer_info = &tx_ring->buffer_info[i];
55aa6985 5233 e1000_put_txbuf(tx_ring, buffer_info);
03b1320d
AD		 5234 }
5235
5236 return 0;
bc7f75fa
AK		 5237}
5238
55aa6985 5239static void e1000_tx_queue(struct e1000_ring *tx_ring, int tx_flags, int count)
bc7f75fa 5240{
55aa6985 5241 struct e1000_adapter *adapter = tx_ring->adapter;
bc7f75fa
AK		 5242 struct e1000_tx_desc *tx_desc = NULL;
5243 struct e1000_buffer *buffer_info;
5244 u32 txd_upper = 0, txd_lower = E1000_TXD_CMD_IFCS;
5245 unsigned int i;
5246
5247 if (tx_flags & E1000_TX_FLAGS_TSO) {
5248 txd_lower |= E1000_TXD_CMD_DEXT | E1000_TXD_DTYP_D |
f0ff4398 5249 E1000_TXD_CMD_TSE;
bc7f75fa
AK		 5250 txd_upper |= E1000_TXD_POPTS_TXSM << 8;
5251
5252 if (tx_flags & E1000_TX_FLAGS_IPV4)
5253 txd_upper |= E1000_TXD_POPTS_IXSM << 8;
5254 }
5255
5256 if (tx_flags & E1000_TX_FLAGS_CSUM) {
5257 txd_lower |= E1000_TXD_CMD_DEXT | E1000_TXD_DTYP_D;
5258 txd_upper |= E1000_TXD_POPTS_TXSM << 8;
5259 }
5260
5261 if (tx_flags & E1000_TX_FLAGS_VLAN) {
5262 txd_lower |= E1000_TXD_CMD_VLE;
5263 txd_upper |= (tx_flags & E1000_TX_FLAGS_VLAN_MASK);
5264 }
5265
943146de
BG		 5266 if (unlikely(tx_flags & E1000_TX_FLAGS_NO_FCS))
5267 txd_lower &= ~(E1000_TXD_CMD_IFCS);
5268
b67e1913
BA		 5269 if (unlikely(tx_flags & E1000_TX_FLAGS_HWTSTAMP)) {
5270 txd_lower |= E1000_TXD_CMD_DEXT | E1000_TXD_DTYP_D;
5271 txd_upper |= E1000_TXD_EXTCMD_TSTAMP;
5272 }
5273
bc7f75fa
AK		 5274 i = tx_ring->next_to_use;
5275
36b973df 5276 do {
bc7f75fa
AK		 5277 buffer_info = &tx_ring->buffer_info[i];
5278 tx_desc = E1000_TX_DESC(*tx_ring, i);
5279 tx_desc->buffer_addr = cpu_to_le64(buffer_info->dma);
f0ff4398
BA		 5280 tx_desc->lower.data = cpu_to_le32(txd_lower |
5281 buffer_info->length);
bc7f75fa
AK		 5282 tx_desc->upper.data = cpu_to_le32(txd_upper);
5283
5284 i++;
5285 if (i == tx_ring->count)
5286 i = 0;
36b973df 5287 } while (--count > 0);
bc7f75fa
AK		 5288
5289 tx_desc->lower.data |= cpu_to_le32(adapter->txd_cmd);
5290
943146de
BG		 5291 /* txd_cmd re-enables FCS, so we'll re-disable it here as desired. */
5292 if (unlikely(tx_flags & E1000_TX_FLAGS_NO_FCS))
5293 tx_desc->lower.data &= ~(cpu_to_le32(E1000_TXD_CMD_IFCS));
5294
e921eb1a 5295 /* Force memory writes to complete before letting h/w
bc7f75fa
AK		 5296 * know there are new descriptors to fetch. (Only
5297 * applicable for weak-ordered memory model archs,
ad68076e
BA		 5298 * such as IA-64).
5299 */
bc7f75fa
AK		 5300 wmb();
5301
5302 tx_ring->next_to_use = i;
c6e7f51e
BA		 5303
5304 if (adapter->flags2 & FLAG2_PCIM2PCI_ARBITER_WA)
55aa6985 5305 e1000e_update_tdt_wa(tx_ring, i);
c6e7f51e 5306 else
c5083cf6 5307 writel(i, tx_ring->tail);
c6e7f51e 5308
e921eb1a 5309 /* we need this if more than one processor can write to our tail
ad68076e
BA		 5310 * at a time, it synchronizes IO on IA64/Altix systems
5311 */
bc7f75fa
AK		 5312 mmiowb();
5313}
5314
5315#define MINIMUM_DHCP_PACKET_SIZE 282
5316static int e1000_transfer_dhcp_info(struct e1000_adapter *adapter,
5317 struct sk_buff *skb)
5318{
5319 struct e1000_hw *hw = &adapter->hw;
5320 u16 length, offset;
5321
d60923c4
BA		 5322 if (vlan_tx_tag_present(skb) &&
5323 !((vlan_tx_tag_get(skb) == adapter->hw.mng_cookie.vlan_id) &&
5324 (adapter->hw.mng_cookie.status &
5325 E1000_MNG_DHCP_COOKIE_STATUS_VLAN)))
5326 return 0;
bc7f75fa
AK		 5327
5328 if (skb->len <= MINIMUM_DHCP_PACKET_SIZE)
5329 return 0;
5330
53aa82da 5331 if (((struct ethhdr *)skb->data)->h_proto != htons(ETH_P_IP))
bc7f75fa
AK		 5332 return 0;
5333
5334 {
362e20ca 5335 const struct iphdr *ip = (struct iphdr *)((u8 *)skb->data + 14);
bc7f75fa
AK		 5336 struct udphdr *udp;
5337
5338 if (ip->protocol != IPPROTO_UDP)
5339 return 0;
5340
5341 udp = (struct udphdr *)((u8 *)ip + (ip->ihl << 2));
5342 if (ntohs(udp->dest) != 67)
5343 return 0;
5344
5345 offset = (u8 *)udp + 8 - skb->data;
5346 length = skb->len - offset;
5347 return e1000e_mng_write_dhcp_info(hw, (u8 *)udp + 8, length);
5348 }
5349
5350 return 0;
5351}
5352
55aa6985 5353static int __e1000_maybe_stop_tx(struct e1000_ring *tx_ring, int size)
bc7f75fa 5354{
55aa6985 5355 struct e1000_adapter *adapter = tx_ring->adapter;
bc7f75fa 5356
55aa6985 5357 netif_stop_queue(adapter->netdev);
e921eb1a 5358 /* Herbert's original patch had:
bc7f75fa 5359 * smp_mb__after_netif_stop_queue();
ad68076e
BA		 5360 * but since that doesn't exist yet, just open code it.
5361 */
bc7f75fa
AK		 5362 smp_mb();
5363
e921eb1a 5364 /* We need to check again in a case another CPU has just
ad68076e
BA		 5365 * made room available.
5366 */
55aa6985 5367 if (e1000_desc_unused(tx_ring) < size)
bc7f75fa
AK		 5368 return -EBUSY;
5369
5370 /* A reprieve! */
55aa6985 5371 netif_start_queue(adapter->netdev);
bc7f75fa
AK		 5372 ++adapter->restart_queue;
5373 return 0;
5374}
5375
55aa6985 5376static int e1000_maybe_stop_tx(struct e1000_ring *tx_ring, int size)
bc7f75fa 5377{
d821a4c4
BA		 5378 BUG_ON(size > tx_ring->count);
5379
55aa6985 5380 if (e1000_desc_unused(tx_ring) >= size)
bc7f75fa 5381 return 0;
55aa6985 5382 return __e1000_maybe_stop_tx(tx_ring, size);
bc7f75fa
AK		 5383}
5384
3b29a56d
SH		 5385static netdev_tx_t e1000_xmit_frame(struct sk_buff *skb,
5386 struct net_device *netdev)
bc7f75fa
AK		 5387{
5388 struct e1000_adapter *adapter = netdev_priv(netdev);
5389 struct e1000_ring *tx_ring = adapter->tx_ring;
5390 unsigned int first;
bc7f75fa 5391 unsigned int tx_flags = 0;
e743d313 5392 unsigned int len = skb_headlen(skb);
4e6c709c
AK		 5393 unsigned int nr_frags;
5394 unsigned int mss;
bc7f75fa
AK		 5395 int count = 0;
5396 int tso;
5397 unsigned int f;
bc7f75fa
AK		 5398
5399 if (test_bit(__E1000_DOWN, &adapter->state)) {
5400 dev_kfree_skb_any(skb);
5401 return NETDEV_TX_OK;
5402 }
5403
5404 if (skb->len <= 0) {
5405 dev_kfree_skb_any(skb);
5406 return NETDEV_TX_OK;
5407 }
5408
e921eb1a 5409 /* The minimum packet size with TCTL.PSP set is 17 bytes so
6e97c170
TD		 5410 * pad skb in order to meet this minimum size requirement
5411 */
5412 if (unlikely(skb->len < 17)) {
5413 if (skb_pad(skb, 17 - skb->len))
5414 return NETDEV_TX_OK;
5415 skb->len = 17;
5416 skb_set_tail_pointer(skb, 17);
5417 }
5418
bc7f75fa 5419 mss = skb_shinfo(skb)->gso_size;
bc7f75fa
AK		 5420 if (mss) {
5421 u8 hdr_len;
bc7f75fa 5422
e921eb1a 5423 /* TSO Workaround for 82571/2/3 Controllers -- if skb->data
ad68076e
BA		 5424 * points to just header, pull a few bytes of payload from
5425 * frags into skb->data
5426 */
bc7f75fa 5427 hdr_len = skb_transport_offset(skb) + tcp_hdrlen(skb);
e921eb1a 5428 /* we do this workaround for ES2LAN, but it is un-necessary,
ad68076e
BA		 5429 * avoiding it could save a lot of cycles
5430 */
4e6c709c 5431 if (skb->data_len && (hdr_len == len)) {
bc7f75fa
AK		 5432 unsigned int pull_size;
5433
a2a5b323 5434 pull_size = min_t(unsigned int, 4, skb->data_len);
bc7f75fa 5435 if (!__pskb_pull_tail(skb, pull_size)) {
44defeb3 5436 e_err("__pskb_pull_tail failed.\n");
bc7f75fa
AK		 5437 dev_kfree_skb_any(skb);
5438 return NETDEV_TX_OK;
5439 }
e743d313 5440 len = skb_headlen(skb);
bc7f75fa
AK		 5441 }
5442 }
5443
5444 /* reserve a descriptor for the offload context */
5445 if ((mss) || (skb->ip_summed == CHECKSUM_PARTIAL))
5446 count++;
5447 count++;
5448
d821a4c4 5449 count += DIV_ROUND_UP(len, adapter->tx_fifo_limit);
bc7f75fa
AK		 5450
5451 nr_frags = skb_shinfo(skb)->nr_frags;
5452 for (f = 0; f < nr_frags; f++)
d821a4c4
BA		 5453 count += DIV_ROUND_UP(skb_frag_size(&skb_shinfo(skb)->frags[f]),
5454 adapter->tx_fifo_limit);
bc7f75fa
AK		 5455
5456 if (adapter->hw.mac.tx_pkt_filtering)
5457 e1000_transfer_dhcp_info(adapter, skb);
5458
e921eb1a 5459 /* need: count + 2 desc gap to keep tail from touching
ad68076e
BA		 5460 * head, otherwise try next time
5461 */
55aa6985 5462 if (e1000_maybe_stop_tx(tx_ring, count + 2))
bc7f75fa 5463 return NETDEV_TX_BUSY;
bc7f75fa 5464
eab6d18d 5465 if (vlan_tx_tag_present(skb)) {
bc7f75fa
AK		 5466 tx_flags |= E1000_TX_FLAGS_VLAN;
5467 tx_flags |= (vlan_tx_tag_get(skb) << E1000_TX_FLAGS_VLAN_SHIFT);
5468 }
5469
5470 first = tx_ring->next_to_use;
5471
55aa6985 5472 tso = e1000_tso(tx_ring, skb);
bc7f75fa
AK		 5473 if (tso < 0) {
5474 dev_kfree_skb_any(skb);
bc7f75fa
AK		 5475 return NETDEV_TX_OK;
5476 }
5477
5478 if (tso)
5479 tx_flags |= E1000_TX_FLAGS_TSO;
55aa6985 5480 else if (e1000_tx_csum(tx_ring, skb))
bc7f75fa
AK		 5481 tx_flags |= E1000_TX_FLAGS_CSUM;
5482
e921eb1a 5483 /* Old method was to assume IPv4 packet by default if TSO was enabled.
bc7f75fa 5484 * 82571 hardware supports TSO capabilities for IPv6 as well...
ad68076e
BA		 5485 * no longer assume, we must.
5486 */
bc7f75fa
AK		 5487 if (skb->protocol == htons(ETH_P_IP))
5488 tx_flags |= E1000_TX_FLAGS_IPV4;
5489
943146de
BG		 5490 if (unlikely(skb->no_fcs))
5491 tx_flags |= E1000_TX_FLAGS_NO_FCS;
5492
25985edc 5493 /* if count is 0 then mapping error has occurred */
d821a4c4
BA		 5494 count = e1000_tx_map(tx_ring, skb, first, adapter->tx_fifo_limit,
5495 nr_frags);
1b7719c4 5496 if (count) {
b67e1913
BA		 5497 if (unlikely((skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) &&
5498 !adapter->tx_hwtstamp_skb)) {
5499 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;
5500 tx_flags |= E1000_TX_FLAGS_HWTSTAMP;
5501 adapter->tx_hwtstamp_skb = skb_get(skb);
5502 schedule_work(&adapter->tx_hwtstamp_work);
5503 } else {
5504 skb_tx_timestamp(skb);
5505 }
80be3129 5506
3f0cfa3b 5507 netdev_sent_queue(netdev, skb->len);
55aa6985 5508 e1000_tx_queue(tx_ring, tx_flags, count);
1b7719c4 5509 /* Make sure there is space in the ring for the next send. */
d821a4c4
BA		 5510 e1000_maybe_stop_tx(tx_ring,
5511 (MAX_SKB_FRAGS *
5512 DIV_ROUND_UP(PAGE_SIZE,
5513 adapter->tx_fifo_limit) + 2));
1b7719c4 5514 } else {
bc7f75fa 5515 dev_kfree_skb_any(skb);
1b7719c4
AD		 5516 tx_ring->buffer_info[first].time_stamp = 0;
5517 tx_ring->next_to_use = first;
bc7f75fa
AK		 5518 }
5519
bc7f75fa
AK		 5520 return NETDEV_TX_OK;
5521}
5522
5523/**
5524 * e1000_tx_timeout - Respond to a Tx Hang
5525 * @netdev: network interface device structure
5526 **/
5527static void e1000_tx_timeout(struct net_device *netdev)
5528{
5529 struct e1000_adapter *adapter = netdev_priv(netdev);
5530
5531 /* Do the reset outside of interrupt context */
5532 adapter->tx_timeout_count++;
5533 schedule_work(&adapter->reset_task);
5534}
5535
5536static void e1000_reset_task(struct work_struct *work)
5537{
5538 struct e1000_adapter *adapter;
5539 adapter = container_of(work, struct e1000_adapter, reset_task);
5540
615b32af
JB		 5541 /* don't run the task if already down */
5542 if (test_bit(__E1000_DOWN, &adapter->state))
5543 return;
5544
12d43f7d 5545 if (!(adapter->flags & FLAG_RESTART_NOW)) {
affa9dfb 5546 e1000e_dump(adapter);
12d43f7d 5547 e_err("Reset adapter unexpectedly\n");
affa9dfb 5548 }
bc7f75fa
AK		 5549 e1000e_reinit_locked(adapter);
5550}
5551
5552/**
67fd4fcb 5553 * e1000_get_stats64 - Get System Network Statistics
bc7f75fa 5554 * @netdev: network interface device structure
67fd4fcb 5555 * @stats: rtnl_link_stats64 pointer
bc7f75fa
AK		 5556 *
5557 * Returns the address of the device statistics structure.
bc7f75fa 5558 **/
67fd4fcb 5559struct rtnl_link_stats64 *e1000e_get_stats64(struct net_device *netdev,
66501f56 5560 struct rtnl_link_stats64 *stats)
bc7f75fa 5561{
67fd4fcb
JK		 5562 struct e1000_adapter *adapter = netdev_priv(netdev);
5563
5564 memset(stats, 0, sizeof(struct rtnl_link_stats64));
5565 spin_lock(&adapter->stats64_lock);
5566 e1000e_update_stats(adapter);
5567 /* Fill out the OS statistics structure */
5568 stats->rx_bytes = adapter->stats.gorc;
5569 stats->rx_packets = adapter->stats.gprc;
5570 stats->tx_bytes = adapter->stats.gotc;
5571 stats->tx_packets = adapter->stats.gptc;
5572 stats->multicast = adapter->stats.mprc;
5573 stats->collisions = adapter->stats.colc;
5574
5575 /* Rx Errors */
5576
e921eb1a 5577 /* RLEC on some newer hardware can be incorrect so build
67fd4fcb
JK		 5578 * our own version based on RUC and ROC
5579 */
5580 stats->rx_errors = adapter->stats.rxerrc +
f0ff4398
BA		 5581 adapter->stats.crcerrs + adapter->stats.algnerrc +
5582 adapter->stats.ruc + adapter->stats.roc + adapter->stats.cexterr;
5583 stats->rx_length_errors = adapter->stats.ruc + adapter->stats.roc;
67fd4fcb
JK		 5584 stats->rx_crc_errors = adapter->stats.crcerrs;
5585 stats->rx_frame_errors = adapter->stats.algnerrc;
5586 stats->rx_missed_errors = adapter->stats.mpc;
5587
5588 /* Tx Errors */
f0ff4398 5589 stats->tx_errors = adapter->stats.ecol + adapter->stats.latecol;
67fd4fcb
JK		 5590 stats->tx_aborted_errors = adapter->stats.ecol;
5591 stats->tx_window_errors = adapter->stats.latecol;
5592 stats->tx_carrier_errors = adapter->stats.tncrs;
5593
5594 /* Tx Dropped needs to be maintained elsewhere */
5595
5596 spin_unlock(&adapter->stats64_lock);
5597 return stats;
bc7f75fa
AK		 5598}
5599
5600/**
5601 * e1000_change_mtu - Change the Maximum Transfer Unit
5602 * @netdev: network interface device structure
5603 * @new_mtu: new value for maximum frame size
5604 *
5605 * Returns 0 on success, negative on failure
5606 **/
5607static int e1000_change_mtu(struct net_device *netdev, int new_mtu)
5608{
5609 struct e1000_adapter *adapter = netdev_priv(netdev);
5610 int max_frame = new_mtu + ETH_HLEN + ETH_FCS_LEN;
5611
2adc55c9 5612 /* Jumbo frame support */
2e1706f2
BA		 5613 if ((max_frame > ETH_FRAME_LEN + ETH_FCS_LEN) &&
5614 !(adapter->flags & FLAG_HAS_JUMBO_FRAMES)) {
5615 e_err("Jumbo Frames not supported.\n");
5616 return -EINVAL;
bc7f75fa
AK		 5617 }
5618
2adc55c9
BA		 5619 /* Supported frame sizes */
5620 if ((new_mtu < ETH_ZLEN + ETH_FCS_LEN + VLAN_HLEN) ||
5621 (max_frame > adapter->max_hw_frame_size)) {
5622 e_err("Unsupported MTU setting\n");
bc7f75fa
AK		 5623 return -EINVAL;
5624 }
5625
2fbe4526
BA		 5626 /* Jumbo frame workaround on 82579 and newer requires CRC be stripped */
5627 if ((adapter->hw.mac.type >= e1000_pch2lan) &&
a1ce6473
BA		 5628 !(adapter->flags2 & FLAG2_CRC_STRIPPING) &&
5629 (new_mtu > ETH_DATA_LEN)) {
2fbe4526 5630 e_err("Jumbo Frames not supported on this device when CRC stripping is disabled.\n");
a1ce6473
BA		 5631 return -EINVAL;
5632 }
5633
bc7f75fa 5634 while (test_and_set_bit(__E1000_RESETTING, &adapter->state))
1bba4386 5635 usleep_range(1000, 2000);
610c9928 5636 /* e1000e_down -> e1000e_reset dependent on max_frame_size & mtu */
318a94d6 5637 adapter->max_frame_size = max_frame;
610c9928
BA		 5638 e_info("changing MTU from %d to %d\n", netdev->mtu, new_mtu);
5639 netdev->mtu = new_mtu;
bc7f75fa
AK		 5640 if (netif_running(netdev))
5641 e1000e_down(adapter);
5642
e921eb1a 5643 /* NOTE: netdev_alloc_skb reserves 16 bytes, and typically NET_IP_ALIGN
bc7f75fa
AK		 5644 * means we reserve 2 more, this pushes us to allocate from the next
5645 * larger slab size.
ad68076e 5646 * i.e. RXBUFFER_2048 --> size-4096 slab
97ac8cae
BA		 5647 * However with the new *_jumbo_rx* routines, jumbo receives will use
5648 * fragmented skbs
ad68076e 5649 */
bc7f75fa 5650
9926146b 5651 if (max_frame <= 2048)
bc7f75fa
AK		 5652 adapter->rx_buffer_len = 2048;
5653 else
5654 adapter->rx_buffer_len = 4096;
5655
5656 /* adjust allocation if LPE protects us, and we aren't using SBP */
5657 if ((max_frame == ETH_FRAME_LEN + ETH_FCS_LEN) ||
17e813ec 5658 (max_frame == ETH_FRAME_LEN + VLAN_HLEN + ETH_FCS_LEN))
bc7f75fa 5659 adapter->rx_buffer_len = ETH_FRAME_LEN + VLAN_HLEN
17e813ec 5660 + ETH_FCS_LEN;
bc7f75fa 5661
bc7f75fa
AK		 5662 if (netif_running(netdev))
5663 e1000e_up(adapter);
5664 else
5665 e1000e_reset(adapter);
5666
5667 clear_bit(__E1000_RESETTING, &adapter->state);
5668
5669 return 0;
5670}
5671
5672static int e1000_mii_ioctl(struct net_device *netdev, struct ifreq *ifr,
5673 int cmd)
5674{
5675 struct e1000_adapter *adapter = netdev_priv(netdev);
5676 struct mii_ioctl_data *data = if_mii(ifr);
bc7f75fa 5677
318a94d6 5678 if (adapter->hw.phy.media_type != e1000_media_type_copper)
bc7f75fa
AK		 5679 return -EOPNOTSUPP;
5680
5681 switch (cmd) {
5682 case SIOCGMIIPHY:
5683 data->phy_id = adapter->hw.phy.addr;
5684 break;
5685 case SIOCGMIIREG:
b16a002e
BA		 5686 e1000_phy_read_status(adapter);
5687
7c25769f
BA		 5688 switch (data->reg_num & 0x1F) {
5689 case MII_BMCR:
5690 data->val_out = adapter->phy_regs.bmcr;
5691 break;
5692 case MII_BMSR:
5693 data->val_out = adapter->phy_regs.bmsr;
5694 break;
5695 case MII_PHYSID1:
5696 data->val_out = (adapter->hw.phy.id >> 16);
5697 break;
5698 case MII_PHYSID2:
5699 data->val_out = (adapter->hw.phy.id & 0xFFFF);
5700 break;
5701 case MII_ADVERTISE:
5702 data->val_out = adapter->phy_regs.advertise;
5703 break;
5704 case MII_LPA:
5705 data->val_out = adapter->phy_regs.lpa;
5706 break;
5707 case MII_EXPANSION:
5708 data->val_out = adapter->phy_regs.expansion;
5709 break;
5710 case MII_CTRL1000:
5711 data->val_out = adapter->phy_regs.ctrl1000;
5712 break;
5713 case MII_STAT1000:
5714 data->val_out = adapter->phy_regs.stat1000;
5715 break;
5716 case MII_ESTATUS:
5717 data->val_out = adapter->phy_regs.estatus;
5718 break;
5719 default:
bc7f75fa
AK		 5720 return -EIO;
5721 }
bc7f75fa
AK		 5722 break;
5723 case SIOCSMIIREG:
5724 default:
5725 return -EOPNOTSUPP;
5726 }
5727 return 0;
5728}
5729
b67e1913
BA		 5730/**
5731 * e1000e_hwtstamp_ioctl - control hardware time stamping
5732 * @netdev: network interface device structure
5733 * @ifreq: interface request
5734 *
5735 * Outgoing time stamping can be enabled and disabled. Play nice and
5736 * disable it when requested, although it shouldn't cause any overhead
5737 * when no packet needs it. At most one packet in the queue may be
5738 * marked for time stamping, otherwise it would be impossible to tell
5739 * for sure to which packet the hardware time stamp belongs.
5740 *
5741 * Incoming time stamping has to be configured via the hardware filters.
5742 * Not all combinations are supported, in particular event type has to be
5743 * specified. Matching the kind of event packet is not supported, with the
5744 * exception of "all V2 events regardless of level 2 or 4".
5745 **/
5746static int e1000e_hwtstamp_ioctl(struct net_device *netdev, struct ifreq *ifr)
5747{
5748 struct e1000_adapter *adapter = netdev_priv(netdev);
5749 struct hwtstamp_config config;
5750 int ret_val;
5751
5752 if (copy_from_user(&config, ifr->ifr_data, sizeof(config)))
5753 return -EFAULT;
5754
5755 adapter->hwtstamp_config = config;
5756
5757 ret_val = e1000e_config_hwtstamp(adapter);
5758 if (ret_val)
5759 return ret_val;
5760
5761 config = adapter->hwtstamp_config;
5762
d89777bf
BA		 5763 switch (config.rx_filter) {
5764 case HWTSTAMP_FILTER_PTP_V2_L4_SYNC:
5765 case HWTSTAMP_FILTER_PTP_V2_L2_SYNC:
5766 case HWTSTAMP_FILTER_PTP_V2_SYNC:
5767 case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ:
5768 case HWTSTAMP_FILTER_PTP_V2_L2_DELAY_REQ:
5769 case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ:
5770 /* With V2 type filters which specify a Sync or Delay Request,
5771 * Path Delay Request/Response messages are also time stamped
5772 * by hardware so notify the caller the requested packets plus
5773 * some others are time stamped.
5774 */
5775 config.rx_filter = HWTSTAMP_FILTER_SOME;
5776 break;
5777 default:
5778 break;
5779 }
5780
b67e1913
BA		 5781 return copy_to_user(ifr->ifr_data, &config,
5782 sizeof(config)) ? -EFAULT : 0;
5783}
5784
bc7f75fa
AK		 5785static int e1000_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd)
5786{
5787 switch (cmd) {
5788 case SIOCGMIIPHY:
5789 case SIOCGMIIREG:
5790 case SIOCSMIIREG:
5791 return e1000_mii_ioctl(netdev, ifr, cmd);
b67e1913
BA		 5792 case SIOCSHWTSTAMP:
5793 return e1000e_hwtstamp_ioctl(netdev, ifr);
bc7f75fa
AK		 5794 default:
5795 return -EOPNOTSUPP;
5796 }
5797}
5798
a4f58f54
BA		 5799static int e1000_init_phy_wakeup(struct e1000_adapter *adapter, u32 wufc)
5800{
5801 struct e1000_hw *hw = &adapter->hw;
5802 u32 i, mac_reg;
2b6b168d 5803 u16 phy_reg, wuc_enable;
70806a7f 5804 int retval;
a4f58f54
BA		 5805
5806 /* copy MAC RARs to PHY RARs */
d3738bb8 5807 e1000_copy_rx_addrs_to_phy_ich8lan(hw);
a4f58f54 5808
2b6b168d
BA		 5809 retval = hw->phy.ops.acquire(hw);
5810 if (retval) {
5811 e_err("Could not acquire PHY\n");
5812 return retval;
5813 }
5814
5815 /* Enable access to wakeup registers on and set page to BM_WUC_PAGE */
5816 retval = e1000_enable_phy_wakeup_reg_access_bm(hw, &wuc_enable);
5817 if (retval)
75ce1532 5818 goto release;
2b6b168d
BA		 5819
5820 /* copy MAC MTA to PHY MTA - only needed for pchlan */
a4f58f54
BA		 5821 for (i = 0; i < adapter->hw.mac.mta_reg_count; i++) {
5822 mac_reg = E1000_READ_REG_ARRAY(hw, E1000_MTA, i);
2b6b168d
BA		 5823 hw->phy.ops.write_reg_page(hw, BM_MTA(i),
5824 (u16)(mac_reg & 0xFFFF));
5825 hw->phy.ops.write_reg_page(hw, BM_MTA(i) + 1,
5826 (u16)((mac_reg >> 16) & 0xFFFF));
a4f58f54
BA		 5827 }
5828
5829 /* configure PHY Rx Control register */
2b6b168d 5830 hw->phy.ops.read_reg_page(&adapter->hw, BM_RCTL, &phy_reg);
a4f58f54
BA		 5831 mac_reg = er32(RCTL);
5832 if (mac_reg & E1000_RCTL_UPE)
5833 phy_reg |= BM_RCTL_UPE;
5834 if (mac_reg & E1000_RCTL_MPE)
5835 phy_reg |= BM_RCTL_MPE;
5836 phy_reg &= ~(BM_RCTL_MO_MASK);
5837 if (mac_reg & E1000_RCTL_MO_3)
5838 phy_reg |= (((mac_reg & E1000_RCTL_MO_3) >> E1000_RCTL_MO_SHIFT)
17e813ec 5839 << BM_RCTL_MO_SHIFT);
a4f58f54
BA		 5840 if (mac_reg & E1000_RCTL_BAM)
5841 phy_reg |= BM_RCTL_BAM;
5842 if (mac_reg & E1000_RCTL_PMCF)
5843 phy_reg |= BM_RCTL_PMCF;
5844 mac_reg = er32(CTRL);
5845 if (mac_reg & E1000_CTRL_RFCE)
5846 phy_reg |= BM_RCTL_RFCE;
2b6b168d 5847 hw->phy.ops.write_reg_page(&adapter->hw, BM_RCTL, phy_reg);
a4f58f54
BA		 5848
5849 /* enable PHY wakeup in MAC register */
5850 ew32(WUFC, wufc);
5851 ew32(WUC, E1000_WUC_PHY_WAKE | E1000_WUC_PME_EN);
5852
5853 /* configure and enable PHY wakeup in PHY registers */
2b6b168d
BA		 5854 hw->phy.ops.write_reg_page(&adapter->hw, BM_WUFC, wufc);
5855 hw->phy.ops.write_reg_page(&adapter->hw, BM_WUC, E1000_WUC_PME_EN);
a4f58f54
BA		 5856
5857 /* activate PHY wakeup */
2b6b168d
BA		 5858 wuc_enable |= BM_WUC_ENABLE_BIT | BM_WUC_HOST_WU_BIT;
5859 retval = e1000_disable_phy_wakeup_reg_access_bm(hw, &wuc_enable);
a4f58f54
BA		 5860 if (retval)
5861 e_err("Could not set PHY Host Wakeup bit\n");
75ce1532 5862release:
94d8186a 5863 hw->phy.ops.release(hw);
a4f58f54
BA		 5864
5865 return retval;
5866}
5867
23606cf5
RW		 5868static int __e1000_shutdown(struct pci_dev *pdev, bool *enable_wake,
5869 bool runtime)
bc7f75fa
AK		 5870{
5871 struct net_device *netdev = pci_get_drvdata(pdev);
5872 struct e1000_adapter *adapter = netdev_priv(netdev);
5873 struct e1000_hw *hw = &adapter->hw;
5874 u32 ctrl, ctrl_ext, rctl, status;
23606cf5
RW		 5875 /* Runtime suspend should only enable wakeup for link changes */
5876 u32 wufc = runtime ? E1000_WUFC_LNKC : adapter->wol;
bc7f75fa
AK		 5877 int retval = 0;
5878
5879 netif_device_detach(netdev);
5880
5881 if (netif_running(netdev)) {
bb9e44d0
BA		 5882 int count = E1000_CHECK_RESET_COUNT;
5883
5884 while (test_bit(__E1000_RESETTING, &adapter->state) && count--)
5885 usleep_range(10000, 20000);
5886
bc7f75fa
AK		 5887 WARN_ON(test_bit(__E1000_RESETTING, &adapter->state));
5888 e1000e_down(adapter);
5889 e1000_free_irq(adapter);
5890 }
4662e82b 5891 e1000e_reset_interrupt_capability(adapter);
bc7f75fa
AK		 5892
5893 retval = pci_save_state(pdev);
5894 if (retval)
5895 return retval;
5896
5897 status = er32(STATUS);
5898 if (status & E1000_STATUS_LU)
5899 wufc &= ~E1000_WUFC_LNKC;
5900
5901 if (wufc) {
5902 e1000_setup_rctl(adapter);
ef9b965a 5903 e1000e_set_rx_mode(netdev);
bc7f75fa
AK		 5904
5905 /* turn on all-multi mode if wake on multicast is enabled */
5906 if (wufc & E1000_WUFC_MC) {
5907 rctl = er32(RCTL);
5908 rctl |= E1000_RCTL_MPE;
5909 ew32(RCTL, rctl);
5910 }
5911
5912 ctrl = er32(CTRL);
a4f58f54
BA		 5913 ctrl |= E1000_CTRL_ADVD3WUC;
5914 if (!(adapter->flags2 & FLAG2_HAS_PHY_WAKEUP))
5915 ctrl |= E1000_CTRL_EN_PHY_PWR_MGMT;
bc7f75fa
AK		 5916 ew32(CTRL, ctrl);
5917
318a94d6
JK		 5918 if (adapter->hw.phy.media_type == e1000_media_type_fiber ||
5919 adapter->hw.phy.media_type ==
5920 e1000_media_type_internal_serdes) {
bc7f75fa
AK		 5921 /* keep the laser running in D3 */
5922 ctrl_ext = er32(CTRL_EXT);
93a23f48 5923 ctrl_ext |= E1000_CTRL_EXT_SDP3_DATA;
bc7f75fa
AK		 5924 ew32(CTRL_EXT, ctrl_ext);
5925 }
5926
97ac8cae 5927 if (adapter->flags & FLAG_IS_ICH)
99730e4c 5928 e1000_suspend_workarounds_ich8lan(&adapter->hw);
97ac8cae 5929
bc7f75fa
AK		 5930 /* Allow time for pending master requests to run */
5931 e1000e_disable_pcie_master(&adapter->hw);
5932
82776a4b 5933 if (adapter->flags2 & FLAG2_HAS_PHY_WAKEUP) {
a4f58f54
BA		 5934 /* enable wakeup by the PHY */
5935 retval = e1000_init_phy_wakeup(adapter, wufc);
5936 if (retval)
5937 return retval;
5938 } else {
5939 /* enable wakeup by the MAC */
5940 ew32(WUFC, wufc);
5941 ew32(WUC, E1000_WUC_PME_EN);
5942 }
bc7f75fa
AK		 5943 } else {
5944 ew32(WUC, 0);
5945 ew32(WUFC, 0);
bc7f75fa
AK		 5946 }
5947
4f9de721
RW		 5948 *enable_wake = !!wufc;
5949
bc7f75fa 5950 /* make sure adapter isn't asleep if manageability is enabled */
82776a4b
BA		 5951 if ((adapter->flags & FLAG_MNG_PT_ENABLED) ||
5952 (hw->mac.ops.check_mng_mode(hw)))
4f9de721 5953 *enable_wake = true;
bc7f75fa
AK		 5954
5955 if (adapter->hw.phy.type == e1000_phy_igp_3)
5956 e1000e_igp3_phy_powerdown_workaround_ich8lan(&adapter->hw);
5957
e921eb1a 5958 /* Release control of h/w to f/w. If f/w is AMT enabled, this
ad68076e
BA		 5959 * would have already happened in close and is redundant.
5960 */
31dbe5b4 5961 e1000e_release_hw_control(adapter);
bc7f75fa
AK		 5962
5963 pci_disable_device(pdev);
5964
4f9de721
RW		 5965 return 0;
5966}
5967
5968static void e1000_power_off(struct pci_dev *pdev, bool sleep, bool wake)
5969{
5970 if (sleep && wake) {
5971 pci_prepare_to_sleep(pdev);
5972 return;
5973 }
5974
5975 pci_wake_from_d3(pdev, wake);
5976 pci_set_power_state(pdev, PCI_D3hot);
5977}
5978
f0ff4398 5979static void e1000_complete_shutdown(struct pci_dev *pdev, bool sleep, bool wake)
4f9de721
RW		 5980{
5981 struct net_device *netdev = pci_get_drvdata(pdev);
5982 struct e1000_adapter *adapter = netdev_priv(netdev);
5983
e921eb1a 5984 /* The pci-e switch on some quad port adapters will report a
005cbdfc
AD		 5985 * correctable error when the MAC transitions from D0 to D3. To
5986 * prevent this we need to mask off the correctable errors on the
5987 * downstream port of the pci-e switch.
5988 */
5989 if (adapter->flags & FLAG_IS_QUAD_PORT) {
5990 struct pci_dev *us_dev = pdev->bus->self;
005cbdfc
AD		 5991 u16 devctl;
5992
f8c0fcac
JL		 5993 pcie_capability_read_word(us_dev, PCI_EXP_DEVCTL, &devctl);
5994 pcie_capability_write_word(us_dev, PCI_EXP_DEVCTL,
5995 (devctl & ~PCI_EXP_DEVCTL_CERE));
005cbdfc 5996
4f9de721 5997 e1000_power_off(pdev, sleep, wake);
005cbdfc 5998
f8c0fcac 5999 pcie_capability_write_word(us_dev, PCI_EXP_DEVCTL, devctl);
005cbdfc 6000 } else {
4f9de721 6001 e1000_power_off(pdev, sleep, wake);
005cbdfc 6002 }
bc7f75fa
AK		 6003}
6004
6f461f6c
BA		 6005#ifdef CONFIG_PCIEASPM
6006static void __e1000e_disable_aspm(struct pci_dev *pdev, u16 state)
6007{
9f728f53 6008 pci_disable_link_state_locked(pdev, state);
6f461f6c
BA		 6009}
6010#else
6011static void __e1000e_disable_aspm(struct pci_dev *pdev, u16 state)
1eae4eb2 6012{
ffe0b2ff
BH		 6013 u16 aspm_ctl = 0;
6014
6015 if (state & PCIE_LINK_STATE_L0S)
6016 aspm_ctl |= PCI_EXP_LNKCTL_ASPM_L0S;
6017 if (state & PCIE_LINK_STATE_L1)
6018 aspm_ctl |= PCI_EXP_LNKCTL_ASPM_L1;
6019
e921eb1a 6020 /* Both device and parent should have the same ASPM setting.
6f461f6c 6021 * Disable ASPM in downstream component first and then upstream.
1eae4eb2 6022 */
ffe0b2ff 6023 pcie_capability_clear_word(pdev, PCI_EXP_LNKCTL, aspm_ctl);
0c75ba22 6024
f8c0fcac
JL		 6025 if (pdev->bus->self)
6026 pcie_capability_clear_word(pdev->bus->self, PCI_EXP_LNKCTL,
ffe0b2ff 6027 aspm_ctl);
6f461f6c
BA		 6028}
6029#endif
78cd29d5 6030static void e1000e_disable_aspm(struct pci_dev *pdev, u16 state)
6f461f6c
BA		 6031{
6032 dev_info(&pdev->dev, "Disabling ASPM %s %s\n",
6033 (state & PCIE_LINK_STATE_L0S) ? "L0s" : "",
6034 (state & PCIE_LINK_STATE_L1) ? "L1" : "");
6035
6036 __e1000e_disable_aspm(pdev, state);
1eae4eb2
AK		 6037}
6038
aa338601 6039#ifdef CONFIG_PM
23606cf5 6040static bool e1000e_pm_ready(struct e1000_adapter *adapter)
4f9de721 6041{
23606cf5 6042 return !!adapter->tx_ring->buffer_info;
4f9de721
RW		 6043}
6044
23606cf5 6045static int __e1000_resume(struct pci_dev *pdev)
bc7f75fa
AK		 6046{
6047 struct net_device *netdev = pci_get_drvdata(pdev);
6048 struct e1000_adapter *adapter = netdev_priv(netdev);
6049 struct e1000_hw *hw = &adapter->hw;
78cd29d5 6050 u16 aspm_disable_flag = 0;
bc7f75fa
AK		 6051 u32 err;
6052
78cd29d5
BA		 6053 if (adapter->flags2 & FLAG2_DISABLE_ASPM_L0S)
6054 aspm_disable_flag = PCIE_LINK_STATE_L0S;
6055 if (adapter->flags2 & FLAG2_DISABLE_ASPM_L1)
6056 aspm_disable_flag |= PCIE_LINK_STATE_L1;
6057 if (aspm_disable_flag)
6058 e1000e_disable_aspm(pdev, aspm_disable_flag);
6059
bc7f75fa
AK		 6060 pci_set_power_state(pdev, PCI_D0);
6061 pci_restore_state(pdev);
28b8f04a 6062 pci_save_state(pdev);
6e4f6f6b 6063
4662e82b 6064 e1000e_set_interrupt_capability(adapter);
bc7f75fa
AK		 6065 if (netif_running(netdev)) {
6066 err = e1000_request_irq(adapter);
6067 if (err)
6068 return err;
6069 }
6070
2fbe4526 6071 if (hw->mac.type >= e1000_pch2lan)
99730e4c
BA		 6072 e1000_resume_workarounds_pchlan(&adapter->hw);
6073
bc7f75fa 6074 e1000e_power_up_phy(adapter);
a4f58f54
BA		 6075
6076 /* report the system wakeup cause from S3/S4 */
6077 if (adapter->flags2 & FLAG2_HAS_PHY_WAKEUP) {
6078 u16 phy_data;
6079
6080 e1e_rphy(&adapter->hw, BM_WUS, &phy_data);
6081 if (phy_data) {
6082 e_info("PHY Wakeup cause - %s\n",
17e813ec
BA		 6083 phy_data & E1000_WUS_EX ? "Unicast Packet" :
6084 phy_data & E1000_WUS_MC ? "Multicast Packet" :
6085 phy_data & E1000_WUS_BC ? "Broadcast Packet" :
6086 phy_data & E1000_WUS_MAG ? "Magic Packet" :
6087 phy_data & E1000_WUS_LNKC ?
6088 "Link Status Change" : "other");
a4f58f54
BA		 6089 }
6090 e1e_wphy(&adapter->hw, BM_WUS, ~0);
6091 } else {
6092 u32 wus = er32(WUS);
6093 if (wus) {
6094 e_info("MAC Wakeup cause - %s\n",
17e813ec
BA		 6095 wus & E1000_WUS_EX ? "Unicast Packet" :
6096 wus & E1000_WUS_MC ? "Multicast Packet" :
6097 wus & E1000_WUS_BC ? "Broadcast Packet" :
6098 wus & E1000_WUS_MAG ? "Magic Packet" :
6099 wus & E1000_WUS_LNKC ? "Link Status Change" :
6100 "other");
a4f58f54
BA		 6101 }
6102 ew32(WUS, ~0);
6103 }
6104
bc7f75fa 6105 e1000e_reset(adapter);
bc7f75fa 6106
cd791618 6107 e1000_init_manageability_pt(adapter);
bc7f75fa
AK		 6108
6109 if (netif_running(netdev))
6110 e1000e_up(adapter);
6111
6112 netif_device_attach(netdev);
6113
e921eb1a 6114 /* If the controller has AMT, do not set DRV_LOAD until the interface
bc7f75fa 6115 * is up. For all other cases, let the f/w know that the h/w is now
ad68076e
BA		 6116 * under the control of the driver.
6117 */
c43bc57e 6118 if (!(adapter->flags & FLAG_HAS_AMT))
31dbe5b4 6119 e1000e_get_hw_control(adapter);
bc7f75fa
AK		 6120
6121 return 0;
6122}
23606cf5 6123
a0340162
RW		 6124#ifdef CONFIG_PM_SLEEP
6125static int e1000_suspend(struct device *dev)
6126{
6127 struct pci_dev *pdev = to_pci_dev(dev);
6128 int retval;
6129 bool wake;
6130
6131 retval = __e1000_shutdown(pdev, &wake, false);
6132 if (!retval)
6133 e1000_complete_shutdown(pdev, true, wake);
6134
6135 return retval;
6136}
6137
23606cf5
RW		 6138static int e1000_resume(struct device *dev)
6139{
6140 struct pci_dev *pdev = to_pci_dev(dev);
6141 struct net_device *netdev = pci_get_drvdata(pdev);
6142 struct e1000_adapter *adapter = netdev_priv(netdev);
6143
6144 if (e1000e_pm_ready(adapter))
6145 adapter->idle_check = true;
6146
6147 return __e1000_resume(pdev);
6148}
a0340162
RW		 6149#endif /* CONFIG_PM_SLEEP */
6150
6151#ifdef CONFIG_PM_RUNTIME
6152static int e1000_runtime_suspend(struct device *dev)
6153{
6154 struct pci_dev *pdev = to_pci_dev(dev);
6155 struct net_device *netdev = pci_get_drvdata(pdev);
6156 struct e1000_adapter *adapter = netdev_priv(netdev);
6157
6158 if (e1000e_pm_ready(adapter)) {
6159 bool wake;
6160
6161 __e1000_shutdown(pdev, &wake, true);
6162 }
6163
6164 return 0;
6165}
6166
6167static int e1000_idle(struct device *dev)
6168{
6169 struct pci_dev *pdev = to_pci_dev(dev);
6170 struct net_device *netdev = pci_get_drvdata(pdev);
6171 struct e1000_adapter *adapter = netdev_priv(netdev);
6172
6173 if (!e1000e_pm_ready(adapter))
6174 return 0;
6175
6176 if (adapter->idle_check) {
6177 adapter->idle_check = false;
6178 if (!e1000e_has_link(adapter))
6179 pm_schedule_suspend(dev, MSEC_PER_SEC);
6180 }
6181
6182 return -EBUSY;
6183}
23606cf5
RW		 6184
6185static int e1000_runtime_resume(struct device *dev)
6186{
6187 struct pci_dev *pdev = to_pci_dev(dev);
6188 struct net_device *netdev = pci_get_drvdata(pdev);
6189 struct e1000_adapter *adapter = netdev_priv(netdev);
6190
6191 if (!e1000e_pm_ready(adapter))
6192 return 0;
6193
6194 adapter->idle_check = !dev->power.runtime_auto;
6195 return __e1000_resume(pdev);
6196}
a0340162 6197#endif /* CONFIG_PM_RUNTIME */
aa338601 6198#endif /* CONFIG_PM */
bc7f75fa
AK		 6199
6200static void e1000_shutdown(struct pci_dev *pdev)
6201{
4f9de721
RW		 6202 bool wake = false;
6203
23606cf5 6204 __e1000_shutdown(pdev, &wake, false);
4f9de721
RW		 6205
6206 if (system_state == SYSTEM_POWER_OFF)
6207 e1000_complete_shutdown(pdev, false, wake);
bc7f75fa
AK		 6208}
6209
6210#ifdef CONFIG_NET_POLL_CONTROLLER
147b2c8c 6211
8bb62869 6212static irqreturn_t e1000_intr_msix(int __always_unused irq, void *data)
147b2c8c
DD		 6213{
6214 struct net_device *netdev = data;
6215 struct e1000_adapter *adapter = netdev_priv(netdev);
147b2c8c
DD		 6216
6217 if (adapter->msix_entries) {
90da0669
BA		 6218 int vector, msix_irq;
6219
147b2c8c
DD		 6220 vector = 0;
6221 msix_irq = adapter->msix_entries[vector].vector;
6222 disable_irq(msix_irq);
6223 e1000_intr_msix_rx(msix_irq, netdev);
6224 enable_irq(msix_irq);
6225
6226 vector++;
6227 msix_irq = adapter->msix_entries[vector].vector;
6228 disable_irq(msix_irq);
6229 e1000_intr_msix_tx(msix_irq, netdev);
6230 enable_irq(msix_irq);
6231
6232 vector++;
6233 msix_irq = adapter->msix_entries[vector].vector;
6234 disable_irq(msix_irq);
6235 e1000_msix_other(msix_irq, netdev);
6236 enable_irq(msix_irq);
6237 }
6238
6239 return IRQ_HANDLED;
6240}
6241
e921eb1a
BA		 6242/**
6243 * e1000_netpoll
6244 * @netdev: network interface device structure
6245 *
bc7f75fa
AK		 6246 * Polling 'interrupt' - used by things like netconsole to send skbs
6247 * without having to re-enable interrupts. It's not called while
6248 * the interrupt routine is executing.
6249 */
6250static void e1000_netpoll(struct net_device *netdev)
6251{
6252 struct e1000_adapter *adapter = netdev_priv(netdev);
6253
147b2c8c
DD		 6254 switch (adapter->int_mode) {
6255 case E1000E_INT_MODE_MSIX:
6256 e1000_intr_msix(adapter->pdev->irq, netdev);
6257 break;
6258 case E1000E_INT_MODE_MSI:
6259 disable_irq(adapter->pdev->irq);
6260 e1000_intr_msi(adapter->pdev->irq, netdev);
6261 enable_irq(adapter->pdev->irq);
6262 break;
6263 default: /* E1000E_INT_MODE_LEGACY */
6264 disable_irq(adapter->pdev->irq);
6265 e1000_intr(adapter->pdev->irq, netdev);
6266 enable_irq(adapter->pdev->irq);
6267 break;
6268 }
bc7f75fa
AK		 6269}
6270#endif
6271
6272/**
6273 * e1000_io_error_detected - called when PCI error is detected
6274 * @pdev: Pointer to PCI device
6275 * @state: The current pci connection state
6276 *
6277 * This function is called after a PCI bus error affecting
6278 * this device has been detected.
6279 */
6280static pci_ers_result_t e1000_io_error_detected(struct pci_dev *pdev,
6281 pci_channel_state_t state)
6282{
6283 struct net_device *netdev = pci_get_drvdata(pdev);
6284 struct e1000_adapter *adapter = netdev_priv(netdev);
6285
6286 netif_device_detach(netdev);
6287
c93b5a76
MM		 6288 if (state == pci_channel_io_perm_failure)
6289 return PCI_ERS_RESULT_DISCONNECT;
6290
bc7f75fa
AK		 6291 if (netif_running(netdev))
6292 e1000e_down(adapter);
6293 pci_disable_device(pdev);
6294
6295 /* Request a slot slot reset. */
6296 return PCI_ERS_RESULT_NEED_RESET;
6297}
6298
6299/**
6300 * e1000_io_slot_reset - called after the pci bus has been reset.
6301 * @pdev: Pointer to PCI device
6302 *
6303 * Restart the card from scratch, as if from a cold-boot. Implementation
6304 * resembles the first-half of the e1000_resume routine.
6305 */
6306static pci_ers_result_t e1000_io_slot_reset(struct pci_dev *pdev)
6307{
6308 struct net_device *netdev = pci_get_drvdata(pdev);
6309 struct e1000_adapter *adapter = netdev_priv(netdev);
6310 struct e1000_hw *hw = &adapter->hw;
78cd29d5 6311 u16 aspm_disable_flag = 0;
6e4f6f6b 6312 int err;
111b9dc5 6313 pci_ers_result_t result;
bc7f75fa 6314
78cd29d5
BA		 6315 if (adapter->flags2 & FLAG2_DISABLE_ASPM_L0S)
6316 aspm_disable_flag = PCIE_LINK_STATE_L0S;
6f461f6c 6317 if (adapter->flags2 & FLAG2_DISABLE_ASPM_L1)
78cd29d5
BA		 6318 aspm_disable_flag |= PCIE_LINK_STATE_L1;
6319 if (aspm_disable_flag)
6320 e1000e_disable_aspm(pdev, aspm_disable_flag);
6321
f0f422e5 6322 err = pci_enable_device_mem(pdev);
6e4f6f6b 6323 if (err) {
bc7f75fa
AK		 6324 dev_err(&pdev->dev,
6325 "Cannot re-enable PCI device after reset.\n");
111b9dc5
JB		 6326 result = PCI_ERS_RESULT_DISCONNECT;
6327 } else {
6328 pci_set_master(pdev);
23606cf5 6329 pdev->state_saved = true;
111b9dc5 6330 pci_restore_state(pdev);
bc7f75fa 6331
111b9dc5
JB		 6332 pci_enable_wake(pdev, PCI_D3hot, 0);
6333 pci_enable_wake(pdev, PCI_D3cold, 0);
bc7f75fa 6334
111b9dc5
JB		 6335 e1000e_reset(adapter);
6336 ew32(WUS, ~0);
6337 result = PCI_ERS_RESULT_RECOVERED;
6338 }
bc7f75fa 6339
111b9dc5
JB		 6340 pci_cleanup_aer_uncorrect_error_status(pdev);
6341
6342 return result;
bc7f75fa
AK		 6343}
6344
6345/**
6346 * e1000_io_resume - called when traffic can start flowing again.
6347 * @pdev: Pointer to PCI device
6348 *
6349 * This callback is called when the error recovery driver tells us that
6350 * its OK to resume normal operation. Implementation resembles the
6351 * second-half of the e1000_resume routine.
6352 */
6353static void e1000_io_resume(struct pci_dev *pdev)
6354{
6355 struct net_device *netdev = pci_get_drvdata(pdev);
6356 struct e1000_adapter *adapter = netdev_priv(netdev);
6357
cd791618 6358 e1000_init_manageability_pt(adapter);
bc7f75fa
AK		 6359
6360 if (netif_running(netdev)) {
6361 if (e1000e_up(adapter)) {
6362 dev_err(&pdev->dev,
6363 "can't bring device back up after reset\n");
6364 return;
6365 }
6366 }
6367
6368 netif_device_attach(netdev);
6369
e921eb1a 6370 /* If the controller has AMT, do not set DRV_LOAD until the interface
bc7f75fa 6371 * is up. For all other cases, let the f/w know that the h/w is now
ad68076e
BA		 6372 * under the control of the driver.
6373 */
c43bc57e 6374 if (!(adapter->flags & FLAG_HAS_AMT))
31dbe5b4 6375 e1000e_get_hw_control(adapter);
bc7f75fa
AK		 6376}
6377
6378static void e1000_print_device_info(struct e1000_adapter *adapter)
6379{
6380 struct e1000_hw *hw = &adapter->hw;
6381 struct net_device *netdev = adapter->netdev;
073287c0
BA		 6382 u32 ret_val;
6383 u8 pba_str[E1000_PBANUM_LENGTH];
bc7f75fa
AK		 6384
6385 /* print bus type/speed/width info */
a5cc7642 6386 e_info("(PCI Express:2.5GT/s:%s) %pM\n",
44defeb3
JK		 6387 /* bus width */
6388 ((hw->bus.width == e1000_bus_width_pcie_x4) ? "Width x4" :
f0ff4398 6389 "Width x1"),
44defeb3 6390 /* MAC address */
7c510e4b 6391 netdev->dev_addr);
44defeb3
JK		 6392 e_info("Intel(R) PRO/%s Network Connection\n",
6393 (hw->phy.type == e1000_phy_ife) ? "10/100" : "1000");
073287c0
BA		 6394 ret_val = e1000_read_pba_string_generic(hw, pba_str,
6395 E1000_PBANUM_LENGTH);
6396 if (ret_val)
f2315bf1 6397 strlcpy((char *)pba_str, "Unknown", sizeof(pba_str));
073287c0
BA		 6398 e_info("MAC: %d, PHY: %d, PBA No: %s\n",
6399 hw->mac.type, hw->phy.type, pba_str);
bc7f75fa
AK		 6400}
6401
10aa4c04
AK		 6402static void e1000_eeprom_checks(struct e1000_adapter *adapter)
6403{
6404 struct e1000_hw *hw = &adapter->hw;
6405 int ret_val;
6406 u16 buf = 0;
6407
6408 if (hw->mac.type != e1000_82573)
6409 return;
6410
6411 ret_val = e1000_read_nvm(hw, NVM_INIT_CONTROL2_REG, 1, &buf);
e885d762
BA		 6412 le16_to_cpus(&buf);
6413 if (!ret_val && (!(buf & (1 << 0)))) {
10aa4c04 6414 /* Deep Smart Power Down (DSPD) */
6c2a9efa
FP		 6415 dev_warn(&adapter->pdev->dev,
6416 "Warning: detected DSPD enabled in EEPROM\n");
10aa4c04 6417 }
10aa4c04
AK		 6418}
6419
c8f44aff 6420static int e1000_set_features(struct net_device *netdev,
70495a50 6421 netdev_features_t features)
dc221294
BA		 6422{
6423 struct e1000_adapter *adapter = netdev_priv(netdev);
c8f44aff 6424 netdev_features_t changed = features ^ netdev->features;
dc221294
BA		 6425
6426 if (changed & (NETIF_F_TSO | NETIF_F_TSO6))
6427 adapter->flags |= FLAG_TSO_FORCE;
6428
6429 if (!(changed & (NETIF_F_HW_VLAN_RX | NETIF_F_HW_VLAN_TX |
cf955e6c
BG		 6430 NETIF_F_RXCSUM | NETIF_F_RXHASH | NETIF_F_RXFCS |
6431 NETIF_F_RXALL)))
dc221294
BA		 6432 return 0;
6433
0184039a
BG		 6434 if (changed & NETIF_F_RXFCS) {
6435 if (features & NETIF_F_RXFCS) {
6436 adapter->flags2 &= ~FLAG2_CRC_STRIPPING;
6437 } else {
6438 /* We need to take it back to defaults, which might mean
6439 * stripping is still disabled at the adapter level.
6440 */
6441 if (adapter->flags2 & FLAG2_DFLT_CRC_STRIPPING)
6442 adapter->flags2 |= FLAG2_CRC_STRIPPING;
6443 else
6444 adapter->flags2 &= ~FLAG2_CRC_STRIPPING;
6445 }
6446 }
6447
70495a50
BA		 6448 netdev->features = features;
6449
dc221294
BA		 6450 if (netif_running(netdev))
6451 e1000e_reinit_locked(adapter);
6452 else
6453 e1000e_reset(adapter);
6454
6455 return 0;
6456}
6457
651c2466
SH		 6458static const struct net_device_ops e1000e_netdev_ops = {
6459 .ndo_open = e1000_open,
6460 .ndo_stop = e1000_close,
00829823 6461 .ndo_start_xmit = e1000_xmit_frame,
67fd4fcb 6462 .ndo_get_stats64 = e1000e_get_stats64,
ef9b965a 6463 .ndo_set_rx_mode = e1000e_set_rx_mode,
651c2466
SH		 6464 .ndo_set_mac_address = e1000_set_mac,
6465 .ndo_change_mtu = e1000_change_mtu,
6466 .ndo_do_ioctl = e1000_ioctl,
6467 .ndo_tx_timeout = e1000_tx_timeout,
6468 .ndo_validate_addr = eth_validate_addr,
6469
651c2466
SH		 6470 .ndo_vlan_rx_add_vid = e1000_vlan_rx_add_vid,
6471 .ndo_vlan_rx_kill_vid = e1000_vlan_rx_kill_vid,
6472#ifdef CONFIG_NET_POLL_CONTROLLER
6473 .ndo_poll_controller = e1000_netpoll,
6474#endif
dc221294 6475 .ndo_set_features = e1000_set_features,
651c2466
SH		 6476};
6477
bc7f75fa
AK		 6478/**
6479 * e1000_probe - Device Initialization Routine
6480 * @pdev: PCI device information struct
6481 * @ent: entry in e1000_pci_tbl
6482 *
6483 * Returns 0 on success, negative on failure
6484 *
6485 * e1000_probe initializes an adapter identified by a pci_dev structure.
6486 * The OS initialization, configuring of the adapter private structure,
6487 * and a hardware reset occur.
6488 **/
1dd06ae8 6489static int e1000_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
bc7f75fa
AK		 6490{
6491 struct net_device *netdev;
6492 struct e1000_adapter *adapter;
6493 struct e1000_hw *hw;
6494 const struct e1000_info *ei = e1000_info_tbl[ent->driver_data];
f47e81fc
BB		 6495 resource_size_t mmio_start, mmio_len;
6496 resource_size_t flash_start, flash_len;
bc7f75fa 6497 static int cards_found;
78cd29d5 6498 u16 aspm_disable_flag = 0;
17e813ec 6499 int bars, i, err, pci_using_dac;
bc7f75fa
AK		 6500 u16 eeprom_data = 0;
6501 u16 eeprom_apme_mask = E1000_EEPROM_APME;
6502
78cd29d5
BA		 6503 if (ei->flags2 & FLAG2_DISABLE_ASPM_L0S)
6504 aspm_disable_flag = PCIE_LINK_STATE_L0S;
6f461f6c 6505 if (ei->flags2 & FLAG2_DISABLE_ASPM_L1)
78cd29d5
BA		 6506 aspm_disable_flag |= PCIE_LINK_STATE_L1;
6507 if (aspm_disable_flag)
6508 e1000e_disable_aspm(pdev, aspm_disable_flag);
6e4f6f6b 6509
f0f422e5 6510 err = pci_enable_device_mem(pdev);
bc7f75fa
AK		 6511 if (err)
6512 return err;
6513
6514 pci_using_dac = 0;
0be3f55f 6515 err = dma_set_mask(&pdev->dev, DMA_BIT_MASK(64));
bc7f75fa 6516 if (!err) {
0be3f55f 6517 err = dma_set_coherent_mask(&pdev->dev, DMA_BIT_MASK(64));
bc7f75fa
AK		 6518 if (!err)
6519 pci_using_dac = 1;
6520 } else {
0be3f55f 6521 err = dma_set_mask(&pdev->dev, DMA_BIT_MASK(32));
bc7f75fa 6522 if (err) {
0be3f55f
NN		 6523 err = dma_set_coherent_mask(&pdev->dev,
6524 DMA_BIT_MASK(32));
bc7f75fa 6525 if (err) {
f0ff4398
BA		 6526 dev_err(&pdev->dev,
6527 "No usable DMA configuration, aborting\n");
bc7f75fa
AK		 6528 goto err_dma;
6529 }
6530 }
6531 }
6532
17e813ec
BA		 6533 bars = pci_select_bars(pdev, IORESOURCE_MEM);
6534 err = pci_request_selected_regions_exclusive(pdev, bars,
6535 e1000e_driver_name);
bc7f75fa
AK		 6536 if (err)
6537 goto err_pci_reg;
6538
68eac460 6539 /* AER (Advanced Error Reporting) hooks */
19d5afd4 6540 pci_enable_pcie_error_reporting(pdev);
68eac460 6541
bc7f75fa 6542 pci_set_master(pdev);
438b365a
BA		 6543 /* PCI config space info */
6544 err = pci_save_state(pdev);
6545 if (err)
6546 goto err_alloc_etherdev;
bc7f75fa
AK		 6547
6548 err = -ENOMEM;
6549 netdev = alloc_etherdev(sizeof(struct e1000_adapter));
6550 if (!netdev)
6551 goto err_alloc_etherdev;
6552
bc7f75fa
AK		 6553 SET_NETDEV_DEV(netdev, &pdev->dev);
6554
f85e4dfa
TH		 6555 netdev->irq = pdev->irq;
6556
bc7f75fa
AK		 6557 pci_set_drvdata(pdev, netdev);
6558 adapter = netdev_priv(netdev);
6559 hw = &adapter->hw;
6560 adapter->netdev = netdev;
6561 adapter->pdev = pdev;
6562 adapter->ei = ei;
6563 adapter->pba = ei->pba;
6564 adapter->flags = ei->flags;
eb7c3adb 6565 adapter->flags2 = ei->flags2;
bc7f75fa
AK		 6566 adapter->hw.adapter = adapter;
6567 adapter->hw.mac.type = ei->mac;
2adc55c9 6568 adapter->max_hw_frame_size = ei->max_hw_frame_size;
b3f4d599 6569 adapter->msg_enable = netif_msg_init(debug, DEFAULT_MSG_ENABLE);
bc7f75fa
AK		 6570
6571 mmio_start = pci_resource_start(pdev, 0);
6572 mmio_len = pci_resource_len(pdev, 0);
6573
6574 err = -EIO;
6575 adapter->hw.hw_addr = ioremap(mmio_start, mmio_len);
6576 if (!adapter->hw.hw_addr)
6577 goto err_ioremap;
6578
6579 if ((adapter->flags & FLAG_HAS_FLASH) &&
6580 (pci_resource_flags(pdev, 1) & IORESOURCE_MEM)) {
6581 flash_start = pci_resource_start(pdev, 1);
6582 flash_len = pci_resource_len(pdev, 1);
6583 adapter->hw.flash_address = ioremap(flash_start, flash_len);
6584 if (!adapter->hw.flash_address)
6585 goto err_flashmap;
6586 }
6587
6588 /* construct the net_device struct */
651c2466 6589 netdev->netdev_ops = &e1000e_netdev_ops;
bc7f75fa 6590 e1000e_set_ethtool_ops(netdev);
bc7f75fa 6591 netdev->watchdog_timeo = 5 * HZ;
c58c8a78 6592 netif_napi_add(netdev, &adapter->napi, e1000e_poll, 64);
f2315bf1 6593 strlcpy(netdev->name, pci_name(pdev), sizeof(netdev->name));
bc7f75fa
AK		 6594
6595 netdev->mem_start = mmio_start;
6596 netdev->mem_end = mmio_start + mmio_len;
6597
6598 adapter->bd_number = cards_found++;
6599
4662e82b
BA		 6600 e1000e_check_options(adapter);
6601
bc7f75fa
AK		 6602 /* setup adapter struct */
6603 err = e1000_sw_init(adapter);
6604 if (err)
6605 goto err_sw_init;
6606
bc7f75fa
AK		 6607 memcpy(&hw->mac.ops, ei->mac_ops, sizeof(hw->mac.ops));
6608 memcpy(&hw->nvm.ops, ei->nvm_ops, sizeof(hw->nvm.ops));
6609 memcpy(&hw->phy.ops, ei->phy_ops, sizeof(hw->phy.ops));
6610
69e3fd8c 6611 err = ei->get_variants(adapter);
bc7f75fa
AK		 6612 if (err)
6613 goto err_hw_init;
6614
4a770358
BA		 6615 if ((adapter->flags & FLAG_IS_ICH) &&
6616 (adapter->flags & FLAG_READ_ONLY_NVM))
6617 e1000e_write_protect_nvm_ich8lan(&adapter->hw);
6618
bc7f75fa
AK		 6619 hw->mac.ops.get_bus_info(&adapter->hw);
6620
318a94d6 6621 adapter->hw.phy.autoneg_wait_to_complete = 0;
bc7f75fa
AK		 6622
6623 /* Copper options */
318a94d6 6624 if (adapter->hw.phy.media_type == e1000_media_type_copper) {
bc7f75fa
AK		 6625 adapter->hw.phy.mdix = AUTO_ALL_MODES;
6626 adapter->hw.phy.disable_polarity_correction = 0;
6627 adapter->hw.phy.ms_type = e1000_ms_hw_default;
6628 }
6629
470a5420 6630 if (hw->phy.ops.check_reset_block && hw->phy.ops.check_reset_block(hw))
185095fb
BA		 6631 dev_info(&pdev->dev,
6632 "PHY reset is blocked due to SOL/IDER session.\n");
bc7f75fa 6633
dc221294
BA		 6634 /* Set initial default active device features */
6635 netdev->features = (NETIF_F_SG |
6636 NETIF_F_HW_VLAN_RX |
6637 NETIF_F_HW_VLAN_TX |
6638 NETIF_F_TSO |
6639 NETIF_F_TSO6 |
70495a50 6640 NETIF_F_RXHASH |
dc221294
BA		 6641 NETIF_F_RXCSUM |
6642 NETIF_F_HW_CSUM);
6643
6644 /* Set user-changeable features (subset of all device features) */
6645 netdev->hw_features = netdev->features;
0184039a 6646 netdev->hw_features |= NETIF_F_RXFCS;
943146de 6647 netdev->priv_flags |= IFF_SUPP_NOFCS;
cf955e6c 6648 netdev->hw_features |= NETIF_F_RXALL;
bc7f75fa
AK		 6649
6650 if (adapter->flags & FLAG_HAS_HW_VLAN_FILTER)
6651 netdev->features |= NETIF_F_HW_VLAN_FILTER;
6652
dc221294
BA		 6653 netdev->vlan_features |= (NETIF_F_SG |
6654 NETIF_F_TSO |
6655 NETIF_F_TSO6 |
6656 NETIF_F_HW_CSUM);
a5136e23 6657
ef9b965a
JB		 6658 netdev->priv_flags |= IFF_UNICAST_FLT;
6659
7b872a55 6660 if (pci_using_dac) {
bc7f75fa 6661 netdev->features |= NETIF_F_HIGHDMA;
7b872a55
YZ		 6662 netdev->vlan_features |= NETIF_F_HIGHDMA;
6663 }
bc7f75fa 6664
bc7f75fa
AK		 6665 if (e1000e_enable_mng_pass_thru(&adapter->hw))
6666 adapter->flags |= FLAG_MNG_PT_ENABLED;
6667
e921eb1a 6668 /* before reading the NVM, reset the controller to
ad68076e
BA		 6669 * put the device in a known good starting state
6670 */
bc7f75fa
AK		 6671 adapter->hw.mac.ops.reset_hw(&adapter->hw);
6672
e921eb1a 6673 /* systems with ASPM and others may see the checksum fail on the first
bc7f75fa
AK		 6674 * attempt. Let's give it a few tries
6675 */
6676 for (i = 0;; i++) {
6677 if (e1000_validate_nvm_checksum(&adapter->hw) >= 0)
6678 break;
6679 if (i == 2) {
185095fb 6680 dev_err(&pdev->dev, "The NVM Checksum Is Not Valid\n");
bc7f75fa
AK		 6681 err = -EIO;
6682 goto err_eeprom;
6683 }
6684 }
6685
10aa4c04
AK		 6686 e1000_eeprom_checks(adapter);
6687
608f8a0d 6688 /* copy the MAC address */
bc7f75fa 6689 if (e1000e_read_mac_addr(&adapter->hw))
185095fb
BA		 6690 dev_err(&pdev->dev,
6691 "NVM Read Error while reading MAC address\n");
bc7f75fa
AK		 6692
6693 memcpy(netdev->dev_addr, adapter->hw.mac.addr, netdev->addr_len);
bc7f75fa 6694
aaeb6cdf 6695 if (!is_valid_ether_addr(netdev->dev_addr)) {
185095fb 6696 dev_err(&pdev->dev, "Invalid MAC Address: %pM\n",
aaeb6cdf 6697 netdev->dev_addr);
bc7f75fa
AK		 6698 err = -EIO;
6699 goto err_eeprom;
6700 }
6701
6702 init_timer(&adapter->watchdog_timer);
c061b18d 6703 adapter->watchdog_timer.function = e1000_watchdog;
53aa82da 6704 adapter->watchdog_timer.data = (unsigned long)adapter;
bc7f75fa
AK		 6705
6706 init_timer(&adapter->phy_info_timer);
c061b18d 6707 adapter->phy_info_timer.function = e1000_update_phy_info;
53aa82da 6708 adapter->phy_info_timer.data = (unsigned long)adapter;
bc7f75fa
AK		 6709
6710 INIT_WORK(&adapter->reset_task, e1000_reset_task);
6711 INIT_WORK(&adapter->watchdog_task, e1000_watchdog_task);
a8f88ff5
JB		 6712 INIT_WORK(&adapter->downshift_task, e1000e_downshift_workaround);
6713 INIT_WORK(&adapter->update_phy_task, e1000e_update_phy_task);
41cec6f1 6714 INIT_WORK(&adapter->print_hang_task, e1000_print_hw_hang);
bc7f75fa 6715
bc7f75fa
AK		 6716 /* Initialize link parameters. User can change them with ethtool */
6717 adapter->hw.mac.autoneg = 1;
3db1cd5c 6718 adapter->fc_autoneg = true;
5c48ef3e
BA		 6719 adapter->hw.fc.requested_mode = e1000_fc_default;
6720 adapter->hw.fc.current_mode = e1000_fc_default;
bc7f75fa
AK		 6721 adapter->hw.phy.autoneg_advertised = 0x2f;
6722
6723 /* ring size defaults */
d821a4c4
BA		 6724 adapter->rx_ring->count = E1000_DEFAULT_RXD;
6725 adapter->tx_ring->count = E1000_DEFAULT_TXD;
bc7f75fa 6726
e921eb1a 6727 /* Initial Wake on LAN setting - If APM wake is enabled in
bc7f75fa
AK		 6728 * the EEPROM, enable the ACPI Magic Packet filter
6729 */
6730 if (adapter->flags & FLAG_APME_IN_WUC) {
6731 /* APME bit in EEPROM is mapped to WUC.APME */
6732 eeprom_data = er32(WUC);
6733 eeprom_apme_mask = E1000_WUC_APME;
4def99bb
BA		 6734 if ((hw->mac.type > e1000_ich10lan) &&
6735 (eeprom_data & E1000_WUC_PHY_WAKE))
a4f58f54 6736 adapter->flags2 |= FLAG2_HAS_PHY_WAKEUP;
bc7f75fa
AK		 6737 } else if (adapter->flags & FLAG_APME_IN_CTRL3) {
6738 if (adapter->flags & FLAG_APME_CHECK_PORT_B &&
6739 (adapter->hw.bus.func == 1))
3d3a1676
BA		 6740 e1000_read_nvm(&adapter->hw, NVM_INIT_CONTROL3_PORT_B,
6741 1, &eeprom_data);
bc7f75fa 6742 else
3d3a1676
BA		 6743 e1000_read_nvm(&adapter->hw, NVM_INIT_CONTROL3_PORT_A,
6744 1, &eeprom_data);
bc7f75fa
AK		 6745 }
6746
6747 /* fetch WoL from EEPROM */
6748 if (eeprom_data & eeprom_apme_mask)
6749 adapter->eeprom_wol |= E1000_WUFC_MAG;
6750
e921eb1a 6751 /* now that we have the eeprom settings, apply the special cases
bc7f75fa
AK		 6752 * where the eeprom may be wrong or the board simply won't support
6753 * wake on lan on a particular port
6754 */
6755 if (!(adapter->flags & FLAG_HAS_WOL))
6756 adapter->eeprom_wol = 0;
6757
6758 /* initialize the wol settings based on the eeprom settings */
6759 adapter->wol = adapter->eeprom_wol;
6ff68026 6760 device_set_wakeup_enable(&adapter->pdev->dev, adapter->wol);
bc7f75fa 6761
84527590
BA		 6762 /* save off EEPROM version number */
6763 e1000_read_nvm(&adapter->hw, 5, 1, &adapter->eeprom_vers);
6764
bc7f75fa
AK		 6765 /* reset the hardware with the new settings */
6766 e1000e_reset(adapter);
6767
e921eb1a 6768 /* If the controller has AMT, do not set DRV_LOAD until the interface
bc7f75fa 6769 * is up. For all other cases, let the f/w know that the h/w is now
ad68076e
BA		 6770 * under the control of the driver.
6771 */
c43bc57e 6772 if (!(adapter->flags & FLAG_HAS_AMT))
31dbe5b4 6773 e1000e_get_hw_control(adapter);
bc7f75fa 6774
f2315bf1 6775 strlcpy(netdev->name, "eth%d", sizeof(netdev->name));
bc7f75fa
AK		 6776 err = register_netdev(netdev);
6777 if (err)
6778 goto err_register;
6779
9c563d20
JB		 6780 /* carrier off reporting is important to ethtool even BEFORE open */
6781 netif_carrier_off(netdev);
6782
d89777bf
BA		 6783 /* init PTP hardware clock */
6784 e1000e_ptp_init(adapter);
6785
bc7f75fa
AK		 6786 e1000_print_device_info(adapter);
6787
f3ec4f87
AS		 6788 if (pci_dev_run_wake(pdev))
6789 pm_runtime_put_noidle(&pdev->dev);
23606cf5 6790
bc7f75fa
AK		 6791 return 0;
6792
6793err_register:
c43bc57e 6794 if (!(adapter->flags & FLAG_HAS_AMT))
31dbe5b4 6795 e1000e_release_hw_control(adapter);
bc7f75fa 6796err_eeprom:
470a5420 6797 if (hw->phy.ops.check_reset_block && !hw->phy.ops.check_reset_block(hw))
bc7f75fa 6798 e1000_phy_hw_reset(&adapter->hw);
c43bc57e 6799err_hw_init:
bc7f75fa
AK		 6800 kfree(adapter->tx_ring);
6801 kfree(adapter->rx_ring);
6802err_sw_init:
c43bc57e
JB		 6803 if (adapter->hw.flash_address)
6804 iounmap(adapter->hw.flash_address);
e82f54ba 6805 e1000e_reset_interrupt_capability(adapter);
c43bc57e 6806err_flashmap:
bc7f75fa
AK		 6807 iounmap(adapter->hw.hw_addr);
6808err_ioremap:
6809 free_netdev(netdev);
6810err_alloc_etherdev:
f0f422e5 6811 pci_release_selected_regions(pdev,
f0ff4398 6812 pci_select_bars(pdev, IORESOURCE_MEM));
bc7f75fa
AK		 6813err_pci_reg:
6814err_dma:
6815 pci_disable_device(pdev);
6816 return err;
6817}
6818
6819/**
6820 * e1000_remove - Device Removal Routine
6821 * @pdev: PCI device information struct
6822 *
6823 * e1000_remove is called by the PCI subsystem to alert the driver
6824 * that it should release a PCI device. The could be caused by a
6825 * Hot-Plug event, or because the driver is going to be removed from
6826 * memory.
6827 **/
9f9a12f8 6828static void e1000_remove(struct pci_dev *pdev)
bc7f75fa
AK		 6829{
6830 struct net_device *netdev = pci_get_drvdata(pdev);
6831 struct e1000_adapter *adapter = netdev_priv(netdev);
23606cf5
RW		 6832 bool down = test_bit(__E1000_DOWN, &adapter->state);
6833
d89777bf
BA		 6834 e1000e_ptp_remove(adapter);
6835
e921eb1a 6836 /* The timers may be rescheduled, so explicitly disable them
23f333a2 6837 * from being rescheduled.
ad68076e 6838 */
23606cf5
RW		 6839 if (!down)
6840 set_bit(__E1000_DOWN, &adapter->state);
bc7f75fa
AK		 6841 del_timer_sync(&adapter->watchdog_timer);
6842 del_timer_sync(&adapter->phy_info_timer);
6843
41cec6f1
BA		 6844 cancel_work_sync(&adapter->reset_task);
6845 cancel_work_sync(&adapter->watchdog_task);
6846 cancel_work_sync(&adapter->downshift_task);
6847 cancel_work_sync(&adapter->update_phy_task);
6848 cancel_work_sync(&adapter->print_hang_task);
bc7f75fa 6849
b67e1913
BA		 6850 if (adapter->flags & FLAG_HAS_HW_TIMESTAMP) {
6851 cancel_work_sync(&adapter->tx_hwtstamp_work);
6852 if (adapter->tx_hwtstamp_skb) {
6853 dev_kfree_skb_any(adapter->tx_hwtstamp_skb);
6854 adapter->tx_hwtstamp_skb = NULL;
6855 }
6856 }
6857
17f208de
BA		 6858 if (!(netdev->flags & IFF_UP))
6859 e1000_power_down_phy(adapter);
6860
23606cf5
RW		 6861 /* Don't lie to e1000_close() down the road. */
6862 if (!down)
6863 clear_bit(__E1000_DOWN, &adapter->state);
17f208de
BA		 6864 unregister_netdev(netdev);
6865
f3ec4f87
AS		 6866 if (pci_dev_run_wake(pdev))
6867 pm_runtime_get_noresume(&pdev->dev);
23606cf5 6868
e921eb1a 6869 /* Release control of h/w to f/w. If f/w is AMT enabled, this
ad68076e
BA		 6870 * would have already happened in close and is redundant.
6871 */
31dbe5b4 6872 e1000e_release_hw_control(adapter);
bc7f75fa 6873
4662e82b 6874 e1000e_reset_interrupt_capability(adapter);
bc7f75fa
AK		 6875 kfree(adapter->tx_ring);
6876 kfree(adapter->rx_ring);
6877
6878 iounmap(adapter->hw.hw_addr);
6879 if (adapter->hw.flash_address)
6880 iounmap(adapter->hw.flash_address);
f0f422e5 6881 pci_release_selected_regions(pdev,
f0ff4398 6882 pci_select_bars(pdev, IORESOURCE_MEM));
bc7f75fa
AK		 6883
6884 free_netdev(netdev);
6885
111b9dc5 6886 /* AER disable */
19d5afd4 6887 pci_disable_pcie_error_reporting(pdev);
111b9dc5 6888
bc7f75fa
AK		 6889 pci_disable_device(pdev);
6890}
6891
6892/* PCI Error Recovery (ERS) */
3646f0e5 6893static const struct pci_error_handlers e1000_err_handler = {
bc7f75fa
AK		 6894 .error_detected = e1000_io_error_detected,
6895 .slot_reset = e1000_io_slot_reset,
6896 .resume = e1000_io_resume,
6897};
6898
a3aa1884 6899static DEFINE_PCI_DEVICE_TABLE(e1000_pci_tbl) = {
bc7f75fa
AK		 6900 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82571EB_COPPER), board_82571 },
6901 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82571EB_FIBER), board_82571 },
6902 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82571EB_QUAD_COPPER), board_82571 },
c29c3ba5
BA		 6903 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82571EB_QUAD_COPPER_LP),
6904 board_82571 },
bc7f75fa
AK		 6905 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82571EB_QUAD_FIBER), board_82571 },
6906 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82571EB_SERDES), board_82571 },
040babf9
AK		 6907 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82571EB_SERDES_DUAL), board_82571 },
6908 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82571EB_SERDES_QUAD), board_82571 },
6909 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82571PT_QUAD_COPPER), board_82571 },
ad68076e 6910
bc7f75fa
AK		 6911 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82572EI), board_82572 },
6912 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82572EI_COPPER), board_82572 },
6913 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82572EI_FIBER), board_82572 },
6914 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82572EI_SERDES), board_82572 },
ad68076e 6915
bc7f75fa
AK		 6916 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82573E), board_82573 },
6917 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82573E_IAMT), board_82573 },
6918 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82573L), board_82573 },
ad68076e 6919
4662e82b 6920 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82574L), board_82574 },
bef28b11 6921 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82574LA), board_82574 },
8c81c9c3 6922 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82583V), board_82583 },
4662e82b 6923
bc7f75fa
AK		 6924 { PCI_VDEVICE(INTEL, E1000_DEV_ID_80003ES2LAN_COPPER_DPT),
6925 board_80003es2lan },
6926 { PCI_VDEVICE(INTEL, E1000_DEV_ID_80003ES2LAN_COPPER_SPT),
6927 board_80003es2lan },
6928 { PCI_VDEVICE(INTEL, E1000_DEV_ID_80003ES2LAN_SERDES_DPT),
6929 board_80003es2lan },
6930 { PCI_VDEVICE(INTEL, E1000_DEV_ID_80003ES2LAN_SERDES_SPT),
6931 board_80003es2lan },
ad68076e 6932
bc7f75fa
AK		 6933 { PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH8_IFE), board_ich8lan },
6934 { PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH8_IFE_G), board_ich8lan },
6935 { PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH8_IFE_GT), board_ich8lan },
6936 { PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH8_IGP_AMT), board_ich8lan },
6937 { PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH8_IGP_C), board_ich8lan },
6938 { PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH8_IGP_M), board_ich8lan },
6939 { PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH8_IGP_M_AMT), board_ich8lan },
9e135a2e 6940 { PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH8_82567V_3), board_ich8lan },
ad68076e 6941
bc7f75fa
AK		 6942 { PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH9_IFE), board_ich9lan },
6943 { PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH9_IFE_G), board_ich9lan },
6944 { PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH9_IFE_GT), board_ich9lan },
6945 { PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH9_IGP_AMT), board_ich9lan },
6946 { PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH9_IGP_C), board_ich9lan },
2f15f9d6 6947 { PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH9_BM), board_ich9lan },
97ac8cae
BA		 6948 { PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH9_IGP_M), board_ich9lan },
6949 { PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH9_IGP_M_AMT), board_ich9lan },
6950 { PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH9_IGP_M_V), board_ich9lan },
6951
6952 { PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH10_R_BM_LM), board_ich9lan },
6953 { PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH10_R_BM_LF), board_ich9lan },
6954 { PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH10_R_BM_V), board_ich9lan },
bc7f75fa 6955
f4187b56
BA		 6956 { PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH10_D_BM_LM), board_ich10lan },
6957 { PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH10_D_BM_LF), board_ich10lan },
10df0b91 6958 { PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH10_D_BM_V), board_ich10lan },
f4187b56 6959
a4f58f54
BA		 6960 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_M_HV_LM), board_pchlan },
6961 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_M_HV_LC), board_pchlan },
6962 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_D_HV_DM), board_pchlan },
6963 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_D_HV_DC), board_pchlan },
6964
d3738bb8
BA		 6965 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH2_LV_LM), board_pch2lan },
6966 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH2_LV_V), board_pch2lan },
6967
2fbe4526
BA		 6968 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_LPT_I217_LM), board_pch_lpt },
6969 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_LPT_I217_V), board_pch_lpt },
16e310ae
BA		 6970 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_LPTLP_I218_LM), board_pch_lpt },
6971 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_LPTLP_I218_V), board_pch_lpt },
2fbe4526 6972
f36bb6ca 6973 { 0, 0, 0, 0, 0, 0, 0 } /* terminate list */
bc7f75fa
AK		 6974};
6975MODULE_DEVICE_TABLE(pci, e1000_pci_tbl);
6976
aa338601 6977#ifdef CONFIG_PM
23606cf5 6978static const struct dev_pm_ops e1000_pm_ops = {
a0340162 6979 SET_SYSTEM_SLEEP_PM_OPS(e1000_suspend, e1000_resume)
17e813ec
BA		 6980 SET_RUNTIME_PM_OPS(e1000_runtime_suspend, e1000_runtime_resume,
6981 e1000_idle)
23606cf5 6982};
e50208a0 6983#endif
23606cf5 6984
bc7f75fa
AK		 6985/* PCI Device API Driver */
6986static struct pci_driver e1000_driver = {
6987 .name = e1000e_driver_name,
6988 .id_table = e1000_pci_tbl,
6989 .probe = e1000_probe,
9f9a12f8 6990 .remove = e1000_remove,
aa338601 6991#ifdef CONFIG_PM
f36bb6ca
BA		 6992 .driver = {
6993 .pm = &e1000_pm_ops,
6994 },
bc7f75fa
AK		 6995#endif
6996 .shutdown = e1000_shutdown,
6997 .err_handler = &e1000_err_handler
6998};
6999
7000/**
7001 * e1000_init_module - Driver Registration Routine
7002 *
7003 * e1000_init_module is the first routine called when the driver is
7004 * loaded. All it does is register with the PCI subsystem.
7005 **/
7006static int __init e1000_init_module(void)
7007{
7008 int ret;
8544b9f7
BA		 7009 pr_info("Intel(R) PRO/1000 Network Driver - %s\n",
7010 e1000e_driver_version);
bf67044b 7011 pr_info("Copyright(c) 1999 - 2013 Intel Corporation.\n");
bc7f75fa 7012 ret = pci_register_driver(&e1000_driver);
53ec5498 7013
bc7f75fa
AK		 7014 return ret;
7015}
7016module_init(e1000_init_module);
7017
7018/**
7019 * e1000_exit_module - Driver Exit Cleanup Routine
7020 *
7021 * e1000_exit_module is called just before the driver is removed
7022 * from memory.
7023 **/
7024static void __exit e1000_exit_module(void)
7025{
7026 pci_unregister_driver(&e1000_driver);
7027}
7028module_exit(e1000_exit_module);
7029
7030
7031MODULE_AUTHOR("Intel Corporation, <linux.nics@intel.com>");
7032MODULE_DESCRIPTION("Intel(R) PRO/1000 Network Driver");
7033MODULE_LICENSE("GPL");
7034MODULE_VERSION(DRV_VERSION);
7035
06c24b91 7036/* netdev.c */
Linux kernel - Deliverables
This page took 1.756896 seconds and 5 git commands to generate.