projects / deliverable / linux.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
e1000e: cleanup CODE_INDENT checkpatch errors
[deliverable/linux.git] / drivers / net / ethernet / intel / e1000e / netdev.c
CommitLineData
bc7f75fa
AK		 1/*******************************************************************************
2
3 Intel PRO/1000 Linux driver
bf67044b 4 Copyright(c) 1999 - 2013 Intel Corporation.
bc7f75fa
AK		 5
6 This program is free software; you can redistribute it and/or modify it
7 under the terms and conditions of the GNU General Public License,
8 version 2, as published by the Free Software Foundation.
9
10 This program is distributed in the hope it will be useful, but WITHOUT
11 ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
13 more details.
14
15 You should have received a copy of the GNU General Public License along with
16 this program; if not, write to the Free Software Foundation, Inc.,
17 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
18
19 The full GNU General Public License is included in this distribution in
20 the file called "COPYING".
21
22 Contact Information:
23 Linux NICS <linux.nics@intel.com>
24 e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
25 Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
26
27*******************************************************************************/
28
8544b9f7
BA		 29#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
30
bc7f75fa
AK		 31#include <linux/module.h>
32#include <linux/types.h>
33#include <linux/init.h>
34#include <linux/pci.h>
35#include <linux/vmalloc.h>
36#include <linux/pagemap.h>
37#include <linux/delay.h>
38#include <linux/netdevice.h>
9fb7a5f7 39#include <linux/interrupt.h>
bc7f75fa
AK		 40#include <linux/tcp.h>
41#include <linux/ipv6.h>
5a0e3ad6 42#include <linux/slab.h>
bc7f75fa
AK		 43#include <net/checksum.h>
44#include <net/ip6_checksum.h>
bc7f75fa
AK		 45#include <linux/ethtool.h>
46#include <linux/if_vlan.h>
47#include <linux/cpu.h>
48#include <linux/smp.h>
e8db0be1 49#include <linux/pm_qos.h>
23606cf5 50#include <linux/pm_runtime.h>
111b9dc5 51#include <linux/aer.h>
70c71606 52#include <linux/prefetch.h>
bc7f75fa
AK		 53
54#include "e1000.h"
55
b3ccf267 56#define DRV_EXTRAVERSION "-k"
c14c643b 57
9e019901 58#define DRV_VERSION "2.2.14" DRV_EXTRAVERSION
bc7f75fa
AK		 59char e1000e_driver_name[] = "e1000e";
60const char e1000e_driver_version[] = DRV_VERSION;
61
b3f4d599 62#define DEFAULT_MSG_ENABLE (NETIF_MSG_DRV|NETIF_MSG_PROBE|NETIF_MSG_LINK)
63static int debug = -1;
64module_param(debug, int, 0);
65MODULE_PARM_DESC(debug, "Debug level (0=none,...,16=all)");
66
78cd29d5
BA		 67static void e1000e_disable_aspm(struct pci_dev *pdev, u16 state);
68
bc7f75fa
AK		 69static const struct e1000_info *e1000_info_tbl[] = {
70 [board_82571] = &e1000_82571_info,
71 [board_82572] = &e1000_82572_info,
72 [board_82573] = &e1000_82573_info,
4662e82b 73 [board_82574] = &e1000_82574_info,
8c81c9c3 74 [board_82583] = &e1000_82583_info,
bc7f75fa
AK		 75 [board_80003es2lan] = &e1000_es2_info,
76 [board_ich8lan] = &e1000_ich8_info,
77 [board_ich9lan] = &e1000_ich9_info,
f4187b56 78 [board_ich10lan] = &e1000_ich10_info,
a4f58f54 79 [board_pchlan] = &e1000_pch_info,
d3738bb8 80 [board_pch2lan] = &e1000_pch2_info,
2fbe4526 81 [board_pch_lpt] = &e1000_pch_lpt_info,
bc7f75fa
AK		 82};
83
84f4ee90
TI		 84struct e1000_reg_info {
85 u32 ofs;
86 char *name;
87};
88
84f4ee90 89static const struct e1000_reg_info e1000_reg_info_tbl[] = {
84f4ee90
TI		 90 /* General Registers */
91 {E1000_CTRL, "CTRL"},
92 {E1000_STATUS, "STATUS"},
93 {E1000_CTRL_EXT, "CTRL_EXT"},
94
95 /* Interrupt Registers */
96 {E1000_ICR, "ICR"},
97
af667a29 98 /* Rx Registers */
84f4ee90 99 {E1000_RCTL, "RCTL"},
1e36052e
BA		 100 {E1000_RDLEN(0), "RDLEN"},
101 {E1000_RDH(0), "RDH"},
102 {E1000_RDT(0), "RDT"},
84f4ee90
TI		 103 {E1000_RDTR, "RDTR"},
104 {E1000_RXDCTL(0), "RXDCTL"},
105 {E1000_ERT, "ERT"},
1e36052e
BA		 106 {E1000_RDBAL(0), "RDBAL"},
107 {E1000_RDBAH(0), "RDBAH"},
84f4ee90
TI		 108 {E1000_RDFH, "RDFH"},
109 {E1000_RDFT, "RDFT"},
110 {E1000_RDFHS, "RDFHS"},
111 {E1000_RDFTS, "RDFTS"},
112 {E1000_RDFPC, "RDFPC"},
113
af667a29 114 /* Tx Registers */
84f4ee90 115 {E1000_TCTL, "TCTL"},
1e36052e
BA		 116 {E1000_TDBAL(0), "TDBAL"},
117 {E1000_TDBAH(0), "TDBAH"},
118 {E1000_TDLEN(0), "TDLEN"},
119 {E1000_TDH(0), "TDH"},
120 {E1000_TDT(0), "TDT"},
84f4ee90
TI		 121 {E1000_TIDV, "TIDV"},
122 {E1000_TXDCTL(0), "TXDCTL"},
123 {E1000_TADV, "TADV"},
124 {E1000_TARC(0), "TARC"},
125 {E1000_TDFH, "TDFH"},
126 {E1000_TDFT, "TDFT"},
127 {E1000_TDFHS, "TDFHS"},
128 {E1000_TDFTS, "TDFTS"},
129 {E1000_TDFPC, "TDFPC"},
130
131 /* List Terminator */
f36bb6ca 132 {0, NULL}
84f4ee90
TI		 133};
134
e921eb1a 135/**
84f4ee90 136 * e1000_regdump - register printout routine
e921eb1a
BA		 137 * @hw: pointer to the HW structure
138 * @reginfo: pointer to the register info table
139 **/
84f4ee90
TI		 140static void e1000_regdump(struct e1000_hw *hw, struct e1000_reg_info *reginfo)
141{
142 int n = 0;
143 char rname[16];
144 u32 regs[8];
145
146 switch (reginfo->ofs) {
147 case E1000_RXDCTL(0):
148 for (n = 0; n < 2; n++)
149 regs[n] = __er32(hw, E1000_RXDCTL(n));
150 break;
151 case E1000_TXDCTL(0):
152 for (n = 0; n < 2; n++)
153 regs[n] = __er32(hw, E1000_TXDCTL(n));
154 break;
155 case E1000_TARC(0):
156 for (n = 0; n < 2; n++)
157 regs[n] = __er32(hw, E1000_TARC(n));
158 break;
159 default:
ef456f85
JK		 160 pr_info("%-15s %08x\n",
161 reginfo->name, __er32(hw, reginfo->ofs));
84f4ee90
TI		 162 return;
163 }
164
165 snprintf(rname, 16, "%s%s", reginfo->name, "[0-1]");
ef456f85 166 pr_info("%-15s %08x %08x\n", rname, regs[0], regs[1]);
84f4ee90
TI		 167}
168
f0c5dadf
ET		 169static void e1000e_dump_ps_pages(struct e1000_adapter *adapter,
170 struct e1000_buffer *bi)
171{
172 int i;
173 struct e1000_ps_page *ps_page;
174
175 for (i = 0; i < adapter->rx_ps_pages; i++) {
176 ps_page = &bi->ps_pages[i];
177
178 if (ps_page->page) {
179 pr_info("packet dump for ps_page %d:\n", i);
180 print_hex_dump(KERN_INFO, "", DUMP_PREFIX_ADDRESS,
181 16, 1, page_address(ps_page->page),
182 PAGE_SIZE, true);
183 }
184 }
185}
186
e921eb1a 187/**
af667a29 188 * e1000e_dump - Print registers, Tx-ring and Rx-ring
e921eb1a
BA		 189 * @adapter: board private structure
190 **/
84f4ee90
TI		 191static void e1000e_dump(struct e1000_adapter *adapter)
192{
193 struct net_device *netdev = adapter->netdev;
194 struct e1000_hw *hw = &adapter->hw;
195 struct e1000_reg_info *reginfo;
196 struct e1000_ring *tx_ring = adapter->tx_ring;
197 struct e1000_tx_desc *tx_desc;
af667a29 198 struct my_u0 {
e885d762
BA		 199 __le64 a;
200 __le64 b;
af667a29 201 } *u0;
84f4ee90
TI		 202 struct e1000_buffer *buffer_info;
203 struct e1000_ring *rx_ring = adapter->rx_ring;
204 union e1000_rx_desc_packet_split *rx_desc_ps;
5f450212 205 union e1000_rx_desc_extended *rx_desc;
af667a29 206 struct my_u1 {
e885d762
BA		 207 __le64 a;
208 __le64 b;
209 __le64 c;
210 __le64 d;
af667a29 211 } *u1;
84f4ee90
TI		 212 u32 staterr;
213 int i = 0;
214
215 if (!netif_msg_hw(adapter))
216 return;
217
218 /* Print netdevice Info */
219 if (netdev) {
220 dev_info(&adapter->pdev->dev, "Net device Info\n");
ef456f85
JK		 221 pr_info("Device Name state trans_start last_rx\n");
222 pr_info("%-15s %016lX %016lX %016lX\n",
223 netdev->name, netdev->state, netdev->trans_start,
224 netdev->last_rx);
84f4ee90
TI		 225 }
226
227 /* Print Registers */
228 dev_info(&adapter->pdev->dev, "Register Dump\n");
ef456f85 229 pr_info(" Register Name Value\n");
84f4ee90
TI		 230 for (reginfo = (struct e1000_reg_info *)e1000_reg_info_tbl;
231 reginfo->name; reginfo++) {
232 e1000_regdump(hw, reginfo);
233 }
234
af667a29 235 /* Print Tx Ring Summary */
84f4ee90 236 if (!netdev || !netif_running(netdev))
fe1e980f 237 return;
84f4ee90 238
af667a29 239 dev_info(&adapter->pdev->dev, "Tx Ring Summary\n");
ef456f85 240 pr_info("Queue [NTU] [NTC] [bi(ntc)->dma ] leng ntw timestamp\n");
84f4ee90 241 buffer_info = &tx_ring->buffer_info[tx_ring->next_to_clean];
ef456f85
JK		 242 pr_info(" %5d %5X %5X %016llX %04X %3X %016llX\n",
243 0, tx_ring->next_to_use, tx_ring->next_to_clean,
244 (unsigned long long)buffer_info->dma,
245 buffer_info->length,
246 buffer_info->next_to_watch,
247 (unsigned long long)buffer_info->time_stamp);
84f4ee90 248
af667a29 249 /* Print Tx Ring */
84f4ee90
TI		 250 if (!netif_msg_tx_done(adapter))
251 goto rx_ring_summary;
252
af667a29 253 dev_info(&adapter->pdev->dev, "Tx Ring Dump\n");
84f4ee90
TI		 254
255 /* Transmit Descriptor Formats - DEXT[29] is 0 (Legacy) or 1 (Extended)
256 *
257 * Legacy Transmit Descriptor
258 * +--------------------------------------------------------------+
259 * 0 | Buffer Address [63:0] (Reserved on Write Back) |
260 * +--------------------------------------------------------------+
261 * 8 | Special | CSS | Status | CMD | CSO | Length |
262 * +--------------------------------------------------------------+
263 * 63 48 47 36 35 32 31 24 23 16 15 0
264 *
265 * Extended Context Descriptor (DTYP=0x0) for TSO or checksum offload
266 * 63 48 47 40 39 32 31 16 15 8 7 0
267 * +----------------------------------------------------------------+
268 * 0 | TUCSE | TUCS0 | TUCSS | IPCSE | IPCS0 | IPCSS |
269 * +----------------------------------------------------------------+
270 * 8 | MSS | HDRLEN | RSV | STA | TUCMD | DTYP | PAYLEN |
271 * +----------------------------------------------------------------+
272 * 63 48 47 40 39 36 35 32 31 24 23 20 19 0
273 *
274 * Extended Data Descriptor (DTYP=0x1)
275 * +----------------------------------------------------------------+
276 * 0 | Buffer Address [63:0] |
277 * +----------------------------------------------------------------+
278 * 8 | VLAN tag | POPTS | Rsvd | Status | Command | DTYP | DTALEN |
279 * +----------------------------------------------------------------+
280 * 63 48 47 40 39 36 35 32 31 24 23 20 19 0
281 */
ef456f85
JK		 282 pr_info("Tl[desc] [address 63:0 ] [SpeCssSCmCsLen] [bi->dma ] leng ntw timestamp bi->skb <-- Legacy format\n");
283 pr_info("Tc[desc] [Ce CoCsIpceCoS] [MssHlRSCm0Plen] [bi->dma ] leng ntw timestamp bi->skb <-- Ext Context format\n");
284 pr_info("Td[desc] [address 63:0 ] [VlaPoRSCm1Dlen] [bi->dma ] leng ntw timestamp bi->skb <-- Ext Data format\n");
84f4ee90 285 for (i = 0; tx_ring->desc && (i < tx_ring->count); i++) {
ef456f85 286 const char *next_desc;
84f4ee90
TI		 287 tx_desc = E1000_TX_DESC(*tx_ring, i);
288 buffer_info = &tx_ring->buffer_info[i];
289 u0 = (struct my_u0 *)tx_desc;
84f4ee90 290 if (i == tx_ring->next_to_use && i == tx_ring->next_to_clean)
ef456f85 291 next_desc = " NTC/U";
84f4ee90 292 else if (i == tx_ring->next_to_use)
ef456f85 293 next_desc = " NTU";
84f4ee90 294 else if (i == tx_ring->next_to_clean)
ef456f85 295 next_desc = " NTC";
84f4ee90 296 else
ef456f85
JK		 297 next_desc = "";
298 pr_info("T%c[0x%03X] %016llX %016llX %016llX %04X %3X %016llX %p%s\n",
299 (!(le64_to_cpu(u0->b) & (1 << 29)) ? 'l' :
300 ((le64_to_cpu(u0->b) & (1 << 20)) ? 'd' : 'c')),
301 i,
302 (unsigned long long)le64_to_cpu(u0->a),
303 (unsigned long long)le64_to_cpu(u0->b),
304 (unsigned long long)buffer_info->dma,
305 buffer_info->length, buffer_info->next_to_watch,
306 (unsigned long long)buffer_info->time_stamp,
307 buffer_info->skb, next_desc);
84f4ee90 308
f0c5dadf 309 if (netif_msg_pktdata(adapter) && buffer_info->skb)
84f4ee90 310 print_hex_dump(KERN_INFO, "", DUMP_PREFIX_ADDRESS,
f0c5dadf
ET		 311 16, 1, buffer_info->skb->data,
312 buffer_info->skb->len, true);
84f4ee90
TI		 313 }
314
af667a29 315 /* Print Rx Ring Summary */
84f4ee90 316rx_ring_summary:
af667a29 317 dev_info(&adapter->pdev->dev, "Rx Ring Summary\n");
ef456f85
JK		 318 pr_info("Queue [NTU] [NTC]\n");
319 pr_info(" %5d %5X %5X\n",
320 0, rx_ring->next_to_use, rx_ring->next_to_clean);
84f4ee90 321
af667a29 322 /* Print Rx Ring */
84f4ee90 323 if (!netif_msg_rx_status(adapter))
fe1e980f 324 return;
84f4ee90 325
af667a29 326 dev_info(&adapter->pdev->dev, "Rx Ring Dump\n");
84f4ee90
TI		 327 switch (adapter->rx_ps_pages) {
328 case 1:
329 case 2:
330 case 3:
331 /* [Extended] Packet Split Receive Descriptor Format
332 *
333 * +-----------------------------------------------------+
334 * 0 | Buffer Address 0 [63:0] |
335 * +-----------------------------------------------------+
336 * 8 | Buffer Address 1 [63:0] |
337 * +-----------------------------------------------------+
338 * 16 | Buffer Address 2 [63:0] |
339 * +-----------------------------------------------------+
340 * 24 | Buffer Address 3 [63:0] |
341 * +-----------------------------------------------------+
342 */
ef456f85 343 pr_info("R [desc] [buffer 0 63:0 ] [buffer 1 63:0 ] [buffer 2 63:0 ] [buffer 3 63:0 ] [bi->dma ] [bi->skb] <-- Ext Pkt Split format\n");
84f4ee90
TI		 344 /* [Extended] Receive Descriptor (Write-Back) Format
345 *
346 * 63 48 47 32 31 13 12 8 7 4 3 0
347 * +------------------------------------------------------+
348 * 0 | Packet | IP | Rsvd | MRQ | Rsvd | MRQ RSS |
349 * | Checksum | Ident | | Queue | | Type |
350 * +------------------------------------------------------+
351 * 8 | VLAN Tag | Length | Extended Error | Extended Status |
352 * +------------------------------------------------------+
353 * 63 48 47 32 31 20 19 0
354 */
ef456f85 355 pr_info("RWB[desc] [ck ipid mrqhsh] [vl l0 ee es] [ l3 l2 l1 hs] [reserved ] ---------------- [bi->skb] <-- Ext Rx Write-Back format\n");
84f4ee90 356 for (i = 0; i < rx_ring->count; i++) {
ef456f85 357 const char *next_desc;
84f4ee90
TI		 358 buffer_info = &rx_ring->buffer_info[i];
359 rx_desc_ps = E1000_RX_DESC_PS(*rx_ring, i);
360 u1 = (struct my_u1 *)rx_desc_ps;
361 staterr =
af667a29 362 le32_to_cpu(rx_desc_ps->wb.middle.status_error);
ef456f85
JK		 363
364 if (i == rx_ring->next_to_use)
365 next_desc = " NTU";
366 else if (i == rx_ring->next_to_clean)
367 next_desc = " NTC";
368 else
369 next_desc = "";
370
84f4ee90
TI		 371 if (staterr & E1000_RXD_STAT_DD) {
372 /* Descriptor Done */
ef456f85
JK		 373 pr_info("%s[0x%03X] %016llX %016llX %016llX %016llX ---------------- %p%s\n",
374 "RWB", i,
375 (unsigned long long)le64_to_cpu(u1->a),
376 (unsigned long long)le64_to_cpu(u1->b),
377 (unsigned long long)le64_to_cpu(u1->c),
378 (unsigned long long)le64_to_cpu(u1->d),
379 buffer_info->skb, next_desc);
84f4ee90 380 } else {
ef456f85
JK		 381 pr_info("%s[0x%03X] %016llX %016llX %016llX %016llX %016llX %p%s\n",
382 "R ", i,
383 (unsigned long long)le64_to_cpu(u1->a),
384 (unsigned long long)le64_to_cpu(u1->b),
385 (unsigned long long)le64_to_cpu(u1->c),
386 (unsigned long long)le64_to_cpu(u1->d),
387 (unsigned long long)buffer_info->dma,
388 buffer_info->skb, next_desc);
84f4ee90
TI		 389
390 if (netif_msg_pktdata(adapter))
f0c5dadf
ET		 391 e1000e_dump_ps_pages(adapter,
392 buffer_info);
84f4ee90 393 }
84f4ee90
TI		 394 }
395 break;
396 default:
397 case 0:
5f450212 398 /* Extended Receive Descriptor (Read) Format
84f4ee90 399 *
5f450212
BA		 400 * +-----------------------------------------------------+
401 * 0 | Buffer Address [63:0] |
402 * +-----------------------------------------------------+
403 * 8 | Reserved |
404 * +-----------------------------------------------------+
84f4ee90 405 */
ef456f85 406 pr_info("R [desc] [buf addr 63:0 ] [reserved 63:0 ] [bi->dma ] [bi->skb] <-- Ext (Read) format\n");
5f450212
BA		 407 /* Extended Receive Descriptor (Write-Back) Format
408 *
409 * 63 48 47 32 31 24 23 4 3 0
410 * +------------------------------------------------------+
411 * | RSS Hash | | | |
412 * 0 +-------------------+ Rsvd | Reserved | MRQ RSS |
413 * | Packet | IP | | | Type |
414 * | Checksum | Ident | | | |
415 * +------------------------------------------------------+
416 * 8 | VLAN Tag | Length | Extended Error | Extended Status |
417 * +------------------------------------------------------+
418 * 63 48 47 32 31 20 19 0
419 */
ef456f85 420 pr_info("RWB[desc] [cs ipid mrq] [vt ln xe xs] [bi->skb] <-- Ext (Write-Back) format\n");
5f450212
BA		 421
422 for (i = 0; i < rx_ring->count; i++) {
ef456f85
JK		 423 const char *next_desc;
424
84f4ee90 425 buffer_info = &rx_ring->buffer_info[i];
5f450212
BA		 426 rx_desc = E1000_RX_DESC_EXT(*rx_ring, i);
427 u1 = (struct my_u1 *)rx_desc;
428 staterr = le32_to_cpu(rx_desc->wb.upper.status_error);
ef456f85
JK		 429
430 if (i == rx_ring->next_to_use)
431 next_desc = " NTU";
432 else if (i == rx_ring->next_to_clean)
433 next_desc = " NTC";
434 else
435 next_desc = "";
436
5f450212
BA		 437 if (staterr & E1000_RXD_STAT_DD) {
438 /* Descriptor Done */
ef456f85
JK		 439 pr_info("%s[0x%03X] %016llX %016llX ---------------- %p%s\n",
440 "RWB", i,
441 (unsigned long long)le64_to_cpu(u1->a),
442 (unsigned long long)le64_to_cpu(u1->b),
443 buffer_info->skb, next_desc);
5f450212 444 } else {
ef456f85
JK		 445 pr_info("%s[0x%03X] %016llX %016llX %016llX %p%s\n",
446 "R ", i,
447 (unsigned long long)le64_to_cpu(u1->a),
448 (unsigned long long)le64_to_cpu(u1->b),
449 (unsigned long long)buffer_info->dma,
450 buffer_info->skb, next_desc);
5f450212 451
f0c5dadf
ET		 452 if (netif_msg_pktdata(adapter) &&
453 buffer_info->skb)
5f450212
BA		 454 print_hex_dump(KERN_INFO, "",
455 DUMP_PREFIX_ADDRESS, 16,
456 1,
f0c5dadf 457 buffer_info->skb->data,
5f450212
BA		 458 adapter->rx_buffer_len,
459 true);
460 }
84f4ee90
TI		 461 }
462 }
84f4ee90
TI		 463}
464
bc7f75fa
AK		 465/**
466 * e1000_desc_unused - calculate if we have unused descriptors
467 **/
468static int e1000_desc_unused(struct e1000_ring *ring)
469{
470 if (ring->next_to_clean > ring->next_to_use)
471 return ring->next_to_clean - ring->next_to_use - 1;
472
473 return ring->count + ring->next_to_clean - ring->next_to_use - 1;
474}
475
b67e1913
BA		 476/**
477 * e1000e_systim_to_hwtstamp - convert system time value to hw time stamp
478 * @adapter: board private structure
479 * @hwtstamps: time stamp structure to update
480 * @systim: unsigned 64bit system time value.
481 *
482 * Convert the system time value stored in the RX/TXSTMP registers into a
483 * hwtstamp which can be used by the upper level time stamping functions.
484 *
485 * The 'systim_lock' spinlock is used to protect the consistency of the
486 * system time value. This is needed because reading the 64 bit time
487 * value involves reading two 32 bit registers. The first read latches the
488 * value.
489 **/
490static void e1000e_systim_to_hwtstamp(struct e1000_adapter *adapter,
491 struct skb_shared_hwtstamps *hwtstamps,
492 u64 systim)
493{
494 u64 ns;
495 unsigned long flags;
496
497 spin_lock_irqsave(&adapter->systim_lock, flags);
498 ns = timecounter_cyc2time(&adapter->tc, systim);
499 spin_unlock_irqrestore(&adapter->systim_lock, flags);
500
501 memset(hwtstamps, 0, sizeof(*hwtstamps));
502 hwtstamps->hwtstamp = ns_to_ktime(ns);
503}
504
505/**
506 * e1000e_rx_hwtstamp - utility function which checks for Rx time stamp
507 * @adapter: board private structure
508 * @status: descriptor extended error and status field
509 * @skb: particular skb to include time stamp
510 *
511 * If the time stamp is valid, convert it into the timecounter ns value
512 * and store that result into the shhwtstamps structure which is passed
513 * up the network stack.
514 **/
515static void e1000e_rx_hwtstamp(struct e1000_adapter *adapter, u32 status,
516 struct sk_buff *skb)
517{
518 struct e1000_hw *hw = &adapter->hw;
519 u64 rxstmp;
520
521 if (!(adapter->flags & FLAG_HAS_HW_TIMESTAMP) ||
522 !(status & E1000_RXDEXT_STATERR_TST) ||
523 !(er32(TSYNCRXCTL) & E1000_TSYNCRXCTL_VALID))
524 return;
525
526 /* The Rx time stamp registers contain the time stamp. No other
527 * received packet will be time stamped until the Rx time stamp
528 * registers are read. Because only one packet can be time stamped
529 * at a time, the register values must belong to this packet and
530 * therefore none of the other additional attributes need to be
531 * compared.
532 */
533 rxstmp = (u64)er32(RXSTMPL);
534 rxstmp |= (u64)er32(RXSTMPH) << 32;
535 e1000e_systim_to_hwtstamp(adapter, skb_hwtstamps(skb), rxstmp);
536
537 adapter->flags2 &= ~FLAG2_CHECK_RX_HWTSTAMP;
538}
539
bc7f75fa 540/**
ad68076e 541 * e1000_receive_skb - helper function to handle Rx indications
bc7f75fa 542 * @adapter: board private structure
b67e1913 543 * @staterr: descriptor extended error and status field as written by hardware
bc7f75fa
AK		 544 * @vlan: descriptor vlan field as written by hardware (no le/be conversion)
545 * @skb: pointer to sk_buff to be indicated to stack
546 **/
547static void e1000_receive_skb(struct e1000_adapter *adapter,
af667a29 548 struct net_device *netdev, struct sk_buff *skb,
b67e1913 549 u32 staterr, __le16 vlan)
bc7f75fa 550{
86d70e53 551 u16 tag = le16_to_cpu(vlan);
b67e1913
BA		 552
553 e1000e_rx_hwtstamp(adapter, staterr, skb);
554
bc7f75fa
AK		 555 skb->protocol = eth_type_trans(skb, netdev);
556
b67e1913 557 if (staterr & E1000_RXD_STAT_VP)
86d70e53
JK		 558 __vlan_hwaccel_put_tag(skb, tag);
559
560 napi_gro_receive(&adapter->napi, skb);
bc7f75fa
AK		 561}
562
563/**
af667a29 564 * e1000_rx_checksum - Receive Checksum Offload
afd12939
BA		 565 * @adapter: board private structure
566 * @status_err: receive descriptor status and error fields
567 * @csum: receive descriptor csum field
568 * @sk_buff: socket buffer with received data
bc7f75fa
AK		 569 **/
570static void e1000_rx_checksum(struct e1000_adapter *adapter, u32 status_err,
2e1706f2 571 struct sk_buff *skb)
bc7f75fa
AK		 572{
573 u16 status = (u16)status_err;
574 u8 errors = (u8)(status_err >> 24);
bc8acf2c
ED		 575
576 skb_checksum_none_assert(skb);
bc7f75fa 577
afd12939
BA		 578 /* Rx checksum disabled */
579 if (!(adapter->netdev->features & NETIF_F_RXCSUM))
580 return;
581
bc7f75fa
AK		 582 /* Ignore Checksum bit is set */
583 if (status & E1000_RXD_STAT_IXSM)
584 return;
afd12939 585
2e1706f2
BA		 586 /* TCP/UDP checksum error bit or IP checksum error bit is set */
587 if (errors & (E1000_RXD_ERR_TCPE | E1000_RXD_ERR_IPE)) {
bc7f75fa
AK		 588 /* let the stack verify checksum errors */
589 adapter->hw_csum_err++;
590 return;
591 }
592
593 /* TCP/UDP Checksum has not been calculated */
594 if (!(status & (E1000_RXD_STAT_TCPCS | E1000_RXD_STAT_UDPCS)))
595 return;
596
597 /* It must be a TCP or UDP packet with a valid checksum */
2e1706f2 598 skb->ip_summed = CHECKSUM_UNNECESSARY;
bc7f75fa
AK		 599 adapter->hw_csum_good++;
600}
601
55aa6985 602static void e1000e_update_rdt_wa(struct e1000_ring *rx_ring, unsigned int i)
c6e7f51e 603{
55aa6985 604 struct e1000_adapter *adapter = rx_ring->adapter;
c6e7f51e 605 struct e1000_hw *hw = &adapter->hw;
bdc125f7
BA		 606 s32 ret_val = __ew32_prepare(hw);
607
608 writel(i, rx_ring->tail);
c6e7f51e 609
bdc125f7 610 if (unlikely(!ret_val && (i != readl(rx_ring->tail)))) {
c6e7f51e
BA		 611 u32 rctl = er32(RCTL);
612 ew32(RCTL, rctl & ~E1000_RCTL_EN);
613 e_err("ME firmware caused invalid RDT - resetting\n");
614 schedule_work(&adapter->reset_task);
615 }
616}
617
55aa6985 618static void e1000e_update_tdt_wa(struct e1000_ring *tx_ring, unsigned int i)
c6e7f51e 619{
55aa6985 620 struct e1000_adapter *adapter = tx_ring->adapter;
c6e7f51e 621 struct e1000_hw *hw = &adapter->hw;
bdc125f7 622 s32 ret_val = __ew32_prepare(hw);
c6e7f51e 623
bdc125f7
BA		 624 writel(i, tx_ring->tail);
625
626 if (unlikely(!ret_val && (i != readl(tx_ring->tail)))) {
c6e7f51e
BA		 627 u32 tctl = er32(TCTL);
628 ew32(TCTL, tctl & ~E1000_TCTL_EN);
629 e_err("ME firmware caused invalid TDT - resetting\n");
630 schedule_work(&adapter->reset_task);
631 }
632}
633
bc7f75fa 634/**
5f450212 635 * e1000_alloc_rx_buffers - Replace used receive buffers
55aa6985 636 * @rx_ring: Rx descriptor ring
bc7f75fa 637 **/
55aa6985 638static void e1000_alloc_rx_buffers(struct e1000_ring *rx_ring,
c2fed996 639 int cleaned_count, gfp_t gfp)
bc7f75fa 640{
55aa6985 641 struct e1000_adapter *adapter = rx_ring->adapter;
bc7f75fa
AK		 642 struct net_device *netdev = adapter->netdev;
643 struct pci_dev *pdev = adapter->pdev;
5f450212 644 union e1000_rx_desc_extended *rx_desc;
bc7f75fa
AK		 645 struct e1000_buffer *buffer_info;
646 struct sk_buff *skb;
647 unsigned int i;
89d71a66 648 unsigned int bufsz = adapter->rx_buffer_len;
bc7f75fa
AK		 649
650 i = rx_ring->next_to_use;
651 buffer_info = &rx_ring->buffer_info[i];
652
653 while (cleaned_count--) {
654 skb = buffer_info->skb;
655 if (skb) {
656 skb_trim(skb, 0);
657 goto map_skb;
658 }
659
c2fed996 660 skb = __netdev_alloc_skb_ip_align(netdev, bufsz, gfp);
bc7f75fa
AK		 661 if (!skb) {
662 /* Better luck next round */
663 adapter->alloc_rx_buff_failed++;
664 break;
665 }
666
bc7f75fa
AK		 667 buffer_info->skb = skb;
668map_skb:
0be3f55f 669 buffer_info->dma = dma_map_single(&pdev->dev, skb->data,
bc7f75fa 670 adapter->rx_buffer_len,
0be3f55f
NN		 671 DMA_FROM_DEVICE);
672 if (dma_mapping_error(&pdev->dev, buffer_info->dma)) {
af667a29 673 dev_err(&pdev->dev, "Rx DMA map failed\n");
bc7f75fa
AK		 674 adapter->rx_dma_failed++;
675 break;
676 }
677
5f450212
BA		 678 rx_desc = E1000_RX_DESC_EXT(*rx_ring, i);
679 rx_desc->read.buffer_addr = cpu_to_le64(buffer_info->dma);
bc7f75fa 680
50849d79 681 if (unlikely(!(i & (E1000_RX_BUFFER_WRITE - 1)))) {
e921eb1a 682 /* Force memory writes to complete before letting h/w
50849d79
TH		 683 * know there are new descriptors to fetch. (Only
684 * applicable for weak-ordered memory model archs,
685 * such as IA-64).
686 */
687 wmb();
c6e7f51e 688 if (adapter->flags2 & FLAG2_PCIM2PCI_ARBITER_WA)
55aa6985 689 e1000e_update_rdt_wa(rx_ring, i);
c6e7f51e 690 else
c5083cf6 691 writel(i, rx_ring->tail);
50849d79 692 }
bc7f75fa
AK		 693 i++;
694 if (i == rx_ring->count)
695 i = 0;
696 buffer_info = &rx_ring->buffer_info[i];
697 }
698
50849d79 699 rx_ring->next_to_use = i;
bc7f75fa
AK		 700}
701
702/**
703 * e1000_alloc_rx_buffers_ps - Replace used receive buffers; packet split
55aa6985 704 * @rx_ring: Rx descriptor ring
bc7f75fa 705 **/
55aa6985 706static void e1000_alloc_rx_buffers_ps(struct e1000_ring *rx_ring,
c2fed996 707 int cleaned_count, gfp_t gfp)
bc7f75fa 708{
55aa6985 709 struct e1000_adapter *adapter = rx_ring->adapter;
bc7f75fa
AK		 710 struct net_device *netdev = adapter->netdev;
711 struct pci_dev *pdev = adapter->pdev;
712 union e1000_rx_desc_packet_split *rx_desc;
bc7f75fa
AK		 713 struct e1000_buffer *buffer_info;
714 struct e1000_ps_page *ps_page;
715 struct sk_buff *skb;
716 unsigned int i, j;
717
718 i = rx_ring->next_to_use;
719 buffer_info = &rx_ring->buffer_info[i];
720
721 while (cleaned_count--) {
722 rx_desc = E1000_RX_DESC_PS(*rx_ring, i);
723
724 for (j = 0; j < PS_PAGE_BUFFERS; j++) {
47f44e40
AK		 725 ps_page = &buffer_info->ps_pages[j];
726 if (j >= adapter->rx_ps_pages) {
727 /* all unused desc entries get hw null ptr */
af667a29
BA		 728 rx_desc->read.buffer_addr[j + 1] =
729 ~cpu_to_le64(0);
47f44e40
AK		 730 continue;
731 }
732 if (!ps_page->page) {
c2fed996 733 ps_page->page = alloc_page(gfp);
bc7f75fa 734 if (!ps_page->page) {
47f44e40
AK		 735 adapter->alloc_rx_buff_failed++;
736 goto no_buffers;
737 }
0be3f55f
NN		 738 ps_page->dma = dma_map_page(&pdev->dev,
739 ps_page->page,
740 0, PAGE_SIZE,
741 DMA_FROM_DEVICE);
742 if (dma_mapping_error(&pdev->dev,
743 ps_page->dma)) {
47f44e40 744 dev_err(&adapter->pdev->dev,
af667a29 745 "Rx DMA page map failed\n");
47f44e40
AK		 746 adapter->rx_dma_failed++;
747 goto no_buffers;
bc7f75fa 748 }
bc7f75fa 749 }
e921eb1a 750 /* Refresh the desc even if buffer_addrs
47f44e40
AK		 751 * didn't change because each write-back
752 * erases this info.
753 */
af667a29
BA		 754 rx_desc->read.buffer_addr[j + 1] =
755 cpu_to_le64(ps_page->dma);
bc7f75fa
AK		 756 }
757
c2fed996
JK		 758 skb = __netdev_alloc_skb_ip_align(netdev,
759 adapter->rx_ps_bsize0,
760 gfp);
bc7f75fa
AK		 761
762 if (!skb) {
763 adapter->alloc_rx_buff_failed++;
764 break;
765 }
766
bc7f75fa 767 buffer_info->skb = skb;
0be3f55f 768 buffer_info->dma = dma_map_single(&pdev->dev, skb->data,
bc7f75fa 769 adapter->rx_ps_bsize0,
0be3f55f
NN		 770 DMA_FROM_DEVICE);
771 if (dma_mapping_error(&pdev->dev, buffer_info->dma)) {
af667a29 772 dev_err(&pdev->dev, "Rx DMA map failed\n");
bc7f75fa
AK		 773 adapter->rx_dma_failed++;
774 /* cleanup skb */
775 dev_kfree_skb_any(skb);
776 buffer_info->skb = NULL;
777 break;
778 }
779
780 rx_desc->read.buffer_addr[0] = cpu_to_le64(buffer_info->dma);
781
50849d79 782 if (unlikely(!(i & (E1000_RX_BUFFER_WRITE - 1)))) {
e921eb1a 783 /* Force memory writes to complete before letting h/w
50849d79
TH		 784 * know there are new descriptors to fetch. (Only
785 * applicable for weak-ordered memory model archs,
786 * such as IA-64).
787 */
788 wmb();
c6e7f51e 789 if (adapter->flags2 & FLAG2_PCIM2PCI_ARBITER_WA)
55aa6985 790 e1000e_update_rdt_wa(rx_ring, i << 1);
c6e7f51e 791 else
c5083cf6 792 writel(i << 1, rx_ring->tail);
50849d79
TH		 793 }
794
bc7f75fa
AK		 795 i++;
796 if (i == rx_ring->count)
797 i = 0;
798 buffer_info = &rx_ring->buffer_info[i];
799 }
800
801no_buffers:
50849d79 802 rx_ring->next_to_use = i;
bc7f75fa
AK		 803}
804
97ac8cae
BA		 805/**
806 * e1000_alloc_jumbo_rx_buffers - Replace used jumbo receive buffers
55aa6985 807 * @rx_ring: Rx descriptor ring
97ac8cae
BA		 808 * @cleaned_count: number of buffers to allocate this pass
809 **/
810
55aa6985 811static void e1000_alloc_jumbo_rx_buffers(struct e1000_ring *rx_ring,
c2fed996 812 int cleaned_count, gfp_t gfp)
97ac8cae 813{
55aa6985 814 struct e1000_adapter *adapter = rx_ring->adapter;
97ac8cae
BA		 815 struct net_device *netdev = adapter->netdev;
816 struct pci_dev *pdev = adapter->pdev;
5f450212 817 union e1000_rx_desc_extended *rx_desc;
97ac8cae
BA		 818 struct e1000_buffer *buffer_info;
819 struct sk_buff *skb;
820 unsigned int i;
2a2293b9 821 unsigned int bufsz = 256 - 16; /* for skb_reserve */
97ac8cae
BA		 822
823 i = rx_ring->next_to_use;
824 buffer_info = &rx_ring->buffer_info[i];
825
826 while (cleaned_count--) {
827 skb = buffer_info->skb;
828 if (skb) {
829 skb_trim(skb, 0);
830 goto check_page;
831 }
832
c2fed996 833 skb = __netdev_alloc_skb_ip_align(netdev, bufsz, gfp);
97ac8cae
BA		 834 if (unlikely(!skb)) {
835 /* Better luck next round */
836 adapter->alloc_rx_buff_failed++;
837 break;
838 }
839
97ac8cae
BA		 840 buffer_info->skb = skb;
841check_page:
842 /* allocate a new page if necessary */
843 if (!buffer_info->page) {
c2fed996 844 buffer_info->page = alloc_page(gfp);
97ac8cae
BA		 845 if (unlikely(!buffer_info->page)) {
846 adapter->alloc_rx_buff_failed++;
847 break;
848 }
849 }
850
851 if (!buffer_info->dma)
0be3f55f 852 buffer_info->dma = dma_map_page(&pdev->dev,
f0ff4398
BA		 853 buffer_info->page, 0,
854 PAGE_SIZE,
0be3f55f 855 DMA_FROM_DEVICE);
97ac8cae 856
5f450212
BA		 857 rx_desc = E1000_RX_DESC_EXT(*rx_ring, i);
858 rx_desc->read.buffer_addr = cpu_to_le64(buffer_info->dma);
97ac8cae
BA		 859
860 if (unlikely(++i == rx_ring->count))
861 i = 0;
862 buffer_info = &rx_ring->buffer_info[i];
863 }
864
865 if (likely(rx_ring->next_to_use != i)) {
866 rx_ring->next_to_use = i;
867 if (unlikely(i-- == 0))
868 i = (rx_ring->count - 1);
869
870 /* Force memory writes to complete before letting h/w
871 * know there are new descriptors to fetch. (Only
872 * applicable for weak-ordered memory model archs,
e921eb1a
BA		 873 * such as IA-64).
874 */
97ac8cae 875 wmb();
c6e7f51e 876 if (adapter->flags2 & FLAG2_PCIM2PCI_ARBITER_WA)
55aa6985 877 e1000e_update_rdt_wa(rx_ring, i);
c6e7f51e 878 else
c5083cf6 879 writel(i, rx_ring->tail);
97ac8cae
BA		 880 }
881}
882
70495a50
BA		 883static inline void e1000_rx_hash(struct net_device *netdev, __le32 rss,
884 struct sk_buff *skb)
885{
886 if (netdev->features & NETIF_F_RXHASH)
887 skb->rxhash = le32_to_cpu(rss);
888}
889
bc7f75fa 890/**
55aa6985
BA		 891 * e1000_clean_rx_irq - Send received data up the network stack
892 * @rx_ring: Rx descriptor ring
bc7f75fa
AK		 893 *
894 * the return value indicates whether actual cleaning was done, there
895 * is no guarantee that everything was cleaned
896 **/
55aa6985
BA		 897static bool e1000_clean_rx_irq(struct e1000_ring *rx_ring, int *work_done,
898 int work_to_do)
bc7f75fa 899{
55aa6985 900 struct e1000_adapter *adapter = rx_ring->adapter;
bc7f75fa
AK		 901 struct net_device *netdev = adapter->netdev;
902 struct pci_dev *pdev = adapter->pdev;
3bb99fe2 903 struct e1000_hw *hw = &adapter->hw;
5f450212 904 union e1000_rx_desc_extended *rx_desc, *next_rxd;
bc7f75fa 905 struct e1000_buffer *buffer_info, *next_buffer;
5f450212 906 u32 length, staterr;
bc7f75fa
AK		 907 unsigned int i;
908 int cleaned_count = 0;
3db1cd5c 909 bool cleaned = false;
bc7f75fa
AK		 910 unsigned int total_rx_bytes = 0, total_rx_packets = 0;
911
912 i = rx_ring->next_to_clean;
5f450212
BA		 913 rx_desc = E1000_RX_DESC_EXT(*rx_ring, i);
914 staterr = le32_to_cpu(rx_desc->wb.upper.status_error);
bc7f75fa
AK		 915 buffer_info = &rx_ring->buffer_info[i];
916
5f450212 917 while (staterr & E1000_RXD_STAT_DD) {
bc7f75fa 918 struct sk_buff *skb;
bc7f75fa
AK		 919
920 if (*work_done >= work_to_do)
921 break;
922 (*work_done)++;
2d0bb1c1 923 rmb(); /* read descriptor and rx_buffer_info after status DD */
bc7f75fa 924
bc7f75fa
AK		 925 skb = buffer_info->skb;
926 buffer_info->skb = NULL;
927
928 prefetch(skb->data - NET_IP_ALIGN);
929
930 i++;
931 if (i == rx_ring->count)
932 i = 0;
5f450212 933 next_rxd = E1000_RX_DESC_EXT(*rx_ring, i);
bc7f75fa
AK		 934 prefetch(next_rxd);
935
936 next_buffer = &rx_ring->buffer_info[i];
937
3db1cd5c 938 cleaned = true;
bc7f75fa 939 cleaned_count++;
0be3f55f 940 dma_unmap_single(&pdev->dev,
bc7f75fa
AK		 941 buffer_info->dma,
942 adapter->rx_buffer_len,
0be3f55f 943 DMA_FROM_DEVICE);
bc7f75fa
AK		 944 buffer_info->dma = 0;
945
5f450212 946 length = le16_to_cpu(rx_desc->wb.upper.length);
bc7f75fa 947
e921eb1a 948 /* !EOP means multiple descriptors were used to store a single
b94b5028
JB		 949 * packet, if that's the case we need to toss it. In fact, we
950 * need to toss every packet with the EOP bit clear and the
951 * next frame that _does_ have the EOP bit set, as it is by
952 * definition only a frame fragment
953 */
5f450212 954 if (unlikely(!(staterr & E1000_RXD_STAT_EOP)))
b94b5028
JB		 955 adapter->flags2 |= FLAG2_IS_DISCARDING;
956
957 if (adapter->flags2 & FLAG2_IS_DISCARDING) {
bc7f75fa 958 /* All receives must fit into a single buffer */
3bb99fe2 959 e_dbg("Receive packet consumed multiple buffers\n");
bc7f75fa
AK		 960 /* recycle */
961 buffer_info->skb = skb;
5f450212 962 if (staterr & E1000_RXD_STAT_EOP)
b94b5028 963 adapter->flags2 &= ~FLAG2_IS_DISCARDING;
bc7f75fa
AK		 964 goto next_desc;
965 }
966
cf955e6c
BG		 967 if (unlikely((staterr & E1000_RXDEXT_ERR_FRAME_ERR_MASK) &&
968 !(netdev->features & NETIF_F_RXALL))) {
bc7f75fa
AK		 969 /* recycle */
970 buffer_info->skb = skb;
971 goto next_desc;
972 }
973
eb7c3adb 974 /* adjust length to remove Ethernet CRC */
0184039a
BG		 975 if (!(adapter->flags2 & FLAG2_CRC_STRIPPING)) {
976 /* If configured to store CRC, don't subtract FCS,
977 * but keep the FCS bytes out of the total_rx_bytes
978 * counter
979 */
980 if (netdev->features & NETIF_F_RXFCS)
981 total_rx_bytes -= 4;
982 else
983 length -= 4;
984 }
eb7c3adb 985
bc7f75fa
AK		 986 total_rx_bytes += length;
987 total_rx_packets++;
988
e921eb1a 989 /* code added for copybreak, this should improve
bc7f75fa 990 * performance for small packets with large amounts
ad68076e
BA		 991 * of reassembly being done in the stack
992 */
bc7f75fa
AK		 993 if (length < copybreak) {
994 struct sk_buff *new_skb =
89d71a66 995 netdev_alloc_skb_ip_align(netdev, length);
bc7f75fa 996 if (new_skb) {
808ff676
BA		 997 skb_copy_to_linear_data_offset(new_skb,
998 -NET_IP_ALIGN,
999 (skb->data -
1000 NET_IP_ALIGN),
1001 (length +
1002 NET_IP_ALIGN));
bc7f75fa
AK		 1003 /* save the skb in buffer_info as good */
1004 buffer_info->skb = skb;
1005 skb = new_skb;
1006 }
1007 /* else just continue with the old one */
1008 }
1009 /* end copybreak code */
1010 skb_put(skb, length);
1011
1012 /* Receive Checksum Offload */
2e1706f2 1013 e1000_rx_checksum(adapter, staterr, skb);
bc7f75fa 1014
70495a50
BA		 1015 e1000_rx_hash(netdev, rx_desc->wb.lower.hi_dword.rss, skb);
1016
5f450212
BA		 1017 e1000_receive_skb(adapter, netdev, skb, staterr,
1018 rx_desc->wb.upper.vlan);
bc7f75fa
AK		 1019
1020next_desc:
5f450212 1021 rx_desc->wb.upper.status_error &= cpu_to_le32(~0xFF);
bc7f75fa
AK		 1022
1023 /* return some buffers to hardware, one at a time is too slow */
1024 if (cleaned_count >= E1000_RX_BUFFER_WRITE) {
55aa6985 1025 adapter->alloc_rx_buf(rx_ring, cleaned_count,
c2fed996 1026 GFP_ATOMIC);
bc7f75fa
AK		 1027 cleaned_count = 0;
1028 }
1029
1030 /* use prefetched values */
1031 rx_desc = next_rxd;
1032 buffer_info = next_buffer;
5f450212
BA		 1033
1034 staterr = le32_to_cpu(rx_desc->wb.upper.status_error);
bc7f75fa
AK		 1035 }
1036 rx_ring->next_to_clean = i;
1037
1038 cleaned_count = e1000_desc_unused(rx_ring);
1039 if (cleaned_count)
55aa6985 1040 adapter->alloc_rx_buf(rx_ring, cleaned_count, GFP_ATOMIC);
bc7f75fa 1041
bc7f75fa 1042 adapter->total_rx_bytes += total_rx_bytes;
7c25769f 1043 adapter->total_rx_packets += total_rx_packets;
bc7f75fa
AK		 1044 return cleaned;
1045}
1046
55aa6985
BA		 1047static void e1000_put_txbuf(struct e1000_ring *tx_ring,
1048 struct e1000_buffer *buffer_info)
bc7f75fa 1049{
55aa6985
BA		 1050 struct e1000_adapter *adapter = tx_ring->adapter;
1051
03b1320d
AD		 1052 if (buffer_info->dma) {
1053 if (buffer_info->mapped_as_page)
0be3f55f
NN		 1054 dma_unmap_page(&adapter->pdev->dev, buffer_info->dma,
1055 buffer_info->length, DMA_TO_DEVICE);
03b1320d 1056 else
0be3f55f
NN		 1057 dma_unmap_single(&adapter->pdev->dev, buffer_info->dma,
1058 buffer_info->length, DMA_TO_DEVICE);
03b1320d
AD		 1059 buffer_info->dma = 0;
1060 }
bc7f75fa
AK		 1061 if (buffer_info->skb) {
1062 dev_kfree_skb_any(buffer_info->skb);
1063 buffer_info->skb = NULL;
1064 }
1b7719c4 1065 buffer_info->time_stamp = 0;
bc7f75fa
AK		 1066}
1067
41cec6f1 1068static void e1000_print_hw_hang(struct work_struct *work)
bc7f75fa 1069{
41cec6f1 1070 struct e1000_adapter *adapter = container_of(work,
f0ff4398
BA		 1071 struct e1000_adapter,
1072 print_hang_task);
09357b00 1073 struct net_device *netdev = adapter->netdev;
bc7f75fa
AK		 1074 struct e1000_ring *tx_ring = adapter->tx_ring;
1075 unsigned int i = tx_ring->next_to_clean;
1076 unsigned int eop = tx_ring->buffer_info[i].next_to_watch;
1077 struct e1000_tx_desc *eop_desc = E1000_TX_DESC(*tx_ring, eop);
41cec6f1
BA		 1078 struct e1000_hw *hw = &adapter->hw;
1079 u16 phy_status, phy_1000t_status, phy_ext_status;
1080 u16 pci_status;
1081
615b32af
JB		 1082 if (test_bit(__E1000_DOWN, &adapter->state))
1083 return;
1084
09357b00
JK		 1085 if (!adapter->tx_hang_recheck &&
1086 (adapter->flags2 & FLAG2_DMA_BURST)) {
e921eb1a 1087 /* May be block on write-back, flush and detect again
09357b00
JK		 1088 * flush pending descriptor writebacks to memory
1089 */
1090 ew32(TIDV, adapter->tx_int_delay | E1000_TIDV_FPD);
1091 /* execute the writes immediately */
1092 e1e_flush();
e921eb1a 1093 /* Due to rare timing issues, write to TIDV again to ensure
bf03085f
MV		 1094 * the write is successful
1095 */
1096 ew32(TIDV, adapter->tx_int_delay | E1000_TIDV_FPD);
1097 /* execute the writes immediately */
1098 e1e_flush();
09357b00
JK		 1099 adapter->tx_hang_recheck = true;
1100 return;
1101 }
1102 /* Real hang detected */
1103 adapter->tx_hang_recheck = false;
1104 netif_stop_queue(netdev);
1105
c2ade1a4
BA		 1106 e1e_rphy(hw, MII_BMSR, &phy_status);
1107 e1e_rphy(hw, MII_STAT1000, &phy_1000t_status);
1108 e1e_rphy(hw, MII_ESTATUS, &phy_ext_status);
bc7f75fa 1109
41cec6f1
BA		 1110 pci_read_config_word(adapter->pdev, PCI_STATUS, &pci_status);
1111
1112 /* detected Hardware unit hang */
1113 e_err("Detected Hardware Unit Hang:\n"
44defeb3
JK		 1114 " TDH <%x>\n"
1115 " TDT <%x>\n"
1116 " next_to_use <%x>\n"
1117 " next_to_clean <%x>\n"
1118 "buffer_info[next_to_clean]:\n"
1119 " time_stamp <%lx>\n"
1120 " next_to_watch <%x>\n"
1121 " jiffies <%lx>\n"
41cec6f1
BA		 1122 " next_to_watch.status <%x>\n"
1123 "MAC Status <%x>\n"
1124 "PHY Status <%x>\n"
1125 "PHY 1000BASE-T Status <%x>\n"
1126 "PHY Extended Status <%x>\n"
1127 "PCI Status <%x>\n",
c5083cf6
BA		 1128 readl(tx_ring->head),
1129 readl(tx_ring->tail),
44defeb3
JK		 1130 tx_ring->next_to_use,
1131 tx_ring->next_to_clean,
1132 tx_ring->buffer_info[eop].time_stamp,
1133 eop,
1134 jiffies,
41cec6f1
BA		 1135 eop_desc->upper.fields.status,
1136 er32(STATUS),
1137 phy_status,
1138 phy_1000t_status,
1139 phy_ext_status,
1140 pci_status);
7c0427ee
BA		 1141
1142 /* Suggest workaround for known h/w issue */
1143 if ((hw->mac.type == e1000_pchlan) && (er32(CTRL) & E1000_CTRL_TFCE))
1144 e_err("Try turning off Tx pause (flow control) via ethtool\n");
bc7f75fa
AK		 1145}
1146
b67e1913
BA		 1147/**
1148 * e1000e_tx_hwtstamp_work - check for Tx time stamp
1149 * @work: pointer to work struct
1150 *
1151 * This work function polls the TSYNCTXCTL valid bit to determine when a
1152 * timestamp has been taken for the current stored skb. The timestamp must
1153 * be for this skb because only one such packet is allowed in the queue.
1154 */
1155static void e1000e_tx_hwtstamp_work(struct work_struct *work)
1156{
1157 struct e1000_adapter *adapter = container_of(work, struct e1000_adapter,
1158 tx_hwtstamp_work);
1159 struct e1000_hw *hw = &adapter->hw;
1160
1161 if (!adapter->tx_hwtstamp_skb)
1162 return;
1163
1164 if (er32(TSYNCTXCTL) & E1000_TSYNCTXCTL_VALID) {
1165 struct skb_shared_hwtstamps shhwtstamps;
1166 u64 txstmp;
1167
1168 txstmp = er32(TXSTMPL);
1169 txstmp |= (u64)er32(TXSTMPH) << 32;
1170
1171 e1000e_systim_to_hwtstamp(adapter, &shhwtstamps, txstmp);
1172
1173 skb_tstamp_tx(adapter->tx_hwtstamp_skb, &shhwtstamps);
1174 dev_kfree_skb_any(adapter->tx_hwtstamp_skb);
1175 adapter->tx_hwtstamp_skb = NULL;
1176 } else {
1177 /* reschedule to check later */
1178 schedule_work(&adapter->tx_hwtstamp_work);
1179 }
1180}
1181
bc7f75fa
AK		 1182/**
1183 * e1000_clean_tx_irq - Reclaim resources after transmit completes
55aa6985 1184 * @tx_ring: Tx descriptor ring
bc7f75fa
AK		 1185 *
1186 * the return value indicates whether actual cleaning was done, there
1187 * is no guarantee that everything was cleaned
1188 **/
55aa6985 1189static bool e1000_clean_tx_irq(struct e1000_ring *tx_ring)
bc7f75fa 1190{
55aa6985 1191 struct e1000_adapter *adapter = tx_ring->adapter;
bc7f75fa
AK		 1192 struct net_device *netdev = adapter->netdev;
1193 struct e1000_hw *hw = &adapter->hw;
bc7f75fa
AK		 1194 struct e1000_tx_desc *tx_desc, *eop_desc;
1195 struct e1000_buffer *buffer_info;
1196 unsigned int i, eop;
1197 unsigned int count = 0;
bc7f75fa 1198 unsigned int total_tx_bytes = 0, total_tx_packets = 0;
3f0cfa3b 1199 unsigned int bytes_compl = 0, pkts_compl = 0;
bc7f75fa
AK		 1200
1201 i = tx_ring->next_to_clean;
1202 eop = tx_ring->buffer_info[i].next_to_watch;
1203 eop_desc = E1000_TX_DESC(*tx_ring, eop);
1204
12d04a3c
AD		 1205 while ((eop_desc->upper.data & cpu_to_le32(E1000_TXD_STAT_DD)) &&
1206 (count < tx_ring->count)) {
a86043c2 1207 bool cleaned = false;
2d0bb1c1 1208 rmb(); /* read buffer_info after eop_desc */
a86043c2 1209 for (; !cleaned; count++) {
bc7f75fa
AK		 1210 tx_desc = E1000_TX_DESC(*tx_ring, i);
1211 buffer_info = &tx_ring->buffer_info[i];
1212 cleaned = (i == eop);
1213
1214 if (cleaned) {
9ed318d5
TH		 1215 total_tx_packets += buffer_info->segs;
1216 total_tx_bytes += buffer_info->bytecount;
3f0cfa3b
TH		 1217 if (buffer_info->skb) {
1218 bytes_compl += buffer_info->skb->len;
1219 pkts_compl++;
1220 }
bc7f75fa
AK		 1221 }
1222
55aa6985 1223 e1000_put_txbuf(tx_ring, buffer_info);
bc7f75fa
AK		 1224 tx_desc->upper.data = 0;
1225
1226 i++;
1227 if (i == tx_ring->count)
1228 i = 0;
1229 }
1230
dac87619
TL		 1231 if (i == tx_ring->next_to_use)
1232 break;
bc7f75fa
AK		 1233 eop = tx_ring->buffer_info[i].next_to_watch;
1234 eop_desc = E1000_TX_DESC(*tx_ring, eop);
bc7f75fa
AK		 1235 }
1236
1237 tx_ring->next_to_clean = i;
1238
3f0cfa3b
TH		 1239 netdev_completed_queue(netdev, pkts_compl, bytes_compl);
1240
bc7f75fa 1241#define TX_WAKE_THRESHOLD 32
a86043c2
JB		 1242 if (count && netif_carrier_ok(netdev) &&
1243 e1000_desc_unused(tx_ring) >= TX_WAKE_THRESHOLD) {
bc7f75fa
AK		 1244 /* Make sure that anybody stopping the queue after this
1245 * sees the new next_to_clean.
1246 */
1247 smp_mb();
1248
1249 if (netif_queue_stopped(netdev) &&
1250 !(test_bit(__E1000_DOWN, &adapter->state))) {
1251 netif_wake_queue(netdev);
1252 ++adapter->restart_queue;
1253 }
1254 }
1255
1256 if (adapter->detect_tx_hung) {
e921eb1a 1257 /* Detect a transmit hang in hardware, this serializes the
41cec6f1
BA		 1258 * check with the clearing of time_stamp and movement of i
1259 */
3db1cd5c 1260 adapter->detect_tx_hung = false;
12d04a3c
AD		 1261 if (tx_ring->buffer_info[i].time_stamp &&
1262 time_after(jiffies, tx_ring->buffer_info[i].time_stamp
8e95a202 1263 + (adapter->tx_timeout_factor * HZ)) &&
09357b00 1264 !(er32(STATUS) & E1000_STATUS_TXOFF))
41cec6f1 1265 schedule_work(&adapter->print_hang_task);
09357b00
JK		 1266 else
1267 adapter->tx_hang_recheck = false;
bc7f75fa
AK		 1268 }
1269 adapter->total_tx_bytes += total_tx_bytes;
1270 adapter->total_tx_packets += total_tx_packets;
807540ba 1271 return count < tx_ring->count;
bc7f75fa
AK		 1272}
1273
bc7f75fa
AK		 1274/**
1275 * e1000_clean_rx_irq_ps - Send received data up the network stack; packet split
55aa6985 1276 * @rx_ring: Rx descriptor ring
bc7f75fa
AK		 1277 *
1278 * the return value indicates whether actual cleaning was done, there
1279 * is no guarantee that everything was cleaned
1280 **/
55aa6985
BA		 1281static bool e1000_clean_rx_irq_ps(struct e1000_ring *rx_ring, int *work_done,
1282 int work_to_do)
bc7f75fa 1283{
55aa6985 1284 struct e1000_adapter *adapter = rx_ring->adapter;
3bb99fe2 1285 struct e1000_hw *hw = &adapter->hw;
bc7f75fa
AK		 1286 union e1000_rx_desc_packet_split *rx_desc, *next_rxd;
1287 struct net_device *netdev = adapter->netdev;
1288 struct pci_dev *pdev = adapter->pdev;
bc7f75fa
AK		 1289 struct e1000_buffer *buffer_info, *next_buffer;
1290 struct e1000_ps_page *ps_page;
1291 struct sk_buff *skb;
1292 unsigned int i, j;
1293 u32 length, staterr;
1294 int cleaned_count = 0;
3db1cd5c 1295 bool cleaned = false;
bc7f75fa
AK		 1296 unsigned int total_rx_bytes = 0, total_rx_packets = 0;
1297
1298 i = rx_ring->next_to_clean;
1299 rx_desc = E1000_RX_DESC_PS(*rx_ring, i);
1300 staterr = le32_to_cpu(rx_desc->wb.middle.status_error);
1301 buffer_info = &rx_ring->buffer_info[i];
1302
1303 while (staterr & E1000_RXD_STAT_DD) {
1304 if (*work_done >= work_to_do)
1305 break;
1306 (*work_done)++;
1307 skb = buffer_info->skb;
2d0bb1c1 1308 rmb(); /* read descriptor and rx_buffer_info after status DD */
bc7f75fa
AK		 1309
1310 /* in the packet split case this is header only */
1311 prefetch(skb->data - NET_IP_ALIGN);
1312
1313 i++;
1314 if (i == rx_ring->count)
1315 i = 0;
1316 next_rxd = E1000_RX_DESC_PS(*rx_ring, i);
1317 prefetch(next_rxd);
1318
1319 next_buffer = &rx_ring->buffer_info[i];
1320
3db1cd5c 1321 cleaned = true;
bc7f75fa 1322 cleaned_count++;
0be3f55f 1323 dma_unmap_single(&pdev->dev, buffer_info->dma,
af667a29 1324 adapter->rx_ps_bsize0, DMA_FROM_DEVICE);
bc7f75fa
AK		 1325 buffer_info->dma = 0;
1326
af667a29 1327 /* see !EOP comment in other Rx routine */
b94b5028
JB		 1328 if (!(staterr & E1000_RXD_STAT_EOP))
1329 adapter->flags2 |= FLAG2_IS_DISCARDING;
1330
1331 if (adapter->flags2 & FLAG2_IS_DISCARDING) {
ef456f85 1332 e_dbg("Packet Split buffers didn't pick up the full packet\n");
bc7f75fa 1333 dev_kfree_skb_irq(skb);
b94b5028
JB		 1334 if (staterr & E1000_RXD_STAT_EOP)
1335 adapter->flags2 &= ~FLAG2_IS_DISCARDING;
bc7f75fa
AK		 1336 goto next_desc;
1337 }
1338
cf955e6c
BG		 1339 if (unlikely((staterr & E1000_RXDEXT_ERR_FRAME_ERR_MASK) &&
1340 !(netdev->features & NETIF_F_RXALL))) {
bc7f75fa
AK		 1341 dev_kfree_skb_irq(skb);
1342 goto next_desc;
1343 }
1344
1345 length = le16_to_cpu(rx_desc->wb.middle.length0);
1346
1347 if (!length) {
ef456f85 1348 e_dbg("Last part of the packet spanning multiple descriptors\n");
bc7f75fa
AK		 1349 dev_kfree_skb_irq(skb);
1350 goto next_desc;
1351 }
1352
1353 /* Good Receive */
1354 skb_put(skb, length);
1355
1356 {
e921eb1a 1357 /* this looks ugly, but it seems compiler issues make
0e15df49
BA		 1358 * it more efficient than reusing j
1359 */
1360 int l1 = le16_to_cpu(rx_desc->wb.upper.length[0]);
bc7f75fa 1361
e921eb1a 1362 /* page alloc/put takes too long and effects small
0e15df49
BA		 1363 * packet throughput, so unsplit small packets and
1364 * save the alloc/put only valid in softirq (napi)
1365 * context to call kmap_*
ad68076e 1366 */
0e15df49
BA		 1367 if (l1 && (l1 <= copybreak) &&
1368 ((length + l1) <= adapter->rx_ps_bsize0)) {
1369 u8 *vaddr;
1370
1371 ps_page = &buffer_info->ps_pages[0];
1372
e921eb1a 1373 /* there is no documentation about how to call
0e15df49
BA		 1374 * kmap_atomic, so we can't hold the mapping
1375 * very long
1376 */
1377 dma_sync_single_for_cpu(&pdev->dev,
1378 ps_page->dma,
1379 PAGE_SIZE,
1380 DMA_FROM_DEVICE);
9f393834 1381 vaddr = kmap_atomic(ps_page->page);
0e15df49 1382 memcpy(skb_tail_pointer(skb), vaddr, l1);
9f393834 1383 kunmap_atomic(vaddr);
0e15df49
BA		 1384 dma_sync_single_for_device(&pdev->dev,
1385 ps_page->dma,
1386 PAGE_SIZE,
1387 DMA_FROM_DEVICE);
1388
1389 /* remove the CRC */
0184039a
BG		 1390 if (!(adapter->flags2 & FLAG2_CRC_STRIPPING)) {
1391 if (!(netdev->features & NETIF_F_RXFCS))
1392 l1 -= 4;
1393 }
0e15df49
BA		 1394
1395 skb_put(skb, l1);
1396 goto copydone;
1397 } /* if */
bc7f75fa
AK		 1398 }
1399
1400 for (j = 0; j < PS_PAGE_BUFFERS; j++) {
1401 length = le16_to_cpu(rx_desc->wb.upper.length[j]);
1402 if (!length)
1403 break;
1404
47f44e40 1405 ps_page = &buffer_info->ps_pages[j];
0be3f55f
NN		 1406 dma_unmap_page(&pdev->dev, ps_page->dma, PAGE_SIZE,
1407 DMA_FROM_DEVICE);
bc7f75fa
AK		 1408 ps_page->dma = 0;
1409 skb_fill_page_desc(skb, j, ps_page->page, 0, length);
1410 ps_page->page = NULL;
1411 skb->len += length;
1412 skb->data_len += length;
98a045d7 1413 skb->truesize += PAGE_SIZE;
bc7f75fa
AK		 1414 }
1415
eb7c3adb
JK		 1416 /* strip the ethernet crc, problem is we're using pages now so
1417 * this whole operation can get a little cpu intensive
1418 */
0184039a
BG		 1419 if (!(adapter->flags2 & FLAG2_CRC_STRIPPING)) {
1420 if (!(netdev->features & NETIF_F_RXFCS))
1421 pskb_trim(skb, skb->len - 4);
1422 }
eb7c3adb 1423
bc7f75fa
AK		 1424copydone:
1425 total_rx_bytes += skb->len;
1426 total_rx_packets++;
1427
2e1706f2 1428 e1000_rx_checksum(adapter, staterr, skb);
bc7f75fa 1429
70495a50
BA		 1430 e1000_rx_hash(netdev, rx_desc->wb.lower.hi_dword.rss, skb);
1431
bc7f75fa
AK		 1432 if (rx_desc->wb.upper.header_status &
1433 cpu_to_le16(E1000_RXDPS_HDRSTAT_HDRSP))
1434 adapter->rx_hdr_split++;
1435
b67e1913
BA		 1436 e1000_receive_skb(adapter, netdev, skb, staterr,
1437 rx_desc->wb.middle.vlan);
bc7f75fa
AK		 1438
1439next_desc:
1440 rx_desc->wb.middle.status_error &= cpu_to_le32(~0xFF);
1441 buffer_info->skb = NULL;
1442
1443 /* return some buffers to hardware, one at a time is too slow */
1444 if (cleaned_count >= E1000_RX_BUFFER_WRITE) {
55aa6985 1445 adapter->alloc_rx_buf(rx_ring, cleaned_count,
c2fed996 1446 GFP_ATOMIC);
bc7f75fa
AK		 1447 cleaned_count = 0;
1448 }
1449
1450 /* use prefetched values */
1451 rx_desc = next_rxd;
1452 buffer_info = next_buffer;
1453
1454 staterr = le32_to_cpu(rx_desc->wb.middle.status_error);
1455 }
1456 rx_ring->next_to_clean = i;
1457
1458 cleaned_count = e1000_desc_unused(rx_ring);
1459 if (cleaned_count)
55aa6985 1460 adapter->alloc_rx_buf(rx_ring, cleaned_count, GFP_ATOMIC);
bc7f75fa 1461
bc7f75fa 1462 adapter->total_rx_bytes += total_rx_bytes;
7c25769f 1463 adapter->total_rx_packets += total_rx_packets;
bc7f75fa
AK		 1464 return cleaned;
1465}
1466
97ac8cae
BA		 1467/**
1468 * e1000_consume_page - helper function
1469 **/
1470static void e1000_consume_page(struct e1000_buffer *bi, struct sk_buff *skb,
1471 u16 length)
1472{
1473 bi->page = NULL;
1474 skb->len += length;
1475 skb->data_len += length;
98a045d7 1476 skb->truesize += PAGE_SIZE;
97ac8cae
BA		 1477}
1478
1479/**
1480 * e1000_clean_jumbo_rx_irq - Send received data up the network stack; legacy
1481 * @adapter: board private structure
1482 *
1483 * the return value indicates whether actual cleaning was done, there
1484 * is no guarantee that everything was cleaned
1485 **/
55aa6985
BA		 1486static bool e1000_clean_jumbo_rx_irq(struct e1000_ring *rx_ring, int *work_done,
1487 int work_to_do)
97ac8cae 1488{
55aa6985 1489 struct e1000_adapter *adapter = rx_ring->adapter;
97ac8cae
BA		 1490 struct net_device *netdev = adapter->netdev;
1491 struct pci_dev *pdev = adapter->pdev;
5f450212 1492 union e1000_rx_desc_extended *rx_desc, *next_rxd;
97ac8cae 1493 struct e1000_buffer *buffer_info, *next_buffer;
5f450212 1494 u32 length, staterr;
97ac8cae
BA		 1495 unsigned int i;
1496 int cleaned_count = 0;
1497 bool cleaned = false;
1498 unsigned int total_rx_bytes=0, total_rx_packets=0;
1499
1500 i = rx_ring->next_to_clean;
5f450212
BA		 1501 rx_desc = E1000_RX_DESC_EXT(*rx_ring, i);
1502 staterr = le32_to_cpu(rx_desc->wb.upper.status_error);
97ac8cae
BA		 1503 buffer_info = &rx_ring->buffer_info[i];
1504
5f450212 1505 while (staterr & E1000_RXD_STAT_DD) {
97ac8cae 1506 struct sk_buff *skb;
97ac8cae
BA		 1507
1508 if (*work_done >= work_to_do)
1509 break;
1510 (*work_done)++;
2d0bb1c1 1511 rmb(); /* read descriptor and rx_buffer_info after status DD */
97ac8cae 1512
97ac8cae
BA		 1513 skb = buffer_info->skb;
1514 buffer_info->skb = NULL;
1515
1516 ++i;
1517 if (i == rx_ring->count)
1518 i = 0;
5f450212 1519 next_rxd = E1000_RX_DESC_EXT(*rx_ring, i);
97ac8cae
BA		 1520 prefetch(next_rxd);
1521
1522 next_buffer = &rx_ring->buffer_info[i];
1523
1524 cleaned = true;
1525 cleaned_count++;
0be3f55f
NN		 1526 dma_unmap_page(&pdev->dev, buffer_info->dma, PAGE_SIZE,
1527 DMA_FROM_DEVICE);
97ac8cae
BA		 1528 buffer_info->dma = 0;
1529
5f450212 1530 length = le16_to_cpu(rx_desc->wb.upper.length);
97ac8cae
BA		 1531
1532 /* errors is only valid for DD + EOP descriptors */
5f450212 1533 if (unlikely((staterr & E1000_RXD_STAT_EOP) &&
cf955e6c
BG		 1534 ((staterr & E1000_RXDEXT_ERR_FRAME_ERR_MASK) &&
1535 !(netdev->features & NETIF_F_RXALL)))) {
5f450212
BA		 1536 /* recycle both page and skb */
1537 buffer_info->skb = skb;
1538 /* an error means any chain goes out the window too */
1539 if (rx_ring->rx_skb_top)
1540 dev_kfree_skb_irq(rx_ring->rx_skb_top);
1541 rx_ring->rx_skb_top = NULL;
1542 goto next_desc;
97ac8cae
BA		 1543 }
1544
f0f1a172 1545#define rxtop (rx_ring->rx_skb_top)
5f450212 1546 if (!(staterr & E1000_RXD_STAT_EOP)) {
97ac8cae
BA		 1547 /* this descriptor is only the beginning (or middle) */
1548 if (!rxtop) {
1549 /* this is the beginning of a chain */
1550 rxtop = skb;
1551 skb_fill_page_desc(rxtop, 0, buffer_info->page,
f0ff4398 1552 0, length);
97ac8cae
BA		 1553 } else {
1554 /* this is the middle of a chain */
1555 skb_fill_page_desc(rxtop,
1556 skb_shinfo(rxtop)->nr_frags,
1557 buffer_info->page, 0, length);
1558 /* re-use the skb, only consumed the page */
1559 buffer_info->skb = skb;
1560 }
1561 e1000_consume_page(buffer_info, rxtop, length);
1562 goto next_desc;
1563 } else {
1564 if (rxtop) {
1565 /* end of the chain */
1566 skb_fill_page_desc(rxtop,
1567 skb_shinfo(rxtop)->nr_frags,
1568 buffer_info->page, 0, length);
1569 /* re-use the current skb, we only consumed the
e921eb1a
BA		 1570 * page
1571 */
97ac8cae
BA		 1572 buffer_info->skb = skb;
1573 skb = rxtop;
1574 rxtop = NULL;
1575 e1000_consume_page(buffer_info, skb, length);
1576 } else {
1577 /* no chain, got EOP, this buf is the packet
e921eb1a
BA		 1578 * copybreak to save the put_page/alloc_page
1579 */
97ac8cae
BA		 1580 if (length <= copybreak &&
1581 skb_tailroom(skb) >= length) {
1582 u8 *vaddr;
4679026d 1583 vaddr = kmap_atomic(buffer_info->page);
97ac8cae
BA		 1584 memcpy(skb_tail_pointer(skb), vaddr,
1585 length);
4679026d 1586 kunmap_atomic(vaddr);
97ac8cae 1587 /* re-use the page, so don't erase
e921eb1a
BA		 1588 * buffer_info->page
1589 */
97ac8cae
BA		 1590 skb_put(skb, length);
1591 } else {
1592 skb_fill_page_desc(skb, 0,
f0ff4398
BA		 1593 buffer_info->page, 0,
1594 length);
97ac8cae 1595 e1000_consume_page(buffer_info, skb,
f0ff4398 1596 length);
97ac8cae
BA		 1597 }
1598 }
1599 }
1600
2e1706f2
BA		 1601 /* Receive Checksum Offload */
1602 e1000_rx_checksum(adapter, staterr, skb);
97ac8cae 1603
70495a50
BA		 1604 e1000_rx_hash(netdev, rx_desc->wb.lower.hi_dword.rss, skb);
1605
97ac8cae
BA		 1606 /* probably a little skewed due to removing CRC */
1607 total_rx_bytes += skb->len;
1608 total_rx_packets++;
1609
1610 /* eth type trans needs skb->data to point to something */
1611 if (!pskb_may_pull(skb, ETH_HLEN)) {
44defeb3 1612 e_err("pskb_may_pull failed.\n");
ef5ab89c 1613 dev_kfree_skb_irq(skb);
97ac8cae
BA		 1614 goto next_desc;
1615 }
1616
5f450212
BA		 1617 e1000_receive_skb(adapter, netdev, skb, staterr,
1618 rx_desc->wb.upper.vlan);
97ac8cae
BA		 1619
1620next_desc:
5f450212 1621 rx_desc->wb.upper.status_error &= cpu_to_le32(~0xFF);
97ac8cae
BA		 1622
1623 /* return some buffers to hardware, one at a time is too slow */
1624 if (unlikely(cleaned_count >= E1000_RX_BUFFER_WRITE)) {
55aa6985 1625 adapter->alloc_rx_buf(rx_ring, cleaned_count,
c2fed996 1626 GFP_ATOMIC);
97ac8cae
BA		 1627 cleaned_count = 0;
1628 }
1629
1630 /* use prefetched values */
1631 rx_desc = next_rxd;
1632 buffer_info = next_buffer;
5f450212
BA		 1633
1634 staterr = le32_to_cpu(rx_desc->wb.upper.status_error);
97ac8cae
BA		 1635 }
1636 rx_ring->next_to_clean = i;
1637
1638 cleaned_count = e1000_desc_unused(rx_ring);
1639 if (cleaned_count)
55aa6985 1640 adapter->alloc_rx_buf(rx_ring, cleaned_count, GFP_ATOMIC);
97ac8cae
BA		 1641
1642 adapter->total_rx_bytes += total_rx_bytes;
1643 adapter->total_rx_packets += total_rx_packets;
97ac8cae
BA		 1644 return cleaned;
1645}
1646
bc7f75fa
AK		 1647/**
1648 * e1000_clean_rx_ring - Free Rx Buffers per Queue
55aa6985 1649 * @rx_ring: Rx descriptor ring
bc7f75fa 1650 **/
55aa6985 1651static void e1000_clean_rx_ring(struct e1000_ring *rx_ring)
bc7f75fa 1652{
55aa6985 1653 struct e1000_adapter *adapter = rx_ring->adapter;
bc7f75fa
AK		 1654 struct e1000_buffer *buffer_info;
1655 struct e1000_ps_page *ps_page;
1656 struct pci_dev *pdev = adapter->pdev;
bc7f75fa
AK		 1657 unsigned int i, j;
1658
1659 /* Free all the Rx ring sk_buffs */
1660 for (i = 0; i < rx_ring->count; i++) {
1661 buffer_info = &rx_ring->buffer_info[i];
1662 if (buffer_info->dma) {
1663 if (adapter->clean_rx == e1000_clean_rx_irq)
0be3f55f 1664 dma_unmap_single(&pdev->dev, buffer_info->dma,
bc7f75fa 1665 adapter->rx_buffer_len,
0be3f55f 1666 DMA_FROM_DEVICE);
97ac8cae 1667 else if (adapter->clean_rx == e1000_clean_jumbo_rx_irq)
0be3f55f 1668 dma_unmap_page(&pdev->dev, buffer_info->dma,
f0ff4398 1669 PAGE_SIZE, DMA_FROM_DEVICE);
bc7f75fa 1670 else if (adapter->clean_rx == e1000_clean_rx_irq_ps)
0be3f55f 1671 dma_unmap_single(&pdev->dev, buffer_info->dma,
bc7f75fa 1672 adapter->rx_ps_bsize0,
0be3f55f 1673 DMA_FROM_DEVICE);
bc7f75fa
AK		 1674 buffer_info->dma = 0;
1675 }
1676
97ac8cae
BA		 1677 if (buffer_info->page) {
1678 put_page(buffer_info->page);
1679 buffer_info->page = NULL;
1680 }
1681
bc7f75fa
AK		 1682 if (buffer_info->skb) {
1683 dev_kfree_skb(buffer_info->skb);
1684 buffer_info->skb = NULL;
1685 }
1686
1687 for (j = 0; j < PS_PAGE_BUFFERS; j++) {
47f44e40 1688 ps_page = &buffer_info->ps_pages[j];
bc7f75fa
AK		 1689 if (!ps_page->page)
1690 break;
0be3f55f
NN		 1691 dma_unmap_page(&pdev->dev, ps_page->dma, PAGE_SIZE,
1692 DMA_FROM_DEVICE);
bc7f75fa
AK		 1693 ps_page->dma = 0;
1694 put_page(ps_page->page);
1695 ps_page->page = NULL;
1696 }
1697 }
1698
1699 /* there also may be some cached data from a chained receive */
1700 if (rx_ring->rx_skb_top) {
1701 dev_kfree_skb(rx_ring->rx_skb_top);
1702 rx_ring->rx_skb_top = NULL;
1703 }
1704
bc7f75fa
AK		 1705 /* Zero out the descriptor ring */
1706 memset(rx_ring->desc, 0, rx_ring->size);
1707
1708 rx_ring->next_to_clean = 0;
1709 rx_ring->next_to_use = 0;
b94b5028 1710 adapter->flags2 &= ~FLAG2_IS_DISCARDING;
bc7f75fa 1711
c5083cf6 1712 writel(0, rx_ring->head);
bdc125f7
BA		 1713 if (rx_ring->adapter->flags2 & FLAG2_PCIM2PCI_ARBITER_WA)
1714 e1000e_update_rdt_wa(rx_ring, 0);
1715 else
1716 writel(0, rx_ring->tail);
bc7f75fa
AK		 1717}
1718
a8f88ff5
JB		 1719static void e1000e_downshift_workaround(struct work_struct *work)
1720{
1721 struct e1000_adapter *adapter = container_of(work,
1722 struct e1000_adapter, downshift_task);
1723
615b32af
JB		 1724 if (test_bit(__E1000_DOWN, &adapter->state))
1725 return;
1726
a8f88ff5
JB		 1727 e1000e_gig_downshift_workaround_ich8lan(&adapter->hw);
1728}
1729
bc7f75fa
AK		 1730/**
1731 * e1000_intr_msi - Interrupt Handler
1732 * @irq: interrupt number
1733 * @data: pointer to a network interface device structure
1734 **/
8bb62869 1735static irqreturn_t e1000_intr_msi(int __always_unused irq, void *data)
bc7f75fa
AK		 1736{
1737 struct net_device *netdev = data;
1738 struct e1000_adapter *adapter = netdev_priv(netdev);
1739 struct e1000_hw *hw = &adapter->hw;
1740 u32 icr = er32(ICR);
1741
e921eb1a 1742 /* read ICR disables interrupts using IAM */
573cca8c 1743 if (icr & E1000_ICR_LSC) {
f92518dd 1744 hw->mac.get_link_status = true;
e921eb1a 1745 /* ICH8 workaround-- Call gig speed drop workaround on cable
ad68076e
BA		 1746 * disconnect (LSC) before accessing any PHY registers
1747 */
bc7f75fa
AK		 1748 if ((adapter->flags & FLAG_LSC_GIG_SPEED_DROP) &&
1749 (!(er32(STATUS) & E1000_STATUS_LU)))
a8f88ff5 1750 schedule_work(&adapter->downshift_task);
bc7f75fa 1751
e921eb1a 1752 /* 80003ES2LAN workaround-- For packet buffer work-around on
bc7f75fa 1753 * link down event; disable receives here in the ISR and reset
ad68076e
BA		 1754 * adapter in watchdog
1755 */
bc7f75fa
AK		 1756 if (netif_carrier_ok(netdev) &&
1757 adapter->flags & FLAG_RX_NEEDS_RESTART) {
1758 /* disable receives */
1759 u32 rctl = er32(RCTL);
1760 ew32(RCTL, rctl & ~E1000_RCTL_EN);
12d43f7d 1761 adapter->flags |= FLAG_RESTART_NOW;
bc7f75fa
AK		 1762 }
1763 /* guard against interrupt when we're going down */
1764 if (!test_bit(__E1000_DOWN, &adapter->state))
1765 mod_timer(&adapter->watchdog_timer, jiffies + 1);
1766 }
1767
94fb848b
BA		 1768 /* Reset on uncorrectable ECC error */
1769 if ((icr & E1000_ICR_ECCER) && (hw->mac.type == e1000_pch_lpt)) {
1770 u32 pbeccsts = er32(PBECCSTS);
1771
1772 adapter->corr_errors +=
1773 pbeccsts & E1000_PBECCSTS_CORR_ERR_CNT_MASK;
1774 adapter->uncorr_errors +=
1775 (pbeccsts & E1000_PBECCSTS_UNCORR_ERR_CNT_MASK) >>
1776 E1000_PBECCSTS_UNCORR_ERR_CNT_SHIFT;
1777
1778 /* Do the reset outside of interrupt context */
1779 schedule_work(&adapter->reset_task);
1780
1781 /* return immediately since reset is imminent */
1782 return IRQ_HANDLED;
1783 }
1784
288379f0 1785 if (napi_schedule_prep(&adapter->napi)) {
bc7f75fa
AK		 1786 adapter->total_tx_bytes = 0;
1787 adapter->total_tx_packets = 0;
1788 adapter->total_rx_bytes = 0;
1789 adapter->total_rx_packets = 0;
288379f0 1790 __napi_schedule(&adapter->napi);
bc7f75fa
AK		 1791 }
1792
1793 return IRQ_HANDLED;
1794}
1795
1796/**
1797 * e1000_intr - Interrupt Handler
1798 * @irq: interrupt number
1799 * @data: pointer to a network interface device structure
1800 **/
8bb62869 1801static irqreturn_t e1000_intr(int __always_unused irq, void *data)
bc7f75fa
AK		 1802{
1803 struct net_device *netdev = data;
1804 struct e1000_adapter *adapter = netdev_priv(netdev);
1805 struct e1000_hw *hw = &adapter->hw;
bc7f75fa 1806 u32 rctl, icr = er32(ICR);
4662e82b 1807
a68ea775 1808 if (!icr || test_bit(__E1000_DOWN, &adapter->state))
bc7f75fa
AK		 1809 return IRQ_NONE; /* Not our interrupt */
1810
e921eb1a 1811 /* IMS will not auto-mask if INT_ASSERTED is not set, and if it is
ad68076e
BA		 1812 * not set, then the adapter didn't send an interrupt
1813 */
bc7f75fa
AK		 1814 if (!(icr & E1000_ICR_INT_ASSERTED))
1815 return IRQ_NONE;
1816
e921eb1a 1817 /* Interrupt Auto-Mask...upon reading ICR,
ad68076e
BA		 1818 * interrupts are masked. No need for the
1819 * IMC write
1820 */
bc7f75fa 1821
573cca8c 1822 if (icr & E1000_ICR_LSC) {
f92518dd 1823 hw->mac.get_link_status = true;
e921eb1a 1824 /* ICH8 workaround-- Call gig speed drop workaround on cable
ad68076e
BA		 1825 * disconnect (LSC) before accessing any PHY registers
1826 */
bc7f75fa
AK		 1827 if ((adapter->flags & FLAG_LSC_GIG_SPEED_DROP) &&
1828 (!(er32(STATUS) & E1000_STATUS_LU)))
a8f88ff5 1829 schedule_work(&adapter->downshift_task);
bc7f75fa 1830
e921eb1a 1831 /* 80003ES2LAN workaround--
bc7f75fa
AK		 1832 * For packet buffer work-around on link down event;
1833 * disable receives here in the ISR and
1834 * reset adapter in watchdog
1835 */
1836 if (netif_carrier_ok(netdev) &&
1837 (adapter->flags & FLAG_RX_NEEDS_RESTART)) {
1838 /* disable receives */
1839 rctl = er32(RCTL);
1840 ew32(RCTL, rctl & ~E1000_RCTL_EN);
12d43f7d 1841 adapter->flags |= FLAG_RESTART_NOW;
bc7f75fa
AK		 1842 }
1843 /* guard against interrupt when we're going down */
1844 if (!test_bit(__E1000_DOWN, &adapter->state))
1845 mod_timer(&adapter->watchdog_timer, jiffies + 1);
1846 }
1847
94fb848b
BA		 1848 /* Reset on uncorrectable ECC error */
1849 if ((icr & E1000_ICR_ECCER) && (hw->mac.type == e1000_pch_lpt)) {
1850 u32 pbeccsts = er32(PBECCSTS);
1851
1852 adapter->corr_errors +=
1853 pbeccsts & E1000_PBECCSTS_CORR_ERR_CNT_MASK;
1854 adapter->uncorr_errors +=
1855 (pbeccsts & E1000_PBECCSTS_UNCORR_ERR_CNT_MASK) >>
1856 E1000_PBECCSTS_UNCORR_ERR_CNT_SHIFT;
1857
1858 /* Do the reset outside of interrupt context */
1859 schedule_work(&adapter->reset_task);
1860
1861 /* return immediately since reset is imminent */
1862 return IRQ_HANDLED;
1863 }
1864
288379f0 1865 if (napi_schedule_prep(&adapter->napi)) {
bc7f75fa
AK		 1866 adapter->total_tx_bytes = 0;
1867 adapter->total_tx_packets = 0;
1868 adapter->total_rx_bytes = 0;
1869 adapter->total_rx_packets = 0;
288379f0 1870 __napi_schedule(&adapter->napi);
bc7f75fa
AK		 1871 }
1872
1873 return IRQ_HANDLED;
1874}
1875
8bb62869 1876static irqreturn_t e1000_msix_other(int __always_unused irq, void *data)
4662e82b
BA		 1877{
1878 struct net_device *netdev = data;
1879 struct e1000_adapter *adapter = netdev_priv(netdev);
1880 struct e1000_hw *hw = &adapter->hw;
1881 u32 icr = er32(ICR);
1882
1883 if (!(icr & E1000_ICR_INT_ASSERTED)) {
a3c69fef
JB		 1884 if (!test_bit(__E1000_DOWN, &adapter->state))
1885 ew32(IMS, E1000_IMS_OTHER);
4662e82b
BA		 1886 return IRQ_NONE;
1887 }
1888
1889 if (icr & adapter->eiac_mask)
1890 ew32(ICS, (icr & adapter->eiac_mask));
1891
1892 if (icr & E1000_ICR_OTHER) {
1893 if (!(icr & E1000_ICR_LSC))
1894 goto no_link_interrupt;
f92518dd 1895 hw->mac.get_link_status = true;
4662e82b
BA		 1896 /* guard against interrupt when we're going down */
1897 if (!test_bit(__E1000_DOWN, &adapter->state))
1898 mod_timer(&adapter->watchdog_timer, jiffies + 1);
1899 }
1900
1901no_link_interrupt:
a3c69fef
JB		 1902 if (!test_bit(__E1000_DOWN, &adapter->state))
1903 ew32(IMS, E1000_IMS_LSC | E1000_IMS_OTHER);
4662e82b
BA		 1904
1905 return IRQ_HANDLED;
1906}
1907
8bb62869 1908static irqreturn_t e1000_intr_msix_tx(int __always_unused irq, void *data)
4662e82b
BA		 1909{
1910 struct net_device *netdev = data;
1911 struct e1000_adapter *adapter = netdev_priv(netdev);
1912 struct e1000_hw *hw = &adapter->hw;
1913 struct e1000_ring *tx_ring = adapter->tx_ring;
1914
1915
1916 adapter->total_tx_bytes = 0;
1917 adapter->total_tx_packets = 0;
1918
55aa6985 1919 if (!e1000_clean_tx_irq(tx_ring))
4662e82b
BA		 1920 /* Ring was not completely cleaned, so fire another interrupt */
1921 ew32(ICS, tx_ring->ims_val);
1922
1923 return IRQ_HANDLED;
1924}
1925
8bb62869 1926static irqreturn_t e1000_intr_msix_rx(int __always_unused irq, void *data)
4662e82b
BA		 1927{
1928 struct net_device *netdev = data;
1929 struct e1000_adapter *adapter = netdev_priv(netdev);
55aa6985 1930 struct e1000_ring *rx_ring = adapter->rx_ring;
4662e82b
BA		 1931
1932 /* Write the ITR value calculated at the end of the
1933 * previous interrupt.
1934 */
55aa6985
BA		 1935 if (rx_ring->set_itr) {
1936 writel(1000000000 / (rx_ring->itr_val * 256),
1937 rx_ring->itr_register);
1938 rx_ring->set_itr = 0;
4662e82b
BA		 1939 }
1940
288379f0 1941 if (napi_schedule_prep(&adapter->napi)) {
4662e82b
BA		 1942 adapter->total_rx_bytes = 0;
1943 adapter->total_rx_packets = 0;
288379f0 1944 __napi_schedule(&adapter->napi);
4662e82b
BA		 1945 }
1946 return IRQ_HANDLED;
1947}
1948
1949/**
1950 * e1000_configure_msix - Configure MSI-X hardware
1951 *
1952 * e1000_configure_msix sets up the hardware to properly
1953 * generate MSI-X interrupts.
1954 **/
1955static void e1000_configure_msix(struct e1000_adapter *adapter)
1956{
1957 struct e1000_hw *hw = &adapter->hw;
1958 struct e1000_ring *rx_ring = adapter->rx_ring;
1959 struct e1000_ring *tx_ring = adapter->tx_ring;
1960 int vector = 0;
1961 u32 ctrl_ext, ivar = 0;
1962
1963 adapter->eiac_mask = 0;
1964
1965 /* Workaround issue with spurious interrupts on 82574 in MSI-X mode */
1966 if (hw->mac.type == e1000_82574) {
1967 u32 rfctl = er32(RFCTL);
1968 rfctl |= E1000_RFCTL_ACK_DIS;
1969 ew32(RFCTL, rfctl);
1970 }
1971
1972#define E1000_IVAR_INT_ALLOC_VALID 0x8
1973 /* Configure Rx vector */
1974 rx_ring->ims_val = E1000_IMS_RXQ0;
1975 adapter->eiac_mask |= rx_ring->ims_val;
1976 if (rx_ring->itr_val)
1977 writel(1000000000 / (rx_ring->itr_val * 256),
c5083cf6 1978 rx_ring->itr_register);
4662e82b 1979 else
c5083cf6 1980 writel(1, rx_ring->itr_register);
4662e82b
BA		 1981 ivar = E1000_IVAR_INT_ALLOC_VALID | vector;
1982
1983 /* Configure Tx vector */
1984 tx_ring->ims_val = E1000_IMS_TXQ0;
1985 vector++;
1986 if (tx_ring->itr_val)
1987 writel(1000000000 / (tx_ring->itr_val * 256),
c5083cf6 1988 tx_ring->itr_register);
4662e82b 1989 else
c5083cf6 1990 writel(1, tx_ring->itr_register);
4662e82b
BA		 1991 adapter->eiac_mask |= tx_ring->ims_val;
1992 ivar |= ((E1000_IVAR_INT_ALLOC_VALID | vector) << 8);
1993
1994 /* set vector for Other Causes, e.g. link changes */
1995 vector++;
1996 ivar |= ((E1000_IVAR_INT_ALLOC_VALID | vector) << 16);
1997 if (rx_ring->itr_val)
1998 writel(1000000000 / (rx_ring->itr_val * 256),
1999 hw->hw_addr + E1000_EITR_82574(vector));
2000 else
2001 writel(1, hw->hw_addr + E1000_EITR_82574(vector));
2002
2003 /* Cause Tx interrupts on every write back */
2004 ivar |= (1 << 31);
2005
2006 ew32(IVAR, ivar);
2007
2008 /* enable MSI-X PBA support */
2009 ctrl_ext = er32(CTRL_EXT);
2010 ctrl_ext |= E1000_CTRL_EXT_PBA_CLR;
2011
2012 /* Auto-Mask Other interrupts upon ICR read */
4662e82b
BA		 2013 ew32(IAM, ~E1000_EIAC_MASK_82574 | E1000_IMS_OTHER);
2014 ctrl_ext |= E1000_CTRL_EXT_EIAME;
2015 ew32(CTRL_EXT, ctrl_ext);
2016 e1e_flush();
2017}
2018
2019void e1000e_reset_interrupt_capability(struct e1000_adapter *adapter)
2020{
2021 if (adapter->msix_entries) {
2022 pci_disable_msix(adapter->pdev);
2023 kfree(adapter->msix_entries);
2024 adapter->msix_entries = NULL;
2025 } else if (adapter->flags & FLAG_MSI_ENABLED) {
2026 pci_disable_msi(adapter->pdev);
2027 adapter->flags &= ~FLAG_MSI_ENABLED;
2028 }
4662e82b
BA		 2029}
2030
2031/**
2032 * e1000e_set_interrupt_capability - set MSI or MSI-X if supported
2033 *
2034 * Attempt to configure interrupts using the best available
2035 * capabilities of the hardware and kernel.
2036 **/
2037void e1000e_set_interrupt_capability(struct e1000_adapter *adapter)
2038{
2039 int err;
8e86acd7 2040 int i;
4662e82b
BA		 2041
2042 switch (adapter->int_mode) {
2043 case E1000E_INT_MODE_MSIX:
2044 if (adapter->flags & FLAG_HAS_MSIX) {
8e86acd7
JK		 2045 adapter->num_vectors = 3; /* RxQ0, TxQ0 and other */
2046 adapter->msix_entries = kcalloc(adapter->num_vectors,
4662e82b
BA		 2047 sizeof(struct msix_entry),
2048 GFP_KERNEL);
2049 if (adapter->msix_entries) {
8e86acd7 2050 for (i = 0; i < adapter->num_vectors; i++)
4662e82b
BA		 2051 adapter->msix_entries[i].entry = i;
2052
2053 err = pci_enable_msix(adapter->pdev,
2054 adapter->msix_entries,
8e86acd7 2055 adapter->num_vectors);
b1cdfead 2056 if (err == 0)
4662e82b
BA		 2057 return;
2058 }
2059 /* MSI-X failed, so fall through and try MSI */
ef456f85 2060 e_err("Failed to initialize MSI-X interrupts. Falling back to MSI interrupts.\n");
4662e82b
BA		 2061 e1000e_reset_interrupt_capability(adapter);
2062 }
2063 adapter->int_mode = E1000E_INT_MODE_MSI;
2064 /* Fall through */
2065 case E1000E_INT_MODE_MSI:
2066 if (!pci_enable_msi(adapter->pdev)) {
2067 adapter->flags |= FLAG_MSI_ENABLED;
2068 } else {
2069 adapter->int_mode = E1000E_INT_MODE_LEGACY;
ef456f85 2070 e_err("Failed to initialize MSI interrupts. Falling back to legacy interrupts.\n");
4662e82b
BA		 2071 }
2072 /* Fall through */
2073 case E1000E_INT_MODE_LEGACY:
2074 /* Don't do anything; this is the system default */
2075 break;
2076 }
8e86acd7
JK		 2077
2078 /* store the number of vectors being used */
2079 adapter->num_vectors = 1;
4662e82b
BA		 2080}
2081
2082/**
2083 * e1000_request_msix - Initialize MSI-X interrupts
2084 *
2085 * e1000_request_msix allocates MSI-X vectors and requests interrupts from the
2086 * kernel.
2087 **/
2088static int e1000_request_msix(struct e1000_adapter *adapter)
2089{
2090 struct net_device *netdev = adapter->netdev;
2091 int err = 0, vector = 0;
2092
2093 if (strlen(netdev->name) < (IFNAMSIZ - 5))
79f5e840
BA		 2094 snprintf(adapter->rx_ring->name,
2095 sizeof(adapter->rx_ring->name) - 1,
2096 "%s-rx-0", netdev->name);
4662e82b
BA		 2097 else
2098 memcpy(adapter->rx_ring->name, netdev->name, IFNAMSIZ);
2099 err = request_irq(adapter->msix_entries[vector].vector,
a0607fd3 2100 e1000_intr_msix_rx, 0, adapter->rx_ring->name,
4662e82b
BA		 2101 netdev);
2102 if (err)
5015e53a 2103 return err;
c5083cf6
BA		 2104 adapter->rx_ring->itr_register = adapter->hw.hw_addr +
2105 E1000_EITR_82574(vector);
4662e82b
BA		 2106 adapter->rx_ring->itr_val = adapter->itr;
2107 vector++;
2108
2109 if (strlen(netdev->name) < (IFNAMSIZ - 5))
79f5e840
BA		 2110 snprintf(adapter->tx_ring->name,
2111 sizeof(adapter->tx_ring->name) - 1,
2112 "%s-tx-0", netdev->name);
4662e82b
BA		 2113 else
2114 memcpy(adapter->tx_ring->name, netdev->name, IFNAMSIZ);
2115 err = request_irq(adapter->msix_entries[vector].vector,
a0607fd3 2116 e1000_intr_msix_tx, 0, adapter->tx_ring->name,
4662e82b
BA		 2117 netdev);
2118 if (err)
5015e53a 2119 return err;
c5083cf6
BA		 2120 adapter->tx_ring->itr_register = adapter->hw.hw_addr +
2121 E1000_EITR_82574(vector);
4662e82b
BA		 2122 adapter->tx_ring->itr_val = adapter->itr;
2123 vector++;
2124
2125 err = request_irq(adapter->msix_entries[vector].vector,
a0607fd3 2126 e1000_msix_other, 0, netdev->name, netdev);
4662e82b 2127 if (err)
5015e53a 2128 return err;
4662e82b
BA		 2129
2130 e1000_configure_msix(adapter);
5015e53a 2131
4662e82b 2132 return 0;
4662e82b
BA		 2133}
2134
f8d59f78
BA		 2135/**
2136 * e1000_request_irq - initialize interrupts
2137 *
2138 * Attempts to configure interrupts using the best available
2139 * capabilities of the hardware and kernel.
2140 **/
bc7f75fa
AK		 2141static int e1000_request_irq(struct e1000_adapter *adapter)
2142{
2143 struct net_device *netdev = adapter->netdev;
bc7f75fa
AK		 2144 int err;
2145
4662e82b
BA		 2146 if (adapter->msix_entries) {
2147 err = e1000_request_msix(adapter);
2148 if (!err)
2149 return err;
2150 /* fall back to MSI */
2151 e1000e_reset_interrupt_capability(adapter);
2152 adapter->int_mode = E1000E_INT_MODE_MSI;
2153 e1000e_set_interrupt_capability(adapter);
bc7f75fa 2154 }
4662e82b 2155 if (adapter->flags & FLAG_MSI_ENABLED) {
a0607fd3 2156 err = request_irq(adapter->pdev->irq, e1000_intr_msi, 0,
4662e82b
BA		 2157 netdev->name, netdev);
2158 if (!err)
2159 return err;
bc7f75fa 2160
4662e82b
BA		 2161 /* fall back to legacy interrupt */
2162 e1000e_reset_interrupt_capability(adapter);
2163 adapter->int_mode = E1000E_INT_MODE_LEGACY;
bc7f75fa
AK		 2164 }
2165
a0607fd3 2166 err = request_irq(adapter->pdev->irq, e1000_intr, IRQF_SHARED,
4662e82b
BA		 2167 netdev->name, netdev);
2168 if (err)
2169 e_err("Unable to allocate interrupt, Error: %d\n", err);
2170
bc7f75fa
AK		 2171 return err;
2172}
2173
2174static void e1000_free_irq(struct e1000_adapter *adapter)
2175{
2176 struct net_device *netdev = adapter->netdev;
2177
4662e82b
BA		 2178 if (adapter->msix_entries) {
2179 int vector = 0;
2180
2181 free_irq(adapter->msix_entries[vector].vector, netdev);
2182 vector++;
2183
2184 free_irq(adapter->msix_entries[vector].vector, netdev);
2185 vector++;
2186
2187 /* Other Causes interrupt vector */
2188 free_irq(adapter->msix_entries[vector].vector, netdev);
2189 return;
bc7f75fa 2190 }
4662e82b
BA		 2191
2192 free_irq(adapter->pdev->irq, netdev);
bc7f75fa
AK		 2193}
2194
2195/**
2196 * e1000_irq_disable - Mask off interrupt generation on the NIC
2197 **/
2198static void e1000_irq_disable(struct e1000_adapter *adapter)
2199{
2200 struct e1000_hw *hw = &adapter->hw;
2201
bc7f75fa 2202 ew32(IMC, ~0);
4662e82b
BA		 2203 if (adapter->msix_entries)
2204 ew32(EIAC_82574, 0);
bc7f75fa 2205 e1e_flush();
8e86acd7
JK		 2206
2207 if (adapter->msix_entries) {
2208 int i;
2209 for (i = 0; i < adapter->num_vectors; i++)
2210 synchronize_irq(adapter->msix_entries[i].vector);
2211 } else {
2212 synchronize_irq(adapter->pdev->irq);
2213 }
bc7f75fa
AK		 2214}
2215
2216/**
2217 * e1000_irq_enable - Enable default interrupt generation settings
2218 **/
2219static void e1000_irq_enable(struct e1000_adapter *adapter)
2220{
2221 struct e1000_hw *hw = &adapter->hw;
2222
4662e82b
BA		 2223 if (adapter->msix_entries) {
2224 ew32(EIAC_82574, adapter->eiac_mask & E1000_EIAC_MASK_82574);
2225 ew32(IMS, adapter->eiac_mask | E1000_IMS_OTHER | E1000_IMS_LSC);
94fb848b
BA		 2226 } else if (hw->mac.type == e1000_pch_lpt) {
2227 ew32(IMS, IMS_ENABLE_MASK | E1000_IMS_ECCER);
4662e82b
BA		 2228 } else {
2229 ew32(IMS, IMS_ENABLE_MASK);
2230 }
74ef9c39 2231 e1e_flush();
bc7f75fa
AK		 2232}
2233
2234/**
31dbe5b4 2235 * e1000e_get_hw_control - get control of the h/w from f/w
bc7f75fa
AK		 2236 * @adapter: address of board private structure
2237 *
31dbe5b4 2238 * e1000e_get_hw_control sets {CTRL_EXT|SWSM}:DRV_LOAD bit.
bc7f75fa
AK		 2239 * For ASF and Pass Through versions of f/w this means that
2240 * the driver is loaded. For AMT version (only with 82573)
2241 * of the f/w this means that the network i/f is open.
2242 **/
31dbe5b4 2243void e1000e_get_hw_control(struct e1000_adapter *adapter)
bc7f75fa
AK		 2244{
2245 struct e1000_hw *hw = &adapter->hw;
2246 u32 ctrl_ext;
2247 u32 swsm;
2248
2249 /* Let firmware know the driver has taken over */
2250 if (adapter->flags & FLAG_HAS_SWSM_ON_LOAD) {
2251 swsm = er32(SWSM);
2252 ew32(SWSM, swsm | E1000_SWSM_DRV_LOAD);
2253 } else if (adapter->flags & FLAG_HAS_CTRLEXT_ON_LOAD) {
2254 ctrl_ext = er32(CTRL_EXT);
ad68076e 2255 ew32(CTRL_EXT, ctrl_ext | E1000_CTRL_EXT_DRV_LOAD);
bc7f75fa
AK		 2256 }
2257}
2258
2259/**
31dbe5b4 2260 * e1000e_release_hw_control - release control of the h/w to f/w
bc7f75fa
AK		 2261 * @adapter: address of board private structure
2262 *
31dbe5b4 2263 * e1000e_release_hw_control resets {CTRL_EXT|SWSM}:DRV_LOAD bit.
bc7f75fa
AK		 2264 * For ASF and Pass Through versions of f/w this means that the
2265 * driver is no longer loaded. For AMT version (only with 82573) i
2266 * of the f/w this means that the network i/f is closed.
2267 *
2268 **/
31dbe5b4 2269void e1000e_release_hw_control(struct e1000_adapter *adapter)
bc7f75fa
AK		 2270{
2271 struct e1000_hw *hw = &adapter->hw;
2272 u32 ctrl_ext;
2273 u32 swsm;
2274
2275 /* Let firmware taken over control of h/w */
2276 if (adapter->flags & FLAG_HAS_SWSM_ON_LOAD) {
2277 swsm = er32(SWSM);
2278 ew32(SWSM, swsm & ~E1000_SWSM_DRV_LOAD);
2279 } else if (adapter->flags & FLAG_HAS_CTRLEXT_ON_LOAD) {
2280 ctrl_ext = er32(CTRL_EXT);
ad68076e 2281 ew32(CTRL_EXT, ctrl_ext & ~E1000_CTRL_EXT_DRV_LOAD);
bc7f75fa
AK		 2282 }
2283}
2284
bc7f75fa 2285/**
49ce9c2c 2286 * e1000_alloc_ring_dma - allocate memory for a ring structure
bc7f75fa
AK		 2287 **/
2288static int e1000_alloc_ring_dma(struct e1000_adapter *adapter,
2289 struct e1000_ring *ring)
2290{
2291 struct pci_dev *pdev = adapter->pdev;
2292
2293 ring->desc = dma_alloc_coherent(&pdev->dev, ring->size, &ring->dma,
2294 GFP_KERNEL);
2295 if (!ring->desc)
2296 return -ENOMEM;
2297
2298 return 0;
2299}
2300
2301/**
2302 * e1000e_setup_tx_resources - allocate Tx resources (Descriptors)
55aa6985 2303 * @tx_ring: Tx descriptor ring
bc7f75fa
AK		 2304 *
2305 * Return 0 on success, negative on failure
2306 **/
55aa6985 2307int e1000e_setup_tx_resources(struct e1000_ring *tx_ring)
bc7f75fa 2308{
55aa6985 2309 struct e1000_adapter *adapter = tx_ring->adapter;
bc7f75fa
AK		 2310 int err = -ENOMEM, size;
2311
2312 size = sizeof(struct e1000_buffer) * tx_ring->count;
89bf67f1 2313 tx_ring->buffer_info = vzalloc(size);
bc7f75fa
AK		 2314 if (!tx_ring->buffer_info)
2315 goto err;
bc7f75fa
AK		 2316
2317 /* round up to nearest 4K */
2318 tx_ring->size = tx_ring->count * sizeof(struct e1000_tx_desc);
2319 tx_ring->size = ALIGN(tx_ring->size, 4096);
2320
2321 err = e1000_alloc_ring_dma(adapter, tx_ring);
2322 if (err)
2323 goto err;
2324
2325 tx_ring->next_to_use = 0;
2326 tx_ring->next_to_clean = 0;
bc7f75fa
AK		 2327
2328 return 0;
2329err:
2330 vfree(tx_ring->buffer_info);
44defeb3 2331 e_err("Unable to allocate memory for the transmit descriptor ring\n");
bc7f75fa
AK		 2332 return err;
2333}
2334
2335/**
2336 * e1000e_setup_rx_resources - allocate Rx resources (Descriptors)
55aa6985 2337 * @rx_ring: Rx descriptor ring
bc7f75fa
AK		 2338 *
2339 * Returns 0 on success, negative on failure
2340 **/
55aa6985 2341int e1000e_setup_rx_resources(struct e1000_ring *rx_ring)
bc7f75fa 2342{
55aa6985 2343 struct e1000_adapter *adapter = rx_ring->adapter;
47f44e40
AK		 2344 struct e1000_buffer *buffer_info;
2345 int i, size, desc_len, err = -ENOMEM;
bc7f75fa
AK		 2346
2347 size = sizeof(struct e1000_buffer) * rx_ring->count;
89bf67f1 2348 rx_ring->buffer_info = vzalloc(size);
bc7f75fa
AK		 2349 if (!rx_ring->buffer_info)
2350 goto err;
bc7f75fa 2351
47f44e40
AK		 2352 for (i = 0; i < rx_ring->count; i++) {
2353 buffer_info = &rx_ring->buffer_info[i];
2354 buffer_info->ps_pages = kcalloc(PS_PAGE_BUFFERS,
2355 sizeof(struct e1000_ps_page),
2356 GFP_KERNEL);
2357 if (!buffer_info->ps_pages)
2358 goto err_pages;
2359 }
bc7f75fa
AK		 2360
2361 desc_len = sizeof(union e1000_rx_desc_packet_split);
2362
2363 /* Round up to nearest 4K */
2364 rx_ring->size = rx_ring->count * desc_len;
2365 rx_ring->size = ALIGN(rx_ring->size, 4096);
2366
2367 err = e1000_alloc_ring_dma(adapter, rx_ring);
2368 if (err)
47f44e40 2369 goto err_pages;
bc7f75fa
AK		 2370
2371 rx_ring->next_to_clean = 0;
2372 rx_ring->next_to_use = 0;
2373 rx_ring->rx_skb_top = NULL;
2374
2375 return 0;
47f44e40
AK		 2376
2377err_pages:
2378 for (i = 0; i < rx_ring->count; i++) {
2379 buffer_info = &rx_ring->buffer_info[i];
2380 kfree(buffer_info->ps_pages);
2381 }
bc7f75fa
AK		 2382err:
2383 vfree(rx_ring->buffer_info);
e9262447 2384 e_err("Unable to allocate memory for the receive descriptor ring\n");
bc7f75fa
AK		 2385 return err;
2386}
2387
2388/**
2389 * e1000_clean_tx_ring - Free Tx Buffers
55aa6985 2390 * @tx_ring: Tx descriptor ring
bc7f75fa 2391 **/
55aa6985 2392static void e1000_clean_tx_ring(struct e1000_ring *tx_ring)
bc7f75fa 2393{
55aa6985 2394 struct e1000_adapter *adapter = tx_ring->adapter;
bc7f75fa
AK		 2395 struct e1000_buffer *buffer_info;
2396 unsigned long size;
2397 unsigned int i;
2398
2399 for (i = 0; i < tx_ring->count; i++) {
2400 buffer_info = &tx_ring->buffer_info[i];
55aa6985 2401 e1000_put_txbuf(tx_ring, buffer_info);
bc7f75fa
AK		 2402 }
2403
3f0cfa3b 2404 netdev_reset_queue(adapter->netdev);
bc7f75fa
AK		 2405 size = sizeof(struct e1000_buffer) * tx_ring->count;
2406 memset(tx_ring->buffer_info, 0, size);
2407
2408 memset(tx_ring->desc, 0, tx_ring->size);
2409
2410 tx_ring->next_to_use = 0;
2411 tx_ring->next_to_clean = 0;
2412
c5083cf6 2413 writel(0, tx_ring->head);
bdc125f7
BA		 2414 if (tx_ring->adapter->flags2 & FLAG2_PCIM2PCI_ARBITER_WA)
2415 e1000e_update_tdt_wa(tx_ring, 0);
2416 else
2417 writel(0, tx_ring->tail);
bc7f75fa
AK		 2418}
2419
2420/**
2421 * e1000e_free_tx_resources - Free Tx Resources per Queue
55aa6985 2422 * @tx_ring: Tx descriptor ring
bc7f75fa
AK		 2423 *
2424 * Free all transmit software resources
2425 **/
55aa6985 2426void e1000e_free_tx_resources(struct e1000_ring *tx_ring)
bc7f75fa 2427{
55aa6985 2428 struct e1000_adapter *adapter = tx_ring->adapter;
bc7f75fa 2429 struct pci_dev *pdev = adapter->pdev;
bc7f75fa 2430
55aa6985 2431 e1000_clean_tx_ring(tx_ring);
bc7f75fa
AK		 2432
2433 vfree(tx_ring->buffer_info);
2434 tx_ring->buffer_info = NULL;
2435
2436 dma_free_coherent(&pdev->dev, tx_ring->size, tx_ring->desc,
2437 tx_ring->dma);
2438 tx_ring->desc = NULL;
2439}
2440
2441/**
2442 * e1000e_free_rx_resources - Free Rx Resources
55aa6985 2443 * @rx_ring: Rx descriptor ring
bc7f75fa
AK		 2444 *
2445 * Free all receive software resources
2446 **/
55aa6985 2447void e1000e_free_rx_resources(struct e1000_ring *rx_ring)
bc7f75fa 2448{
55aa6985 2449 struct e1000_adapter *adapter = rx_ring->adapter;
bc7f75fa 2450 struct pci_dev *pdev = adapter->pdev;
47f44e40 2451 int i;
bc7f75fa 2452
55aa6985 2453 e1000_clean_rx_ring(rx_ring);
bc7f75fa 2454
b1cdfead 2455 for (i = 0; i < rx_ring->count; i++)
47f44e40 2456 kfree(rx_ring->buffer_info[i].ps_pages);
47f44e40 2457
bc7f75fa
AK		 2458 vfree(rx_ring->buffer_info);
2459 rx_ring->buffer_info = NULL;
2460
bc7f75fa
AK		 2461 dma_free_coherent(&pdev->dev, rx_ring->size, rx_ring->desc,
2462 rx_ring->dma);
2463 rx_ring->desc = NULL;
2464}
2465
2466/**
2467 * e1000_update_itr - update the dynamic ITR value based on statistics
489815ce
AK		 2468 * @adapter: pointer to adapter
2469 * @itr_setting: current adapter->itr
2470 * @packets: the number of packets during this measurement interval
2471 * @bytes: the number of bytes during this measurement interval
2472 *
bc7f75fa
AK		 2473 * Stores a new ITR value based on packets and byte
2474 * counts during the last interrupt. The advantage of per interrupt
2475 * computation is faster updates and more accurate ITR for the current
2476 * traffic pattern. Constants in this function were computed
2477 * based on theoretical maximum wire speed and thresholds were set based
2478 * on testing data as well as attempting to minimize response time
4662e82b
BA		 2479 * while increasing bulk throughput. This functionality is controlled
2480 * by the InterruptThrottleRate module parameter.
bc7f75fa 2481 **/
8bb62869 2482static unsigned int e1000_update_itr(u16 itr_setting, int packets, int bytes)
bc7f75fa
AK		 2483{
2484 unsigned int retval = itr_setting;
2485
2486 if (packets == 0)
5015e53a 2487 return itr_setting;
bc7f75fa
AK		 2488
2489 switch (itr_setting) {
2490 case lowest_latency:
2491 /* handle TSO and jumbo frames */
2492 if (bytes/packets > 8000)
2493 retval = bulk_latency;
b1cdfead 2494 else if ((packets < 5) && (bytes > 512))
bc7f75fa 2495 retval = low_latency;
bc7f75fa
AK		 2496 break;
2497 case low_latency: /* 50 usec aka 20000 ints/s */
2498 if (bytes > 10000) {
2499 /* this if handles the TSO accounting */
b1cdfead 2500 if (bytes/packets > 8000)
bc7f75fa 2501 retval = bulk_latency;
b1cdfead 2502 else if ((packets < 10) || ((bytes/packets) > 1200))
bc7f75fa 2503 retval = bulk_latency;
b1cdfead 2504 else if ((packets > 35))
bc7f75fa 2505 retval = lowest_latency;
bc7f75fa
AK		 2506 } else if (bytes/packets > 2000) {
2507 retval = bulk_latency;
2508 } else if (packets <= 2 && bytes < 512) {
2509 retval = lowest_latency;
2510 }
2511 break;
2512 case bulk_latency: /* 250 usec aka 4000 ints/s */
2513 if (bytes > 25000) {
b1cdfead 2514 if (packets > 35)
bc7f75fa 2515 retval = low_latency;
bc7f75fa
AK		 2516 } else if (bytes < 6000) {
2517 retval = low_latency;
2518 }
2519 break;
2520 }
2521
bc7f75fa
AK		 2522 return retval;
2523}
2524
2525static void e1000_set_itr(struct e1000_adapter *adapter)
2526{
bc7f75fa
AK		 2527 u16 current_itr;
2528 u32 new_itr = adapter->itr;
2529
2530 /* for non-gigabit speeds, just fix the interrupt rate at 4000 */
2531 if (adapter->link_speed != SPEED_1000) {
2532 current_itr = 0;
2533 new_itr = 4000;
2534 goto set_itr_now;
2535 }
2536
828bac87
BA		 2537 if (adapter->flags2 & FLAG2_DISABLE_AIM) {
2538 new_itr = 0;
2539 goto set_itr_now;
2540 }
2541
8bb62869
BA		 2542 adapter->tx_itr = e1000_update_itr(adapter->tx_itr,
2543 adapter->total_tx_packets,
2544 adapter->total_tx_bytes);
bc7f75fa
AK		 2545 /* conservative mode (itr 3) eliminates the lowest_latency setting */
2546 if (adapter->itr_setting == 3 && adapter->tx_itr == lowest_latency)
2547 adapter->tx_itr = low_latency;
2548
8bb62869
BA		 2549 adapter->rx_itr = e1000_update_itr(adapter->rx_itr,
2550 adapter->total_rx_packets,
2551 adapter->total_rx_bytes);
bc7f75fa
AK		 2552 /* conservative mode (itr 3) eliminates the lowest_latency setting */
2553 if (adapter->itr_setting == 3 && adapter->rx_itr == lowest_latency)
2554 adapter->rx_itr = low_latency;
2555
2556 current_itr = max(adapter->rx_itr, adapter->tx_itr);
2557
2558 switch (current_itr) {
2559 /* counts and packets in update_itr are dependent on these numbers */
2560 case lowest_latency:
2561 new_itr = 70000;
2562 break;
2563 case low_latency:
2564 new_itr = 20000; /* aka hwitr = ~200 */
2565 break;
2566 case bulk_latency:
2567 new_itr = 4000;
2568 break;
2569 default:
2570 break;
2571 }
2572
2573set_itr_now:
2574 if (new_itr != adapter->itr) {
e921eb1a 2575 /* this attempts to bias the interrupt rate towards Bulk
bc7f75fa 2576 * by adding intermediate steps when interrupt rate is
ad68076e
BA		 2577 * increasing
2578 */
bc7f75fa 2579 new_itr = new_itr > adapter->itr ?
f0ff4398 2580 min(adapter->itr + (new_itr >> 2), new_itr) : new_itr;
bc7f75fa 2581 adapter->itr = new_itr;
4662e82b
BA		 2582 adapter->rx_ring->itr_val = new_itr;
2583 if (adapter->msix_entries)
2584 adapter->rx_ring->set_itr = 1;
2585 else
e3d14b08 2586 e1000e_write_itr(adapter, new_itr);
bc7f75fa
AK		 2587 }
2588}
2589
22a4cca2
MV		 2590/**
2591 * e1000e_write_itr - write the ITR value to the appropriate registers
2592 * @adapter: address of board private structure
2593 * @itr: new ITR value to program
2594 *
2595 * e1000e_write_itr determines if the adapter is in MSI-X mode
2596 * and, if so, writes the EITR registers with the ITR value.
2597 * Otherwise, it writes the ITR value into the ITR register.
2598 **/
2599void e1000e_write_itr(struct e1000_adapter *adapter, u32 itr)
2600{
2601 struct e1000_hw *hw = &adapter->hw;
2602 u32 new_itr = itr ? 1000000000 / (itr * 256) : 0;
2603
2604 if (adapter->msix_entries) {
2605 int vector;
2606
2607 for (vector = 0; vector < adapter->num_vectors; vector++)
2608 writel(new_itr, hw->hw_addr + E1000_EITR_82574(vector));
2609 } else {
2610 ew32(ITR, new_itr);
2611 }
2612}
2613
4662e82b
BA		 2614/**
2615 * e1000_alloc_queues - Allocate memory for all rings
2616 * @adapter: board private structure to initialize
2617 **/
9f9a12f8 2618static int e1000_alloc_queues(struct e1000_adapter *adapter)
4662e82b 2619{
55aa6985
BA		 2620 int size = sizeof(struct e1000_ring);
2621
2622 adapter->tx_ring = kzalloc(size, GFP_KERNEL);
4662e82b
BA		 2623 if (!adapter->tx_ring)
2624 goto err;
55aa6985
BA		 2625 adapter->tx_ring->count = adapter->tx_ring_count;
2626 adapter->tx_ring->adapter = adapter;
4662e82b 2627
55aa6985 2628 adapter->rx_ring = kzalloc(size, GFP_KERNEL);
4662e82b
BA		 2629 if (!adapter->rx_ring)
2630 goto err;
55aa6985
BA		 2631 adapter->rx_ring->count = adapter->rx_ring_count;
2632 adapter->rx_ring->adapter = adapter;
4662e82b
BA		 2633
2634 return 0;
2635err:
2636 e_err("Unable to allocate memory for queues\n");
2637 kfree(adapter->rx_ring);
2638 kfree(adapter->tx_ring);
2639 return -ENOMEM;
2640}
2641
bc7f75fa 2642/**
c58c8a78 2643 * e1000e_poll - NAPI Rx polling callback
ad68076e 2644 * @napi: struct associated with this polling callback
c58c8a78 2645 * @weight: number of packets driver is allowed to process this poll
bc7f75fa 2646 **/
c58c8a78 2647static int e1000e_poll(struct napi_struct *napi, int weight)
bc7f75fa 2648{
c58c8a78
BA		 2649 struct e1000_adapter *adapter = container_of(napi, struct e1000_adapter,
2650 napi);
4662e82b 2651 struct e1000_hw *hw = &adapter->hw;
bc7f75fa 2652 struct net_device *poll_dev = adapter->netdev;
679e8a0f 2653 int tx_cleaned = 1, work_done = 0;
bc7f75fa 2654
4cf1653a 2655 adapter = netdev_priv(poll_dev);
bc7f75fa 2656
c58c8a78
BA		 2657 if (!adapter->msix_entries ||
2658 (adapter->rx_ring->ims_val & adapter->tx_ring->ims_val))
2659 tx_cleaned = e1000_clean_tx_irq(adapter->tx_ring);
4662e82b 2660
c58c8a78 2661 adapter->clean_rx(adapter->rx_ring, &work_done, weight);
d2c7ddd6 2662
12d04a3c 2663 if (!tx_cleaned)
c58c8a78 2664 work_done = weight;
bc7f75fa 2665
c58c8a78
BA		 2666 /* If weight not fully consumed, exit the polling mode */
2667 if (work_done < weight) {
bc7f75fa
AK		 2668 if (adapter->itr_setting & 3)
2669 e1000_set_itr(adapter);
288379f0 2670 napi_complete(napi);
a3c69fef
JB		 2671 if (!test_bit(__E1000_DOWN, &adapter->state)) {
2672 if (adapter->msix_entries)
2673 ew32(IMS, adapter->rx_ring->ims_val);
2674 else
2675 e1000_irq_enable(adapter);
2676 }
bc7f75fa
AK		 2677 }
2678
2679 return work_done;
2680}
2681
8e586137 2682static int e1000_vlan_rx_add_vid(struct net_device *netdev, u16 vid)
bc7f75fa
AK		 2683{
2684 struct e1000_adapter *adapter = netdev_priv(netdev);
2685 struct e1000_hw *hw = &adapter->hw;
2686 u32 vfta, index;
2687
2688 /* don't update vlan cookie if already programmed */
2689 if ((adapter->hw.mng_cookie.status &
2690 E1000_MNG_DHCP_COOKIE_STATUS_VLAN) &&
2691 (vid == adapter->mng_vlan_id))
8e586137 2692 return 0;
caaddaf8 2693
bc7f75fa 2694 /* add VID to filter table */
caaddaf8
BA		 2695 if (adapter->flags & FLAG_HAS_HW_VLAN_FILTER) {
2696 index = (vid >> 5) & 0x7F;
2697 vfta = E1000_READ_REG_ARRAY(hw, E1000_VFTA, index);
2698 vfta |= (1 << (vid & 0x1F));
2699 hw->mac.ops.write_vfta(hw, index, vfta);
2700 }
86d70e53
JK		 2701
2702 set_bit(vid, adapter->active_vlans);
8e586137
JP		 2703
2704 return 0;
bc7f75fa
AK		 2705}
2706
8e586137 2707static int e1000_vlan_rx_kill_vid(struct net_device *netdev, u16 vid)
bc7f75fa
AK		 2708{
2709 struct e1000_adapter *adapter = netdev_priv(netdev);
2710 struct e1000_hw *hw = &adapter->hw;
2711 u32 vfta, index;
2712
bc7f75fa
AK		 2713 if ((adapter->hw.mng_cookie.status &
2714 E1000_MNG_DHCP_COOKIE_STATUS_VLAN) &&
2715 (vid == adapter->mng_vlan_id)) {
2716 /* release control to f/w */
31dbe5b4 2717 e1000e_release_hw_control(adapter);
8e586137 2718 return 0;
bc7f75fa
AK		 2719 }
2720
2721 /* remove VID from filter table */
caaddaf8
BA		 2722 if (adapter->flags & FLAG_HAS_HW_VLAN_FILTER) {
2723 index = (vid >> 5) & 0x7F;
2724 vfta = E1000_READ_REG_ARRAY(hw, E1000_VFTA, index);
2725 vfta &= ~(1 << (vid & 0x1F));
2726 hw->mac.ops.write_vfta(hw, index, vfta);
2727 }
86d70e53
JK		 2728
2729 clear_bit(vid, adapter->active_vlans);
8e586137
JP		 2730
2731 return 0;
bc7f75fa
AK		 2732}
2733
86d70e53
JK		 2734/**
2735 * e1000e_vlan_filter_disable - helper to disable hw VLAN filtering
2736 * @adapter: board private structure to initialize
2737 **/
2738static void e1000e_vlan_filter_disable(struct e1000_adapter *adapter)
bc7f75fa
AK		 2739{
2740 struct net_device *netdev = adapter->netdev;
86d70e53
JK		 2741 struct e1000_hw *hw = &adapter->hw;
2742 u32 rctl;
bc7f75fa 2743
86d70e53
JK		 2744 if (adapter->flags & FLAG_HAS_HW_VLAN_FILTER) {
2745 /* disable VLAN receive filtering */
2746 rctl = er32(RCTL);
2747 rctl &= ~(E1000_RCTL_VFE | E1000_RCTL_CFIEN);
2748 ew32(RCTL, rctl);
2749
2750 if (adapter->mng_vlan_id != (u16)E1000_MNG_VLAN_NONE) {
2751 e1000_vlan_rx_kill_vid(netdev, adapter->mng_vlan_id);
2752 adapter->mng_vlan_id = E1000_MNG_VLAN_NONE;
bc7f75fa 2753 }
bc7f75fa
AK		 2754 }
2755}
2756
86d70e53
JK		 2757/**
2758 * e1000e_vlan_filter_enable - helper to enable HW VLAN filtering
2759 * @adapter: board private structure to initialize
2760 **/
2761static void e1000e_vlan_filter_enable(struct e1000_adapter *adapter)
2762{
2763 struct e1000_hw *hw = &adapter->hw;
2764 u32 rctl;
2765
2766 if (adapter->flags & FLAG_HAS_HW_VLAN_FILTER) {
2767 /* enable VLAN receive filtering */
2768 rctl = er32(RCTL);
2769 rctl |= E1000_RCTL_VFE;
2770 rctl &= ~E1000_RCTL_CFIEN;
2771 ew32(RCTL, rctl);
2772 }
2773}
bc7f75fa 2774
86d70e53
JK		 2775/**
2776 * e1000e_vlan_strip_enable - helper to disable HW VLAN stripping
2777 * @adapter: board private structure to initialize
2778 **/
2779static void e1000e_vlan_strip_disable(struct e1000_adapter *adapter)
bc7f75fa 2780{
bc7f75fa 2781 struct e1000_hw *hw = &adapter->hw;
86d70e53 2782 u32 ctrl;
bc7f75fa 2783
86d70e53
JK		 2784 /* disable VLAN tag insert/strip */
2785 ctrl = er32(CTRL);
2786 ctrl &= ~E1000_CTRL_VME;
2787 ew32(CTRL, ctrl);
2788}
bc7f75fa 2789
86d70e53
JK		 2790/**
2791 * e1000e_vlan_strip_enable - helper to enable HW VLAN stripping
2792 * @adapter: board private structure to initialize
2793 **/
2794static void e1000e_vlan_strip_enable(struct e1000_adapter *adapter)
2795{
2796 struct e1000_hw *hw = &adapter->hw;
2797 u32 ctrl;
bc7f75fa 2798
86d70e53
JK		 2799 /* enable VLAN tag insert/strip */
2800 ctrl = er32(CTRL);
2801 ctrl |= E1000_CTRL_VME;
2802 ew32(CTRL, ctrl);
2803}
bc7f75fa 2804
86d70e53
JK		 2805static void e1000_update_mng_vlan(struct e1000_adapter *adapter)
2806{
2807 struct net_device *netdev = adapter->netdev;
2808 u16 vid = adapter->hw.mng_cookie.vlan_id;
2809 u16 old_vid = adapter->mng_vlan_id;
2810
2811 if (adapter->hw.mng_cookie.status &
2812 E1000_MNG_DHCP_COOKIE_STATUS_VLAN) {
2813 e1000_vlan_rx_add_vid(netdev, vid);
2814 adapter->mng_vlan_id = vid;
bc7f75fa
AK		 2815 }
2816
86d70e53
JK		 2817 if ((old_vid != (u16)E1000_MNG_VLAN_NONE) && (vid != old_vid))
2818 e1000_vlan_rx_kill_vid(netdev, old_vid);
bc7f75fa
AK		 2819}
2820
2821static void e1000_restore_vlan(struct e1000_adapter *adapter)
2822{
2823 u16 vid;
2824
86d70e53 2825 e1000_vlan_rx_add_vid(adapter->netdev, 0);
bc7f75fa 2826
86d70e53 2827 for_each_set_bit(vid, adapter->active_vlans, VLAN_N_VID)
f0ff4398 2828 e1000_vlan_rx_add_vid(adapter->netdev, vid);
bc7f75fa
AK		 2829}
2830
cd791618 2831static void e1000_init_manageability_pt(struct e1000_adapter *adapter)
bc7f75fa
AK		 2832{
2833 struct e1000_hw *hw = &adapter->hw;
cd791618 2834 u32 manc, manc2h, mdef, i, j;
bc7f75fa
AK		 2835
2836 if (!(adapter->flags & FLAG_MNG_PT_ENABLED))
2837 return;
2838
2839 manc = er32(MANC);
2840
e921eb1a 2841 /* enable receiving management packets to the host. this will probably
bc7f75fa 2842 * generate destination unreachable messages from the host OS, but
ad68076e
BA		 2843 * the packets will be handled on SMBUS
2844 */
bc7f75fa
AK		 2845 manc |= E1000_MANC_EN_MNG2HOST;
2846 manc2h = er32(MANC2H);
cd791618
BA		 2847
2848 switch (hw->mac.type) {
2849 default:
2850 manc2h |= (E1000_MANC2H_PORT_623 | E1000_MANC2H_PORT_664);
2851 break;
2852 case e1000_82574:
2853 case e1000_82583:
e921eb1a 2854 /* Check if IPMI pass-through decision filter already exists;
cd791618
BA		 2855 * if so, enable it.
2856 */
2857 for (i = 0, j = 0; i < 8; i++) {
2858 mdef = er32(MDEF(i));
2859
2860 /* Ignore filters with anything other than IPMI ports */
3b21b508 2861 if (mdef & ~(E1000_MDEF_PORT_623 | E1000_MDEF_PORT_664))
cd791618
BA		 2862 continue;
2863
2864 /* Enable this decision filter in MANC2H */
2865 if (mdef)
2866 manc2h |= (1 << i);
2867
2868 j |= mdef;
2869 }
2870
2871 if (j == (E1000_MDEF_PORT_623 | E1000_MDEF_PORT_664))
2872 break;
2873
2874 /* Create new decision filter in an empty filter */
2875 for (i = 0, j = 0; i < 8; i++)
2876 if (er32(MDEF(i)) == 0) {
2877 ew32(MDEF(i), (E1000_MDEF_PORT_623 |
2878 E1000_MDEF_PORT_664));
2879 manc2h |= (1 << 1);
2880 j++;
2881 break;
2882 }
2883
2884 if (!j)
2885 e_warn("Unable to create IPMI pass-through filter\n");
2886 break;
2887 }
2888
bc7f75fa
AK		 2889 ew32(MANC2H, manc2h);
2890 ew32(MANC, manc);
2891}
2892
2893/**
af667a29 2894 * e1000_configure_tx - Configure Transmit Unit after Reset
bc7f75fa
AK		 2895 * @adapter: board private structure
2896 *
2897 * Configure the Tx unit of the MAC after a reset.
2898 **/
2899static void e1000_configure_tx(struct e1000_adapter *adapter)
2900{
2901 struct e1000_hw *hw = &adapter->hw;
2902 struct e1000_ring *tx_ring = adapter->tx_ring;
2903 u64 tdba;
c550b121 2904 u32 tdlen, tarc;
bc7f75fa
AK		 2905
2906 /* Setup the HW Tx Head and Tail descriptor pointers */
2907 tdba = tx_ring->dma;
2908 tdlen = tx_ring->count * sizeof(struct e1000_tx_desc);
1e36052e
BA		 2909 ew32(TDBAL(0), (tdba & DMA_BIT_MASK(32)));
2910 ew32(TDBAH(0), (tdba >> 32));
2911 ew32(TDLEN(0), tdlen);
2912 ew32(TDH(0), 0);
2913 ew32(TDT(0), 0);
2914 tx_ring->head = adapter->hw.hw_addr + E1000_TDH(0);
2915 tx_ring->tail = adapter->hw.hw_addr + E1000_TDT(0);
bc7f75fa 2916
bc7f75fa
AK		 2917 /* Set the Tx Interrupt Delay register */
2918 ew32(TIDV, adapter->tx_int_delay);
ad68076e 2919 /* Tx irq moderation */
bc7f75fa
AK		 2920 ew32(TADV, adapter->tx_abs_int_delay);
2921
3a3b7586
JB		 2922 if (adapter->flags2 & FLAG2_DMA_BURST) {
2923 u32 txdctl = er32(TXDCTL(0));
2924 txdctl &= ~(E1000_TXDCTL_PTHRESH | E1000_TXDCTL_HTHRESH |
2925 E1000_TXDCTL_WTHRESH);
e921eb1a 2926 /* set up some performance related parameters to encourage the
3a3b7586
JB		 2927 * hardware to use the bus more efficiently in bursts, depends
2928 * on the tx_int_delay to be enabled,
8edc0e62 2929 * wthresh = 1 ==> burst write is disabled to avoid Tx stalls
3a3b7586
JB		 2930 * hthresh = 1 ==> prefetch when one or more available
2931 * pthresh = 0x1f ==> prefetch if internal cache 31 or less
2932 * BEWARE: this seems to work but should be considered first if
af667a29 2933 * there are Tx hangs or other Tx related bugs
3a3b7586
JB		 2934 */
2935 txdctl |= E1000_TXDCTL_DMA_BURST_ENABLE;
2936 ew32(TXDCTL(0), txdctl);
3a3b7586 2937 }
56032be7
BA		 2938 /* erratum work around: set txdctl the same for both queues */
2939 ew32(TXDCTL(1), er32(TXDCTL(0)));
3a3b7586 2940
bc7f75fa 2941 if (adapter->flags & FLAG_TARC_SPEED_MODE_BIT) {
e9ec2c0f 2942 tarc = er32(TARC(0));
e921eb1a 2943 /* set the speed mode bit, we'll clear it if we're not at
ad68076e
BA		 2944 * gigabit link later
2945 */
bc7f75fa
AK		 2946#define SPEED_MODE_BIT (1 << 21)
2947 tarc |= SPEED_MODE_BIT;
e9ec2c0f 2948 ew32(TARC(0), tarc);
bc7f75fa
AK		 2949 }
2950
2951 /* errata: program both queues to unweighted RR */
2952 if (adapter->flags & FLAG_TARC_SET_BIT_ZERO) {
e9ec2c0f 2953 tarc = er32(TARC(0));
bc7f75fa 2954 tarc |= 1;
e9ec2c0f
JK		 2955 ew32(TARC(0), tarc);
2956 tarc = er32(TARC(1));
bc7f75fa 2957 tarc |= 1;
e9ec2c0f 2958 ew32(TARC(1), tarc);
bc7f75fa
AK		 2959 }
2960
bc7f75fa
AK		 2961 /* Setup Transmit Descriptor Settings for eop descriptor */
2962 adapter->txd_cmd = E1000_TXD_CMD_EOP | E1000_TXD_CMD_IFCS;
2963
2964 /* only set IDE if we are delaying interrupts using the timers */
2965 if (adapter->tx_int_delay)
2966 adapter->txd_cmd |= E1000_TXD_CMD_IDE;
2967
2968 /* enable Report Status bit */
2969 adapter->txd_cmd |= E1000_TXD_CMD_RS;
2970
57cde763 2971 hw->mac.ops.config_collision_dist(hw);
bc7f75fa
AK		 2972}
2973
2974/**
2975 * e1000_setup_rctl - configure the receive control registers
2976 * @adapter: Board private structure
2977 **/
2978#define PAGE_USE_COUNT(S) (((S) >> PAGE_SHIFT) + \
2979 (((S) & (PAGE_SIZE - 1)) ? 1 : 0))
2980static void e1000_setup_rctl(struct e1000_adapter *adapter)
2981{
2982 struct e1000_hw *hw = &adapter->hw;
2983 u32 rctl, rfctl;
bc7f75fa
AK		 2984 u32 pages = 0;
2985
2fbe4526
BA		 2986 /* Workaround Si errata on PCHx - configure jumbo frame flow */
2987 if (hw->mac.type >= e1000_pch2lan) {
a1ce6473
BA		 2988 s32 ret_val;
2989
2990 if (adapter->netdev->mtu > ETH_DATA_LEN)
2991 ret_val = e1000_lv_jumbo_workaround_ich8lan(hw, true);
2992 else
2993 ret_val = e1000_lv_jumbo_workaround_ich8lan(hw, false);
dd93f95e
BA		 2994
2995 if (ret_val)
2996 e_dbg("failed to enable jumbo frame workaround mode\n");
a1ce6473
BA		 2997 }
2998
bc7f75fa
AK		 2999 /* Program MC offset vector base */
3000 rctl = er32(RCTL);
3001 rctl &= ~(3 << E1000_RCTL_MO_SHIFT);
3002 rctl |= E1000_RCTL_EN | E1000_RCTL_BAM |
f0ff4398
BA		 3003 E1000_RCTL_LBM_NO | E1000_RCTL_RDMTS_HALF |
3004 (adapter->hw.mac.mc_filter_type << E1000_RCTL_MO_SHIFT);
bc7f75fa
AK		 3005
3006 /* Do not Store bad packets */
3007 rctl &= ~E1000_RCTL_SBP;
3008
3009 /* Enable Long Packet receive */
3010 if (adapter->netdev->mtu <= ETH_DATA_LEN)
3011 rctl &= ~E1000_RCTL_LPE;
3012 else
3013 rctl |= E1000_RCTL_LPE;
3014
eb7c3adb
JK		 3015 /* Some systems expect that the CRC is included in SMBUS traffic. The
3016 * hardware strips the CRC before sending to both SMBUS (BMC) and to
3017 * host memory when this is enabled
3018 */
3019 if (adapter->flags2 & FLAG2_CRC_STRIPPING)
3020 rctl |= E1000_RCTL_SECRC;
5918bd88 3021
a4f58f54
BA		 3022 /* Workaround Si errata on 82577 PHY - configure IPG for jumbos */
3023 if ((hw->phy.type == e1000_phy_82577) && (rctl & E1000_RCTL_LPE)) {
3024 u16 phy_data;
3025
3026 e1e_rphy(hw, PHY_REG(770, 26), &phy_data);
3027 phy_data &= 0xfff8;
3028 phy_data |= (1 << 2);
3029 e1e_wphy(hw, PHY_REG(770, 26), phy_data);
3030
3031 e1e_rphy(hw, 22, &phy_data);
3032 phy_data &= 0x0fff;
3033 phy_data |= (1 << 14);
3034 e1e_wphy(hw, 0x10, 0x2823);
3035 e1e_wphy(hw, 0x11, 0x0003);
3036 e1e_wphy(hw, 22, phy_data);
3037 }
3038
bc7f75fa
AK		 3039 /* Setup buffer sizes */
3040 rctl &= ~E1000_RCTL_SZ_4096;
3041 rctl |= E1000_RCTL_BSEX;
3042 switch (adapter->rx_buffer_len) {
bc7f75fa
AK		 3043 case 2048:
3044 default:
3045 rctl |= E1000_RCTL_SZ_2048;
3046 rctl &= ~E1000_RCTL_BSEX;
3047 break;
3048 case 4096:
3049 rctl |= E1000_RCTL_SZ_4096;
3050 break;
3051 case 8192:
3052 rctl |= E1000_RCTL_SZ_8192;
3053 break;
3054 case 16384:
3055 rctl |= E1000_RCTL_SZ_16384;
3056 break;
3057 }
3058
5f450212
BA		 3059 /* Enable Extended Status in all Receive Descriptors */
3060 rfctl = er32(RFCTL);
3061 rfctl |= E1000_RFCTL_EXTEN;
f6bd5577 3062 ew32(RFCTL, rfctl);
5f450212 3063
e921eb1a 3064 /* 82571 and greater support packet-split where the protocol
bc7f75fa
AK		 3065 * header is placed in skb->data and the packet data is
3066 * placed in pages hanging off of skb_shinfo(skb)->nr_frags.
3067 * In the case of a non-split, skb->data is linearly filled,
3068 * followed by the page buffers. Therefore, skb->data is
3069 * sized to hold the largest protocol header.
3070 *
3071 * allocations using alloc_page take too long for regular MTU
3072 * so only enable packet split for jumbo frames
3073 *
3074 * Using pages when the page size is greater than 16k wastes
3075 * a lot of memory, since we allocate 3 pages at all times
3076 * per packet.
3077 */
bc7f75fa 3078 pages = PAGE_USE_COUNT(adapter->netdev->mtu);
79d4e908 3079 if ((pages <= 3) && (PAGE_SIZE <= 16384) && (rctl & E1000_RCTL_LPE))
bc7f75fa 3080 adapter->rx_ps_pages = pages;
97ac8cae
BA		 3081 else
3082 adapter->rx_ps_pages = 0;
bc7f75fa
AK		 3083
3084 if (adapter->rx_ps_pages) {
90da0669
BA		 3085 u32 psrctl = 0;
3086
140a7480
AK		 3087 /* Enable Packet split descriptors */
3088 rctl |= E1000_RCTL_DTYP_PS;
bc7f75fa
AK		 3089
3090 psrctl |= adapter->rx_ps_bsize0 >>
3091 E1000_PSRCTL_BSIZE0_SHIFT;
3092
3093 switch (adapter->rx_ps_pages) {
3094 case 3:
3095 psrctl |= PAGE_SIZE <<
3096 E1000_PSRCTL_BSIZE3_SHIFT;
3097 case 2:
3098 psrctl |= PAGE_SIZE <<
3099 E1000_PSRCTL_BSIZE2_SHIFT;
3100 case 1:
3101 psrctl |= PAGE_SIZE >>
3102 E1000_PSRCTL_BSIZE1_SHIFT;
3103 break;
3104 }
3105
3106 ew32(PSRCTL, psrctl);
3107 }
3108
cf955e6c
BG		 3109 /* This is useful for sniffing bad packets. */
3110 if (adapter->netdev->features & NETIF_F_RXALL) {
3111 /* UPE and MPE will be handled by normal PROMISC logic
e921eb1a
BA		 3112 * in e1000e_set_rx_mode
3113 */
cf955e6c
BG		 3114 rctl |= (E1000_RCTL_SBP | /* Receive bad packets */
3115 E1000_RCTL_BAM | /* RX All Bcast Pkts */
3116 E1000_RCTL_PMCF); /* RX All MAC Ctrl Pkts */
3117
3118 rctl &= ~(E1000_RCTL_VFE | /* Disable VLAN filter */
3119 E1000_RCTL_DPF | /* Allow filtered pause */
3120 E1000_RCTL_CFIEN); /* Dis VLAN CFIEN Filter */
3121 /* Do not mess with E1000_CTRL_VME, it affects transmit as well,
3122 * and that breaks VLANs.
3123 */
3124 }
3125
bc7f75fa 3126 ew32(RCTL, rctl);
318a94d6 3127 /* just started the receive unit, no need to restart */
12d43f7d 3128 adapter->flags &= ~FLAG_RESTART_NOW;
bc7f75fa
AK		 3129}
3130
3131/**
3132 * e1000_configure_rx - Configure Receive Unit after Reset
3133 * @adapter: board private structure
3134 *
3135 * Configure the Rx unit of the MAC after a reset.
3136 **/
3137static void e1000_configure_rx(struct e1000_adapter *adapter)
3138{
3139 struct e1000_hw *hw = &adapter->hw;
3140 struct e1000_ring *rx_ring = adapter->rx_ring;
3141 u64 rdba;
3142 u32 rdlen, rctl, rxcsum, ctrl_ext;
3143
3144 if (adapter->rx_ps_pages) {
3145 /* this is a 32 byte descriptor */
3146 rdlen = rx_ring->count *
af667a29 3147 sizeof(union e1000_rx_desc_packet_split);
bc7f75fa
AK		 3148 adapter->clean_rx = e1000_clean_rx_irq_ps;
3149 adapter->alloc_rx_buf = e1000_alloc_rx_buffers_ps;
97ac8cae 3150 } else if (adapter->netdev->mtu > ETH_FRAME_LEN + ETH_FCS_LEN) {
5f450212 3151 rdlen = rx_ring->count * sizeof(union e1000_rx_desc_extended);
97ac8cae
BA		 3152 adapter->clean_rx = e1000_clean_jumbo_rx_irq;
3153 adapter->alloc_rx_buf = e1000_alloc_jumbo_rx_buffers;
bc7f75fa 3154 } else {
5f450212 3155 rdlen = rx_ring->count * sizeof(union e1000_rx_desc_extended);
bc7f75fa
AK		 3156 adapter->clean_rx = e1000_clean_rx_irq;
3157 adapter->alloc_rx_buf = e1000_alloc_rx_buffers;
3158 }
3159
3160 /* disable receives while setting up the descriptors */
3161 rctl = er32(RCTL);
7f99ae63
BA		 3162 if (!(adapter->flags2 & FLAG2_NO_DISABLE_RX))
3163 ew32(RCTL, rctl & ~E1000_RCTL_EN);
bc7f75fa 3164 e1e_flush();
1bba4386 3165 usleep_range(10000, 20000);
bc7f75fa 3166
3a3b7586 3167 if (adapter->flags2 & FLAG2_DMA_BURST) {
e921eb1a 3168 /* set the writeback threshold (only takes effect if the RDTR
3a3b7586 3169 * is set). set GRAN=1 and write back up to 0x4 worth, and
af667a29 3170 * enable prefetching of 0x20 Rx descriptors
3a3b7586
JB		 3171 * granularity = 01
3172 * wthresh = 04,
3173 * hthresh = 04,
3174 * pthresh = 0x20
3175 */
3176 ew32(RXDCTL(0), E1000_RXDCTL_DMA_BURST_ENABLE);
3177 ew32(RXDCTL(1), E1000_RXDCTL_DMA_BURST_ENABLE);
3178
e921eb1a 3179 /* override the delay timers for enabling bursting, only if
3a3b7586
JB		 3180 * the value was not set by the user via module options
3181 */
3182 if (adapter->rx_int_delay == DEFAULT_RDTR)
3183 adapter->rx_int_delay = BURST_RDTR;
3184 if (adapter->rx_abs_int_delay == DEFAULT_RADV)
3185 adapter->rx_abs_int_delay = BURST_RADV;
3186 }
3187
bc7f75fa
AK		 3188 /* set the Receive Delay Timer Register */
3189 ew32(RDTR, adapter->rx_int_delay);
3190
3191 /* irq moderation */
3192 ew32(RADV, adapter->rx_abs_int_delay);
828bac87 3193 if ((adapter->itr_setting != 0) && (adapter->itr != 0))
22a4cca2 3194 e1000e_write_itr(adapter, adapter->itr);
bc7f75fa
AK		 3195
3196 ctrl_ext = er32(CTRL_EXT);
bc7f75fa
AK		 3197 /* Auto-Mask interrupts upon ICR access */
3198 ctrl_ext |= E1000_CTRL_EXT_IAME;
3199 ew32(IAM, 0xffffffff);
3200 ew32(CTRL_EXT, ctrl_ext);
3201 e1e_flush();
3202
e921eb1a 3203 /* Setup the HW Rx Head and Tail Descriptor Pointers and
ad68076e
BA		 3204 * the Base and Length of the Rx Descriptor Ring
3205 */
bc7f75fa 3206 rdba = rx_ring->dma;
1e36052e
BA		 3207 ew32(RDBAL(0), (rdba & DMA_BIT_MASK(32)));
3208 ew32(RDBAH(0), (rdba >> 32));
3209 ew32(RDLEN(0), rdlen);
3210 ew32(RDH(0), 0);
3211 ew32(RDT(0), 0);
3212 rx_ring->head = adapter->hw.hw_addr + E1000_RDH(0);
3213 rx_ring->tail = adapter->hw.hw_addr + E1000_RDT(0);
bc7f75fa
AK		 3214
3215 /* Enable Receive Checksum Offload for TCP and UDP */
3216 rxcsum = er32(RXCSUM);
2e1706f2 3217 if (adapter->netdev->features & NETIF_F_RXCSUM)
bc7f75fa 3218 rxcsum |= E1000_RXCSUM_TUOFL;
2e1706f2 3219 else
bc7f75fa 3220 rxcsum &= ~E1000_RXCSUM_TUOFL;
bc7f75fa
AK		 3221 ew32(RXCSUM, rxcsum);
3222
3e35d991
BA		 3223 /* With jumbo frames, excessive C-state transition latencies result
3224 * in dropped transactions.
3225 */
3226 if (adapter->netdev->mtu > ETH_DATA_LEN) {
3227 u32 lat =
3228 ((er32(PBA) & E1000_PBA_RXA_MASK) * 1024 -
3229 adapter->max_frame_size) * 8 / 1000;
3230
3231 if (adapter->flags & FLAG_IS_ICH) {
53ec5498
BA		 3232 u32 rxdctl = er32(RXDCTL(0));
3233 ew32(RXDCTL(0), rxdctl | 0x3);
53ec5498 3234 }
3e35d991
BA		 3235
3236 pm_qos_update_request(&adapter->netdev->pm_qos_req, lat);
3237 } else {
3238 pm_qos_update_request(&adapter->netdev->pm_qos_req,
3239 PM_QOS_DEFAULT_VALUE);
97ac8cae 3240 }
bc7f75fa
AK		 3241
3242 /* Enable Receives */
3243 ew32(RCTL, rctl);
3244}
3245
3246/**
ef9b965a
JB		 3247 * e1000e_write_mc_addr_list - write multicast addresses to MTA
3248 * @netdev: network interface device structure
bc7f75fa 3249 *
ef9b965a
JB		 3250 * Writes multicast address list to the MTA hash table.
3251 * Returns: -ENOMEM on failure
3252 * 0 on no addresses written
3253 * X on writing X addresses to MTA
3254 */
3255static int e1000e_write_mc_addr_list(struct net_device *netdev)
3256{
3257 struct e1000_adapter *adapter = netdev_priv(netdev);
3258 struct e1000_hw *hw = &adapter->hw;
3259 struct netdev_hw_addr *ha;
3260 u8 *mta_list;
3261 int i;
3262
3263 if (netdev_mc_empty(netdev)) {
3264 /* nothing to program, so clear mc list */
3265 hw->mac.ops.update_mc_addr_list(hw, NULL, 0);
3266 return 0;
3267 }
3268
3269 mta_list = kzalloc(netdev_mc_count(netdev) * ETH_ALEN, GFP_ATOMIC);
3270 if (!mta_list)
3271 return -ENOMEM;
3272
3273 /* update_mc_addr_list expects a packed array of only addresses. */
3274 i = 0;
3275 netdev_for_each_mc_addr(ha, netdev)
f0ff4398 3276 memcpy(mta_list + (i++ * ETH_ALEN), ha->addr, ETH_ALEN);
ef9b965a
JB		 3277
3278 hw->mac.ops.update_mc_addr_list(hw, mta_list, i);
3279 kfree(mta_list);
3280
3281 return netdev_mc_count(netdev);
3282}
3283
3284/**
3285 * e1000e_write_uc_addr_list - write unicast addresses to RAR table
3286 * @netdev: network interface device structure
bc7f75fa 3287 *
ef9b965a
JB		 3288 * Writes unicast address list to the RAR table.
3289 * Returns: -ENOMEM on failure/insufficient address space
3290 * 0 on no addresses written
3291 * X on writing X addresses to the RAR table
bc7f75fa 3292 **/
ef9b965a 3293static int e1000e_write_uc_addr_list(struct net_device *netdev)
bc7f75fa 3294{
ef9b965a
JB		 3295 struct e1000_adapter *adapter = netdev_priv(netdev);
3296 struct e1000_hw *hw = &adapter->hw;
3297 unsigned int rar_entries = hw->mac.rar_entry_count;
3298 int count = 0;
3299
3300 /* save a rar entry for our hardware address */
3301 rar_entries--;
3302
3303 /* save a rar entry for the LAA workaround */
3304 if (adapter->flags & FLAG_RESET_OVERWRITES_LAA)
3305 rar_entries--;
3306
3307 /* return ENOMEM indicating insufficient memory for addresses */
3308 if (netdev_uc_count(netdev) > rar_entries)
3309 return -ENOMEM;
3310
3311 if (!netdev_uc_empty(netdev) && rar_entries) {
3312 struct netdev_hw_addr *ha;
3313
e921eb1a 3314 /* write the addresses in reverse order to avoid write
ef9b965a
JB		 3315 * combining
3316 */
3317 netdev_for_each_uc_addr(ha, netdev) {
3318 if (!rar_entries)
3319 break;
69e1e019 3320 hw->mac.ops.rar_set(hw, ha->addr, rar_entries--);
ef9b965a
JB		 3321 count++;
3322 }
3323 }
3324
3325 /* zero out the remaining RAR entries not used above */
3326 for (; rar_entries > 0; rar_entries--) {
3327 ew32(RAH(rar_entries), 0);
3328 ew32(RAL(rar_entries), 0);
3329 }
3330 e1e_flush();
3331
3332 return count;
bc7f75fa
AK		 3333}
3334
3335/**
ef9b965a 3336 * e1000e_set_rx_mode - secondary unicast, Multicast and Promiscuous mode set
bc7f75fa
AK		 3337 * @netdev: network interface device structure
3338 *
ef9b965a
JB		 3339 * The ndo_set_rx_mode entry point is called whenever the unicast or multicast
3340 * address list or the network interface flags are updated. This routine is
3341 * responsible for configuring the hardware for proper unicast, multicast,
bc7f75fa
AK		 3342 * promiscuous mode, and all-multi behavior.
3343 **/
ef9b965a 3344static void e1000e_set_rx_mode(struct net_device *netdev)
bc7f75fa
AK		 3345{
3346 struct e1000_adapter *adapter = netdev_priv(netdev);
3347 struct e1000_hw *hw = &adapter->hw;
bc7f75fa 3348 u32 rctl;
bc7f75fa
AK		 3349
3350 /* Check for Promiscuous and All Multicast modes */
bc7f75fa
AK		 3351 rctl = er32(RCTL);
3352
ef9b965a
JB		 3353 /* clear the affected bits */
3354 rctl &= ~(E1000_RCTL_UPE | E1000_RCTL_MPE);
3355
bc7f75fa
AK		 3356 if (netdev->flags & IFF_PROMISC) {
3357 rctl |= (E1000_RCTL_UPE | E1000_RCTL_MPE);
86d70e53
JK		 3358 /* Do not hardware filter VLANs in promisc mode */
3359 e1000e_vlan_filter_disable(adapter);
bc7f75fa 3360 } else {
ef9b965a 3361 int count;
3d3a1676 3362
746b9f02
PM		 3363 if (netdev->flags & IFF_ALLMULTI) {
3364 rctl |= E1000_RCTL_MPE;
746b9f02 3365 } else {
e921eb1a 3366 /* Write addresses to the MTA, if the attempt fails
ef9b965a
JB		 3367 * then we should just turn on promiscuous mode so
3368 * that we can at least receive multicast traffic
3369 */
3370 count = e1000e_write_mc_addr_list(netdev);
3371 if (count < 0)
3372 rctl |= E1000_RCTL_MPE;
746b9f02 3373 }
86d70e53 3374 e1000e_vlan_filter_enable(adapter);
e921eb1a 3375 /* Write addresses to available RAR registers, if there is not
ef9b965a
JB		 3376 * sufficient space to store all the addresses then enable
3377 * unicast promiscuous mode
bc7f75fa 3378 */
ef9b965a
JB		 3379 count = e1000e_write_uc_addr_list(netdev);
3380 if (count < 0)
3381 rctl |= E1000_RCTL_UPE;
bc7f75fa 3382 }
86d70e53 3383
ef9b965a
JB		 3384 ew32(RCTL, rctl);
3385
86d70e53
JK		 3386 if (netdev->features & NETIF_F_HW_VLAN_RX)
3387 e1000e_vlan_strip_enable(adapter);
3388 else
3389 e1000e_vlan_strip_disable(adapter);
bc7f75fa
AK		 3390}
3391
70495a50
BA		 3392static void e1000e_setup_rss_hash(struct e1000_adapter *adapter)
3393{
3394 struct e1000_hw *hw = &adapter->hw;
3395 u32 mrqc, rxcsum;
3396 int i;
3397 static const u32 rsskey[10] = {
3398 0xda565a6d, 0xc20e5b25, 0x3d256741, 0xb08fa343, 0xcb2bcad0,
3399 0xb4307bae, 0xa32dcb77, 0x0cf23080, 0x3bb7426a, 0xfa01acbe
3400 };
3401
3402 /* Fill out hash function seed */
3403 for (i = 0; i < 10; i++)
3404 ew32(RSSRK(i), rsskey[i]);
3405
3406 /* Direct all traffic to queue 0 */
3407 for (i = 0; i < 32; i++)
3408 ew32(RETA(i), 0);
3409
e921eb1a 3410 /* Disable raw packet checksumming so that RSS hash is placed in
70495a50
BA		 3411 * descriptor on writeback.
3412 */
3413 rxcsum = er32(RXCSUM);
3414 rxcsum |= E1000_RXCSUM_PCSD;
3415
3416 ew32(RXCSUM, rxcsum);
3417
3418 mrqc = (E1000_MRQC_RSS_FIELD_IPV4 |
3419 E1000_MRQC_RSS_FIELD_IPV4_TCP |
3420 E1000_MRQC_RSS_FIELD_IPV6 |
3421 E1000_MRQC_RSS_FIELD_IPV6_TCP |
3422 E1000_MRQC_RSS_FIELD_IPV6_TCP_EX);
3423
3424 ew32(MRQC, mrqc);
3425}
3426
b67e1913
BA		 3427/**
3428 * e1000e_get_base_timinca - get default SYSTIM time increment attributes
3429 * @adapter: board private structure
3430 * @timinca: pointer to returned time increment attributes
3431 *
3432 * Get attributes for incrementing the System Time Register SYSTIML/H at
3433 * the default base frequency, and set the cyclecounter shift value.
3434 **/
d89777bf 3435s32 e1000e_get_base_timinca(struct e1000_adapter *adapter, u32 *timinca)
b67e1913
BA		 3436{
3437 struct e1000_hw *hw = &adapter->hw;
3438 u32 incvalue, incperiod, shift;
3439
3440 /* Make sure clock is enabled on I217 before checking the frequency */
3441 if ((hw->mac.type == e1000_pch_lpt) &&
3442 !(er32(TSYNCTXCTL) & E1000_TSYNCTXCTL_ENABLED) &&
3443 !(er32(TSYNCRXCTL) & E1000_TSYNCRXCTL_ENABLED)) {
3444 u32 fextnvm7 = er32(FEXTNVM7);
3445
3446 if (!(fextnvm7 & (1 << 0))) {
3447 ew32(FEXTNVM7, fextnvm7 | (1 << 0));
3448 e1e_flush();
3449 }
3450 }
3451
3452 switch (hw->mac.type) {
3453 case e1000_pch2lan:
3454 case e1000_pch_lpt:
3455 /* On I217, the clock frequency is 25MHz or 96MHz as
3456 * indicated by the System Clock Frequency Indication
3457 */
3458 if ((hw->mac.type != e1000_pch_lpt) ||
3459 (er32(TSYNCRXCTL) & E1000_TSYNCRXCTL_SYSCFI)) {
3460 /* Stable 96MHz frequency */
3461 incperiod = INCPERIOD_96MHz;
3462 incvalue = INCVALUE_96MHz;
3463 shift = INCVALUE_SHIFT_96MHz;
3464 adapter->cc.shift = shift + INCPERIOD_SHIFT_96MHz;
3465 break;
3466 }
3467 /* fall-through */
3468 case e1000_82574:
3469 case e1000_82583:
3470 /* Stable 25MHz frequency */
3471 incperiod = INCPERIOD_25MHz;
3472 incvalue = INCVALUE_25MHz;
3473 shift = INCVALUE_SHIFT_25MHz;
3474 adapter->cc.shift = shift;
3475 break;
3476 default:
3477 return -EINVAL;
3478 }
3479
3480 *timinca = ((incperiod << E1000_TIMINCA_INCPERIOD_SHIFT) |
3481 ((incvalue << shift) & E1000_TIMINCA_INCVALUE_MASK));
3482
3483 return 0;
3484}
3485
3486/**
3487 * e1000e_config_hwtstamp - configure the hwtstamp registers and enable/disable
3488 * @adapter: board private structure
3489 *
3490 * Outgoing time stamping can be enabled and disabled. Play nice and
3491 * disable it when requested, although it shouldn't cause any overhead
3492 * when no packet needs it. At most one packet in the queue may be
3493 * marked for time stamping, otherwise it would be impossible to tell
3494 * for sure to which packet the hardware time stamp belongs.
3495 *
3496 * Incoming time stamping has to be configured via the hardware filters.
3497 * Not all combinations are supported, in particular event type has to be
3498 * specified. Matching the kind of event packet is not supported, with the
3499 * exception of "all V2 events regardless of level 2 or 4".
3500 **/
3501static int e1000e_config_hwtstamp(struct e1000_adapter *adapter)
3502{
3503 struct e1000_hw *hw = &adapter->hw;
3504 struct hwtstamp_config *config = &adapter->hwtstamp_config;
3505 u32 tsync_tx_ctl = E1000_TSYNCTXCTL_ENABLED;
3506 u32 tsync_rx_ctl = E1000_TSYNCRXCTL_ENABLED;
d89777bf
BA		 3507 u32 rxmtrl = 0;
3508 u16 rxudp = 0;
3509 bool is_l4 = false;
3510 bool is_l2 = false;
b67e1913
BA		 3511 u32 regval;
3512 s32 ret_val;
3513
3514 if (!(adapter->flags & FLAG_HAS_HW_TIMESTAMP))
3515 return -EINVAL;
3516
3517 /* flags reserved for future extensions - must be zero */
3518 if (config->flags)
3519 return -EINVAL;
3520
3521 switch (config->tx_type) {
3522 case HWTSTAMP_TX_OFF:
3523 tsync_tx_ctl = 0;
3524 break;
3525 case HWTSTAMP_TX_ON:
3526 break;
3527 default:
3528 return -ERANGE;
3529 }
3530
3531 switch (config->rx_filter) {
3532 case HWTSTAMP_FILTER_NONE:
3533 tsync_rx_ctl = 0;
3534 break;
d89777bf
BA		 3535 case HWTSTAMP_FILTER_PTP_V1_L4_SYNC:
3536 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_L4_V1;
3537 rxmtrl = E1000_RXMTRL_PTP_V1_SYNC_MESSAGE;
3538 is_l4 = true;
3539 break;
3540 case HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ:
3541 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_L4_V1;
3542 rxmtrl = E1000_RXMTRL_PTP_V1_DELAY_REQ_MESSAGE;
3543 is_l4 = true;
3544 break;
3545 case HWTSTAMP_FILTER_PTP_V2_L2_SYNC:
3546 /* Also time stamps V2 L2 Path Delay Request/Response */
3547 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_L2_V2;
3548 rxmtrl = E1000_RXMTRL_PTP_V2_SYNC_MESSAGE;
3549 is_l2 = true;
3550 break;
3551 case HWTSTAMP_FILTER_PTP_V2_L2_DELAY_REQ:
3552 /* Also time stamps V2 L2 Path Delay Request/Response. */
3553 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_L2_V2;
3554 rxmtrl = E1000_RXMTRL_PTP_V2_DELAY_REQ_MESSAGE;
3555 is_l2 = true;
3556 break;
3557 case HWTSTAMP_FILTER_PTP_V2_L4_SYNC:
3558 /* Hardware cannot filter just V2 L4 Sync messages;
3559 * fall-through to V2 (both L2 and L4) Sync.
3560 */
3561 case HWTSTAMP_FILTER_PTP_V2_SYNC:
3562 /* Also time stamps V2 Path Delay Request/Response. */
3563 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_L2_L4_V2;
3564 rxmtrl = E1000_RXMTRL_PTP_V2_SYNC_MESSAGE;
3565 is_l2 = true;
3566 is_l4 = true;
3567 break;
3568 case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ:
3569 /* Hardware cannot filter just V2 L4 Delay Request messages;
3570 * fall-through to V2 (both L2 and L4) Delay Request.
3571 */
3572 case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ:
3573 /* Also time stamps V2 Path Delay Request/Response. */
3574 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_L2_L4_V2;
3575 rxmtrl = E1000_RXMTRL_PTP_V2_DELAY_REQ_MESSAGE;
3576 is_l2 = true;
3577 is_l4 = true;
3578 break;
3579 case HWTSTAMP_FILTER_PTP_V2_L4_EVENT:
3580 case HWTSTAMP_FILTER_PTP_V2_L2_EVENT:
3581 /* Hardware cannot filter just V2 L4 or L2 Event messages;
3582 * fall-through to all V2 (both L2 and L4) Events.
3583 */
3584 case HWTSTAMP_FILTER_PTP_V2_EVENT:
3585 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_EVENT_V2;
3586 config->rx_filter = HWTSTAMP_FILTER_PTP_V2_EVENT;
3587 is_l2 = true;
3588 is_l4 = true;
3589 break;
3590 case HWTSTAMP_FILTER_PTP_V1_L4_EVENT:
3591 /* For V1, the hardware can only filter Sync messages or
3592 * Delay Request messages but not both so fall-through to
3593 * time stamp all packets.
3594 */
b67e1913 3595 case HWTSTAMP_FILTER_ALL:
d89777bf
BA		 3596 is_l2 = true;
3597 is_l4 = true;
b67e1913
BA		 3598 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_ALL;
3599 config->rx_filter = HWTSTAMP_FILTER_ALL;
3600 break;
3601 default:
3602 return -ERANGE;
3603 }
3604
3605 /* enable/disable Tx h/w time stamping */
3606 regval = er32(TSYNCTXCTL);
3607 regval &= ~E1000_TSYNCTXCTL_ENABLED;
3608 regval |= tsync_tx_ctl;
3609 ew32(TSYNCTXCTL, regval);
3610 if ((er32(TSYNCTXCTL) & E1000_TSYNCTXCTL_ENABLED) !=
3611 (regval & E1000_TSYNCTXCTL_ENABLED)) {
3612 e_err("Timesync Tx Control register not set as expected\n");
3613 return -EAGAIN;
3614 }
3615
3616 /* enable/disable Rx h/w time stamping */
3617 regval = er32(TSYNCRXCTL);
3618 regval &= ~(E1000_TSYNCRXCTL_ENABLED | E1000_TSYNCRXCTL_TYPE_MASK);
3619 regval |= tsync_rx_ctl;
3620 ew32(TSYNCRXCTL, regval);
3621 if ((er32(TSYNCRXCTL) & (E1000_TSYNCRXCTL_ENABLED |
3622 E1000_TSYNCRXCTL_TYPE_MASK)) !=
3623 (regval & (E1000_TSYNCRXCTL_ENABLED |
3624 E1000_TSYNCRXCTL_TYPE_MASK))) {
3625 e_err("Timesync Rx Control register not set as expected\n");
3626 return -EAGAIN;
3627 }
3628
d89777bf
BA		 3629 /* L2: define ethertype filter for time stamped packets */
3630 if (is_l2)
3631 rxmtrl |= ETH_P_1588;
3632
3633 /* define which PTP packets get time stamped */
3634 ew32(RXMTRL, rxmtrl);
3635
3636 /* Filter by destination port */
3637 if (is_l4) {
3638 rxudp = PTP_EV_PORT;
3639 cpu_to_be16s(&rxudp);
3640 }
3641 ew32(RXUDP, rxudp);
3642
3643 e1e_flush();
3644
b67e1913 3645 /* Clear TSYNCRXCTL_VALID & TSYNCTXCTL_VALID bit */
70806a7f
BA		 3646 er32(RXSTMPH);
3647 er32(TXSTMPH);
b67e1913
BA		 3648
3649 /* Get and set the System Time Register SYSTIM base frequency */
3650 ret_val = e1000e_get_base_timinca(adapter, &regval);
3651 if (ret_val)
3652 return ret_val;
3653 ew32(TIMINCA, regval);
3654
3655 /* reset the ns time counter */
3656 timecounter_init(&adapter->tc, &adapter->cc,
3657 ktime_to_ns(ktime_get_real()));
3658
3659 return 0;
3660}
3661
bc7f75fa 3662/**
ad68076e 3663 * e1000_configure - configure the hardware for Rx and Tx
bc7f75fa
AK		 3664 * @adapter: private board structure
3665 **/
3666static void e1000_configure(struct e1000_adapter *adapter)
3667{
55aa6985
BA		 3668 struct e1000_ring *rx_ring = adapter->rx_ring;
3669
ef9b965a 3670 e1000e_set_rx_mode(adapter->netdev);
bc7f75fa
AK		 3671
3672 e1000_restore_vlan(adapter);
cd791618 3673 e1000_init_manageability_pt(adapter);
bc7f75fa
AK		 3674
3675 e1000_configure_tx(adapter);
70495a50
BA		 3676
3677 if (adapter->netdev->features & NETIF_F_RXHASH)
3678 e1000e_setup_rss_hash(adapter);
bc7f75fa
AK		 3679 e1000_setup_rctl(adapter);
3680 e1000_configure_rx(adapter);
55aa6985 3681 adapter->alloc_rx_buf(rx_ring, e1000_desc_unused(rx_ring), GFP_KERNEL);
bc7f75fa
AK		 3682}
3683
3684/**
3685 * e1000e_power_up_phy - restore link in case the phy was powered down
3686 * @adapter: address of board private structure
3687 *
3688 * The phy may be powered down to save power and turn off link when the
3689 * driver is unloaded and wake on lan is not enabled (among others)
3690 * *** this routine MUST be followed by a call to e1000e_reset ***
3691 **/
3692void e1000e_power_up_phy(struct e1000_adapter *adapter)
3693{
17f208de
BA		 3694 if (adapter->hw.phy.ops.power_up)
3695 adapter->hw.phy.ops.power_up(&adapter->hw);
bc7f75fa
AK		 3696
3697 adapter->hw.mac.ops.setup_link(&adapter->hw);
3698}
3699
3700/**
3701 * e1000_power_down_phy - Power down the PHY
3702 *
17f208de
BA		 3703 * Power down the PHY so no link is implied when interface is down.
3704 * The PHY cannot be powered down if management or WoL is active.
bc7f75fa
AK		 3705 */
3706static void e1000_power_down_phy(struct e1000_adapter *adapter)
3707{
bc7f75fa 3708 /* WoL is enabled */
23b66e2b 3709 if (adapter->wol)
bc7f75fa
AK		 3710 return;
3711
17f208de
BA		 3712 if (adapter->hw.phy.ops.power_down)
3713 adapter->hw.phy.ops.power_down(&adapter->hw);
bc7f75fa
AK		 3714}
3715
3716/**
3717 * e1000e_reset - bring the hardware into a known good state
3718 *
3719 * This function boots the hardware and enables some settings that
3720 * require a configuration cycle of the hardware - those cannot be
3721 * set/changed during runtime. After reset the device needs to be
ad68076e 3722 * properly configured for Rx, Tx etc.
bc7f75fa
AK		 3723 */
3724void e1000e_reset(struct e1000_adapter *adapter)
3725{
3726 struct e1000_mac_info *mac = &adapter->hw.mac;
318a94d6 3727 struct e1000_fc_info *fc = &adapter->hw.fc;
bc7f75fa
AK		 3728 struct e1000_hw *hw = &adapter->hw;
3729 u32 tx_space, min_tx_space, min_rx_space;
318a94d6 3730 u32 pba = adapter->pba;
bc7f75fa
AK		 3731 u16 hwm;
3732
ad68076e 3733 /* reset Packet Buffer Allocation to default */
318a94d6 3734 ew32(PBA, pba);
df762464 3735
318a94d6 3736 if (adapter->max_frame_size > ETH_FRAME_LEN + ETH_FCS_LEN) {
e921eb1a 3737 /* To maintain wire speed transmits, the Tx FIFO should be
bc7f75fa
AK		 3738 * large enough to accommodate two full transmit packets,
3739 * rounded up to the next 1KB and expressed in KB. Likewise,
3740 * the Rx FIFO should be large enough to accommodate at least
3741 * one full receive packet and is similarly rounded up and
ad68076e
BA		 3742 * expressed in KB.
3743 */
df762464 3744 pba = er32(PBA);
bc7f75fa 3745 /* upper 16 bits has Tx packet buffer allocation size in KB */
df762464 3746 tx_space = pba >> 16;
bc7f75fa 3747 /* lower 16 bits has Rx packet buffer allocation size in KB */
df762464 3748 pba &= 0xffff;
e921eb1a 3749 /* the Tx fifo also stores 16 bytes of information about the Tx
ad68076e 3750 * but don't include ethernet FCS because hardware appends it
318a94d6
JK		 3751 */
3752 min_tx_space = (adapter->max_frame_size +
bc7f75fa
AK		 3753 sizeof(struct e1000_tx_desc) -
3754 ETH_FCS_LEN) * 2;
3755 min_tx_space = ALIGN(min_tx_space, 1024);
3756 min_tx_space >>= 10;
3757 /* software strips receive CRC, so leave room for it */
318a94d6 3758 min_rx_space = adapter->max_frame_size;
bc7f75fa
AK		 3759 min_rx_space = ALIGN(min_rx_space, 1024);
3760 min_rx_space >>= 10;
3761
e921eb1a 3762 /* If current Tx allocation is less than the min Tx FIFO size,
bc7f75fa 3763 * and the min Tx FIFO size is less than the current Rx FIFO
ad68076e
BA		 3764 * allocation, take space away from current Rx allocation
3765 */
df762464
AK		 3766 if ((tx_space < min_tx_space) &&
3767 ((min_tx_space - tx_space) < pba)) {
3768 pba -= min_tx_space - tx_space;
bc7f75fa 3769
e921eb1a 3770 /* if short on Rx space, Rx wins and must trump Tx
419e551c 3771 * adjustment
ad68076e 3772 */
79d4e908 3773 if (pba < min_rx_space)
df762464 3774 pba = min_rx_space;
bc7f75fa 3775 }
df762464
AK		 3776
3777 ew32(PBA, pba);
bc7f75fa
AK		 3778 }
3779
e921eb1a 3780 /* flow control settings
ad68076e 3781 *
38eb394e 3782 * The high water mark must be low enough to fit one full frame
bc7f75fa
AK		 3783 * (or the size used for early receive) above it in the Rx FIFO.
3784 * Set it to the lower of:
3785 * - 90% of the Rx FIFO size, and
38eb394e 3786 * - the full Rx FIFO size minus one full frame
ad68076e 3787 */
d3738bb8
BA		 3788 if (adapter->flags & FLAG_DISABLE_FC_PAUSE_TIME)
3789 fc->pause_time = 0xFFFF;
3790 else
3791 fc->pause_time = E1000_FC_PAUSE_TIME;
b20caa80 3792 fc->send_xon = true;
d3738bb8
BA		 3793 fc->current_mode = fc->requested_mode;
3794
3795 switch (hw->mac.type) {
79d4e908
BA		 3796 case e1000_ich9lan:
3797 case e1000_ich10lan:
3798 if (adapter->netdev->mtu > ETH_DATA_LEN) {
3799 pba = 14;
3800 ew32(PBA, pba);
3801 fc->high_water = 0x2800;
3802 fc->low_water = fc->high_water - 8;
3803 break;
3804 }
3805 /* fall-through */
d3738bb8 3806 default:
79d4e908
BA		 3807 hwm = min(((pba << 10) * 9 / 10),
3808 ((pba << 10) - adapter->max_frame_size));
d3738bb8
BA		 3809
3810 fc->high_water = hwm & E1000_FCRTH_RTH; /* 8-byte granularity */
3811 fc->low_water = fc->high_water - 8;
3812 break;
3813 case e1000_pchlan:
e921eb1a 3814 /* Workaround PCH LOM adapter hangs with certain network
38eb394e
BA		 3815 * loads. If hangs persist, try disabling Tx flow control.
3816 */
3817 if (adapter->netdev->mtu > ETH_DATA_LEN) {
3818 fc->high_water = 0x3500;
3819 fc->low_water = 0x1500;
3820 } else {
3821 fc->high_water = 0x5000;
3822 fc->low_water = 0x3000;
3823 }
a305595b 3824 fc->refresh_time = 0x1000;
d3738bb8
BA		 3825 break;
3826 case e1000_pch2lan:
2fbe4526 3827 case e1000_pch_lpt:
d3738bb8 3828 fc->refresh_time = 0x0400;
347b5201
BA		 3829
3830 if (adapter->netdev->mtu <= ETH_DATA_LEN) {
3831 fc->high_water = 0x05C20;
3832 fc->low_water = 0x05048;
3833 fc->pause_time = 0x0650;
3834 break;
828bac87 3835 }
347b5201
BA		 3836
3837 fc->high_water = ((pba << 10) * 9 / 10) & E1000_FCRTH_RTH;
3838 fc->low_water = ((pba << 10) * 8 / 10) & E1000_FCRTL_RTL;
d3738bb8 3839 break;
38eb394e 3840 }
bc7f75fa 3841
e921eb1a 3842 /* Alignment of Tx data is on an arbitrary byte boundary with the
d821a4c4
BA		 3843 * maximum size per Tx descriptor limited only to the transmit
3844 * allocation of the packet buffer minus 96 bytes with an upper
3845 * limit of 24KB due to receive synchronization limitations.
3846 */
3847 adapter->tx_fifo_limit = min_t(u32, ((er32(PBA) >> 16) << 10) - 96,
3848 24 << 10);
3849
e921eb1a 3850 /* Disable Adaptive Interrupt Moderation if 2 full packets cannot
79d4e908 3851 * fit in receive buffer.
828bac87
BA		 3852 */
3853 if (adapter->itr_setting & 0x3) {
79d4e908 3854 if ((adapter->max_frame_size * 2) > (pba << 10)) {
828bac87
BA		 3855 if (!(adapter->flags2 & FLAG2_DISABLE_AIM)) {
3856 dev_info(&adapter->pdev->dev,
3857 "Interrupt Throttle Rate turned off\n");
3858 adapter->flags2 |= FLAG2_DISABLE_AIM;
22a4cca2 3859 e1000e_write_itr(adapter, 0);
828bac87
BA		 3860 }
3861 } else if (adapter->flags2 & FLAG2_DISABLE_AIM) {
3862 dev_info(&adapter->pdev->dev,
3863 "Interrupt Throttle Rate turned on\n");
3864 adapter->flags2 &= ~FLAG2_DISABLE_AIM;
3865 adapter->itr = 20000;
22a4cca2 3866 e1000e_write_itr(adapter, adapter->itr);
828bac87
BA		 3867 }
3868 }
3869
bc7f75fa
AK		 3870 /* Allow time for pending master requests to run */
3871 mac->ops.reset_hw(hw);
97ac8cae 3872
e921eb1a 3873 /* For parts with AMT enabled, let the firmware know
97ac8cae
BA		 3874 * that the network interface is in control
3875 */
c43bc57e 3876 if (adapter->flags & FLAG_HAS_AMT)
31dbe5b4 3877 e1000e_get_hw_control(adapter);
97ac8cae 3878
bc7f75fa
AK		 3879 ew32(WUC, 0);
3880
3881 if (mac->ops.init_hw(hw))
44defeb3 3882 e_err("Hardware Error\n");
bc7f75fa
AK		 3883
3884 e1000_update_mng_vlan(adapter);
3885
3886 /* Enable h/w to recognize an 802.1Q VLAN Ethernet packet */
3887 ew32(VET, ETH_P_8021Q);
3888
3889 e1000e_reset_adaptive(hw);
31dbe5b4 3890
b67e1913
BA		 3891 /* initialize systim and reset the ns time counter */
3892 e1000e_config_hwtstamp(adapter);
3893
31dbe5b4
BA		 3894 if (!netif_running(adapter->netdev) &&
3895 !test_bit(__E1000_TESTING, &adapter->state)) {
3896 e1000_power_down_phy(adapter);
3897 return;
3898 }
3899
bc7f75fa
AK		 3900 e1000_get_phy_info(hw);
3901
918d7197
BA		 3902 if ((adapter->flags & FLAG_HAS_SMART_POWER_DOWN) &&
3903 !(adapter->flags & FLAG_SMART_POWER_DOWN)) {
bc7f75fa 3904 u16 phy_data = 0;
e921eb1a 3905 /* speed up time to link by disabling smart power down, ignore
bc7f75fa 3906 * the return value of this function because there is nothing
ad68076e
BA		 3907 * different we would do if it failed
3908 */
bc7f75fa
AK		 3909 e1e_rphy(hw, IGP02E1000_PHY_POWER_MGMT, &phy_data);
3910 phy_data &= ~IGP02E1000_PM_SPD;
3911 e1e_wphy(hw, IGP02E1000_PHY_POWER_MGMT, phy_data);
3912 }
bc7f75fa
AK		 3913}
3914
3915int e1000e_up(struct e1000_adapter *adapter)
3916{
3917 struct e1000_hw *hw = &adapter->hw;
3918
3919 /* hardware has been reset, we need to reload some things */
3920 e1000_configure(adapter);
3921
3922 clear_bit(__E1000_DOWN, &adapter->state);
3923
4662e82b
BA		 3924 if (adapter->msix_entries)
3925 e1000_configure_msix(adapter);
bc7f75fa
AK		 3926 e1000_irq_enable(adapter);
3927
400484fa 3928 netif_start_queue(adapter->netdev);
4cb9be7a 3929
bc7f75fa 3930 /* fire a link change interrupt to start the watchdog */
52a9b231
BA		 3931 if (adapter->msix_entries)
3932 ew32(ICS, E1000_ICS_LSC | E1000_ICR_OTHER);
3933 else
3934 ew32(ICS, E1000_ICS_LSC);
3935
bc7f75fa
AK		 3936 return 0;
3937}
3938
713b3c9e
JB		 3939static void e1000e_flush_descriptors(struct e1000_adapter *adapter)
3940{
3941 struct e1000_hw *hw = &adapter->hw;
3942
3943 if (!(adapter->flags2 & FLAG2_DMA_BURST))
3944 return;
3945
3946 /* flush pending descriptor writebacks to memory */
3947 ew32(TIDV, adapter->tx_int_delay | E1000_TIDV_FPD);
3948 ew32(RDTR, adapter->rx_int_delay | E1000_RDTR_FPD);
3949
3950 /* execute the writes immediately */
3951 e1e_flush();
bf03085f 3952
e921eb1a 3953 /* due to rare timing issues, write to TIDV/RDTR again to ensure the
bf03085f
MV		 3954 * write is successful
3955 */
3956 ew32(TIDV, adapter->tx_int_delay | E1000_TIDV_FPD);
3957 ew32(RDTR, adapter->rx_int_delay | E1000_RDTR_FPD);
713b3c9e
JB		 3958
3959 /* execute the writes immediately */
3960 e1e_flush();
3961}
3962
67fd4fcb
JK		 3963static void e1000e_update_stats(struct e1000_adapter *adapter);
3964
bc7f75fa
AK		 3965void e1000e_down(struct e1000_adapter *adapter)
3966{
3967 struct net_device *netdev = adapter->netdev;
3968 struct e1000_hw *hw = &adapter->hw;
3969 u32 tctl, rctl;
3970
e921eb1a 3971 /* signal that we're down so the interrupt handler does not
ad68076e
BA		 3972 * reschedule our watchdog timer
3973 */
bc7f75fa
AK		 3974 set_bit(__E1000_DOWN, &adapter->state);
3975
3976 /* disable receives in the hardware */
3977 rctl = er32(RCTL);
7f99ae63
BA		 3978 if (!(adapter->flags2 & FLAG2_NO_DISABLE_RX))
3979 ew32(RCTL, rctl & ~E1000_RCTL_EN);
bc7f75fa
AK		 3980 /* flush and sleep below */
3981
4cb9be7a 3982 netif_stop_queue(netdev);
bc7f75fa
AK		 3983
3984 /* disable transmits in the hardware */
3985 tctl = er32(TCTL);
3986 tctl &= ~E1000_TCTL_EN;
3987 ew32(TCTL, tctl);
7f99ae63 3988
bc7f75fa
AK		 3989 /* flush both disables and wait for them to finish */
3990 e1e_flush();
1bba4386 3991 usleep_range(10000, 20000);
bc7f75fa 3992
bc7f75fa
AK		 3993 e1000_irq_disable(adapter);
3994
3995 del_timer_sync(&adapter->watchdog_timer);
3996 del_timer_sync(&adapter->phy_info_timer);
3997
bc7f75fa 3998 netif_carrier_off(netdev);
67fd4fcb
JK		 3999
4000 spin_lock(&adapter->stats64_lock);
4001 e1000e_update_stats(adapter);
4002 spin_unlock(&adapter->stats64_lock);
4003
400484fa 4004 e1000e_flush_descriptors(adapter);
55aa6985
BA		 4005 e1000_clean_tx_ring(adapter->tx_ring);
4006 e1000_clean_rx_ring(adapter->rx_ring);
400484fa 4007
bc7f75fa
AK		 4008 adapter->link_speed = 0;
4009 adapter->link_duplex = 0;
4010
52cc3086
JK		 4011 if (!pci_channel_offline(adapter->pdev))
4012 e1000e_reset(adapter);
713b3c9e 4013
e921eb1a 4014 /* TODO: for power management, we could drop the link and
bc7f75fa
AK		 4015 * pci_disable_device here.
4016 */
4017}
4018
4019void e1000e_reinit_locked(struct e1000_adapter *adapter)
4020{
4021 might_sleep();
4022 while (test_and_set_bit(__E1000_RESETTING, &adapter->state))
1bba4386 4023 usleep_range(1000, 2000);
bc7f75fa
AK		 4024 e1000e_down(adapter);
4025 e1000e_up(adapter);
4026 clear_bit(__E1000_RESETTING, &adapter->state);
4027}
4028
b67e1913
BA		 4029/**
4030 * e1000e_cyclecounter_read - read raw cycle counter (used by time counter)
4031 * @cc: cyclecounter structure
4032 **/
4033static cycle_t e1000e_cyclecounter_read(const struct cyclecounter *cc)
4034{
4035 struct e1000_adapter *adapter = container_of(cc, struct e1000_adapter,
4036 cc);
4037 struct e1000_hw *hw = &adapter->hw;
4038 cycle_t systim;
4039
4040 /* latch SYSTIMH on read of SYSTIML */
4041 systim = (cycle_t)er32(SYSTIML);
4042 systim |= (cycle_t)er32(SYSTIMH) << 32;
4043
4044 return systim;
4045}
4046
bc7f75fa
AK		 4047/**
4048 * e1000_sw_init - Initialize general software structures (struct e1000_adapter)
4049 * @adapter: board private structure to initialize
4050 *
4051 * e1000_sw_init initializes the Adapter private data structure.
4052 * Fields are initialized based on PCI device information and
4053 * OS network device settings (MTU size).
4054 **/
9f9a12f8 4055static int e1000_sw_init(struct e1000_adapter *adapter)
bc7f75fa 4056{
bc7f75fa
AK		 4057 struct net_device *netdev = adapter->netdev;
4058
4059 adapter->rx_buffer_len = ETH_FRAME_LEN + VLAN_HLEN + ETH_FCS_LEN;
4060 adapter->rx_ps_bsize0 = 128;
318a94d6
JK		 4061 adapter->max_frame_size = netdev->mtu + ETH_HLEN + ETH_FCS_LEN;
4062 adapter->min_frame_size = ETH_ZLEN + ETH_FCS_LEN;
55aa6985
BA		 4063 adapter->tx_ring_count = E1000_DEFAULT_TXD;
4064 adapter->rx_ring_count = E1000_DEFAULT_RXD;
bc7f75fa 4065
67fd4fcb
JK		 4066 spin_lock_init(&adapter->stats64_lock);
4067
4662e82b 4068 e1000e_set_interrupt_capability(adapter);
bc7f75fa 4069
4662e82b
BA		 4070 if (e1000_alloc_queues(adapter))
4071 return -ENOMEM;
bc7f75fa 4072
b67e1913
BA		 4073 /* Setup hardware time stamping cyclecounter */
4074 if (adapter->flags & FLAG_HAS_HW_TIMESTAMP) {
4075 adapter->cc.read = e1000e_cyclecounter_read;
4076 adapter->cc.mask = CLOCKSOURCE_MASK(64);
4077 adapter->cc.mult = 1;
4078 /* cc.shift set in e1000e_get_base_tininca() */
4079
4080 spin_lock_init(&adapter->systim_lock);
4081 INIT_WORK(&adapter->tx_hwtstamp_work, e1000e_tx_hwtstamp_work);
4082 }
4083
bc7f75fa 4084 /* Explicitly disable IRQ since the NIC can be in any state. */
bc7f75fa
AK		 4085 e1000_irq_disable(adapter);
4086
bc7f75fa
AK		 4087 set_bit(__E1000_DOWN, &adapter->state);
4088 return 0;
bc7f75fa
AK		 4089}
4090
f8d59f78
BA		 4091/**
4092 * e1000_intr_msi_test - Interrupt Handler
4093 * @irq: interrupt number
4094 * @data: pointer to a network interface device structure
4095 **/
8bb62869 4096static irqreturn_t e1000_intr_msi_test(int __always_unused irq, void *data)
f8d59f78
BA		 4097{
4098 struct net_device *netdev = data;
4099 struct e1000_adapter *adapter = netdev_priv(netdev);
4100 struct e1000_hw *hw = &adapter->hw;
4101 u32 icr = er32(ICR);
4102
3bb99fe2 4103 e_dbg("icr is %08X\n", icr);
f8d59f78
BA		 4104 if (icr & E1000_ICR_RXSEQ) {
4105 adapter->flags &= ~FLAG_MSI_TEST_FAILED;
e921eb1a 4106 /* Force memory writes to complete before acknowledging the
bc76329d
BA		 4107 * interrupt is handled.
4108 */
f8d59f78
BA		 4109 wmb();
4110 }
4111
4112 return IRQ_HANDLED;
4113}
4114
4115/**
4116 * e1000_test_msi_interrupt - Returns 0 for successful test
4117 * @adapter: board private struct
4118 *
4119 * code flow taken from tg3.c
4120 **/
4121static int e1000_test_msi_interrupt(struct e1000_adapter *adapter)
4122{
4123 struct net_device *netdev = adapter->netdev;
4124 struct e1000_hw *hw = &adapter->hw;
4125 int err;
4126
4127 /* poll_enable hasn't been called yet, so don't need disable */
4128 /* clear any pending events */
4129 er32(ICR);
4130
4131 /* free the real vector and request a test handler */
4132 e1000_free_irq(adapter);
4662e82b 4133 e1000e_reset_interrupt_capability(adapter);
f8d59f78
BA		 4134
4135 /* Assume that the test fails, if it succeeds then the test
e921eb1a
BA		 4136 * MSI irq handler will unset this flag
4137 */
f8d59f78
BA		 4138 adapter->flags |= FLAG_MSI_TEST_FAILED;
4139
4140 err = pci_enable_msi(adapter->pdev);
4141 if (err)
4142 goto msi_test_failed;
4143
a0607fd3 4144 err = request_irq(adapter->pdev->irq, e1000_intr_msi_test, 0,
f8d59f78
BA		 4145 netdev->name, netdev);
4146 if (err) {
4147 pci_disable_msi(adapter->pdev);
4148 goto msi_test_failed;
4149 }
4150
e921eb1a 4151 /* Force memory writes to complete before enabling and firing an
bc76329d
BA		 4152 * interrupt.
4153 */
f8d59f78
BA		 4154 wmb();
4155
4156 e1000_irq_enable(adapter);
4157
4158 /* fire an unusual interrupt on the test handler */
4159 ew32(ICS, E1000_ICS_RXSEQ);
4160 e1e_flush();
569a3aff 4161 msleep(100);
f8d59f78
BA		 4162
4163 e1000_irq_disable(adapter);
4164
bc76329d 4165 rmb(); /* read flags after interrupt has been fired */
f8d59f78
BA		 4166
4167 if (adapter->flags & FLAG_MSI_TEST_FAILED) {
4662e82b 4168 adapter->int_mode = E1000E_INT_MODE_LEGACY;
068e8a30 4169 e_info("MSI interrupt test failed, using legacy interrupt.\n");
24b706b2 4170 } else {
068e8a30 4171 e_dbg("MSI interrupt test succeeded!\n");
24b706b2 4172 }
f8d59f78
BA		 4173
4174 free_irq(adapter->pdev->irq, netdev);
4175 pci_disable_msi(adapter->pdev);
4176
f8d59f78 4177msi_test_failed:
4662e82b 4178 e1000e_set_interrupt_capability(adapter);
068e8a30 4179 return e1000_request_irq(adapter);
f8d59f78
BA		 4180}
4181
4182/**
4183 * e1000_test_msi - Returns 0 if MSI test succeeds or INTx mode is restored
4184 * @adapter: board private struct
4185 *
4186 * code flow taken from tg3.c, called with e1000 interrupts disabled.
4187 **/
4188static int e1000_test_msi(struct e1000_adapter *adapter)
4189{
4190 int err;
4191 u16 pci_cmd;
4192
4193 if (!(adapter->flags & FLAG_MSI_ENABLED))
4194 return 0;
4195
4196 /* disable SERR in case the MSI write causes a master abort */
4197 pci_read_config_word(adapter->pdev, PCI_COMMAND, &pci_cmd);
36f2407f
DN		 4198 if (pci_cmd & PCI_COMMAND_SERR)
4199 pci_write_config_word(adapter->pdev, PCI_COMMAND,
4200 pci_cmd & ~PCI_COMMAND_SERR);
f8d59f78
BA		 4201
4202 err = e1000_test_msi_interrupt(adapter);
4203
36f2407f
DN		 4204 /* re-enable SERR */
4205 if (pci_cmd & PCI_COMMAND_SERR) {
4206 pci_read_config_word(adapter->pdev, PCI_COMMAND, &pci_cmd);
4207 pci_cmd |= PCI_COMMAND_SERR;
4208 pci_write_config_word(adapter->pdev, PCI_COMMAND, pci_cmd);
4209 }
f8d59f78 4210
f8d59f78
BA		 4211 return err;
4212}
4213
bc7f75fa
AK		 4214/**
4215 * e1000_open - Called when a network interface is made active
4216 * @netdev: network interface device structure
4217 *
4218 * Returns 0 on success, negative value on failure
4219 *
4220 * The open entry point is called when a network interface is made
4221 * active by the system (IFF_UP). At this point all resources needed
4222 * for transmit and receive operations are allocated, the interrupt
4223 * handler is registered with the OS, the watchdog timer is started,
4224 * and the stack is notified that the interface is ready.
4225 **/
4226static int e1000_open(struct net_device *netdev)
4227{
4228 struct e1000_adapter *adapter = netdev_priv(netdev);
4229 struct e1000_hw *hw = &adapter->hw;
23606cf5 4230 struct pci_dev *pdev = adapter->pdev;
bc7f75fa
AK		 4231 int err;
4232
4233 /* disallow open during test */
4234 if (test_bit(__E1000_TESTING, &adapter->state))
4235 return -EBUSY;
4236
23606cf5
RW		 4237 pm_runtime_get_sync(&pdev->dev);
4238
9c563d20
JB		 4239 netif_carrier_off(netdev);
4240
bc7f75fa 4241 /* allocate transmit descriptors */
55aa6985 4242 err = e1000e_setup_tx_resources(adapter->tx_ring);
bc7f75fa
AK		 4243 if (err)
4244 goto err_setup_tx;
4245
4246 /* allocate receive descriptors */
55aa6985 4247 err = e1000e_setup_rx_resources(adapter->rx_ring);
bc7f75fa
AK		 4248 if (err)
4249 goto err_setup_rx;
4250
e921eb1a 4251 /* If AMT is enabled, let the firmware know that the network
11b08be8
BA		 4252 * interface is now open and reset the part to a known state.
4253 */
4254 if (adapter->flags & FLAG_HAS_AMT) {
31dbe5b4 4255 e1000e_get_hw_control(adapter);
11b08be8
BA		 4256 e1000e_reset(adapter);
4257 }
4258
bc7f75fa
AK		 4259 e1000e_power_up_phy(adapter);
4260
4261 adapter->mng_vlan_id = E1000_MNG_VLAN_NONE;
4262 if ((adapter->hw.mng_cookie.status &
4263 E1000_MNG_DHCP_COOKIE_STATUS_VLAN))
4264 e1000_update_mng_vlan(adapter);
4265
79d4e908 4266 /* DMA latency requirement to workaround jumbo issue */
3e35d991
BA		 4267 pm_qos_add_request(&adapter->netdev->pm_qos_req, PM_QOS_CPU_DMA_LATENCY,
4268 PM_QOS_DEFAULT_VALUE);
c128ec29 4269
e921eb1a 4270 /* before we allocate an interrupt, we must be ready to handle it.
bc7f75fa
AK		 4271 * Setting DEBUG_SHIRQ in the kernel makes it fire an interrupt
4272 * as soon as we call pci_request_irq, so we have to setup our
ad68076e
BA		 4273 * clean_rx handler before we do so.
4274 */
bc7f75fa
AK		 4275 e1000_configure(adapter);
4276
4277 err = e1000_request_irq(adapter);
4278 if (err)
4279 goto err_req_irq;
4280
e921eb1a 4281 /* Work around PCIe errata with MSI interrupts causing some chipsets to
f8d59f78
BA		 4282 * ignore e1000e MSI messages, which means we need to test our MSI
4283 * interrupt now
4284 */
4662e82b 4285 if (adapter->int_mode != E1000E_INT_MODE_LEGACY) {
f8d59f78
BA		 4286 err = e1000_test_msi(adapter);
4287 if (err) {
4288 e_err("Interrupt allocation failed\n");
4289 goto err_req_irq;
4290 }
4291 }
4292
bc7f75fa
AK		 4293 /* From here on the code is the same as e1000e_up() */
4294 clear_bit(__E1000_DOWN, &adapter->state);
4295
4296 napi_enable(&adapter->napi);
4297
4298 e1000_irq_enable(adapter);
4299
09357b00 4300 adapter->tx_hang_recheck = false;
4cb9be7a 4301 netif_start_queue(netdev);
d55b53ff 4302
23606cf5
RW		 4303 adapter->idle_check = true;
4304 pm_runtime_put(&pdev->dev);
4305
bc7f75fa 4306 /* fire a link status change interrupt to start the watchdog */
52a9b231
BA		 4307 if (adapter->msix_entries)
4308 ew32(ICS, E1000_ICS_LSC | E1000_ICR_OTHER);
4309 else
4310 ew32(ICS, E1000_ICS_LSC);
bc7f75fa
AK		 4311
4312 return 0;
4313
4314err_req_irq:
31dbe5b4 4315 e1000e_release_hw_control(adapter);
bc7f75fa 4316 e1000_power_down_phy(adapter);
55aa6985 4317 e1000e_free_rx_resources(adapter->rx_ring);
bc7f75fa 4318err_setup_rx:
55aa6985 4319 e1000e_free_tx_resources(adapter->tx_ring);
bc7f75fa
AK		 4320err_setup_tx:
4321 e1000e_reset(adapter);
23606cf5 4322 pm_runtime_put_sync(&pdev->dev);
bc7f75fa
AK		 4323
4324 return err;
4325}
4326
4327/**
4328 * e1000_close - Disables a network interface
4329 * @netdev: network interface device structure
4330 *
4331 * Returns 0, this is not allowed to fail
4332 *
4333 * The close entry point is called when an interface is de-activated
4334 * by the OS. The hardware is still under the drivers control, but
4335 * needs to be disabled. A global MAC reset is issued to stop the
4336 * hardware, and all transmit and receive resources are freed.
4337 **/
4338static int e1000_close(struct net_device *netdev)
4339{
4340 struct e1000_adapter *adapter = netdev_priv(netdev);
23606cf5 4341 struct pci_dev *pdev = adapter->pdev;
bb9e44d0
BA		 4342 int count = E1000_CHECK_RESET_COUNT;
4343
4344 while (test_bit(__E1000_RESETTING, &adapter->state) && count--)
4345 usleep_range(10000, 20000);
bc7f75fa
AK		 4346
4347 WARN_ON(test_bit(__E1000_RESETTING, &adapter->state));
23606cf5
RW		 4348
4349 pm_runtime_get_sync(&pdev->dev);
4350
5f4a780d
BA		 4351 napi_disable(&adapter->napi);
4352
23606cf5
RW		 4353 if (!test_bit(__E1000_DOWN, &adapter->state)) {
4354 e1000e_down(adapter);
4355 e1000_free_irq(adapter);
4356 }
bc7f75fa 4357 e1000_power_down_phy(adapter);
bc7f75fa 4358
55aa6985
BA		 4359 e1000e_free_tx_resources(adapter->tx_ring);
4360 e1000e_free_rx_resources(adapter->rx_ring);
bc7f75fa 4361
e921eb1a 4362 /* kill manageability vlan ID if supported, but not if a vlan with
ad68076e
BA		 4363 * the same ID is registered on the host OS (let 8021q kill it)
4364 */
86d70e53
JK		 4365 if (adapter->hw.mng_cookie.status &
4366 E1000_MNG_DHCP_COOKIE_STATUS_VLAN)
bc7f75fa
AK		 4367 e1000_vlan_rx_kill_vid(netdev, adapter->mng_vlan_id);
4368
e921eb1a 4369 /* If AMT is enabled, let the firmware know that the network
ad68076e
BA		 4370 * interface is now closed
4371 */
31dbe5b4
BA		 4372 if ((adapter->flags & FLAG_HAS_AMT) &&
4373 !test_bit(__E1000_TESTING, &adapter->state))
4374 e1000e_release_hw_control(adapter);
bc7f75fa 4375
3e35d991 4376 pm_qos_remove_request(&adapter->netdev->pm_qos_req);
c128ec29 4377
23606cf5
RW		 4378 pm_runtime_put_sync(&pdev->dev);
4379
bc7f75fa
AK		 4380 return 0;
4381}
4382/**
4383 * e1000_set_mac - Change the Ethernet Address of the NIC
4384 * @netdev: network interface device structure
4385 * @p: pointer to an address structure
4386 *
4387 * Returns 0 on success, negative on failure
4388 **/
4389static int e1000_set_mac(struct net_device *netdev, void *p)
4390{
4391 struct e1000_adapter *adapter = netdev_priv(netdev);
69e1e019 4392 struct e1000_hw *hw = &adapter->hw;
bc7f75fa
AK		 4393 struct sockaddr *addr = p;
4394
4395 if (!is_valid_ether_addr(addr->sa_data))
4396 return -EADDRNOTAVAIL;
4397
4398 memcpy(netdev->dev_addr, addr->sa_data, netdev->addr_len);
4399 memcpy(adapter->hw.mac.addr, addr->sa_data, netdev->addr_len);
4400
69e1e019 4401 hw->mac.ops.rar_set(&adapter->hw, adapter->hw.mac.addr, 0);
bc7f75fa
AK		 4402
4403 if (adapter->flags & FLAG_RESET_OVERWRITES_LAA) {
4404 /* activate the work around */
4405 e1000e_set_laa_state_82571(&adapter->hw, 1);
4406
e921eb1a 4407 /* Hold a copy of the LAA in RAR[14] This is done so that
bc7f75fa
AK		 4408 * between the time RAR[0] gets clobbered and the time it
4409 * gets fixed (in e1000_watchdog), the actual LAA is in one
4410 * of the RARs and no incoming packets directed to this port
4411 * are dropped. Eventually the LAA will be in RAR[0] and
ad68076e
BA		 4412 * RAR[14]
4413 */
69e1e019
BA		 4414 hw->mac.ops.rar_set(&adapter->hw, adapter->hw.mac.addr,
4415 adapter->hw.mac.rar_entry_count - 1);
bc7f75fa
AK		 4416 }
4417
4418 return 0;
4419}
4420
a8f88ff5
JB		 4421/**
4422 * e1000e_update_phy_task - work thread to update phy
4423 * @work: pointer to our work struct
4424 *
4425 * this worker thread exists because we must acquire a
4426 * semaphore to read the phy, which we could msleep while
4427 * waiting for it, and we can't msleep in a timer.
4428 **/
4429static void e1000e_update_phy_task(struct work_struct *work)
4430{
4431 struct e1000_adapter *adapter = container_of(work,
4432 struct e1000_adapter, update_phy_task);
615b32af
JB		 4433
4434 if (test_bit(__E1000_DOWN, &adapter->state))
4435 return;
4436
a8f88ff5
JB		 4437 e1000_get_phy_info(&adapter->hw);
4438}
4439
e921eb1a
BA		 4440/**
4441 * e1000_update_phy_info - timre call-back to update PHY info
4442 * @data: pointer to adapter cast into an unsigned long
4443 *
ad68076e
BA		 4444 * Need to wait a few seconds after link up to get diagnostic information from
4445 * the phy
e921eb1a 4446 **/
bc7f75fa
AK		 4447static void e1000_update_phy_info(unsigned long data)
4448{
4449 struct e1000_adapter *adapter = (struct e1000_adapter *) data;
615b32af
JB		 4450
4451 if (test_bit(__E1000_DOWN, &adapter->state))
4452 return;
4453
a8f88ff5 4454 schedule_work(&adapter->update_phy_task);
bc7f75fa
AK		 4455}
4456
8c7bbb92
BA		 4457/**
4458 * e1000e_update_phy_stats - Update the PHY statistics counters
4459 * @adapter: board private structure
2b6b168d
BA		 4460 *
4461 * Read/clear the upper 16-bit PHY registers and read/accumulate lower
8c7bbb92
BA		 4462 **/
4463static void e1000e_update_phy_stats(struct e1000_adapter *adapter)
4464{
4465 struct e1000_hw *hw = &adapter->hw;
4466 s32 ret_val;
4467 u16 phy_data;
4468
4469 ret_val = hw->phy.ops.acquire(hw);
4470 if (ret_val)
4471 return;
4472
e921eb1a 4473 /* A page set is expensive so check if already on desired page.
8c7bbb92
BA		 4474 * If not, set to the page with the PHY status registers.
4475 */
2b6b168d 4476 hw->phy.addr = 1;
8c7bbb92
BA		 4477 ret_val = e1000e_read_phy_reg_mdic(hw, IGP01E1000_PHY_PAGE_SELECT,
4478 &phy_data);
4479 if (ret_val)
4480 goto release;
2b6b168d
BA		 4481 if (phy_data != (HV_STATS_PAGE << IGP_PAGE_SHIFT)) {
4482 ret_val = hw->phy.ops.set_page(hw,
4483 HV_STATS_PAGE << IGP_PAGE_SHIFT);
8c7bbb92
BA		 4484 if (ret_val)
4485 goto release;
4486 }
4487
8c7bbb92 4488 /* Single Collision Count */
2b6b168d
BA		 4489 hw->phy.ops.read_reg_page(hw, HV_SCC_UPPER, &phy_data);
4490 ret_val = hw->phy.ops.read_reg_page(hw, HV_SCC_LOWER, &phy_data);
8c7bbb92
BA		 4491 if (!ret_val)
4492 adapter->stats.scc += phy_data;
4493
4494 /* Excessive Collision Count */
2b6b168d
BA		 4495 hw->phy.ops.read_reg_page(hw, HV_ECOL_UPPER, &phy_data);
4496 ret_val = hw->phy.ops.read_reg_page(hw, HV_ECOL_LOWER, &phy_data);
8c7bbb92
BA		 4497 if (!ret_val)
4498 adapter->stats.ecol += phy_data;
4499
4500 /* Multiple Collision Count */
2b6b168d
BA		 4501 hw->phy.ops.read_reg_page(hw, HV_MCC_UPPER, &phy_data);
4502 ret_val = hw->phy.ops.read_reg_page(hw, HV_MCC_LOWER, &phy_data);
8c7bbb92
BA		 4503 if (!ret_val)
4504 adapter->stats.mcc += phy_data;
4505
4506 /* Late Collision Count */
2b6b168d
BA		 4507 hw->phy.ops.read_reg_page(hw, HV_LATECOL_UPPER, &phy_data);
4508 ret_val = hw->phy.ops.read_reg_page(hw, HV_LATECOL_LOWER, &phy_data);
8c7bbb92
BA		 4509 if (!ret_val)
4510 adapter->stats.latecol += phy_data;
4511
4512 /* Collision Count - also used for adaptive IFS */
2b6b168d
BA		 4513 hw->phy.ops.read_reg_page(hw, HV_COLC_UPPER, &phy_data);
4514 ret_val = hw->phy.ops.read_reg_page(hw, HV_COLC_LOWER, &phy_data);
8c7bbb92
BA		 4515 if (!ret_val)
4516 hw->mac.collision_delta = phy_data;
4517
4518 /* Defer Count */
2b6b168d
BA		 4519 hw->phy.ops.read_reg_page(hw, HV_DC_UPPER, &phy_data);
4520 ret_val = hw->phy.ops.read_reg_page(hw, HV_DC_LOWER, &phy_data);
8c7bbb92
BA		 4521 if (!ret_val)
4522 adapter->stats.dc += phy_data;
4523
4524 /* Transmit with no CRS */
2b6b168d
BA		 4525 hw->phy.ops.read_reg_page(hw, HV_TNCRS_UPPER, &phy_data);
4526 ret_val = hw->phy.ops.read_reg_page(hw, HV_TNCRS_LOWER, &phy_data);
8c7bbb92
BA		 4527 if (!ret_val)
4528 adapter->stats.tncrs += phy_data;
4529
4530release:
4531 hw->phy.ops.release(hw);
4532}
4533
bc7f75fa
AK		 4534/**
4535 * e1000e_update_stats - Update the board statistics counters
4536 * @adapter: board private structure
4537 **/
67fd4fcb 4538static void e1000e_update_stats(struct e1000_adapter *adapter)
bc7f75fa 4539{
7274c20f 4540 struct net_device *netdev = adapter->netdev;
bc7f75fa
AK		 4541 struct e1000_hw *hw = &adapter->hw;
4542 struct pci_dev *pdev = adapter->pdev;
bc7f75fa 4543
e921eb1a 4544 /* Prevent stats update while adapter is being reset, or if the pci
bc7f75fa
AK		 4545 * connection is down.
4546 */
4547 if (adapter->link_speed == 0)
4548 return;
4549 if (pci_channel_offline(pdev))
4550 return;
4551
bc7f75fa
AK		 4552 adapter->stats.crcerrs += er32(CRCERRS);
4553 adapter->stats.gprc += er32(GPRC);
7c25769f
BA		 4554 adapter->stats.gorc += er32(GORCL);
4555 er32(GORCH); /* Clear gorc */
bc7f75fa
AK		 4556 adapter->stats.bprc += er32(BPRC);
4557 adapter->stats.mprc += er32(MPRC);
4558 adapter->stats.roc += er32(ROC);
4559
bc7f75fa 4560 adapter->stats.mpc += er32(MPC);
8c7bbb92
BA		 4561
4562 /* Half-duplex statistics */
4563 if (adapter->link_duplex == HALF_DUPLEX) {
4564 if (adapter->flags2 & FLAG2_HAS_PHY_STATS) {
4565 e1000e_update_phy_stats(adapter);
4566 } else {
4567 adapter->stats.scc += er32(SCC);
4568 adapter->stats.ecol += er32(ECOL);
4569 adapter->stats.mcc += er32(MCC);
4570 adapter->stats.latecol += er32(LATECOL);
4571 adapter->stats.dc += er32(DC);
4572
4573 hw->mac.collision_delta = er32(COLC);
4574
4575 if ((hw->mac.type != e1000_82574) &&
4576 (hw->mac.type != e1000_82583))
4577 adapter->stats.tncrs += er32(TNCRS);
4578 }
4579 adapter->stats.colc += hw->mac.collision_delta;
a4f58f54 4580 }
8c7bbb92 4581
bc7f75fa
AK		 4582 adapter->stats.xonrxc += er32(XONRXC);
4583 adapter->stats.xontxc += er32(XONTXC);
4584 adapter->stats.xoffrxc += er32(XOFFRXC);
4585 adapter->stats.xofftxc += er32(XOFFTXC);
bc7f75fa 4586 adapter->stats.gptc += er32(GPTC);
7c25769f
BA		 4587 adapter->stats.gotc += er32(GOTCL);
4588 er32(GOTCH); /* Clear gotc */
bc7f75fa
AK		 4589 adapter->stats.rnbc += er32(RNBC);
4590 adapter->stats.ruc += er32(RUC);
bc7f75fa
AK		 4591
4592 adapter->stats.mptc += er32(MPTC);
4593 adapter->stats.bptc += er32(BPTC);
4594
4595 /* used for adaptive IFS */
4596
4597 hw->mac.tx_packet_delta = er32(TPT);
4598 adapter->stats.tpt += hw->mac.tx_packet_delta;
bc7f75fa
AK		 4599
4600 adapter->stats.algnerrc += er32(ALGNERRC);
4601 adapter->stats.rxerrc += er32(RXERRC);
bc7f75fa
AK		 4602 adapter->stats.cexterr += er32(CEXTERR);
4603 adapter->stats.tsctc += er32(TSCTC);
4604 adapter->stats.tsctfc += er32(TSCTFC);
4605
bc7f75fa 4606 /* Fill out the OS statistics structure */
7274c20f
AK		 4607 netdev->stats.multicast = adapter->stats.mprc;
4608 netdev->stats.collisions = adapter->stats.colc;
bc7f75fa
AK		 4609
4610 /* Rx Errors */
4611
e921eb1a 4612 /* RLEC on some newer hardware can be incorrect so build
ad68076e
BA		 4613 * our own version based on RUC and ROC
4614 */
7274c20f 4615 netdev->stats.rx_errors = adapter->stats.rxerrc +
f0ff4398
BA		 4616 adapter->stats.crcerrs + adapter->stats.algnerrc +
4617 adapter->stats.ruc + adapter->stats.roc + adapter->stats.cexterr;
7274c20f 4618 netdev->stats.rx_length_errors = adapter->stats.ruc +
f0ff4398 4619 adapter->stats.roc;
7274c20f
AK		 4620 netdev->stats.rx_crc_errors = adapter->stats.crcerrs;
4621 netdev->stats.rx_frame_errors = adapter->stats.algnerrc;
4622 netdev->stats.rx_missed_errors = adapter->stats.mpc;
bc7f75fa
AK		 4623
4624 /* Tx Errors */
f0ff4398 4625 netdev->stats.tx_errors = adapter->stats.ecol + adapter->stats.latecol;
7274c20f
AK		 4626 netdev->stats.tx_aborted_errors = adapter->stats.ecol;
4627 netdev->stats.tx_window_errors = adapter->stats.latecol;
4628 netdev->stats.tx_carrier_errors = adapter->stats.tncrs;
bc7f75fa
AK		 4629
4630 /* Tx Dropped needs to be maintained elsewhere */
4631
bc7f75fa
AK		 4632 /* Management Stats */
4633 adapter->stats.mgptc += er32(MGTPTC);
4634 adapter->stats.mgprc += er32(MGTPRC);
4635 adapter->stats.mgpdc += er32(MGTPDC);
94fb848b
BA		 4636
4637 /* Correctable ECC Errors */
4638 if (hw->mac.type == e1000_pch_lpt) {
4639 u32 pbeccsts = er32(PBECCSTS);
4640 adapter->corr_errors +=
4641 pbeccsts & E1000_PBECCSTS_CORR_ERR_CNT_MASK;
4642 adapter->uncorr_errors +=
4643 (pbeccsts & E1000_PBECCSTS_UNCORR_ERR_CNT_MASK) >>
4644 E1000_PBECCSTS_UNCORR_ERR_CNT_SHIFT;
4645 }
bc7f75fa
AK		 4646}
4647
7c25769f
BA		 4648/**
4649 * e1000_phy_read_status - Update the PHY register status snapshot
4650 * @adapter: board private structure
4651 **/
4652static void e1000_phy_read_status(struct e1000_adapter *adapter)
4653{
4654 struct e1000_hw *hw = &adapter->hw;
4655 struct e1000_phy_regs *phy = &adapter->phy_regs;
7c25769f
BA		 4656
4657 if ((er32(STATUS) & E1000_STATUS_LU) &&
4658 (adapter->hw.phy.media_type == e1000_media_type_copper)) {
90da0669
BA		 4659 int ret_val;
4660
c2ade1a4
BA		 4661 ret_val = e1e_rphy(hw, MII_BMCR, &phy->bmcr);
4662 ret_val |= e1e_rphy(hw, MII_BMSR, &phy->bmsr);
4663 ret_val |= e1e_rphy(hw, MII_ADVERTISE, &phy->advertise);
4664 ret_val |= e1e_rphy(hw, MII_LPA, &phy->lpa);
4665 ret_val |= e1e_rphy(hw, MII_EXPANSION, &phy->expansion);
4666 ret_val |= e1e_rphy(hw, MII_CTRL1000, &phy->ctrl1000);
4667 ret_val |= e1e_rphy(hw, MII_STAT1000, &phy->stat1000);
4668 ret_val |= e1e_rphy(hw, MII_ESTATUS, &phy->estatus);
7c25769f 4669 if (ret_val)
44defeb3 4670 e_warn("Error reading PHY register\n");
7c25769f 4671 } else {
e921eb1a 4672 /* Do not read PHY registers if link is not up
7c25769f
BA		 4673 * Set values to typical power-on defaults
4674 */
4675 phy->bmcr = (BMCR_SPEED1000 | BMCR_ANENABLE | BMCR_FULLDPLX);
4676 phy->bmsr = (BMSR_100FULL | BMSR_100HALF | BMSR_10FULL |
4677 BMSR_10HALF | BMSR_ESTATEN | BMSR_ANEGCAPABLE |
4678 BMSR_ERCAP);
4679 phy->advertise = (ADVERTISE_PAUSE_ASYM | ADVERTISE_PAUSE_CAP |
4680 ADVERTISE_ALL | ADVERTISE_CSMA);
4681 phy->lpa = 0;
4682 phy->expansion = EXPANSION_ENABLENPAGE;
4683 phy->ctrl1000 = ADVERTISE_1000FULL;
4684 phy->stat1000 = 0;
4685 phy->estatus = (ESTATUS_1000_TFULL | ESTATUS_1000_THALF);
4686 }
7c25769f
BA		 4687}
4688
bc7f75fa
AK		 4689static void e1000_print_link_info(struct e1000_adapter *adapter)
4690{
bc7f75fa
AK		 4691 struct e1000_hw *hw = &adapter->hw;
4692 u32 ctrl = er32(CTRL);
4693
8f12fe86 4694 /* Link status message must follow this format for user tools */
7dbc1672
BA		 4695 pr_info("%s NIC Link is Up %d Mbps %s Duplex, Flow Control: %s\n",
4696 adapter->netdev->name, adapter->link_speed,
ef456f85
JK		 4697 adapter->link_duplex == FULL_DUPLEX ? "Full" : "Half",
4698 (ctrl & E1000_CTRL_TFCE) && (ctrl & E1000_CTRL_RFCE) ? "Rx/Tx" :
4699 (ctrl & E1000_CTRL_RFCE) ? "Rx" :
4700 (ctrl & E1000_CTRL_TFCE) ? "Tx" : "None");
bc7f75fa
AK		 4701}
4702
0c6bdb30 4703static bool e1000e_has_link(struct e1000_adapter *adapter)
318a94d6
JK		 4704{
4705 struct e1000_hw *hw = &adapter->hw;
3db1cd5c 4706 bool link_active = false;
318a94d6
JK		 4707 s32 ret_val = 0;
4708
e921eb1a 4709 /* get_link_status is set on LSC (link status) interrupt or
318a94d6
JK		 4710 * Rx sequence error interrupt. get_link_status will stay
4711 * false until the check_for_link establishes link
4712 * for copper adapters ONLY
4713 */
4714 switch (hw->phy.media_type) {
4715 case e1000_media_type_copper:
4716 if (hw->mac.get_link_status) {
4717 ret_val = hw->mac.ops.check_for_link(hw);
4718 link_active = !hw->mac.get_link_status;
4719 } else {
3db1cd5c 4720 link_active = true;
318a94d6
JK		 4721 }
4722 break;
4723 case e1000_media_type_fiber:
4724 ret_val = hw->mac.ops.check_for_link(hw);
4725 link_active = !!(er32(STATUS) & E1000_STATUS_LU);
4726 break;
4727 case e1000_media_type_internal_serdes:
4728 ret_val = hw->mac.ops.check_for_link(hw);
4729 link_active = adapter->hw.mac.serdes_has_link;
4730 break;
4731 default:
4732 case e1000_media_type_unknown:
4733 break;
4734 }
4735
4736 if ((ret_val == E1000_ERR_PHY) && (hw->phy.type == e1000_phy_igp_3) &&
4737 (er32(CTRL) & E1000_PHY_CTRL_GBE_DISABLE)) {
4738 /* See e1000_kmrn_lock_loss_workaround_ich8lan() */
44defeb3 4739 e_info("Gigabit has been disabled, downgrading speed\n");
318a94d6
JK		 4740 }
4741
4742 return link_active;
4743}
4744
4745static void e1000e_enable_receives(struct e1000_adapter *adapter)
4746{
4747 /* make sure the receive unit is started */
4748 if ((adapter->flags & FLAG_RX_NEEDS_RESTART) &&
12d43f7d 4749 (adapter->flags & FLAG_RESTART_NOW)) {
318a94d6
JK		 4750 struct e1000_hw *hw = &adapter->hw;
4751 u32 rctl = er32(RCTL);
4752 ew32(RCTL, rctl | E1000_RCTL_EN);
12d43f7d 4753 adapter->flags &= ~FLAG_RESTART_NOW;
318a94d6
JK		 4754 }
4755}
4756
ff10e13c
CW		 4757static void e1000e_check_82574_phy_workaround(struct e1000_adapter *adapter)
4758{
4759 struct e1000_hw *hw = &adapter->hw;
4760
e921eb1a 4761 /* With 82574 controllers, PHY needs to be checked periodically
ff10e13c
CW		 4762 * for hung state and reset, if two calls return true
4763 */
4764 if (e1000_check_phy_82574(hw))
4765 adapter->phy_hang_count++;
4766 else
4767 adapter->phy_hang_count = 0;
4768
4769 if (adapter->phy_hang_count > 1) {
4770 adapter->phy_hang_count = 0;
4771 schedule_work(&adapter->reset_task);
4772 }
4773}
4774
bc7f75fa
AK		 4775/**
4776 * e1000_watchdog - Timer Call-back
4777 * @data: pointer to adapter cast into an unsigned long
4778 **/
4779static void e1000_watchdog(unsigned long data)
4780{
4781 struct e1000_adapter *adapter = (struct e1000_adapter *) data;
4782
4783 /* Do the rest outside of interrupt context */
4784 schedule_work(&adapter->watchdog_task);
4785
4786 /* TODO: make this use queue_delayed_work() */
4787}
4788
4789static void e1000_watchdog_task(struct work_struct *work)
4790{
4791 struct e1000_adapter *adapter = container_of(work,
4792 struct e1000_adapter, watchdog_task);
bc7f75fa
AK		 4793 struct net_device *netdev = adapter->netdev;
4794 struct e1000_mac_info *mac = &adapter->hw.mac;
75eb0fad 4795 struct e1000_phy_info *phy = &adapter->hw.phy;
bc7f75fa
AK		 4796 struct e1000_ring *tx_ring = adapter->tx_ring;
4797 struct e1000_hw *hw = &adapter->hw;
4798 u32 link, tctl;
bc7f75fa 4799
615b32af
JB		 4800 if (test_bit(__E1000_DOWN, &adapter->state))
4801 return;
4802
b405e8df 4803 link = e1000e_has_link(adapter);
318a94d6 4804 if ((netif_carrier_ok(netdev)) && link) {
23606cf5
RW		 4805 /* Cancel scheduled suspend requests. */
4806 pm_runtime_resume(netdev->dev.parent);
4807
318a94d6 4808 e1000e_enable_receives(adapter);
bc7f75fa 4809 goto link_up;
bc7f75fa
AK		 4810 }
4811
4812 if ((e1000e_enable_tx_pkt_filtering(hw)) &&
4813 (adapter->mng_vlan_id != adapter->hw.mng_cookie.vlan_id))
4814 e1000_update_mng_vlan(adapter);
4815
bc7f75fa
AK		 4816 if (link) {
4817 if (!netif_carrier_ok(netdev)) {
3db1cd5c 4818 bool txb2b = true;
23606cf5
RW		 4819
4820 /* Cancel scheduled suspend requests. */
4821 pm_runtime_resume(netdev->dev.parent);
4822
318a94d6 4823 /* update snapshot of PHY registers on LSC */
7c25769f 4824 e1000_phy_read_status(adapter);
bc7f75fa
AK		 4825 mac->ops.get_link_up_info(&adapter->hw,
4826 &adapter->link_speed,
4827 &adapter->link_duplex);
4828 e1000_print_link_info(adapter);
e792cd91
KS		 4829
4830 /* check if SmartSpeed worked */
4831 e1000e_check_downshift(hw);
4832 if (phy->speed_downgraded)
4833 netdev_warn(netdev,
4834 "Link Speed was downgraded by SmartSpeed\n");
4835
e921eb1a 4836 /* On supported PHYs, check for duplex mismatch only
f4187b56
BA		 4837 * if link has autonegotiated at 10/100 half
4838 */
4839 if ((hw->phy.type == e1000_phy_igp_3 ||
4840 hw->phy.type == e1000_phy_bm) &&
4841 (hw->mac.autoneg == true) &&
4842 (adapter->link_speed == SPEED_10 ||
4843 adapter->link_speed == SPEED_100) &&
4844 (adapter->link_duplex == HALF_DUPLEX)) {
4845 u16 autoneg_exp;
4846
c2ade1a4 4847 e1e_rphy(hw, MII_EXPANSION, &autoneg_exp);
f4187b56 4848
c2ade1a4 4849 if (!(autoneg_exp & EXPANSION_NWAY))
ef456f85 4850 e_info("Autonegotiated half duplex but link partner cannot autoneg. Try forcing full duplex if link gets many collisions.\n");
f4187b56
BA		 4851 }
4852
f49c57e1 4853 /* adjust timeout factor according to speed/duplex */
bc7f75fa
AK		 4854 adapter->tx_timeout_factor = 1;
4855 switch (adapter->link_speed) {
4856 case SPEED_10:
3db1cd5c 4857 txb2b = false;
10f1b492 4858 adapter->tx_timeout_factor = 16;
bc7f75fa
AK		 4859 break;
4860 case SPEED_100:
3db1cd5c 4861 txb2b = false;
4c86e0b9 4862 adapter->tx_timeout_factor = 10;
bc7f75fa
AK		 4863 break;
4864 }
4865
e921eb1a 4866 /* workaround: re-program speed mode bit after
ad68076e
BA		 4867 * link-up event
4868 */
bc7f75fa
AK		 4869 if ((adapter->flags & FLAG_TARC_SPEED_MODE_BIT) &&
4870 !txb2b) {
4871 u32 tarc0;
e9ec2c0f 4872 tarc0 = er32(TARC(0));
bc7f75fa 4873 tarc0 &= ~SPEED_MODE_BIT;
e9ec2c0f 4874 ew32(TARC(0), tarc0);
bc7f75fa
AK		 4875 }
4876
e921eb1a 4877 /* disable TSO for pcie and 10/100 speeds, to avoid
ad68076e
BA		 4878 * some hardware issues
4879 */
bc7f75fa
AK		 4880 if (!(adapter->flags & FLAG_TSO_FORCE)) {
4881 switch (adapter->link_speed) {
4882 case SPEED_10:
4883 case SPEED_100:
44defeb3 4884 e_info("10/100 speed: disabling TSO\n");
bc7f75fa
AK		 4885 netdev->features &= ~NETIF_F_TSO;
4886 netdev->features &= ~NETIF_F_TSO6;
4887 break;
4888 case SPEED_1000:
4889 netdev->features |= NETIF_F_TSO;
4890 netdev->features |= NETIF_F_TSO6;
4891 break;
4892 default:
4893 /* oops */
4894 break;
4895 }
4896 }
4897
e921eb1a 4898 /* enable transmits in the hardware, need to do this
ad68076e
BA		 4899 * after setting TARC(0)
4900 */
bc7f75fa
AK		 4901 tctl = er32(TCTL);
4902 tctl |= E1000_TCTL_EN;
4903 ew32(TCTL, tctl);
4904
e921eb1a 4905 /* Perform any post-link-up configuration before
75eb0fad
BA		 4906 * reporting link up.
4907 */
4908 if (phy->ops.cfg_on_link_up)
4909 phy->ops.cfg_on_link_up(hw);
4910
bc7f75fa 4911 netif_carrier_on(netdev);
bc7f75fa
AK		 4912
4913 if (!test_bit(__E1000_DOWN, &adapter->state))
4914 mod_timer(&adapter->phy_info_timer,
4915 round_jiffies(jiffies + 2 * HZ));
bc7f75fa
AK		 4916 }
4917 } else {
4918 if (netif_carrier_ok(netdev)) {
4919 adapter->link_speed = 0;
4920 adapter->link_duplex = 0;
8f12fe86 4921 /* Link status message must follow this format */
7dbc1672 4922 pr_info("%s NIC Link is Down\n", adapter->netdev->name);
bc7f75fa 4923 netif_carrier_off(netdev);
bc7f75fa
AK		 4924 if (!test_bit(__E1000_DOWN, &adapter->state))
4925 mod_timer(&adapter->phy_info_timer,
4926 round_jiffies(jiffies + 2 * HZ));
4927
12d43f7d
BA		 4928 /* The link is lost so the controller stops DMA.
4929 * If there is queued Tx work that cannot be done
4930 * or if on an 8000ES2LAN which requires a Rx packet
4931 * buffer work-around on link down event, reset the
4932 * controller to flush the Tx/Rx packet buffers.
4933 * (Do the reset outside of interrupt context).
4934 */
4935 if ((adapter->flags & FLAG_RX_NEEDS_RESTART) ||
4936 (e1000_desc_unused(tx_ring) + 1 < tx_ring->count))
4937 adapter->flags |= FLAG_RESTART_NOW;
23606cf5
RW		 4938 else
4939 pm_schedule_suspend(netdev->dev.parent,
4940 LINK_TIMEOUT);
bc7f75fa
AK		 4941 }
4942 }
4943
4944link_up:
67fd4fcb 4945 spin_lock(&adapter->stats64_lock);
bc7f75fa
AK		 4946 e1000e_update_stats(adapter);
4947
4948 mac->tx_packet_delta = adapter->stats.tpt - adapter->tpt_old;
4949 adapter->tpt_old = adapter->stats.tpt;
4950 mac->collision_delta = adapter->stats.colc - adapter->colc_old;
4951 adapter->colc_old = adapter->stats.colc;
4952
7c25769f
BA		 4953 adapter->gorc = adapter->stats.gorc - adapter->gorc_old;
4954 adapter->gorc_old = adapter->stats.gorc;
4955 adapter->gotc = adapter->stats.gotc - adapter->gotc_old;
4956 adapter->gotc_old = adapter->stats.gotc;
2084b114 4957 spin_unlock(&adapter->stats64_lock);
bc7f75fa 4958
12d43f7d 4959 if (adapter->flags & FLAG_RESTART_NOW) {
90da0669
BA		 4960 schedule_work(&adapter->reset_task);
4961 /* return immediately since reset is imminent */
4962 return;
bc7f75fa
AK		 4963 }
4964
12d43f7d
BA		 4965 e1000e_update_adaptive(&adapter->hw);
4966
eab2abf5
JB		 4967 /* Simple mode for Interrupt Throttle Rate (ITR) */
4968 if (adapter->itr_setting == 4) {
e921eb1a 4969 /* Symmetric Tx/Rx gets a reduced ITR=2000;
eab2abf5
JB		 4970 * Total asymmetrical Tx or Rx gets ITR=8000;
4971 * everyone else is between 2000-8000.
4972 */
4973 u32 goc = (adapter->gotc + adapter->gorc) / 10000;
4974 u32 dif = (adapter->gotc > adapter->gorc ?
4975 adapter->gotc - adapter->gorc :
4976 adapter->gorc - adapter->gotc) / 10000;
4977 u32 itr = goc > 0 ? (dif * 6000 / goc + 2000) : 8000;
4978
22a4cca2 4979 e1000e_write_itr(adapter, itr);
eab2abf5
JB		 4980 }
4981
ad68076e 4982 /* Cause software interrupt to ensure Rx ring is cleaned */
4662e82b
BA		 4983 if (adapter->msix_entries)
4984 ew32(ICS, adapter->rx_ring->ims_val);
4985 else
4986 ew32(ICS, E1000_ICS_RXDMT0);
bc7f75fa 4987
713b3c9e
JB		 4988 /* flush pending descriptors to memory before detecting Tx hang */
4989 e1000e_flush_descriptors(adapter);
4990
bc7f75fa 4991 /* Force detection of hung controller every watchdog period */
3db1cd5c 4992 adapter->detect_tx_hung = true;
bc7f75fa 4993
e921eb1a 4994 /* With 82571 controllers, LAA may be overwritten due to controller
ad68076e
BA		 4995 * reset from the other port. Set the appropriate LAA in RAR[0]
4996 */
bc7f75fa 4997 if (e1000e_get_laa_state_82571(hw))
69e1e019 4998 hw->mac.ops.rar_set(hw, adapter->hw.mac.addr, 0);
bc7f75fa 4999
ff10e13c
CW		 5000 if (adapter->flags2 & FLAG2_CHECK_PHY_HANG)
5001 e1000e_check_82574_phy_workaround(adapter);
5002
b67e1913
BA		 5003 /* Clear valid timestamp stuck in RXSTMPL/H due to a Rx error */
5004 if (adapter->hwtstamp_config.rx_filter != HWTSTAMP_FILTER_NONE) {
5005 if ((adapter->flags2 & FLAG2_CHECK_RX_HWTSTAMP) &&
5006 (er32(TSYNCRXCTL) & E1000_TSYNCRXCTL_VALID)) {
5007 er32(RXSTMPH);
5008 adapter->rx_hwtstamp_cleared++;
5009 } else {
5010 adapter->flags2 |= FLAG2_CHECK_RX_HWTSTAMP;
5011 }
5012 }
5013
bc7f75fa
AK		 5014 /* Reset the timer */
5015 if (!test_bit(__E1000_DOWN, &adapter->state))
5016 mod_timer(&adapter->watchdog_timer,
5017 round_jiffies(jiffies + 2 * HZ));
5018}
5019
5020#define E1000_TX_FLAGS_CSUM 0x00000001
5021#define E1000_TX_FLAGS_VLAN 0x00000002
5022#define E1000_TX_FLAGS_TSO 0x00000004
5023#define E1000_TX_FLAGS_IPV4 0x00000008
943146de 5024#define E1000_TX_FLAGS_NO_FCS 0x00000010
b67e1913 5025#define E1000_TX_FLAGS_HWTSTAMP 0x00000020
bc7f75fa
AK		 5026#define E1000_TX_FLAGS_VLAN_MASK 0xffff0000
5027#define E1000_TX_FLAGS_VLAN_SHIFT 16
5028
55aa6985 5029static int e1000_tso(struct e1000_ring *tx_ring, struct sk_buff *skb)
bc7f75fa 5030{
bc7f75fa
AK		 5031 struct e1000_context_desc *context_desc;
5032 struct e1000_buffer *buffer_info;
5033 unsigned int i;
5034 u32 cmd_length = 0;
70443ae9 5035 u16 ipcse = 0, mss;
bc7f75fa 5036 u8 ipcss, ipcso, tucss, tucso, hdr_len;
bc7f75fa 5037
3d5e33c9
BA		 5038 if (!skb_is_gso(skb))
5039 return 0;
bc7f75fa 5040
3d5e33c9 5041 if (skb_header_cloned(skb)) {
90da0669
BA		 5042 int err = pskb_expand_head(skb, 0, 0, GFP_ATOMIC);
5043
3d5e33c9
BA		 5044 if (err)
5045 return err;
bc7f75fa
AK		 5046 }
5047
3d5e33c9
BA		 5048 hdr_len = skb_transport_offset(skb) + tcp_hdrlen(skb);
5049 mss = skb_shinfo(skb)->gso_size;
5050 if (skb->protocol == htons(ETH_P_IP)) {
5051 struct iphdr *iph = ip_hdr(skb);
5052 iph->tot_len = 0;
5053 iph->check = 0;
5054 tcp_hdr(skb)->check = ~csum_tcpudp_magic(iph->saddr, iph->daddr,
f0ff4398 5055 0, IPPROTO_TCP, 0);
3d5e33c9
BA		 5056 cmd_length = E1000_TXD_CMD_IP;
5057 ipcse = skb_transport_offset(skb) - 1;
8e1e8a47 5058 } else if (skb_is_gso_v6(skb)) {
3d5e33c9
BA		 5059 ipv6_hdr(skb)->payload_len = 0;
5060 tcp_hdr(skb)->check = ~csum_ipv6_magic(&ipv6_hdr(skb)->saddr,
f0ff4398
BA		 5061 &ipv6_hdr(skb)->daddr,
5062 0, IPPROTO_TCP, 0);
3d5e33c9
BA		 5063 ipcse = 0;
5064 }
5065 ipcss = skb_network_offset(skb);
5066 ipcso = (void *)&(ip_hdr(skb)->check) - (void *)skb->data;
5067 tucss = skb_transport_offset(skb);
5068 tucso = (void *)&(tcp_hdr(skb)->check) - (void *)skb->data;
3d5e33c9
BA		 5069
5070 cmd_length |= (E1000_TXD_CMD_DEXT | E1000_TXD_CMD_TSE |
f0ff4398 5071 E1000_TXD_CMD_TCP | (skb->len - (hdr_len)));
3d5e33c9
BA		 5072
5073 i = tx_ring->next_to_use;
5074 context_desc = E1000_CONTEXT_DESC(*tx_ring, i);
5075 buffer_info = &tx_ring->buffer_info[i];
5076
5077 context_desc->lower_setup.ip_fields.ipcss = ipcss;
5078 context_desc->lower_setup.ip_fields.ipcso = ipcso;
5079 context_desc->lower_setup.ip_fields.ipcse = cpu_to_le16(ipcse);
5080 context_desc->upper_setup.tcp_fields.tucss = tucss;
5081 context_desc->upper_setup.tcp_fields.tucso = tucso;
70443ae9 5082 context_desc->upper_setup.tcp_fields.tucse = 0;
3d5e33c9
BA		 5083 context_desc->tcp_seg_setup.fields.mss = cpu_to_le16(mss);
5084 context_desc->tcp_seg_setup.fields.hdr_len = hdr_len;
5085 context_desc->cmd_and_length = cpu_to_le32(cmd_length);
5086
5087 buffer_info->time_stamp = jiffies;
5088 buffer_info->next_to_watch = i;
5089
5090 i++;
5091 if (i == tx_ring->count)
5092 i = 0;
5093 tx_ring->next_to_use = i;
5094
5095 return 1;
bc7f75fa
AK		 5096}
5097
55aa6985 5098static bool e1000_tx_csum(struct e1000_ring *tx_ring, struct sk_buff *skb)
bc7f75fa 5099{
55aa6985 5100 struct e1000_adapter *adapter = tx_ring->adapter;
bc7f75fa
AK		 5101 struct e1000_context_desc *context_desc;
5102 struct e1000_buffer *buffer_info;
5103 unsigned int i;
5104 u8 css;
af807c82 5105 u32 cmd_len = E1000_TXD_CMD_DEXT;
5f66f208 5106 __be16 protocol;
bc7f75fa 5107
af807c82
DG		 5108 if (skb->ip_summed != CHECKSUM_PARTIAL)
5109 return 0;
bc7f75fa 5110
5f66f208
AJ		 5111 if (skb->protocol == cpu_to_be16(ETH_P_8021Q))
5112 protocol = vlan_eth_hdr(skb)->h_vlan_encapsulated_proto;
5113 else
5114 protocol = skb->protocol;
5115
3f518390 5116 switch (protocol) {
09640e63 5117 case cpu_to_be16(ETH_P_IP):
af807c82
DG		 5118 if (ip_hdr(skb)->protocol == IPPROTO_TCP)
5119 cmd_len |= E1000_TXD_CMD_TCP;
5120 break;
09640e63 5121 case cpu_to_be16(ETH_P_IPV6):
af807c82
DG		 5122 /* XXX not handling all IPV6 headers */
5123 if (ipv6_hdr(skb)->nexthdr == IPPROTO_TCP)
5124 cmd_len |= E1000_TXD_CMD_TCP;
5125 break;
5126 default:
5127 if (unlikely(net_ratelimit()))
5f66f208
AJ		 5128 e_warn("checksum_partial proto=%x!\n",
5129 be16_to_cpu(protocol));
af807c82 5130 break;
bc7f75fa
AK		 5131 }
5132
0d0b1672 5133 css = skb_checksum_start_offset(skb);
af807c82
DG		 5134
5135 i = tx_ring->next_to_use;
5136 buffer_info = &tx_ring->buffer_info[i];
5137 context_desc = E1000_CONTEXT_DESC(*tx_ring, i);
5138
5139 context_desc->lower_setup.ip_config = 0;
5140 context_desc->upper_setup.tcp_fields.tucss = css;
f0ff4398 5141 context_desc->upper_setup.tcp_fields.tucso = css + skb->csum_offset;
af807c82
DG		 5142 context_desc->upper_setup.tcp_fields.tucse = 0;
5143 context_desc->tcp_seg_setup.data = 0;
5144 context_desc->cmd_and_length = cpu_to_le32(cmd_len);
5145
5146 buffer_info->time_stamp = jiffies;
5147 buffer_info->next_to_watch = i;
5148
5149 i++;
5150 if (i == tx_ring->count)
5151 i = 0;
5152 tx_ring->next_to_use = i;
5153
5154 return 1;
bc7f75fa
AK		 5155}
5156
55aa6985
BA		 5157static int e1000_tx_map(struct e1000_ring *tx_ring, struct sk_buff *skb,
5158 unsigned int first, unsigned int max_per_txd,
d821a4c4 5159 unsigned int nr_frags)
bc7f75fa 5160{
55aa6985 5161 struct e1000_adapter *adapter = tx_ring->adapter;
03b1320d 5162 struct pci_dev *pdev = adapter->pdev;
1b7719c4 5163 struct e1000_buffer *buffer_info;
8ddc951c 5164 unsigned int len = skb_headlen(skb);
03b1320d 5165 unsigned int offset = 0, size, count = 0, i;
9ed318d5 5166 unsigned int f, bytecount, segs;
bc7f75fa
AK		 5167
5168 i = tx_ring->next_to_use;
5169
5170 while (len) {
1b7719c4 5171 buffer_info = &tx_ring->buffer_info[i];
bc7f75fa
AK		 5172 size = min(len, max_per_txd);
5173
bc7f75fa 5174 buffer_info->length = size;
bc7f75fa 5175 buffer_info->time_stamp = jiffies;
bc7f75fa 5176 buffer_info->next_to_watch = i;
0be3f55f
NN		 5177 buffer_info->dma = dma_map_single(&pdev->dev,
5178 skb->data + offset,
af667a29 5179 size, DMA_TO_DEVICE);
03b1320d 5180 buffer_info->mapped_as_page = false;
0be3f55f 5181 if (dma_mapping_error(&pdev->dev, buffer_info->dma))
03b1320d 5182 goto dma_error;
bc7f75fa
AK		 5183
5184 len -= size;
5185 offset += size;
03b1320d 5186 count++;
1b7719c4
AD		 5187
5188 if (len) {
5189 i++;
5190 if (i == tx_ring->count)
5191 i = 0;
5192 }
bc7f75fa
AK		 5193 }
5194
5195 for (f = 0; f < nr_frags; f++) {
9e903e08 5196 const struct skb_frag_struct *frag;
bc7f75fa
AK		 5197
5198 frag = &skb_shinfo(skb)->frags[f];
9e903e08 5199 len = skb_frag_size(frag);
877749bf 5200 offset = 0;
bc7f75fa
AK		 5201
5202 while (len) {
1b7719c4
AD		 5203 i++;
5204 if (i == tx_ring->count)
5205 i = 0;
5206
bc7f75fa
AK		 5207 buffer_info = &tx_ring->buffer_info[i];
5208 size = min(len, max_per_txd);
bc7f75fa
AK		 5209
5210 buffer_info->length = size;
5211 buffer_info->time_stamp = jiffies;
bc7f75fa 5212 buffer_info->next_to_watch = i;
877749bf
IC		 5213 buffer_info->dma = skb_frag_dma_map(&pdev->dev, frag,
5214 offset, size, DMA_TO_DEVICE);
03b1320d 5215 buffer_info->mapped_as_page = true;
0be3f55f 5216 if (dma_mapping_error(&pdev->dev, buffer_info->dma))
03b1320d 5217 goto dma_error;
bc7f75fa
AK		 5218
5219 len -= size;
5220 offset += size;
5221 count++;
bc7f75fa
AK		 5222 }
5223 }
5224
af667a29 5225 segs = skb_shinfo(skb)->gso_segs ? : 1;
9ed318d5
TH		 5226 /* multiply data chunks by size of headers */
5227 bytecount = ((segs - 1) * skb_headlen(skb)) + skb->len;
5228
bc7f75fa 5229 tx_ring->buffer_info[i].skb = skb;
9ed318d5
TH		 5230 tx_ring->buffer_info[i].segs = segs;
5231 tx_ring->buffer_info[i].bytecount = bytecount;
bc7f75fa
AK		 5232 tx_ring->buffer_info[first].next_to_watch = i;
5233
5234 return count;
03b1320d
AD		 5235
5236dma_error:
af667a29 5237 dev_err(&pdev->dev, "Tx DMA map failed\n");
03b1320d 5238 buffer_info->dma = 0;
c1fa347f 5239 if (count)
03b1320d 5240 count--;
c1fa347f
RK		 5241
5242 while (count--) {
af667a29 5243 if (i == 0)
03b1320d 5244 i += tx_ring->count;
c1fa347f 5245 i--;
03b1320d 5246 buffer_info = &tx_ring->buffer_info[i];
55aa6985 5247 e1000_put_txbuf(tx_ring, buffer_info);
03b1320d
AD		 5248 }
5249
5250 return 0;
bc7f75fa
AK		 5251}
5252
55aa6985 5253static void e1000_tx_queue(struct e1000_ring *tx_ring, int tx_flags, int count)
bc7f75fa 5254{
55aa6985 5255 struct e1000_adapter *adapter = tx_ring->adapter;
bc7f75fa
AK		 5256 struct e1000_tx_desc *tx_desc = NULL;
5257 struct e1000_buffer *buffer_info;
5258 u32 txd_upper = 0, txd_lower = E1000_TXD_CMD_IFCS;
5259 unsigned int i;
5260
5261 if (tx_flags & E1000_TX_FLAGS_TSO) {
5262 txd_lower |= E1000_TXD_CMD_DEXT | E1000_TXD_DTYP_D |
f0ff4398 5263 E1000_TXD_CMD_TSE;
bc7f75fa
AK		 5264 txd_upper |= E1000_TXD_POPTS_TXSM << 8;
5265
5266 if (tx_flags & E1000_TX_FLAGS_IPV4)
5267 txd_upper |= E1000_TXD_POPTS_IXSM << 8;
5268 }
5269
5270 if (tx_flags & E1000_TX_FLAGS_CSUM) {
5271 txd_lower |= E1000_TXD_CMD_DEXT | E1000_TXD_DTYP_D;
5272 txd_upper |= E1000_TXD_POPTS_TXSM << 8;
5273 }
5274
5275 if (tx_flags & E1000_TX_FLAGS_VLAN) {
5276 txd_lower |= E1000_TXD_CMD_VLE;
5277 txd_upper |= (tx_flags & E1000_TX_FLAGS_VLAN_MASK);
5278 }
5279
943146de
BG		 5280 if (unlikely(tx_flags & E1000_TX_FLAGS_NO_FCS))
5281 txd_lower &= ~(E1000_TXD_CMD_IFCS);
5282
b67e1913
BA		 5283 if (unlikely(tx_flags & E1000_TX_FLAGS_HWTSTAMP)) {
5284 txd_lower |= E1000_TXD_CMD_DEXT | E1000_TXD_DTYP_D;
5285 txd_upper |= E1000_TXD_EXTCMD_TSTAMP;
5286 }
5287
bc7f75fa
AK		 5288 i = tx_ring->next_to_use;
5289
36b973df 5290 do {
bc7f75fa
AK		 5291 buffer_info = &tx_ring->buffer_info[i];
5292 tx_desc = E1000_TX_DESC(*tx_ring, i);
5293 tx_desc->buffer_addr = cpu_to_le64(buffer_info->dma);
f0ff4398
BA		 5294 tx_desc->lower.data = cpu_to_le32(txd_lower |
5295 buffer_info->length);
bc7f75fa
AK		 5296 tx_desc->upper.data = cpu_to_le32(txd_upper);
5297
5298 i++;
5299 if (i == tx_ring->count)
5300 i = 0;
36b973df 5301 } while (--count > 0);
bc7f75fa
AK		 5302
5303 tx_desc->lower.data |= cpu_to_le32(adapter->txd_cmd);
5304
943146de
BG		 5305 /* txd_cmd re-enables FCS, so we'll re-disable it here as desired. */
5306 if (unlikely(tx_flags & E1000_TX_FLAGS_NO_FCS))
5307 tx_desc->lower.data &= ~(cpu_to_le32(E1000_TXD_CMD_IFCS));
5308
e921eb1a 5309 /* Force memory writes to complete before letting h/w
bc7f75fa
AK		 5310 * know there are new descriptors to fetch. (Only
5311 * applicable for weak-ordered memory model archs,
ad68076e
BA		 5312 * such as IA-64).
5313 */
bc7f75fa
AK		 5314 wmb();
5315
5316 tx_ring->next_to_use = i;
c6e7f51e
BA		 5317
5318 if (adapter->flags2 & FLAG2_PCIM2PCI_ARBITER_WA)
55aa6985 5319 e1000e_update_tdt_wa(tx_ring, i);
c6e7f51e 5320 else
c5083cf6 5321 writel(i, tx_ring->tail);
c6e7f51e 5322
e921eb1a 5323 /* we need this if more than one processor can write to our tail
ad68076e
BA		 5324 * at a time, it synchronizes IO on IA64/Altix systems
5325 */
bc7f75fa
AK		 5326 mmiowb();
5327}
5328
5329#define MINIMUM_DHCP_PACKET_SIZE 282
5330static int e1000_transfer_dhcp_info(struct e1000_adapter *adapter,
5331 struct sk_buff *skb)
5332{
5333 struct e1000_hw *hw = &adapter->hw;
5334 u16 length, offset;
5335
d60923c4
BA		 5336 if (vlan_tx_tag_present(skb) &&
5337 !((vlan_tx_tag_get(skb) == adapter->hw.mng_cookie.vlan_id) &&
5338 (adapter->hw.mng_cookie.status &
5339 E1000_MNG_DHCP_COOKIE_STATUS_VLAN)))
5340 return 0;
bc7f75fa
AK		 5341
5342 if (skb->len <= MINIMUM_DHCP_PACKET_SIZE)
5343 return 0;
5344
5345 if (((struct ethhdr *) skb->data)->h_proto != htons(ETH_P_IP))
5346 return 0;
5347
5348 {
5349 const struct iphdr *ip = (struct iphdr *)((u8 *)skb->data+14);
5350 struct udphdr *udp;
5351
5352 if (ip->protocol != IPPROTO_UDP)
5353 return 0;
5354
5355 udp = (struct udphdr *)((u8 *)ip + (ip->ihl << 2));
5356 if (ntohs(udp->dest) != 67)
5357 return 0;
5358
5359 offset = (u8 *)udp + 8 - skb->data;
5360 length = skb->len - offset;
5361 return e1000e_mng_write_dhcp_info(hw, (u8 *)udp + 8, length);
5362 }
5363
5364 return 0;
5365}
5366
55aa6985 5367static int __e1000_maybe_stop_tx(struct e1000_ring *tx_ring, int size)
bc7f75fa 5368{
55aa6985 5369 struct e1000_adapter *adapter = tx_ring->adapter;
bc7f75fa 5370
55aa6985 5371 netif_stop_queue(adapter->netdev);
e921eb1a 5372 /* Herbert's original patch had:
bc7f75fa 5373 * smp_mb__after_netif_stop_queue();
ad68076e
BA		 5374 * but since that doesn't exist yet, just open code it.
5375 */
bc7f75fa
AK		 5376 smp_mb();
5377
e921eb1a 5378 /* We need to check again in a case another CPU has just
ad68076e
BA		 5379 * made room available.
5380 */
55aa6985 5381 if (e1000_desc_unused(tx_ring) < size)
bc7f75fa
AK		 5382 return -EBUSY;
5383
5384 /* A reprieve! */
55aa6985 5385 netif_start_queue(adapter->netdev);
bc7f75fa
AK		 5386 ++adapter->restart_queue;
5387 return 0;
5388}
5389
55aa6985 5390static int e1000_maybe_stop_tx(struct e1000_ring *tx_ring, int size)
bc7f75fa 5391{
d821a4c4
BA		 5392 BUG_ON(size > tx_ring->count);
5393
55aa6985 5394 if (e1000_desc_unused(tx_ring) >= size)
bc7f75fa 5395 return 0;
55aa6985 5396 return __e1000_maybe_stop_tx(tx_ring, size);
bc7f75fa
AK		 5397}
5398
3b29a56d
SH		 5399static netdev_tx_t e1000_xmit_frame(struct sk_buff *skb,
5400 struct net_device *netdev)
bc7f75fa
AK		 5401{
5402 struct e1000_adapter *adapter = netdev_priv(netdev);
5403 struct e1000_ring *tx_ring = adapter->tx_ring;
5404 unsigned int first;
bc7f75fa 5405 unsigned int tx_flags = 0;
e743d313 5406 unsigned int len = skb_headlen(skb);
4e6c709c
AK		 5407 unsigned int nr_frags;
5408 unsigned int mss;
bc7f75fa
AK		 5409 int count = 0;
5410 int tso;
5411 unsigned int f;
bc7f75fa
AK		 5412
5413 if (test_bit(__E1000_DOWN, &adapter->state)) {
5414 dev_kfree_skb_any(skb);
5415 return NETDEV_TX_OK;
5416 }
5417
5418 if (skb->len <= 0) {
5419 dev_kfree_skb_any(skb);
5420 return NETDEV_TX_OK;
5421 }
5422
e921eb1a 5423 /* The minimum packet size with TCTL.PSP set is 17 bytes so
6e97c170
TD		 5424 * pad skb in order to meet this minimum size requirement
5425 */
5426 if (unlikely(skb->len < 17)) {
5427 if (skb_pad(skb, 17 - skb->len))
5428 return NETDEV_TX_OK;
5429 skb->len = 17;
5430 skb_set_tail_pointer(skb, 17);
5431 }
5432
bc7f75fa 5433 mss = skb_shinfo(skb)->gso_size;
bc7f75fa
AK		 5434 if (mss) {
5435 u8 hdr_len;
bc7f75fa 5436
e921eb1a 5437 /* TSO Workaround for 82571/2/3 Controllers -- if skb->data
ad68076e
BA		 5438 * points to just header, pull a few bytes of payload from
5439 * frags into skb->data
5440 */
bc7f75fa 5441 hdr_len = skb_transport_offset(skb) + tcp_hdrlen(skb);
e921eb1a 5442 /* we do this workaround for ES2LAN, but it is un-necessary,
ad68076e
BA		 5443 * avoiding it could save a lot of cycles
5444 */
4e6c709c 5445 if (skb->data_len && (hdr_len == len)) {
bc7f75fa
AK		 5446 unsigned int pull_size;
5447
a2a5b323 5448 pull_size = min_t(unsigned int, 4, skb->data_len);
bc7f75fa 5449 if (!__pskb_pull_tail(skb, pull_size)) {
44defeb3 5450 e_err("__pskb_pull_tail failed.\n");
bc7f75fa
AK		 5451 dev_kfree_skb_any(skb);
5452 return NETDEV_TX_OK;
5453 }
e743d313 5454 len = skb_headlen(skb);
bc7f75fa
AK		 5455 }
5456 }
5457
5458 /* reserve a descriptor for the offload context */
5459 if ((mss) || (skb->ip_summed == CHECKSUM_PARTIAL))
5460 count++;
5461 count++;
5462
d821a4c4 5463 count += DIV_ROUND_UP(len, adapter->tx_fifo_limit);
bc7f75fa
AK		 5464
5465 nr_frags = skb_shinfo(skb)->nr_frags;
5466 for (f = 0; f < nr_frags; f++)
d821a4c4
BA		 5467 count += DIV_ROUND_UP(skb_frag_size(&skb_shinfo(skb)->frags[f]),
5468 adapter->tx_fifo_limit);
bc7f75fa
AK		 5469
5470 if (adapter->hw.mac.tx_pkt_filtering)
5471 e1000_transfer_dhcp_info(adapter, skb);
5472
e921eb1a 5473 /* need: count + 2 desc gap to keep tail from touching
ad68076e
BA		 5474 * head, otherwise try next time
5475 */
55aa6985 5476 if (e1000_maybe_stop_tx(tx_ring, count + 2))
bc7f75fa 5477 return NETDEV_TX_BUSY;
bc7f75fa 5478
eab6d18d 5479 if (vlan_tx_tag_present(skb)) {
bc7f75fa
AK		 5480 tx_flags |= E1000_TX_FLAGS_VLAN;
5481 tx_flags |= (vlan_tx_tag_get(skb) << E1000_TX_FLAGS_VLAN_SHIFT);
5482 }
5483
5484 first = tx_ring->next_to_use;
5485
55aa6985 5486 tso = e1000_tso(tx_ring, skb);
bc7f75fa
AK		 5487 if (tso < 0) {
5488 dev_kfree_skb_any(skb);
bc7f75fa
AK		 5489 return NETDEV_TX_OK;
5490 }
5491
5492 if (tso)
5493 tx_flags |= E1000_TX_FLAGS_TSO;
55aa6985 5494 else if (e1000_tx_csum(tx_ring, skb))
bc7f75fa
AK		 5495 tx_flags |= E1000_TX_FLAGS_CSUM;
5496
e921eb1a 5497 /* Old method was to assume IPv4 packet by default if TSO was enabled.
bc7f75fa 5498 * 82571 hardware supports TSO capabilities for IPv6 as well...
ad68076e
BA		 5499 * no longer assume, we must.
5500 */
bc7f75fa
AK		 5501 if (skb->protocol == htons(ETH_P_IP))
5502 tx_flags |= E1000_TX_FLAGS_IPV4;
5503
943146de
BG		 5504 if (unlikely(skb->no_fcs))
5505 tx_flags |= E1000_TX_FLAGS_NO_FCS;
5506
25985edc 5507 /* if count is 0 then mapping error has occurred */
d821a4c4
BA		 5508 count = e1000_tx_map(tx_ring, skb, first, adapter->tx_fifo_limit,
5509 nr_frags);
1b7719c4 5510 if (count) {
b67e1913
BA		 5511 if (unlikely((skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) &&
5512 !adapter->tx_hwtstamp_skb)) {
5513 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;
5514 tx_flags |= E1000_TX_FLAGS_HWTSTAMP;
5515 adapter->tx_hwtstamp_skb = skb_get(skb);
5516 schedule_work(&adapter->tx_hwtstamp_work);
5517 } else {
5518 skb_tx_timestamp(skb);
5519 }
80be3129 5520
3f0cfa3b 5521 netdev_sent_queue(netdev, skb->len);
55aa6985 5522 e1000_tx_queue(tx_ring, tx_flags, count);
1b7719c4 5523 /* Make sure there is space in the ring for the next send. */
d821a4c4
BA		 5524 e1000_maybe_stop_tx(tx_ring,
5525 (MAX_SKB_FRAGS *
5526 DIV_ROUND_UP(PAGE_SIZE,
5527 adapter->tx_fifo_limit) + 2));
1b7719c4 5528 } else {
bc7f75fa 5529 dev_kfree_skb_any(skb);
1b7719c4
AD		 5530 tx_ring->buffer_info[first].time_stamp = 0;
5531 tx_ring->next_to_use = first;
bc7f75fa
AK		 5532 }
5533
bc7f75fa
AK		 5534 return NETDEV_TX_OK;
5535}
5536
5537/**
5538 * e1000_tx_timeout - Respond to a Tx Hang
5539 * @netdev: network interface device structure
5540 **/
5541static void e1000_tx_timeout(struct net_device *netdev)
5542{
5543 struct e1000_adapter *adapter = netdev_priv(netdev);
5544
5545 /* Do the reset outside of interrupt context */
5546 adapter->tx_timeout_count++;
5547 schedule_work(&adapter->reset_task);
5548}
5549
5550static void e1000_reset_task(struct work_struct *work)
5551{
5552 struct e1000_adapter *adapter;
5553 adapter = container_of(work, struct e1000_adapter, reset_task);
5554
615b32af
JB		 5555 /* don't run the task if already down */
5556 if (test_bit(__E1000_DOWN, &adapter->state))
5557 return;
5558
12d43f7d 5559 if (!(adapter->flags & FLAG_RESTART_NOW)) {
affa9dfb 5560 e1000e_dump(adapter);
12d43f7d 5561 e_err("Reset adapter unexpectedly\n");
affa9dfb 5562 }
bc7f75fa
AK		 5563 e1000e_reinit_locked(adapter);
5564}
5565
5566/**
67fd4fcb 5567 * e1000_get_stats64 - Get System Network Statistics
bc7f75fa 5568 * @netdev: network interface device structure
67fd4fcb 5569 * @stats: rtnl_link_stats64 pointer
bc7f75fa
AK		 5570 *
5571 * Returns the address of the device statistics structure.
bc7f75fa 5572 **/
67fd4fcb
JK		 5573struct rtnl_link_stats64 *e1000e_get_stats64(struct net_device *netdev,
5574 struct rtnl_link_stats64 *stats)
bc7f75fa 5575{
67fd4fcb
JK		 5576 struct e1000_adapter *adapter = netdev_priv(netdev);
5577
5578 memset(stats, 0, sizeof(struct rtnl_link_stats64));
5579 spin_lock(&adapter->stats64_lock);
5580 e1000e_update_stats(adapter);
5581 /* Fill out the OS statistics structure */
5582 stats->rx_bytes = adapter->stats.gorc;
5583 stats->rx_packets = adapter->stats.gprc;
5584 stats->tx_bytes = adapter->stats.gotc;
5585 stats->tx_packets = adapter->stats.gptc;
5586 stats->multicast = adapter->stats.mprc;
5587 stats->collisions = adapter->stats.colc;
5588
5589 /* Rx Errors */
5590
e921eb1a 5591 /* RLEC on some newer hardware can be incorrect so build
67fd4fcb
JK		 5592 * our own version based on RUC and ROC
5593 */
5594 stats->rx_errors = adapter->stats.rxerrc +
f0ff4398
BA		 5595 adapter->stats.crcerrs + adapter->stats.algnerrc +
5596 adapter->stats.ruc + adapter->stats.roc + adapter->stats.cexterr;
5597 stats->rx_length_errors = adapter->stats.ruc + adapter->stats.roc;
67fd4fcb
JK		 5598 stats->rx_crc_errors = adapter->stats.crcerrs;
5599 stats->rx_frame_errors = adapter->stats.algnerrc;
5600 stats->rx_missed_errors = adapter->stats.mpc;
5601
5602 /* Tx Errors */
f0ff4398 5603 stats->tx_errors = adapter->stats.ecol + adapter->stats.latecol;
67fd4fcb
JK		 5604 stats->tx_aborted_errors = adapter->stats.ecol;
5605 stats->tx_window_errors = adapter->stats.latecol;
5606 stats->tx_carrier_errors = adapter->stats.tncrs;
5607
5608 /* Tx Dropped needs to be maintained elsewhere */
5609
5610 spin_unlock(&adapter->stats64_lock);
5611 return stats;
bc7f75fa
AK		 5612}
5613
5614/**
5615 * e1000_change_mtu - Change the Maximum Transfer Unit
5616 * @netdev: network interface device structure
5617 * @new_mtu: new value for maximum frame size
5618 *
5619 * Returns 0 on success, negative on failure
5620 **/
5621static int e1000_change_mtu(struct net_device *netdev, int new_mtu)
5622{
5623 struct e1000_adapter *adapter = netdev_priv(netdev);
5624 int max_frame = new_mtu + ETH_HLEN + ETH_FCS_LEN;
5625
2adc55c9 5626 /* Jumbo frame support */
2e1706f2
BA		 5627 if ((max_frame > ETH_FRAME_LEN + ETH_FCS_LEN) &&
5628 !(adapter->flags & FLAG_HAS_JUMBO_FRAMES)) {
5629 e_err("Jumbo Frames not supported.\n");
5630 return -EINVAL;
bc7f75fa
AK		 5631 }
5632
2adc55c9
BA		 5633 /* Supported frame sizes */
5634 if ((new_mtu < ETH_ZLEN + ETH_FCS_LEN + VLAN_HLEN) ||
5635 (max_frame > adapter->max_hw_frame_size)) {
5636 e_err("Unsupported MTU setting\n");
bc7f75fa
AK		 5637 return -EINVAL;
5638 }
5639
2fbe4526
BA		 5640 /* Jumbo frame workaround on 82579 and newer requires CRC be stripped */
5641 if ((adapter->hw.mac.type >= e1000_pch2lan) &&
a1ce6473
BA		 5642 !(adapter->flags2 & FLAG2_CRC_STRIPPING) &&
5643 (new_mtu > ETH_DATA_LEN)) {
2fbe4526 5644 e_err("Jumbo Frames not supported on this device when CRC stripping is disabled.\n");
a1ce6473
BA		 5645 return -EINVAL;
5646 }
5647
bc7f75fa 5648 while (test_and_set_bit(__E1000_RESETTING, &adapter->state))
1bba4386 5649 usleep_range(1000, 2000);
610c9928 5650 /* e1000e_down -> e1000e_reset dependent on max_frame_size & mtu */
318a94d6 5651 adapter->max_frame_size = max_frame;
610c9928
BA		 5652 e_info("changing MTU from %d to %d\n", netdev->mtu, new_mtu);
5653 netdev->mtu = new_mtu;
bc7f75fa
AK		 5654 if (netif_running(netdev))
5655 e1000e_down(adapter);
5656
e921eb1a 5657 /* NOTE: netdev_alloc_skb reserves 16 bytes, and typically NET_IP_ALIGN
bc7f75fa
AK		 5658 * means we reserve 2 more, this pushes us to allocate from the next
5659 * larger slab size.
ad68076e 5660 * i.e. RXBUFFER_2048 --> size-4096 slab
97ac8cae
BA		 5661 * However with the new *_jumbo_rx* routines, jumbo receives will use
5662 * fragmented skbs
ad68076e 5663 */
bc7f75fa 5664
9926146b 5665 if (max_frame <= 2048)
bc7f75fa
AK		 5666 adapter->rx_buffer_len = 2048;
5667 else
5668 adapter->rx_buffer_len = 4096;
5669
5670 /* adjust allocation if LPE protects us, and we aren't using SBP */
5671 if ((max_frame == ETH_FRAME_LEN + ETH_FCS_LEN) ||
5672 (max_frame == ETH_FRAME_LEN + VLAN_HLEN + ETH_FCS_LEN))
5673 adapter->rx_buffer_len = ETH_FRAME_LEN + VLAN_HLEN
ad68076e 5674 + ETH_FCS_LEN;
bc7f75fa 5675
bc7f75fa
AK		 5676 if (netif_running(netdev))
5677 e1000e_up(adapter);
5678 else
5679 e1000e_reset(adapter);
5680
5681 clear_bit(__E1000_RESETTING, &adapter->state);
5682
5683 return 0;
5684}
5685
5686static int e1000_mii_ioctl(struct net_device *netdev, struct ifreq *ifr,
5687 int cmd)
5688{
5689 struct e1000_adapter *adapter = netdev_priv(netdev);
5690 struct mii_ioctl_data *data = if_mii(ifr);
bc7f75fa 5691
318a94d6 5692 if (adapter->hw.phy.media_type != e1000_media_type_copper)
bc7f75fa
AK		 5693 return -EOPNOTSUPP;
5694
5695 switch (cmd) {
5696 case SIOCGMIIPHY:
5697 data->phy_id = adapter->hw.phy.addr;
5698 break;
5699 case SIOCGMIIREG:
b16a002e
BA		 5700 e1000_phy_read_status(adapter);
5701
7c25769f
BA		 5702 switch (data->reg_num & 0x1F) {
5703 case MII_BMCR:
5704 data->val_out = adapter->phy_regs.bmcr;
5705 break;
5706 case MII_BMSR:
5707 data->val_out = adapter->phy_regs.bmsr;
5708 break;
5709 case MII_PHYSID1:
5710 data->val_out = (adapter->hw.phy.id >> 16);
5711 break;
5712 case MII_PHYSID2:
5713 data->val_out = (adapter->hw.phy.id & 0xFFFF);
5714 break;
5715 case MII_ADVERTISE:
5716 data->val_out = adapter->phy_regs.advertise;
5717 break;
5718 case MII_LPA:
5719 data->val_out = adapter->phy_regs.lpa;
5720 break;
5721 case MII_EXPANSION:
5722 data->val_out = adapter->phy_regs.expansion;
5723 break;
5724 case MII_CTRL1000:
5725 data->val_out = adapter->phy_regs.ctrl1000;
5726 break;
5727 case MII_STAT1000:
5728 data->val_out = adapter->phy_regs.stat1000;
5729 break;
5730 case MII_ESTATUS:
5731 data->val_out = adapter->phy_regs.estatus;
5732 break;
5733 default:
bc7f75fa
AK		 5734 return -EIO;
5735 }
bc7f75fa
AK		 5736 break;
5737 case SIOCSMIIREG:
5738 default:
5739 return -EOPNOTSUPP;
5740 }
5741 return 0;
5742}
5743
b67e1913
BA		 5744/**
5745 * e1000e_hwtstamp_ioctl - control hardware time stamping
5746 * @netdev: network interface device structure
5747 * @ifreq: interface request
5748 *
5749 * Outgoing time stamping can be enabled and disabled. Play nice and
5750 * disable it when requested, although it shouldn't cause any overhead
5751 * when no packet needs it. At most one packet in the queue may be
5752 * marked for time stamping, otherwise it would be impossible to tell
5753 * for sure to which packet the hardware time stamp belongs.
5754 *
5755 * Incoming time stamping has to be configured via the hardware filters.
5756 * Not all combinations are supported, in particular event type has to be
5757 * specified. Matching the kind of event packet is not supported, with the
5758 * exception of "all V2 events regardless of level 2 or 4".
5759 **/
5760static int e1000e_hwtstamp_ioctl(struct net_device *netdev, struct ifreq *ifr)
5761{
5762 struct e1000_adapter *adapter = netdev_priv(netdev);
5763 struct hwtstamp_config config;
5764 int ret_val;
5765
5766 if (copy_from_user(&config, ifr->ifr_data, sizeof(config)))
5767 return -EFAULT;
5768
5769 adapter->hwtstamp_config = config;
5770
5771 ret_val = e1000e_config_hwtstamp(adapter);
5772 if (ret_val)
5773 return ret_val;
5774
5775 config = adapter->hwtstamp_config;
5776
d89777bf
BA		 5777 switch (config.rx_filter) {
5778 case HWTSTAMP_FILTER_PTP_V2_L4_SYNC:
5779 case HWTSTAMP_FILTER_PTP_V2_L2_SYNC:
5780 case HWTSTAMP_FILTER_PTP_V2_SYNC:
5781 case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ:
5782 case HWTSTAMP_FILTER_PTP_V2_L2_DELAY_REQ:
5783 case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ:
5784 /* With V2 type filters which specify a Sync or Delay Request,
5785 * Path Delay Request/Response messages are also time stamped
5786 * by hardware so notify the caller the requested packets plus
5787 * some others are time stamped.
5788 */
5789 config.rx_filter = HWTSTAMP_FILTER_SOME;
5790 break;
5791 default:
5792 break;
5793 }
5794
b67e1913
BA		 5795 return copy_to_user(ifr->ifr_data, &config,
5796 sizeof(config)) ? -EFAULT : 0;
5797}
5798
bc7f75fa
AK		 5799static int e1000_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd)
5800{
5801 switch (cmd) {
5802 case SIOCGMIIPHY:
5803 case SIOCGMIIREG:
5804 case SIOCSMIIREG:
5805 return e1000_mii_ioctl(netdev, ifr, cmd);
b67e1913
BA		 5806 case SIOCSHWTSTAMP:
5807 return e1000e_hwtstamp_ioctl(netdev, ifr);
bc7f75fa
AK		 5808 default:
5809 return -EOPNOTSUPP;
5810 }
5811}
5812
a4f58f54
BA		 5813static int e1000_init_phy_wakeup(struct e1000_adapter *adapter, u32 wufc)
5814{
5815 struct e1000_hw *hw = &adapter->hw;
5816 u32 i, mac_reg;
2b6b168d 5817 u16 phy_reg, wuc_enable;
70806a7f 5818 int retval;
a4f58f54
BA		 5819
5820 /* copy MAC RARs to PHY RARs */
d3738bb8 5821 e1000_copy_rx_addrs_to_phy_ich8lan(hw);
a4f58f54 5822
2b6b168d
BA		 5823 retval = hw->phy.ops.acquire(hw);
5824 if (retval) {
5825 e_err("Could not acquire PHY\n");
5826 return retval;
5827 }
5828
5829 /* Enable access to wakeup registers on and set page to BM_WUC_PAGE */
5830 retval = e1000_enable_phy_wakeup_reg_access_bm(hw, &wuc_enable);
5831 if (retval)
75ce1532 5832 goto release;
2b6b168d
BA		 5833
5834 /* copy MAC MTA to PHY MTA - only needed for pchlan */
a4f58f54
BA		 5835 for (i = 0; i < adapter->hw.mac.mta_reg_count; i++) {
5836 mac_reg = E1000_READ_REG_ARRAY(hw, E1000_MTA, i);
2b6b168d
BA		 5837 hw->phy.ops.write_reg_page(hw, BM_MTA(i),
5838 (u16)(mac_reg & 0xFFFF));
5839 hw->phy.ops.write_reg_page(hw, BM_MTA(i) + 1,
5840 (u16)((mac_reg >> 16) & 0xFFFF));
a4f58f54
BA		 5841 }
5842
5843 /* configure PHY Rx Control register */
2b6b168d 5844 hw->phy.ops.read_reg_page(&adapter->hw, BM_RCTL, &phy_reg);
a4f58f54
BA		 5845 mac_reg = er32(RCTL);
5846 if (mac_reg & E1000_RCTL_UPE)
5847 phy_reg |= BM_RCTL_UPE;
5848 if (mac_reg & E1000_RCTL_MPE)
5849 phy_reg |= BM_RCTL_MPE;
5850 phy_reg &= ~(BM_RCTL_MO_MASK);
5851 if (mac_reg & E1000_RCTL_MO_3)
5852 phy_reg |= (((mac_reg & E1000_RCTL_MO_3) >> E1000_RCTL_MO_SHIFT)
5853 << BM_RCTL_MO_SHIFT);
5854 if (mac_reg & E1000_RCTL_BAM)
5855 phy_reg |= BM_RCTL_BAM;
5856 if (mac_reg & E1000_RCTL_PMCF)
5857 phy_reg |= BM_RCTL_PMCF;
5858 mac_reg = er32(CTRL);
5859 if (mac_reg & E1000_CTRL_RFCE)
5860 phy_reg |= BM_RCTL_RFCE;
2b6b168d 5861 hw->phy.ops.write_reg_page(&adapter->hw, BM_RCTL, phy_reg);
a4f58f54
BA		 5862
5863 /* enable PHY wakeup in MAC register */
5864 ew32(WUFC, wufc);
5865 ew32(WUC, E1000_WUC_PHY_WAKE | E1000_WUC_PME_EN);
5866
5867 /* configure and enable PHY wakeup in PHY registers */
2b6b168d
BA		 5868 hw->phy.ops.write_reg_page(&adapter->hw, BM_WUFC, wufc);
5869 hw->phy.ops.write_reg_page(&adapter->hw, BM_WUC, E1000_WUC_PME_EN);
a4f58f54
BA		 5870
5871 /* activate PHY wakeup */
2b6b168d
BA		 5872 wuc_enable |= BM_WUC_ENABLE_BIT | BM_WUC_HOST_WU_BIT;
5873 retval = e1000_disable_phy_wakeup_reg_access_bm(hw, &wuc_enable);
a4f58f54
BA		 5874 if (retval)
5875 e_err("Could not set PHY Host Wakeup bit\n");
75ce1532 5876release:
94d8186a 5877 hw->phy.ops.release(hw);
a4f58f54
BA		 5878
5879 return retval;
5880}
5881
23606cf5
RW		 5882static int __e1000_shutdown(struct pci_dev *pdev, bool *enable_wake,
5883 bool runtime)
bc7f75fa
AK		 5884{
5885 struct net_device *netdev = pci_get_drvdata(pdev);
5886 struct e1000_adapter *adapter = netdev_priv(netdev);
5887 struct e1000_hw *hw = &adapter->hw;
5888 u32 ctrl, ctrl_ext, rctl, status;
23606cf5
RW		 5889 /* Runtime suspend should only enable wakeup for link changes */
5890 u32 wufc = runtime ? E1000_WUFC_LNKC : adapter->wol;
bc7f75fa
AK		 5891 int retval = 0;
5892
5893 netif_device_detach(netdev);
5894
5895 if (netif_running(netdev)) {
bb9e44d0
BA		 5896 int count = E1000_CHECK_RESET_COUNT;
5897
5898 while (test_bit(__E1000_RESETTING, &adapter->state) && count--)
5899 usleep_range(10000, 20000);
5900
bc7f75fa
AK		 5901 WARN_ON(test_bit(__E1000_RESETTING, &adapter->state));
5902 e1000e_down(adapter);
5903 e1000_free_irq(adapter);
5904 }
4662e82b 5905 e1000e_reset_interrupt_capability(adapter);
bc7f75fa
AK		 5906
5907 retval = pci_save_state(pdev);
5908 if (retval)
5909 return retval;
5910
5911 status = er32(STATUS);
5912 if (status & E1000_STATUS_LU)
5913 wufc &= ~E1000_WUFC_LNKC;
5914
5915 if (wufc) {
5916 e1000_setup_rctl(adapter);
ef9b965a 5917 e1000e_set_rx_mode(netdev);
bc7f75fa
AK		 5918
5919 /* turn on all-multi mode if wake on multicast is enabled */
5920 if (wufc & E1000_WUFC_MC) {
5921 rctl = er32(RCTL);
5922 rctl |= E1000_RCTL_MPE;
5923 ew32(RCTL, rctl);
5924 }
5925
5926 ctrl = er32(CTRL);
5927 /* advertise wake from D3Cold */
5928 #define E1000_CTRL_ADVD3WUC 0x00100000
5929 /* phy power management enable */
5930 #define E1000_CTRL_EN_PHY_PWR_MGMT 0x00200000
a4f58f54
BA		 5931 ctrl |= E1000_CTRL_ADVD3WUC;
5932 if (!(adapter->flags2 & FLAG2_HAS_PHY_WAKEUP))
5933 ctrl |= E1000_CTRL_EN_PHY_PWR_MGMT;
bc7f75fa
AK		 5934 ew32(CTRL, ctrl);
5935
318a94d6
JK		 5936 if (adapter->hw.phy.media_type == e1000_media_type_fiber ||
5937 adapter->hw.phy.media_type ==
5938 e1000_media_type_internal_serdes) {
bc7f75fa
AK		 5939 /* keep the laser running in D3 */
5940 ctrl_ext = er32(CTRL_EXT);
93a23f48 5941 ctrl_ext |= E1000_CTRL_EXT_SDP3_DATA;
bc7f75fa
AK		 5942 ew32(CTRL_EXT, ctrl_ext);
5943 }
5944
97ac8cae 5945 if (adapter->flags & FLAG_IS_ICH)
99730e4c 5946 e1000_suspend_workarounds_ich8lan(&adapter->hw);
97ac8cae 5947
bc7f75fa
AK		 5948 /* Allow time for pending master requests to run */
5949 e1000e_disable_pcie_master(&adapter->hw);
5950
82776a4b 5951 if (adapter->flags2 & FLAG2_HAS_PHY_WAKEUP) {
a4f58f54
BA		 5952 /* enable wakeup by the PHY */
5953 retval = e1000_init_phy_wakeup(adapter, wufc);
5954 if (retval)
5955 return retval;
5956 } else {
5957 /* enable wakeup by the MAC */
5958 ew32(WUFC, wufc);
5959 ew32(WUC, E1000_WUC_PME_EN);
5960 }
bc7f75fa
AK		 5961 } else {
5962 ew32(WUC, 0);
5963 ew32(WUFC, 0);
bc7f75fa
AK		 5964 }
5965
4f9de721
RW		 5966 *enable_wake = !!wufc;
5967
bc7f75fa 5968 /* make sure adapter isn't asleep if manageability is enabled */
82776a4b
BA		 5969 if ((adapter->flags & FLAG_MNG_PT_ENABLED) ||
5970 (hw->mac.ops.check_mng_mode(hw)))
4f9de721 5971 *enable_wake = true;
bc7f75fa
AK		 5972
5973 if (adapter->hw.phy.type == e1000_phy_igp_3)
5974 e1000e_igp3_phy_powerdown_workaround_ich8lan(&adapter->hw);
5975
e921eb1a 5976 /* Release control of h/w to f/w. If f/w is AMT enabled, this
ad68076e
BA		 5977 * would have already happened in close and is redundant.
5978 */
31dbe5b4 5979 e1000e_release_hw_control(adapter);
bc7f75fa
AK		 5980
5981 pci_disable_device(pdev);
5982
4f9de721
RW		 5983 return 0;
5984}
5985
5986static void e1000_power_off(struct pci_dev *pdev, bool sleep, bool wake)
5987{
5988 if (sleep && wake) {
5989 pci_prepare_to_sleep(pdev);
5990 return;
5991 }
5992
5993 pci_wake_from_d3(pdev, wake);
5994 pci_set_power_state(pdev, PCI_D3hot);
5995}
5996
f0ff4398 5997static void e1000_complete_shutdown(struct pci_dev *pdev, bool sleep, bool wake)
4f9de721
RW		 5998{
5999 struct net_device *netdev = pci_get_drvdata(pdev);
6000 struct e1000_adapter *adapter = netdev_priv(netdev);
6001
e921eb1a 6002 /* The pci-e switch on some quad port adapters will report a
005cbdfc
AD		 6003 * correctable error when the MAC transitions from D0 to D3. To
6004 * prevent this we need to mask off the correctable errors on the
6005 * downstream port of the pci-e switch.
6006 */
6007 if (adapter->flags & FLAG_IS_QUAD_PORT) {
6008 struct pci_dev *us_dev = pdev->bus->self;
005cbdfc
AD		 6009 u16 devctl;
6010
f8c0fcac
JL		 6011 pcie_capability_read_word(us_dev, PCI_EXP_DEVCTL, &devctl);
6012 pcie_capability_write_word(us_dev, PCI_EXP_DEVCTL,
6013 (devctl & ~PCI_EXP_DEVCTL_CERE));
005cbdfc 6014
4f9de721 6015 e1000_power_off(pdev, sleep, wake);
005cbdfc 6016
f8c0fcac 6017 pcie_capability_write_word(us_dev, PCI_EXP_DEVCTL, devctl);
005cbdfc 6018 } else {
4f9de721 6019 e1000_power_off(pdev, sleep, wake);
005cbdfc 6020 }
bc7f75fa
AK		 6021}
6022
6f461f6c
BA		 6023#ifdef CONFIG_PCIEASPM
6024static void __e1000e_disable_aspm(struct pci_dev *pdev, u16 state)
6025{
9f728f53 6026 pci_disable_link_state_locked(pdev, state);
6f461f6c
BA		 6027}
6028#else
6029static void __e1000e_disable_aspm(struct pci_dev *pdev, u16 state)
1eae4eb2 6030{
ffe0b2ff
BH		 6031 u16 aspm_ctl = 0;
6032
6033 if (state & PCIE_LINK_STATE_L0S)
6034 aspm_ctl |= PCI_EXP_LNKCTL_ASPM_L0S;
6035 if (state & PCIE_LINK_STATE_L1)
6036 aspm_ctl |= PCI_EXP_LNKCTL_ASPM_L1;
6037
e921eb1a 6038 /* Both device and parent should have the same ASPM setting.
6f461f6c 6039 * Disable ASPM in downstream component first and then upstream.
1eae4eb2 6040 */
ffe0b2ff 6041 pcie_capability_clear_word(pdev, PCI_EXP_LNKCTL, aspm_ctl);
0c75ba22 6042
f8c0fcac
JL		 6043 if (pdev->bus->self)
6044 pcie_capability_clear_word(pdev->bus->self, PCI_EXP_LNKCTL,
ffe0b2ff 6045 aspm_ctl);
6f461f6c
BA		 6046}
6047#endif
78cd29d5 6048static void e1000e_disable_aspm(struct pci_dev *pdev, u16 state)
6f461f6c
BA		 6049{
6050 dev_info(&pdev->dev, "Disabling ASPM %s %s\n",
6051 (state & PCIE_LINK_STATE_L0S) ? "L0s" : "",
6052 (state & PCIE_LINK_STATE_L1) ? "L1" : "");
6053
6054 __e1000e_disable_aspm(pdev, state);
1eae4eb2
AK		 6055}
6056
aa338601 6057#ifdef CONFIG_PM
23606cf5 6058static bool e1000e_pm_ready(struct e1000_adapter *adapter)
4f9de721 6059{
23606cf5 6060 return !!adapter->tx_ring->buffer_info;
4f9de721
RW		 6061}
6062
23606cf5 6063static int __e1000_resume(struct pci_dev *pdev)
bc7f75fa
AK		 6064{
6065 struct net_device *netdev = pci_get_drvdata(pdev);
6066 struct e1000_adapter *adapter = netdev_priv(netdev);
6067 struct e1000_hw *hw = &adapter->hw;
78cd29d5 6068 u16 aspm_disable_flag = 0;
bc7f75fa
AK		 6069 u32 err;
6070
78cd29d5
BA		 6071 if (adapter->flags2 & FLAG2_DISABLE_ASPM_L0S)
6072 aspm_disable_flag = PCIE_LINK_STATE_L0S;
6073 if (adapter->flags2 & FLAG2_DISABLE_ASPM_L1)
6074 aspm_disable_flag |= PCIE_LINK_STATE_L1;
6075 if (aspm_disable_flag)
6076 e1000e_disable_aspm(pdev, aspm_disable_flag);
6077
bc7f75fa
AK		 6078 pci_set_power_state(pdev, PCI_D0);
6079 pci_restore_state(pdev);
28b8f04a 6080 pci_save_state(pdev);
6e4f6f6b 6081
4662e82b 6082 e1000e_set_interrupt_capability(adapter);
bc7f75fa
AK		 6083 if (netif_running(netdev)) {
6084 err = e1000_request_irq(adapter);
6085 if (err)
6086 return err;
6087 }
6088
2fbe4526 6089 if (hw->mac.type >= e1000_pch2lan)
99730e4c
BA		 6090 e1000_resume_workarounds_pchlan(&adapter->hw);
6091
bc7f75fa 6092 e1000e_power_up_phy(adapter);
a4f58f54
BA		 6093
6094 /* report the system wakeup cause from S3/S4 */
6095 if (adapter->flags2 & FLAG2_HAS_PHY_WAKEUP) {
6096 u16 phy_data;
6097
6098 e1e_rphy(&adapter->hw, BM_WUS, &phy_data);
6099 if (phy_data) {
6100 e_info("PHY Wakeup cause - %s\n",
6101 phy_data & E1000_WUS_EX ? "Unicast Packet" :
6102 phy_data & E1000_WUS_MC ? "Multicast Packet" :
6103 phy_data & E1000_WUS_BC ? "Broadcast Packet" :
6104 phy_data & E1000_WUS_MAG ? "Magic Packet" :
ef456f85
JK		 6105 phy_data & E1000_WUS_LNKC ?
6106 "Link Status Change" : "other");
a4f58f54
BA		 6107 }
6108 e1e_wphy(&adapter->hw, BM_WUS, ~0);
6109 } else {
6110 u32 wus = er32(WUS);
6111 if (wus) {
6112 e_info("MAC Wakeup cause - %s\n",
6113 wus & E1000_WUS_EX ? "Unicast Packet" :
6114 wus & E1000_WUS_MC ? "Multicast Packet" :
6115 wus & E1000_WUS_BC ? "Broadcast Packet" :
6116 wus & E1000_WUS_MAG ? "Magic Packet" :
6117 wus & E1000_WUS_LNKC ? "Link Status Change" :
6118 "other");
6119 }
6120 ew32(WUS, ~0);
6121 }
6122
bc7f75fa 6123 e1000e_reset(adapter);
bc7f75fa 6124
cd791618 6125 e1000_init_manageability_pt(adapter);
bc7f75fa
AK		 6126
6127 if (netif_running(netdev))
6128 e1000e_up(adapter);
6129
6130 netif_device_attach(netdev);
6131
e921eb1a 6132 /* If the controller has AMT, do not set DRV_LOAD until the interface
bc7f75fa 6133 * is up. For all other cases, let the f/w know that the h/w is now
ad68076e
BA		 6134 * under the control of the driver.
6135 */
c43bc57e 6136 if (!(adapter->flags & FLAG_HAS_AMT))
31dbe5b4 6137 e1000e_get_hw_control(adapter);
bc7f75fa
AK		 6138
6139 return 0;
6140}
23606cf5 6141
a0340162
RW		 6142#ifdef CONFIG_PM_SLEEP
6143static int e1000_suspend(struct device *dev)
6144{
6145 struct pci_dev *pdev = to_pci_dev(dev);
6146 int retval;
6147 bool wake;
6148
6149 retval = __e1000_shutdown(pdev, &wake, false);
6150 if (!retval)
6151 e1000_complete_shutdown(pdev, true, wake);
6152
6153 return retval;
6154}
6155
23606cf5
RW		 6156static int e1000_resume(struct device *dev)
6157{
6158 struct pci_dev *pdev = to_pci_dev(dev);
6159 struct net_device *netdev = pci_get_drvdata(pdev);
6160 struct e1000_adapter *adapter = netdev_priv(netdev);
6161
6162 if (e1000e_pm_ready(adapter))
6163 adapter->idle_check = true;
6164
6165 return __e1000_resume(pdev);
6166}
a0340162
RW		 6167#endif /* CONFIG_PM_SLEEP */
6168
6169#ifdef CONFIG_PM_RUNTIME
6170static int e1000_runtime_suspend(struct device *dev)
6171{
6172 struct pci_dev *pdev = to_pci_dev(dev);
6173 struct net_device *netdev = pci_get_drvdata(pdev);
6174 struct e1000_adapter *adapter = netdev_priv(netdev);
6175
6176 if (e1000e_pm_ready(adapter)) {
6177 bool wake;
6178
6179 __e1000_shutdown(pdev, &wake, true);
6180 }
6181
6182 return 0;
6183}
6184
6185static int e1000_idle(struct device *dev)
6186{
6187 struct pci_dev *pdev = to_pci_dev(dev);
6188 struct net_device *netdev = pci_get_drvdata(pdev);
6189 struct e1000_adapter *adapter = netdev_priv(netdev);
6190
6191 if (!e1000e_pm_ready(adapter))
6192 return 0;
6193
6194 if (adapter->idle_check) {
6195 adapter->idle_check = false;
6196 if (!e1000e_has_link(adapter))
6197 pm_schedule_suspend(dev, MSEC_PER_SEC);
6198 }
6199
6200 return -EBUSY;
6201}
23606cf5
RW		 6202
6203static int e1000_runtime_resume(struct device *dev)
6204{
6205 struct pci_dev *pdev = to_pci_dev(dev);
6206 struct net_device *netdev = pci_get_drvdata(pdev);
6207 struct e1000_adapter *adapter = netdev_priv(netdev);
6208
6209 if (!e1000e_pm_ready(adapter))
6210 return 0;
6211
6212 adapter->idle_check = !dev->power.runtime_auto;
6213 return __e1000_resume(pdev);
6214}
a0340162 6215#endif /* CONFIG_PM_RUNTIME */
aa338601 6216#endif /* CONFIG_PM */
bc7f75fa
AK		 6217
6218static void e1000_shutdown(struct pci_dev *pdev)
6219{
4f9de721
RW		 6220 bool wake = false;
6221
23606cf5 6222 __e1000_shutdown(pdev, &wake, false);
4f9de721
RW		 6223
6224 if (system_state == SYSTEM_POWER_OFF)
6225 e1000_complete_shutdown(pdev, false, wake);
bc7f75fa
AK		 6226}
6227
6228#ifdef CONFIG_NET_POLL_CONTROLLER
147b2c8c 6229
8bb62869 6230static irqreturn_t e1000_intr_msix(int __always_unused irq, void *data)
147b2c8c
DD		 6231{
6232 struct net_device *netdev = data;
6233 struct e1000_adapter *adapter = netdev_priv(netdev);
147b2c8c
DD		 6234
6235 if (adapter->msix_entries) {
90da0669
BA		 6236 int vector, msix_irq;
6237
147b2c8c
DD		 6238 vector = 0;
6239 msix_irq = adapter->msix_entries[vector].vector;
6240 disable_irq(msix_irq);
6241 e1000_intr_msix_rx(msix_irq, netdev);
6242 enable_irq(msix_irq);
6243
6244 vector++;
6245 msix_irq = adapter->msix_entries[vector].vector;
6246 disable_irq(msix_irq);
6247 e1000_intr_msix_tx(msix_irq, netdev);
6248 enable_irq(msix_irq);
6249
6250 vector++;
6251 msix_irq = adapter->msix_entries[vector].vector;
6252 disable_irq(msix_irq);
6253 e1000_msix_other(msix_irq, netdev);
6254 enable_irq(msix_irq);
6255 }
6256
6257 return IRQ_HANDLED;
6258}
6259
e921eb1a
BA		 6260/**
6261 * e1000_netpoll
6262 * @netdev: network interface device structure
6263 *
bc7f75fa
AK		 6264 * Polling 'interrupt' - used by things like netconsole to send skbs
6265 * without having to re-enable interrupts. It's not called while
6266 * the interrupt routine is executing.
6267 */
6268static void e1000_netpoll(struct net_device *netdev)
6269{
6270 struct e1000_adapter *adapter = netdev_priv(netdev);
6271
147b2c8c
DD		 6272 switch (adapter->int_mode) {
6273 case E1000E_INT_MODE_MSIX:
6274 e1000_intr_msix(adapter->pdev->irq, netdev);
6275 break;
6276 case E1000E_INT_MODE_MSI:
6277 disable_irq(adapter->pdev->irq);
6278 e1000_intr_msi(adapter->pdev->irq, netdev);
6279 enable_irq(adapter->pdev->irq);
6280 break;
6281 default: /* E1000E_INT_MODE_LEGACY */
6282 disable_irq(adapter->pdev->irq);
6283 e1000_intr(adapter->pdev->irq, netdev);
6284 enable_irq(adapter->pdev->irq);
6285 break;
6286 }
bc7f75fa
AK		 6287}
6288#endif
6289
6290/**
6291 * e1000_io_error_detected - called when PCI error is detected
6292 * @pdev: Pointer to PCI device
6293 * @state: The current pci connection state
6294 *
6295 * This function is called after a PCI bus error affecting
6296 * this device has been detected.
6297 */
6298static pci_ers_result_t e1000_io_error_detected(struct pci_dev *pdev,
6299 pci_channel_state_t state)
6300{
6301 struct net_device *netdev = pci_get_drvdata(pdev);
6302 struct e1000_adapter *adapter = netdev_priv(netdev);
6303
6304 netif_device_detach(netdev);
6305
c93b5a76
MM		 6306 if (state == pci_channel_io_perm_failure)
6307 return PCI_ERS_RESULT_DISCONNECT;
6308
bc7f75fa
AK		 6309 if (netif_running(netdev))
6310 e1000e_down(adapter);
6311 pci_disable_device(pdev);
6312
6313 /* Request a slot slot reset. */
6314 return PCI_ERS_RESULT_NEED_RESET;
6315}
6316
6317/**
6318 * e1000_io_slot_reset - called after the pci bus has been reset.
6319 * @pdev: Pointer to PCI device
6320 *
6321 * Restart the card from scratch, as if from a cold-boot. Implementation
6322 * resembles the first-half of the e1000_resume routine.
6323 */
6324static pci_ers_result_t e1000_io_slot_reset(struct pci_dev *pdev)
6325{
6326 struct net_device *netdev = pci_get_drvdata(pdev);
6327 struct e1000_adapter *adapter = netdev_priv(netdev);
6328 struct e1000_hw *hw = &adapter->hw;
78cd29d5 6329 u16 aspm_disable_flag = 0;
6e4f6f6b 6330 int err;
111b9dc5 6331 pci_ers_result_t result;
bc7f75fa 6332
78cd29d5
BA		 6333 if (adapter->flags2 & FLAG2_DISABLE_ASPM_L0S)
6334 aspm_disable_flag = PCIE_LINK_STATE_L0S;
6f461f6c 6335 if (adapter->flags2 & FLAG2_DISABLE_ASPM_L1)
78cd29d5
BA		 6336 aspm_disable_flag |= PCIE_LINK_STATE_L1;
6337 if (aspm_disable_flag)
6338 e1000e_disable_aspm(pdev, aspm_disable_flag);
6339
f0f422e5 6340 err = pci_enable_device_mem(pdev);
6e4f6f6b 6341 if (err) {
bc7f75fa
AK		 6342 dev_err(&pdev->dev,
6343 "Cannot re-enable PCI device after reset.\n");
111b9dc5
JB		 6344 result = PCI_ERS_RESULT_DISCONNECT;
6345 } else {
6346 pci_set_master(pdev);
23606cf5 6347 pdev->state_saved = true;
111b9dc5 6348 pci_restore_state(pdev);
bc7f75fa 6349
111b9dc5
JB		 6350 pci_enable_wake(pdev, PCI_D3hot, 0);
6351 pci_enable_wake(pdev, PCI_D3cold, 0);
bc7f75fa 6352
111b9dc5
JB		 6353 e1000e_reset(adapter);
6354 ew32(WUS, ~0);
6355 result = PCI_ERS_RESULT_RECOVERED;
6356 }
bc7f75fa 6357
111b9dc5
JB		 6358 pci_cleanup_aer_uncorrect_error_status(pdev);
6359
6360 return result;
bc7f75fa
AK		 6361}
6362
6363/**
6364 * e1000_io_resume - called when traffic can start flowing again.
6365 * @pdev: Pointer to PCI device
6366 *
6367 * This callback is called when the error recovery driver tells us that
6368 * its OK to resume normal operation. Implementation resembles the
6369 * second-half of the e1000_resume routine.
6370 */
6371static void e1000_io_resume(struct pci_dev *pdev)
6372{
6373 struct net_device *netdev = pci_get_drvdata(pdev);
6374 struct e1000_adapter *adapter = netdev_priv(netdev);
6375
cd791618 6376 e1000_init_manageability_pt(adapter);
bc7f75fa
AK		 6377
6378 if (netif_running(netdev)) {
6379 if (e1000e_up(adapter)) {
6380 dev_err(&pdev->dev,
6381 "can't bring device back up after reset\n");
6382 return;
6383 }
6384 }
6385
6386 netif_device_attach(netdev);
6387
e921eb1a 6388 /* If the controller has AMT, do not set DRV_LOAD until the interface
bc7f75fa 6389 * is up. For all other cases, let the f/w know that the h/w is now
ad68076e
BA		 6390 * under the control of the driver.
6391 */
c43bc57e 6392 if (!(adapter->flags & FLAG_HAS_AMT))
31dbe5b4 6393 e1000e_get_hw_control(adapter);
bc7f75fa
AK		 6394}
6395
6396static void e1000_print_device_info(struct e1000_adapter *adapter)
6397{
6398 struct e1000_hw *hw = &adapter->hw;
6399 struct net_device *netdev = adapter->netdev;
073287c0
BA		 6400 u32 ret_val;
6401 u8 pba_str[E1000_PBANUM_LENGTH];
bc7f75fa
AK		 6402
6403 /* print bus type/speed/width info */
a5cc7642 6404 e_info("(PCI Express:2.5GT/s:%s) %pM\n",
44defeb3
JK		 6405 /* bus width */
6406 ((hw->bus.width == e1000_bus_width_pcie_x4) ? "Width x4" :
f0ff4398 6407 "Width x1"),
44defeb3 6408 /* MAC address */
7c510e4b 6409 netdev->dev_addr);
44defeb3
JK		 6410 e_info("Intel(R) PRO/%s Network Connection\n",
6411 (hw->phy.type == e1000_phy_ife) ? "10/100" : "1000");
073287c0
BA		 6412 ret_val = e1000_read_pba_string_generic(hw, pba_str,
6413 E1000_PBANUM_LENGTH);
6414 if (ret_val)
f2315bf1 6415 strlcpy((char *)pba_str, "Unknown", sizeof(pba_str));
073287c0
BA		 6416 e_info("MAC: %d, PHY: %d, PBA No: %s\n",
6417 hw->mac.type, hw->phy.type, pba_str);
bc7f75fa
AK		 6418}
6419
10aa4c04
AK		 6420static void e1000_eeprom_checks(struct e1000_adapter *adapter)
6421{
6422 struct e1000_hw *hw = &adapter->hw;
6423 int ret_val;
6424 u16 buf = 0;
6425
6426 if (hw->mac.type != e1000_82573)
6427 return;
6428
6429 ret_val = e1000_read_nvm(hw, NVM_INIT_CONTROL2_REG, 1, &buf);
e885d762
BA		 6430 le16_to_cpus(&buf);
6431 if (!ret_val && (!(buf & (1 << 0)))) {
10aa4c04 6432 /* Deep Smart Power Down (DSPD) */
6c2a9efa
FP		 6433 dev_warn(&adapter->pdev->dev,
6434 "Warning: detected DSPD enabled in EEPROM\n");
10aa4c04 6435 }
10aa4c04
AK		 6436}
6437
c8f44aff 6438static int e1000_set_features(struct net_device *netdev,
70495a50 6439 netdev_features_t features)
dc221294
BA		 6440{
6441 struct e1000_adapter *adapter = netdev_priv(netdev);
c8f44aff 6442 netdev_features_t changed = features ^ netdev->features;
dc221294
BA		 6443
6444 if (changed & (NETIF_F_TSO | NETIF_F_TSO6))
6445 adapter->flags |= FLAG_TSO_FORCE;
6446
6447 if (!(changed & (NETIF_F_HW_VLAN_RX | NETIF_F_HW_VLAN_TX |
cf955e6c
BG		 6448 NETIF_F_RXCSUM | NETIF_F_RXHASH | NETIF_F_RXFCS |
6449 NETIF_F_RXALL)))
dc221294
BA		 6450 return 0;
6451
0184039a
BG		 6452 if (changed & NETIF_F_RXFCS) {
6453 if (features & NETIF_F_RXFCS) {
6454 adapter->flags2 &= ~FLAG2_CRC_STRIPPING;
6455 } else {
6456 /* We need to take it back to defaults, which might mean
6457 * stripping is still disabled at the adapter level.
6458 */
6459 if (adapter->flags2 & FLAG2_DFLT_CRC_STRIPPING)
6460 adapter->flags2 |= FLAG2_CRC_STRIPPING;
6461 else
6462 adapter->flags2 &= ~FLAG2_CRC_STRIPPING;
6463 }
6464 }
6465
70495a50
BA		 6466 netdev->features = features;
6467
dc221294
BA		 6468 if (netif_running(netdev))
6469 e1000e_reinit_locked(adapter);
6470 else
6471 e1000e_reset(adapter);
6472
6473 return 0;
6474}
6475
651c2466
SH		 6476static const struct net_device_ops e1000e_netdev_ops = {
6477 .ndo_open = e1000_open,
6478 .ndo_stop = e1000_close,
00829823 6479 .ndo_start_xmit = e1000_xmit_frame,
67fd4fcb 6480 .ndo_get_stats64 = e1000e_get_stats64,
ef9b965a 6481 .ndo_set_rx_mode = e1000e_set_rx_mode,
651c2466
SH		 6482 .ndo_set_mac_address = e1000_set_mac,
6483 .ndo_change_mtu = e1000_change_mtu,
6484 .ndo_do_ioctl = e1000_ioctl,
6485 .ndo_tx_timeout = e1000_tx_timeout,
6486 .ndo_validate_addr = eth_validate_addr,
6487
651c2466
SH		 6488 .ndo_vlan_rx_add_vid = e1000_vlan_rx_add_vid,
6489 .ndo_vlan_rx_kill_vid = e1000_vlan_rx_kill_vid,
6490#ifdef CONFIG_NET_POLL_CONTROLLER
6491 .ndo_poll_controller = e1000_netpoll,
6492#endif
dc221294 6493 .ndo_set_features = e1000_set_features,
651c2466
SH		 6494};
6495
bc7f75fa
AK		 6496/**
6497 * e1000_probe - Device Initialization Routine
6498 * @pdev: PCI device information struct
6499 * @ent: entry in e1000_pci_tbl
6500 *
6501 * Returns 0 on success, negative on failure
6502 *
6503 * e1000_probe initializes an adapter identified by a pci_dev structure.
6504 * The OS initialization, configuring of the adapter private structure,
6505 * and a hardware reset occur.
6506 **/
1dd06ae8 6507static int e1000_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
bc7f75fa
AK		 6508{
6509 struct net_device *netdev;
6510 struct e1000_adapter *adapter;
6511 struct e1000_hw *hw;
6512 const struct e1000_info *ei = e1000_info_tbl[ent->driver_data];
f47e81fc
BB		 6513 resource_size_t mmio_start, mmio_len;
6514 resource_size_t flash_start, flash_len;
bc7f75fa 6515 static int cards_found;
78cd29d5 6516 u16 aspm_disable_flag = 0;
bc7f75fa
AK		 6517 int i, err, pci_using_dac;
6518 u16 eeprom_data = 0;
6519 u16 eeprom_apme_mask = E1000_EEPROM_APME;
6520
78cd29d5
BA		 6521 if (ei->flags2 & FLAG2_DISABLE_ASPM_L0S)
6522 aspm_disable_flag = PCIE_LINK_STATE_L0S;
6f461f6c 6523 if (ei->flags2 & FLAG2_DISABLE_ASPM_L1)
78cd29d5
BA		 6524 aspm_disable_flag |= PCIE_LINK_STATE_L1;
6525 if (aspm_disable_flag)
6526 e1000e_disable_aspm(pdev, aspm_disable_flag);
6e4f6f6b 6527
f0f422e5 6528 err = pci_enable_device_mem(pdev);
bc7f75fa
AK		 6529 if (err)
6530 return err;
6531
6532 pci_using_dac = 0;
0be3f55f 6533 err = dma_set_mask(&pdev->dev, DMA_BIT_MASK(64));
bc7f75fa 6534 if (!err) {
0be3f55f 6535 err = dma_set_coherent_mask(&pdev->dev, DMA_BIT_MASK(64));
bc7f75fa
AK		 6536 if (!err)
6537 pci_using_dac = 1;
6538 } else {
0be3f55f 6539 err = dma_set_mask(&pdev->dev, DMA_BIT_MASK(32));
bc7f75fa 6540 if (err) {
0be3f55f
NN		 6541 err = dma_set_coherent_mask(&pdev->dev,
6542 DMA_BIT_MASK(32));
bc7f75fa 6543 if (err) {
f0ff4398
BA		 6544 dev_err(&pdev->dev,
6545 "No usable DMA configuration, aborting\n");
bc7f75fa
AK		 6546 goto err_dma;
6547 }
6548 }
6549 }
6550
e8de1481 6551 err = pci_request_selected_regions_exclusive(pdev,
55c5f55e
BA		 6552 pci_select_bars(pdev, IORESOURCE_MEM),
6553 e1000e_driver_name);
bc7f75fa
AK		 6554 if (err)
6555 goto err_pci_reg;
6556
68eac460 6557 /* AER (Advanced Error Reporting) hooks */
19d5afd4 6558 pci_enable_pcie_error_reporting(pdev);
68eac460 6559
bc7f75fa 6560 pci_set_master(pdev);
438b365a
BA		 6561 /* PCI config space info */
6562 err = pci_save_state(pdev);
6563 if (err)
6564 goto err_alloc_etherdev;
bc7f75fa
AK		 6565
6566 err = -ENOMEM;
6567 netdev = alloc_etherdev(sizeof(struct e1000_adapter));
6568 if (!netdev)
6569 goto err_alloc_etherdev;
6570
bc7f75fa
AK		 6571 SET_NETDEV_DEV(netdev, &pdev->dev);
6572
f85e4dfa
TH		 6573 netdev->irq = pdev->irq;
6574
bc7f75fa
AK		 6575 pci_set_drvdata(pdev, netdev);
6576 adapter = netdev_priv(netdev);
6577 hw = &adapter->hw;
6578 adapter->netdev = netdev;
6579 adapter->pdev = pdev;
6580 adapter->ei = ei;
6581 adapter->pba = ei->pba;
6582 adapter->flags = ei->flags;
eb7c3adb 6583 adapter->flags2 = ei->flags2;
bc7f75fa
AK		 6584 adapter->hw.adapter = adapter;
6585 adapter->hw.mac.type = ei->mac;
2adc55c9 6586 adapter->max_hw_frame_size = ei->max_hw_frame_size;
b3f4d599 6587 adapter->msg_enable = netif_msg_init(debug, DEFAULT_MSG_ENABLE);
bc7f75fa
AK		 6588
6589 mmio_start = pci_resource_start(pdev, 0);
6590 mmio_len = pci_resource_len(pdev, 0);
6591
6592 err = -EIO;
6593 adapter->hw.hw_addr = ioremap(mmio_start, mmio_len);
6594 if (!adapter->hw.hw_addr)
6595 goto err_ioremap;
6596
6597 if ((adapter->flags & FLAG_HAS_FLASH) &&
6598 (pci_resource_flags(pdev, 1) & IORESOURCE_MEM)) {
6599 flash_start = pci_resource_start(pdev, 1);
6600 flash_len = pci_resource_len(pdev, 1);
6601 adapter->hw.flash_address = ioremap(flash_start, flash_len);
6602 if (!adapter->hw.flash_address)
6603 goto err_flashmap;
6604 }
6605
6606 /* construct the net_device struct */
651c2466 6607 netdev->netdev_ops = &e1000e_netdev_ops;
bc7f75fa 6608 e1000e_set_ethtool_ops(netdev);
bc7f75fa 6609 netdev->watchdog_timeo = 5 * HZ;
c58c8a78 6610 netif_napi_add(netdev, &adapter->napi, e1000e_poll, 64);
f2315bf1 6611 strlcpy(netdev->name, pci_name(pdev), sizeof(netdev->name));
bc7f75fa
AK		 6612
6613 netdev->mem_start = mmio_start;
6614 netdev->mem_end = mmio_start + mmio_len;
6615
6616 adapter->bd_number = cards_found++;
6617
4662e82b
BA		 6618 e1000e_check_options(adapter);
6619
bc7f75fa
AK		 6620 /* setup adapter struct */
6621 err = e1000_sw_init(adapter);
6622 if (err)
6623 goto err_sw_init;
6624
bc7f75fa
AK		 6625 memcpy(&hw->mac.ops, ei->mac_ops, sizeof(hw->mac.ops));
6626 memcpy(&hw->nvm.ops, ei->nvm_ops, sizeof(hw->nvm.ops));
6627 memcpy(&hw->phy.ops, ei->phy_ops, sizeof(hw->phy.ops));
6628
69e3fd8c 6629 err = ei->get_variants(adapter);
bc7f75fa
AK		 6630 if (err)
6631 goto err_hw_init;
6632
4a770358
BA		 6633 if ((adapter->flags & FLAG_IS_ICH) &&
6634 (adapter->flags & FLAG_READ_ONLY_NVM))
6635 e1000e_write_protect_nvm_ich8lan(&adapter->hw);
6636
bc7f75fa
AK		 6637 hw->mac.ops.get_bus_info(&adapter->hw);
6638
318a94d6 6639 adapter->hw.phy.autoneg_wait_to_complete = 0;
bc7f75fa
AK		 6640
6641 /* Copper options */
318a94d6 6642 if (adapter->hw.phy.media_type == e1000_media_type_copper) {
bc7f75fa
AK		 6643 adapter->hw.phy.mdix = AUTO_ALL_MODES;
6644 adapter->hw.phy.disable_polarity_correction = 0;
6645 adapter->hw.phy.ms_type = e1000_ms_hw_default;
6646 }
6647
470a5420 6648 if (hw->phy.ops.check_reset_block && hw->phy.ops.check_reset_block(hw))
185095fb
BA		 6649 dev_info(&pdev->dev,
6650 "PHY reset is blocked due to SOL/IDER session.\n");
bc7f75fa 6651
dc221294
BA		 6652 /* Set initial default active device features */
6653 netdev->features = (NETIF_F_SG |
6654 NETIF_F_HW_VLAN_RX |
6655 NETIF_F_HW_VLAN_TX |
6656 NETIF_F_TSO |
6657 NETIF_F_TSO6 |
70495a50 6658 NETIF_F_RXHASH |
dc221294
BA		 6659 NETIF_F_RXCSUM |
6660 NETIF_F_HW_CSUM);
6661
6662 /* Set user-changeable features (subset of all device features) */
6663 netdev->hw_features = netdev->features;
0184039a 6664 netdev->hw_features |= NETIF_F_RXFCS;
943146de 6665 netdev->priv_flags |= IFF_SUPP_NOFCS;
cf955e6c 6666 netdev->hw_features |= NETIF_F_RXALL;
bc7f75fa
AK		 6667
6668 if (adapter->flags & FLAG_HAS_HW_VLAN_FILTER)
6669 netdev->features |= NETIF_F_HW_VLAN_FILTER;
6670
dc221294
BA		 6671 netdev->vlan_features |= (NETIF_F_SG |
6672 NETIF_F_TSO |
6673 NETIF_F_TSO6 |
6674 NETIF_F_HW_CSUM);
a5136e23 6675
ef9b965a
JB		 6676 netdev->priv_flags |= IFF_UNICAST_FLT;
6677
7b872a55 6678 if (pci_using_dac) {
bc7f75fa 6679 netdev->features |= NETIF_F_HIGHDMA;
7b872a55
YZ		 6680 netdev->vlan_features |= NETIF_F_HIGHDMA;
6681 }
bc7f75fa 6682
bc7f75fa
AK		 6683 if (e1000e_enable_mng_pass_thru(&adapter->hw))
6684 adapter->flags |= FLAG_MNG_PT_ENABLED;
6685
e921eb1a 6686 /* before reading the NVM, reset the controller to
ad68076e
BA		 6687 * put the device in a known good starting state
6688 */
bc7f75fa
AK		 6689 adapter->hw.mac.ops.reset_hw(&adapter->hw);
6690
e921eb1a 6691 /* systems with ASPM and others may see the checksum fail on the first
bc7f75fa
AK		 6692 * attempt. Let's give it a few tries
6693 */
6694 for (i = 0;; i++) {
6695 if (e1000_validate_nvm_checksum(&adapter->hw) >= 0)
6696 break;
6697 if (i == 2) {
185095fb 6698 dev_err(&pdev->dev, "The NVM Checksum Is Not Valid\n");
bc7f75fa
AK		 6699 err = -EIO;
6700 goto err_eeprom;
6701 }
6702 }
6703
10aa4c04
AK		 6704 e1000_eeprom_checks(adapter);
6705
608f8a0d 6706 /* copy the MAC address */
bc7f75fa 6707 if (e1000e_read_mac_addr(&adapter->hw))
185095fb
BA		 6708 dev_err(&pdev->dev,
6709 "NVM Read Error while reading MAC address\n");
bc7f75fa
AK		 6710
6711 memcpy(netdev->dev_addr, adapter->hw.mac.addr, netdev->addr_len);
bc7f75fa 6712
aaeb6cdf 6713 if (!is_valid_ether_addr(netdev->dev_addr)) {
185095fb 6714 dev_err(&pdev->dev, "Invalid MAC Address: %pM\n",
aaeb6cdf 6715 netdev->dev_addr);
bc7f75fa
AK		 6716 err = -EIO;
6717 goto err_eeprom;
6718 }
6719
6720 init_timer(&adapter->watchdog_timer);
c061b18d 6721 adapter->watchdog_timer.function = e1000_watchdog;
bc7f75fa
AK		 6722 adapter->watchdog_timer.data = (unsigned long) adapter;
6723
6724 init_timer(&adapter->phy_info_timer);
c061b18d 6725 adapter->phy_info_timer.function = e1000_update_phy_info;
bc7f75fa
AK		 6726 adapter->phy_info_timer.data = (unsigned long) adapter;
6727
6728 INIT_WORK(&adapter->reset_task, e1000_reset_task);
6729 INIT_WORK(&adapter->watchdog_task, e1000_watchdog_task);
a8f88ff5
JB		 6730 INIT_WORK(&adapter->downshift_task, e1000e_downshift_workaround);
6731 INIT_WORK(&adapter->update_phy_task, e1000e_update_phy_task);
41cec6f1 6732 INIT_WORK(&adapter->print_hang_task, e1000_print_hw_hang);
bc7f75fa 6733
bc7f75fa
AK		 6734 /* Initialize link parameters. User can change them with ethtool */
6735 adapter->hw.mac.autoneg = 1;
3db1cd5c 6736 adapter->fc_autoneg = true;
5c48ef3e
BA		 6737 adapter->hw.fc.requested_mode = e1000_fc_default;
6738 adapter->hw.fc.current_mode = e1000_fc_default;
bc7f75fa
AK		 6739 adapter->hw.phy.autoneg_advertised = 0x2f;
6740
6741 /* ring size defaults */
d821a4c4
BA		 6742 adapter->rx_ring->count = E1000_DEFAULT_RXD;
6743 adapter->tx_ring->count = E1000_DEFAULT_TXD;
bc7f75fa 6744
e921eb1a 6745 /* Initial Wake on LAN setting - If APM wake is enabled in
bc7f75fa
AK		 6746 * the EEPROM, enable the ACPI Magic Packet filter
6747 */
6748 if (adapter->flags & FLAG_APME_IN_WUC) {
6749 /* APME bit in EEPROM is mapped to WUC.APME */
6750 eeprom_data = er32(WUC);
6751 eeprom_apme_mask = E1000_WUC_APME;
4def99bb
BA		 6752 if ((hw->mac.type > e1000_ich10lan) &&
6753 (eeprom_data & E1000_WUC_PHY_WAKE))
a4f58f54 6754 adapter->flags2 |= FLAG2_HAS_PHY_WAKEUP;
bc7f75fa
AK		 6755 } else if (adapter->flags & FLAG_APME_IN_CTRL3) {
6756 if (adapter->flags & FLAG_APME_CHECK_PORT_B &&
6757 (adapter->hw.bus.func == 1))
3d3a1676
BA		 6758 e1000_read_nvm(&adapter->hw, NVM_INIT_CONTROL3_PORT_B,
6759 1, &eeprom_data);
bc7f75fa 6760 else
3d3a1676
BA		 6761 e1000_read_nvm(&adapter->hw, NVM_INIT_CONTROL3_PORT_A,
6762 1, &eeprom_data);
bc7f75fa
AK		 6763 }
6764
6765 /* fetch WoL from EEPROM */
6766 if (eeprom_data & eeprom_apme_mask)
6767 adapter->eeprom_wol |= E1000_WUFC_MAG;
6768
e921eb1a 6769 /* now that we have the eeprom settings, apply the special cases
bc7f75fa
AK		 6770 * where the eeprom may be wrong or the board simply won't support
6771 * wake on lan on a particular port
6772 */
6773 if (!(adapter->flags & FLAG_HAS_WOL))
6774 adapter->eeprom_wol = 0;
6775
6776 /* initialize the wol settings based on the eeprom settings */
6777 adapter->wol = adapter->eeprom_wol;
6ff68026 6778 device_set_wakeup_enable(&adapter->pdev->dev, adapter->wol);
bc7f75fa 6779
84527590
BA		 6780 /* save off EEPROM version number */
6781 e1000_read_nvm(&adapter->hw, 5, 1, &adapter->eeprom_vers);
6782
bc7f75fa
AK		 6783 /* reset the hardware with the new settings */
6784 e1000e_reset(adapter);
6785
e921eb1a 6786 /* If the controller has AMT, do not set DRV_LOAD until the interface
bc7f75fa 6787 * is up. For all other cases, let the f/w know that the h/w is now
ad68076e
BA		 6788 * under the control of the driver.
6789 */
c43bc57e 6790 if (!(adapter->flags & FLAG_HAS_AMT))
31dbe5b4 6791 e1000e_get_hw_control(adapter);
bc7f75fa 6792
f2315bf1 6793 strlcpy(netdev->name, "eth%d", sizeof(netdev->name));
bc7f75fa
AK		 6794 err = register_netdev(netdev);
6795 if (err)
6796 goto err_register;
6797
9c563d20
JB		 6798 /* carrier off reporting is important to ethtool even BEFORE open */
6799 netif_carrier_off(netdev);
6800
d89777bf
BA		 6801 /* init PTP hardware clock */
6802 e1000e_ptp_init(adapter);
6803
bc7f75fa
AK		 6804 e1000_print_device_info(adapter);
6805
f3ec4f87
AS		 6806 if (pci_dev_run_wake(pdev))
6807 pm_runtime_put_noidle(&pdev->dev);
23606cf5 6808
bc7f75fa
AK		 6809 return 0;
6810
6811err_register:
c43bc57e 6812 if (!(adapter->flags & FLAG_HAS_AMT))
31dbe5b4 6813 e1000e_release_hw_control(adapter);
bc7f75fa 6814err_eeprom:
470a5420 6815 if (hw->phy.ops.check_reset_block && !hw->phy.ops.check_reset_block(hw))
bc7f75fa 6816 e1000_phy_hw_reset(&adapter->hw);
c43bc57e 6817err_hw_init:
bc7f75fa
AK		 6818 kfree(adapter->tx_ring);
6819 kfree(adapter->rx_ring);
6820err_sw_init:
c43bc57e
JB		 6821 if (adapter->hw.flash_address)
6822 iounmap(adapter->hw.flash_address);
e82f54ba 6823 e1000e_reset_interrupt_capability(adapter);
c43bc57e 6824err_flashmap:
bc7f75fa
AK		 6825 iounmap(adapter->hw.hw_addr);
6826err_ioremap:
6827 free_netdev(netdev);
6828err_alloc_etherdev:
f0f422e5 6829 pci_release_selected_regions(pdev,
f0ff4398 6830 pci_select_bars(pdev, IORESOURCE_MEM));
bc7f75fa
AK		 6831err_pci_reg:
6832err_dma:
6833 pci_disable_device(pdev);
6834 return err;
6835}
6836
6837/**
6838 * e1000_remove - Device Removal Routine
6839 * @pdev: PCI device information struct
6840 *
6841 * e1000_remove is called by the PCI subsystem to alert the driver
6842 * that it should release a PCI device. The could be caused by a
6843 * Hot-Plug event, or because the driver is going to be removed from
6844 * memory.
6845 **/
9f9a12f8 6846static void e1000_remove(struct pci_dev *pdev)
bc7f75fa
AK		 6847{
6848 struct net_device *netdev = pci_get_drvdata(pdev);
6849 struct e1000_adapter *adapter = netdev_priv(netdev);
23606cf5
RW		 6850 bool down = test_bit(__E1000_DOWN, &adapter->state);
6851
d89777bf
BA		 6852 e1000e_ptp_remove(adapter);
6853
e921eb1a 6854 /* The timers may be rescheduled, so explicitly disable them
23f333a2 6855 * from being rescheduled.
ad68076e 6856 */
23606cf5
RW		 6857 if (!down)
6858 set_bit(__E1000_DOWN, &adapter->state);
bc7f75fa
AK		 6859 del_timer_sync(&adapter->watchdog_timer);
6860 del_timer_sync(&adapter->phy_info_timer);
6861
41cec6f1
BA		 6862 cancel_work_sync(&adapter->reset_task);
6863 cancel_work_sync(&adapter->watchdog_task);
6864 cancel_work_sync(&adapter->downshift_task);
6865 cancel_work_sync(&adapter->update_phy_task);
6866 cancel_work_sync(&adapter->print_hang_task);
bc7f75fa 6867
b67e1913
BA		 6868 if (adapter->flags & FLAG_HAS_HW_TIMESTAMP) {
6869 cancel_work_sync(&adapter->tx_hwtstamp_work);
6870 if (adapter->tx_hwtstamp_skb) {
6871 dev_kfree_skb_any(adapter->tx_hwtstamp_skb);
6872 adapter->tx_hwtstamp_skb = NULL;
6873 }
6874 }
6875
17f208de
BA		 6876 if (!(netdev->flags & IFF_UP))
6877 e1000_power_down_phy(adapter);
6878
23606cf5
RW		 6879 /* Don't lie to e1000_close() down the road. */
6880 if (!down)
6881 clear_bit(__E1000_DOWN, &adapter->state);
17f208de
BA		 6882 unregister_netdev(netdev);
6883
f3ec4f87
AS		 6884 if (pci_dev_run_wake(pdev))
6885 pm_runtime_get_noresume(&pdev->dev);
23606cf5 6886
e921eb1a 6887 /* Release control of h/w to f/w. If f/w is AMT enabled, this
ad68076e
BA		 6888 * would have already happened in close and is redundant.
6889 */
31dbe5b4 6890 e1000e_release_hw_control(adapter);
bc7f75fa 6891
4662e82b 6892 e1000e_reset_interrupt_capability(adapter);
bc7f75fa
AK		 6893 kfree(adapter->tx_ring);
6894 kfree(adapter->rx_ring);
6895
6896 iounmap(adapter->hw.hw_addr);
6897 if (adapter->hw.flash_address)
6898 iounmap(adapter->hw.flash_address);
f0f422e5 6899 pci_release_selected_regions(pdev,
f0ff4398 6900 pci_select_bars(pdev, IORESOURCE_MEM));
bc7f75fa
AK		 6901
6902 free_netdev(netdev);
6903
111b9dc5 6904 /* AER disable */
19d5afd4 6905 pci_disable_pcie_error_reporting(pdev);
111b9dc5 6906
bc7f75fa
AK		 6907 pci_disable_device(pdev);
6908}
6909
6910/* PCI Error Recovery (ERS) */
3646f0e5 6911static const struct pci_error_handlers e1000_err_handler = {
bc7f75fa
AK		 6912 .error_detected = e1000_io_error_detected,
6913 .slot_reset = e1000_io_slot_reset,
6914 .resume = e1000_io_resume,
6915};
6916
a3aa1884 6917static DEFINE_PCI_DEVICE_TABLE(e1000_pci_tbl) = {
bc7f75fa
AK		 6918 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82571EB_COPPER), board_82571 },
6919 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82571EB_FIBER), board_82571 },
6920 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82571EB_QUAD_COPPER), board_82571 },
6921 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82571EB_QUAD_COPPER_LP), board_82571 },
6922 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82571EB_QUAD_FIBER), board_82571 },
6923 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82571EB_SERDES), board_82571 },
040babf9
AK		 6924 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82571EB_SERDES_DUAL), board_82571 },
6925 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82571EB_SERDES_QUAD), board_82571 },
6926 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82571PT_QUAD_COPPER), board_82571 },
ad68076e 6927
bc7f75fa
AK		 6928 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82572EI), board_82572 },
6929 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82572EI_COPPER), board_82572 },
6930 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82572EI_FIBER), board_82572 },
6931 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82572EI_SERDES), board_82572 },
ad68076e 6932
bc7f75fa
AK		 6933 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82573E), board_82573 },
6934 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82573E_IAMT), board_82573 },
6935 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82573L), board_82573 },
ad68076e 6936
4662e82b 6937 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82574L), board_82574 },
bef28b11 6938 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82574LA), board_82574 },
8c81c9c3 6939 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82583V), board_82583 },
4662e82b 6940
bc7f75fa
AK		 6941 { PCI_VDEVICE(INTEL, E1000_DEV_ID_80003ES2LAN_COPPER_DPT),
6942 board_80003es2lan },
6943 { PCI_VDEVICE(INTEL, E1000_DEV_ID_80003ES2LAN_COPPER_SPT),
6944 board_80003es2lan },
6945 { PCI_VDEVICE(INTEL, E1000_DEV_ID_80003ES2LAN_SERDES_DPT),
6946 board_80003es2lan },
6947 { PCI_VDEVICE(INTEL, E1000_DEV_ID_80003ES2LAN_SERDES_SPT),
6948 board_80003es2lan },
ad68076e 6949
bc7f75fa
AK		 6950 { PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH8_IFE), board_ich8lan },
6951 { PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH8_IFE_G), board_ich8lan },
6952 { PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH8_IFE_GT), board_ich8lan },
6953 { PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH8_IGP_AMT), board_ich8lan },
6954 { PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH8_IGP_C), board_ich8lan },
6955 { PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH8_IGP_M), board_ich8lan },
6956 { PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH8_IGP_M_AMT), board_ich8lan },
9e135a2e 6957 { PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH8_82567V_3), board_ich8lan },
ad68076e 6958
bc7f75fa
AK		 6959 { PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH9_IFE), board_ich9lan },
6960 { PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH9_IFE_G), board_ich9lan },
6961 { PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH9_IFE_GT), board_ich9lan },
6962 { PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH9_IGP_AMT), board_ich9lan },
6963 { PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH9_IGP_C), board_ich9lan },
2f15f9d6 6964 { PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH9_BM), board_ich9lan },
97ac8cae
BA		 6965 { PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH9_IGP_M), board_ich9lan },
6966 { PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH9_IGP_M_AMT), board_ich9lan },
6967 { PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH9_IGP_M_V), board_ich9lan },
6968
6969 { PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH10_R_BM_LM), board_ich9lan },
6970 { PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH10_R_BM_LF), board_ich9lan },
6971 { PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH10_R_BM_V), board_ich9lan },
bc7f75fa 6972
f4187b56
BA		 6973 { PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH10_D_BM_LM), board_ich10lan },
6974 { PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH10_D_BM_LF), board_ich10lan },
10df0b91 6975 { PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH10_D_BM_V), board_ich10lan },
f4187b56 6976
a4f58f54
BA		 6977 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_M_HV_LM), board_pchlan },
6978 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_M_HV_LC), board_pchlan },
6979 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_D_HV_DM), board_pchlan },
6980 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_D_HV_DC), board_pchlan },
6981
d3738bb8
BA		 6982 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH2_LV_LM), board_pch2lan },
6983 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH2_LV_V), board_pch2lan },
6984
2fbe4526
BA		 6985 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_LPT_I217_LM), board_pch_lpt },
6986 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_LPT_I217_V), board_pch_lpt },
16e310ae
BA		 6987 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_LPTLP_I218_LM), board_pch_lpt },
6988 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_LPTLP_I218_V), board_pch_lpt },
2fbe4526 6989
f36bb6ca 6990 { 0, 0, 0, 0, 0, 0, 0 } /* terminate list */
bc7f75fa
AK		 6991};
6992MODULE_DEVICE_TABLE(pci, e1000_pci_tbl);
6993
aa338601 6994#ifdef CONFIG_PM
23606cf5 6995static const struct dev_pm_ops e1000_pm_ops = {
a0340162
RW		 6996 SET_SYSTEM_SLEEP_PM_OPS(e1000_suspend, e1000_resume)
6997 SET_RUNTIME_PM_OPS(e1000_runtime_suspend,
6998 e1000_runtime_resume, e1000_idle)
23606cf5 6999};
e50208a0 7000#endif
23606cf5 7001
bc7f75fa
AK		 7002/* PCI Device API Driver */
7003static struct pci_driver e1000_driver = {
7004 .name = e1000e_driver_name,
7005 .id_table = e1000_pci_tbl,
7006 .probe = e1000_probe,
9f9a12f8 7007 .remove = e1000_remove,
aa338601 7008#ifdef CONFIG_PM
f36bb6ca
BA		 7009 .driver = {
7010 .pm = &e1000_pm_ops,
7011 },
bc7f75fa
AK		 7012#endif
7013 .shutdown = e1000_shutdown,
7014 .err_handler = &e1000_err_handler
7015};
7016
7017/**
7018 * e1000_init_module - Driver Registration Routine
7019 *
7020 * e1000_init_module is the first routine called when the driver is
7021 * loaded. All it does is register with the PCI subsystem.
7022 **/
7023static int __init e1000_init_module(void)
7024{
7025 int ret;
8544b9f7
BA		 7026 pr_info("Intel(R) PRO/1000 Network Driver - %s\n",
7027 e1000e_driver_version);
bf67044b 7028 pr_info("Copyright(c) 1999 - 2013 Intel Corporation.\n");
bc7f75fa 7029 ret = pci_register_driver(&e1000_driver);
53ec5498 7030
bc7f75fa
AK		 7031 return ret;
7032}
7033module_init(e1000_init_module);
7034
7035/**
7036 * e1000_exit_module - Driver Exit Cleanup Routine
7037 *
7038 * e1000_exit_module is called just before the driver is removed
7039 * from memory.
7040 **/
7041static void __exit e1000_exit_module(void)
7042{
7043 pci_unregister_driver(&e1000_driver);
7044}
7045module_exit(e1000_exit_module);
7046
7047
7048MODULE_AUTHOR("Intel Corporation, <linux.nics@intel.com>");
7049MODULE_DESCRIPTION("Intel(R) PRO/1000 Network Driver");
7050MODULE_LICENSE("GPL");
7051MODULE_VERSION(DRV_VERSION);
7052
06c24b91 7053/* netdev.c */
Linux kernel - Deliverables
This page took 1.495608 seconds and 5 git commands to generate.