projects / deliverable / linux.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
e1000e: cleanup PARENTHESIS_ALIGNMENT checkpatch checks
[deliverable/linux.git] / drivers / net / ethernet / intel / e1000e / netdev.c
CommitLineData
bc7f75fa
AK		 1/*******************************************************************************
2
3 Intel PRO/1000 Linux driver
bf67044b 4 Copyright(c) 1999 - 2013 Intel Corporation.
bc7f75fa
AK		 5
6 This program is free software; you can redistribute it and/or modify it
7 under the terms and conditions of the GNU General Public License,
8 version 2, as published by the Free Software Foundation.
9
10 This program is distributed in the hope it will be useful, but WITHOUT
11 ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
13 more details.
14
15 You should have received a copy of the GNU General Public License along with
16 this program; if not, write to the Free Software Foundation, Inc.,
17 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
18
19 The full GNU General Public License is included in this distribution in
20 the file called "COPYING".
21
22 Contact Information:
23 Linux NICS <linux.nics@intel.com>
24 e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
25 Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
26
27*******************************************************************************/
28
8544b9f7
BA		 29#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
30
bc7f75fa
AK		 31#include <linux/module.h>
32#include <linux/types.h>
33#include <linux/init.h>
34#include <linux/pci.h>
35#include <linux/vmalloc.h>
36#include <linux/pagemap.h>
37#include <linux/delay.h>
38#include <linux/netdevice.h>
9fb7a5f7 39#include <linux/interrupt.h>
bc7f75fa
AK		 40#include <linux/tcp.h>
41#include <linux/ipv6.h>
5a0e3ad6 42#include <linux/slab.h>
bc7f75fa
AK		 43#include <net/checksum.h>
44#include <net/ip6_checksum.h>
bc7f75fa
AK		 45#include <linux/ethtool.h>
46#include <linux/if_vlan.h>
47#include <linux/cpu.h>
48#include <linux/smp.h>
e8db0be1 49#include <linux/pm_qos.h>
23606cf5 50#include <linux/pm_runtime.h>
111b9dc5 51#include <linux/aer.h>
70c71606 52#include <linux/prefetch.h>
bc7f75fa
AK		 53
54#include "e1000.h"
55
b3ccf267 56#define DRV_EXTRAVERSION "-k"
c14c643b 57
9e019901 58#define DRV_VERSION "2.2.14" DRV_EXTRAVERSION
bc7f75fa
AK		 59char e1000e_driver_name[] = "e1000e";
60const char e1000e_driver_version[] = DRV_VERSION;
61
b3f4d599 62#define DEFAULT_MSG_ENABLE (NETIF_MSG_DRV|NETIF_MSG_PROBE|NETIF_MSG_LINK)
63static int debug = -1;
64module_param(debug, int, 0);
65MODULE_PARM_DESC(debug, "Debug level (0=none,...,16=all)");
66
78cd29d5
BA		 67static void e1000e_disable_aspm(struct pci_dev *pdev, u16 state);
68
bc7f75fa
AK		 69static const struct e1000_info *e1000_info_tbl[] = {
70 [board_82571] = &e1000_82571_info,
71 [board_82572] = &e1000_82572_info,
72 [board_82573] = &e1000_82573_info,
4662e82b 73 [board_82574] = &e1000_82574_info,
8c81c9c3 74 [board_82583] = &e1000_82583_info,
bc7f75fa
AK		 75 [board_80003es2lan] = &e1000_es2_info,
76 [board_ich8lan] = &e1000_ich8_info,
77 [board_ich9lan] = &e1000_ich9_info,
f4187b56 78 [board_ich10lan] = &e1000_ich10_info,
a4f58f54 79 [board_pchlan] = &e1000_pch_info,
d3738bb8 80 [board_pch2lan] = &e1000_pch2_info,
2fbe4526 81 [board_pch_lpt] = &e1000_pch_lpt_info,
bc7f75fa
AK		 82};
83
84f4ee90
TI		 84struct e1000_reg_info {
85 u32 ofs;
86 char *name;
87};
88
84f4ee90 89static const struct e1000_reg_info e1000_reg_info_tbl[] = {
84f4ee90
TI		 90 /* General Registers */
91 {E1000_CTRL, "CTRL"},
92 {E1000_STATUS, "STATUS"},
93 {E1000_CTRL_EXT, "CTRL_EXT"},
94
95 /* Interrupt Registers */
96 {E1000_ICR, "ICR"},
97
af667a29 98 /* Rx Registers */
84f4ee90 99 {E1000_RCTL, "RCTL"},
1e36052e
BA		 100 {E1000_RDLEN(0), "RDLEN"},
101 {E1000_RDH(0), "RDH"},
102 {E1000_RDT(0), "RDT"},
84f4ee90
TI		 103 {E1000_RDTR, "RDTR"},
104 {E1000_RXDCTL(0), "RXDCTL"},
105 {E1000_ERT, "ERT"},
1e36052e
BA		 106 {E1000_RDBAL(0), "RDBAL"},
107 {E1000_RDBAH(0), "RDBAH"},
84f4ee90
TI		 108 {E1000_RDFH, "RDFH"},
109 {E1000_RDFT, "RDFT"},
110 {E1000_RDFHS, "RDFHS"},
111 {E1000_RDFTS, "RDFTS"},
112 {E1000_RDFPC, "RDFPC"},
113
af667a29 114 /* Tx Registers */
84f4ee90 115 {E1000_TCTL, "TCTL"},
1e36052e
BA		 116 {E1000_TDBAL(0), "TDBAL"},
117 {E1000_TDBAH(0), "TDBAH"},
118 {E1000_TDLEN(0), "TDLEN"},
119 {E1000_TDH(0), "TDH"},
120 {E1000_TDT(0), "TDT"},
84f4ee90
TI		 121 {E1000_TIDV, "TIDV"},
122 {E1000_TXDCTL(0), "TXDCTL"},
123 {E1000_TADV, "TADV"},
124 {E1000_TARC(0), "TARC"},
125 {E1000_TDFH, "TDFH"},
126 {E1000_TDFT, "TDFT"},
127 {E1000_TDFHS, "TDFHS"},
128 {E1000_TDFTS, "TDFTS"},
129 {E1000_TDFPC, "TDFPC"},
130
131 /* List Terminator */
f36bb6ca 132 {0, NULL}
84f4ee90
TI		 133};
134
e921eb1a 135/**
84f4ee90 136 * e1000_regdump - register printout routine
e921eb1a
BA		 137 * @hw: pointer to the HW structure
138 * @reginfo: pointer to the register info table
139 **/
84f4ee90
TI		 140static void e1000_regdump(struct e1000_hw *hw, struct e1000_reg_info *reginfo)
141{
142 int n = 0;
143 char rname[16];
144 u32 regs[8];
145
146 switch (reginfo->ofs) {
147 case E1000_RXDCTL(0):
148 for (n = 0; n < 2; n++)
149 regs[n] = __er32(hw, E1000_RXDCTL(n));
150 break;
151 case E1000_TXDCTL(0):
152 for (n = 0; n < 2; n++)
153 regs[n] = __er32(hw, E1000_TXDCTL(n));
154 break;
155 case E1000_TARC(0):
156 for (n = 0; n < 2; n++)
157 regs[n] = __er32(hw, E1000_TARC(n));
158 break;
159 default:
ef456f85
JK		 160 pr_info("%-15s %08x\n",
161 reginfo->name, __er32(hw, reginfo->ofs));
84f4ee90
TI		 162 return;
163 }
164
165 snprintf(rname, 16, "%s%s", reginfo->name, "[0-1]");
ef456f85 166 pr_info("%-15s %08x %08x\n", rname, regs[0], regs[1]);
84f4ee90
TI		 167}
168
f0c5dadf
ET		 169static void e1000e_dump_ps_pages(struct e1000_adapter *adapter,
170 struct e1000_buffer *bi)
171{
172 int i;
173 struct e1000_ps_page *ps_page;
174
175 for (i = 0; i < adapter->rx_ps_pages; i++) {
176 ps_page = &bi->ps_pages[i];
177
178 if (ps_page->page) {
179 pr_info("packet dump for ps_page %d:\n", i);
180 print_hex_dump(KERN_INFO, "", DUMP_PREFIX_ADDRESS,
181 16, 1, page_address(ps_page->page),
182 PAGE_SIZE, true);
183 }
184 }
185}
186
e921eb1a 187/**
af667a29 188 * e1000e_dump - Print registers, Tx-ring and Rx-ring
e921eb1a
BA		 189 * @adapter: board private structure
190 **/
84f4ee90
TI		 191static void e1000e_dump(struct e1000_adapter *adapter)
192{
193 struct net_device *netdev = adapter->netdev;
194 struct e1000_hw *hw = &adapter->hw;
195 struct e1000_reg_info *reginfo;
196 struct e1000_ring *tx_ring = adapter->tx_ring;
197 struct e1000_tx_desc *tx_desc;
af667a29 198 struct my_u0 {
e885d762
BA		 199 __le64 a;
200 __le64 b;
af667a29 201 } *u0;
84f4ee90
TI		 202 struct e1000_buffer *buffer_info;
203 struct e1000_ring *rx_ring = adapter->rx_ring;
204 union e1000_rx_desc_packet_split *rx_desc_ps;
5f450212 205 union e1000_rx_desc_extended *rx_desc;
af667a29 206 struct my_u1 {
e885d762
BA		 207 __le64 a;
208 __le64 b;
209 __le64 c;
210 __le64 d;
af667a29 211 } *u1;
84f4ee90
TI		 212 u32 staterr;
213 int i = 0;
214
215 if (!netif_msg_hw(adapter))
216 return;
217
218 /* Print netdevice Info */
219 if (netdev) {
220 dev_info(&adapter->pdev->dev, "Net device Info\n");
ef456f85
JK		 221 pr_info("Device Name state trans_start last_rx\n");
222 pr_info("%-15s %016lX %016lX %016lX\n",
223 netdev->name, netdev->state, netdev->trans_start,
224 netdev->last_rx);
84f4ee90
TI		 225 }
226
227 /* Print Registers */
228 dev_info(&adapter->pdev->dev, "Register Dump\n");
ef456f85 229 pr_info(" Register Name Value\n");
84f4ee90
TI		 230 for (reginfo = (struct e1000_reg_info *)e1000_reg_info_tbl;
231 reginfo->name; reginfo++) {
232 e1000_regdump(hw, reginfo);
233 }
234
af667a29 235 /* Print Tx Ring Summary */
84f4ee90 236 if (!netdev || !netif_running(netdev))
fe1e980f 237 return;
84f4ee90 238
af667a29 239 dev_info(&adapter->pdev->dev, "Tx Ring Summary\n");
ef456f85 240 pr_info("Queue [NTU] [NTC] [bi(ntc)->dma ] leng ntw timestamp\n");
84f4ee90 241 buffer_info = &tx_ring->buffer_info[tx_ring->next_to_clean];
ef456f85
JK		 242 pr_info(" %5d %5X %5X %016llX %04X %3X %016llX\n",
243 0, tx_ring->next_to_use, tx_ring->next_to_clean,
244 (unsigned long long)buffer_info->dma,
245 buffer_info->length,
246 buffer_info->next_to_watch,
247 (unsigned long long)buffer_info->time_stamp);
84f4ee90 248
af667a29 249 /* Print Tx Ring */
84f4ee90
TI		 250 if (!netif_msg_tx_done(adapter))
251 goto rx_ring_summary;
252
af667a29 253 dev_info(&adapter->pdev->dev, "Tx Ring Dump\n");
84f4ee90
TI		 254
255 /* Transmit Descriptor Formats - DEXT[29] is 0 (Legacy) or 1 (Extended)
256 *
257 * Legacy Transmit Descriptor
258 * +--------------------------------------------------------------+
259 * 0 | Buffer Address [63:0] (Reserved on Write Back) |
260 * +--------------------------------------------------------------+
261 * 8 | Special | CSS | Status | CMD | CSO | Length |
262 * +--------------------------------------------------------------+
263 * 63 48 47 36 35 32 31 24 23 16 15 0
264 *
265 * Extended Context Descriptor (DTYP=0x0) for TSO or checksum offload
266 * 63 48 47 40 39 32 31 16 15 8 7 0
267 * +----------------------------------------------------------------+
268 * 0 | TUCSE | TUCS0 | TUCSS | IPCSE | IPCS0 | IPCSS |
269 * +----------------------------------------------------------------+
270 * 8 | MSS | HDRLEN | RSV | STA | TUCMD | DTYP | PAYLEN |
271 * +----------------------------------------------------------------+
272 * 63 48 47 40 39 36 35 32 31 24 23 20 19 0
273 *
274 * Extended Data Descriptor (DTYP=0x1)
275 * +----------------------------------------------------------------+
276 * 0 | Buffer Address [63:0] |
277 * +----------------------------------------------------------------+
278 * 8 | VLAN tag | POPTS | Rsvd | Status | Command | DTYP | DTALEN |
279 * +----------------------------------------------------------------+
280 * 63 48 47 40 39 36 35 32 31 24 23 20 19 0
281 */
ef456f85
JK		 282 pr_info("Tl[desc] [address 63:0 ] [SpeCssSCmCsLen] [bi->dma ] leng ntw timestamp bi->skb <-- Legacy format\n");
283 pr_info("Tc[desc] [Ce CoCsIpceCoS] [MssHlRSCm0Plen] [bi->dma ] leng ntw timestamp bi->skb <-- Ext Context format\n");
284 pr_info("Td[desc] [address 63:0 ] [VlaPoRSCm1Dlen] [bi->dma ] leng ntw timestamp bi->skb <-- Ext Data format\n");
84f4ee90 285 for (i = 0; tx_ring->desc && (i < tx_ring->count); i++) {
ef456f85 286 const char *next_desc;
84f4ee90
TI		 287 tx_desc = E1000_TX_DESC(*tx_ring, i);
288 buffer_info = &tx_ring->buffer_info[i];
289 u0 = (struct my_u0 *)tx_desc;
84f4ee90 290 if (i == tx_ring->next_to_use && i == tx_ring->next_to_clean)
ef456f85 291 next_desc = " NTC/U";
84f4ee90 292 else if (i == tx_ring->next_to_use)
ef456f85 293 next_desc = " NTU";
84f4ee90 294 else if (i == tx_ring->next_to_clean)
ef456f85 295 next_desc = " NTC";
84f4ee90 296 else
ef456f85
JK		 297 next_desc = "";
298 pr_info("T%c[0x%03X] %016llX %016llX %016llX %04X %3X %016llX %p%s\n",
299 (!(le64_to_cpu(u0->b) & (1 << 29)) ? 'l' :
300 ((le64_to_cpu(u0->b) & (1 << 20)) ? 'd' : 'c')),
301 i,
302 (unsigned long long)le64_to_cpu(u0->a),
303 (unsigned long long)le64_to_cpu(u0->b),
304 (unsigned long long)buffer_info->dma,
305 buffer_info->length, buffer_info->next_to_watch,
306 (unsigned long long)buffer_info->time_stamp,
307 buffer_info->skb, next_desc);
84f4ee90 308
f0c5dadf 309 if (netif_msg_pktdata(adapter) && buffer_info->skb)
84f4ee90 310 print_hex_dump(KERN_INFO, "", DUMP_PREFIX_ADDRESS,
f0c5dadf
ET		 311 16, 1, buffer_info->skb->data,
312 buffer_info->skb->len, true);
84f4ee90
TI		 313 }
314
af667a29 315 /* Print Rx Ring Summary */
84f4ee90 316rx_ring_summary:
af667a29 317 dev_info(&adapter->pdev->dev, "Rx Ring Summary\n");
ef456f85
JK		 318 pr_info("Queue [NTU] [NTC]\n");
319 pr_info(" %5d %5X %5X\n",
320 0, rx_ring->next_to_use, rx_ring->next_to_clean);
84f4ee90 321
af667a29 322 /* Print Rx Ring */
84f4ee90 323 if (!netif_msg_rx_status(adapter))
fe1e980f 324 return;
84f4ee90 325
af667a29 326 dev_info(&adapter->pdev->dev, "Rx Ring Dump\n");
84f4ee90
TI		 327 switch (adapter->rx_ps_pages) {
328 case 1:
329 case 2:
330 case 3:
331 /* [Extended] Packet Split Receive Descriptor Format
332 *
333 * +-----------------------------------------------------+
334 * 0 | Buffer Address 0 [63:0] |
335 * +-----------------------------------------------------+
336 * 8 | Buffer Address 1 [63:0] |
337 * +-----------------------------------------------------+
338 * 16 | Buffer Address 2 [63:0] |
339 * +-----------------------------------------------------+
340 * 24 | Buffer Address 3 [63:0] |
341 * +-----------------------------------------------------+
342 */
ef456f85 343 pr_info("R [desc] [buffer 0 63:0 ] [buffer 1 63:0 ] [buffer 2 63:0 ] [buffer 3 63:0 ] [bi->dma ] [bi->skb] <-- Ext Pkt Split format\n");
84f4ee90
TI		 344 /* [Extended] Receive Descriptor (Write-Back) Format
345 *
346 * 63 48 47 32 31 13 12 8 7 4 3 0
347 * +------------------------------------------------------+
348 * 0 | Packet | IP | Rsvd | MRQ | Rsvd | MRQ RSS |
349 * | Checksum | Ident | | Queue | | Type |
350 * +------------------------------------------------------+
351 * 8 | VLAN Tag | Length | Extended Error | Extended Status |
352 * +------------------------------------------------------+
353 * 63 48 47 32 31 20 19 0
354 */
ef456f85 355 pr_info("RWB[desc] [ck ipid mrqhsh] [vl l0 ee es] [ l3 l2 l1 hs] [reserved ] ---------------- [bi->skb] <-- Ext Rx Write-Back format\n");
84f4ee90 356 for (i = 0; i < rx_ring->count; i++) {
ef456f85 357 const char *next_desc;
84f4ee90
TI		 358 buffer_info = &rx_ring->buffer_info[i];
359 rx_desc_ps = E1000_RX_DESC_PS(*rx_ring, i);
360 u1 = (struct my_u1 *)rx_desc_ps;
361 staterr =
af667a29 362 le32_to_cpu(rx_desc_ps->wb.middle.status_error);
ef456f85
JK		 363
364 if (i == rx_ring->next_to_use)
365 next_desc = " NTU";
366 else if (i == rx_ring->next_to_clean)
367 next_desc = " NTC";
368 else
369 next_desc = "";
370
84f4ee90
TI		 371 if (staterr & E1000_RXD_STAT_DD) {
372 /* Descriptor Done */
ef456f85
JK		 373 pr_info("%s[0x%03X] %016llX %016llX %016llX %016llX ---------------- %p%s\n",
374 "RWB", i,
375 (unsigned long long)le64_to_cpu(u1->a),
376 (unsigned long long)le64_to_cpu(u1->b),
377 (unsigned long long)le64_to_cpu(u1->c),
378 (unsigned long long)le64_to_cpu(u1->d),
379 buffer_info->skb, next_desc);
84f4ee90 380 } else {
ef456f85
JK		 381 pr_info("%s[0x%03X] %016llX %016llX %016llX %016llX %016llX %p%s\n",
382 "R ", i,
383 (unsigned long long)le64_to_cpu(u1->a),
384 (unsigned long long)le64_to_cpu(u1->b),
385 (unsigned long long)le64_to_cpu(u1->c),
386 (unsigned long long)le64_to_cpu(u1->d),
387 (unsigned long long)buffer_info->dma,
388 buffer_info->skb, next_desc);
84f4ee90
TI		 389
390 if (netif_msg_pktdata(adapter))
f0c5dadf
ET		 391 e1000e_dump_ps_pages(adapter,
392 buffer_info);
84f4ee90 393 }
84f4ee90
TI		 394 }
395 break;
396 default:
397 case 0:
5f450212 398 /* Extended Receive Descriptor (Read) Format
84f4ee90 399 *
5f450212
BA		 400 * +-----------------------------------------------------+
401 * 0 | Buffer Address [63:0] |
402 * +-----------------------------------------------------+
403 * 8 | Reserved |
404 * +-----------------------------------------------------+
84f4ee90 405 */
ef456f85 406 pr_info("R [desc] [buf addr 63:0 ] [reserved 63:0 ] [bi->dma ] [bi->skb] <-- Ext (Read) format\n");
5f450212
BA		 407 /* Extended Receive Descriptor (Write-Back) Format
408 *
409 * 63 48 47 32 31 24 23 4 3 0
410 * +------------------------------------------------------+
411 * | RSS Hash | | | |
412 * 0 +-------------------+ Rsvd | Reserved | MRQ RSS |
413 * | Packet | IP | | | Type |
414 * | Checksum | Ident | | | |
415 * +------------------------------------------------------+
416 * 8 | VLAN Tag | Length | Extended Error | Extended Status |
417 * +------------------------------------------------------+
418 * 63 48 47 32 31 20 19 0
419 */
ef456f85 420 pr_info("RWB[desc] [cs ipid mrq] [vt ln xe xs] [bi->skb] <-- Ext (Write-Back) format\n");
5f450212
BA		 421
422 for (i = 0; i < rx_ring->count; i++) {
ef456f85
JK		 423 const char *next_desc;
424
84f4ee90 425 buffer_info = &rx_ring->buffer_info[i];
5f450212
BA		 426 rx_desc = E1000_RX_DESC_EXT(*rx_ring, i);
427 u1 = (struct my_u1 *)rx_desc;
428 staterr = le32_to_cpu(rx_desc->wb.upper.status_error);
ef456f85
JK		 429
430 if (i == rx_ring->next_to_use)
431 next_desc = " NTU";
432 else if (i == rx_ring->next_to_clean)
433 next_desc = " NTC";
434 else
435 next_desc = "";
436
5f450212
BA		 437 if (staterr & E1000_RXD_STAT_DD) {
438 /* Descriptor Done */
ef456f85
JK		 439 pr_info("%s[0x%03X] %016llX %016llX ---------------- %p%s\n",
440 "RWB", i,
441 (unsigned long long)le64_to_cpu(u1->a),
442 (unsigned long long)le64_to_cpu(u1->b),
443 buffer_info->skb, next_desc);
5f450212 444 } else {
ef456f85
JK		 445 pr_info("%s[0x%03X] %016llX %016llX %016llX %p%s\n",
446 "R ", i,
447 (unsigned long long)le64_to_cpu(u1->a),
448 (unsigned long long)le64_to_cpu(u1->b),
449 (unsigned long long)buffer_info->dma,
450 buffer_info->skb, next_desc);
5f450212 451
f0c5dadf
ET		 452 if (netif_msg_pktdata(adapter) &&
453 buffer_info->skb)
5f450212
BA		 454 print_hex_dump(KERN_INFO, "",
455 DUMP_PREFIX_ADDRESS, 16,
456 1,
f0c5dadf 457 buffer_info->skb->data,
5f450212
BA		 458 adapter->rx_buffer_len,
459 true);
460 }
84f4ee90
TI		 461 }
462 }
84f4ee90
TI		 463}
464
bc7f75fa
AK		 465/**
466 * e1000_desc_unused - calculate if we have unused descriptors
467 **/
468static int e1000_desc_unused(struct e1000_ring *ring)
469{
470 if (ring->next_to_clean > ring->next_to_use)
471 return ring->next_to_clean - ring->next_to_use - 1;
472
473 return ring->count + ring->next_to_clean - ring->next_to_use - 1;
474}
475
b67e1913
BA		 476/**
477 * e1000e_systim_to_hwtstamp - convert system time value to hw time stamp
478 * @adapter: board private structure
479 * @hwtstamps: time stamp structure to update
480 * @systim: unsigned 64bit system time value.
481 *
482 * Convert the system time value stored in the RX/TXSTMP registers into a
483 * hwtstamp which can be used by the upper level time stamping functions.
484 *
485 * The 'systim_lock' spinlock is used to protect the consistency of the
486 * system time value. This is needed because reading the 64 bit time
487 * value involves reading two 32 bit registers. The first read latches the
488 * value.
489 **/
490static void e1000e_systim_to_hwtstamp(struct e1000_adapter *adapter,
491 struct skb_shared_hwtstamps *hwtstamps,
492 u64 systim)
493{
494 u64 ns;
495 unsigned long flags;
496
497 spin_lock_irqsave(&adapter->systim_lock, flags);
498 ns = timecounter_cyc2time(&adapter->tc, systim);
499 spin_unlock_irqrestore(&adapter->systim_lock, flags);
500
501 memset(hwtstamps, 0, sizeof(*hwtstamps));
502 hwtstamps->hwtstamp = ns_to_ktime(ns);
503}
504
505/**
506 * e1000e_rx_hwtstamp - utility function which checks for Rx time stamp
507 * @adapter: board private structure
508 * @status: descriptor extended error and status field
509 * @skb: particular skb to include time stamp
510 *
511 * If the time stamp is valid, convert it into the timecounter ns value
512 * and store that result into the shhwtstamps structure which is passed
513 * up the network stack.
514 **/
515static void e1000e_rx_hwtstamp(struct e1000_adapter *adapter, u32 status,
516 struct sk_buff *skb)
517{
518 struct e1000_hw *hw = &adapter->hw;
519 u64 rxstmp;
520
521 if (!(adapter->flags & FLAG_HAS_HW_TIMESTAMP) ||
522 !(status & E1000_RXDEXT_STATERR_TST) ||
523 !(er32(TSYNCRXCTL) & E1000_TSYNCRXCTL_VALID))
524 return;
525
526 /* The Rx time stamp registers contain the time stamp. No other
527 * received packet will be time stamped until the Rx time stamp
528 * registers are read. Because only one packet can be time stamped
529 * at a time, the register values must belong to this packet and
530 * therefore none of the other additional attributes need to be
531 * compared.
532 */
533 rxstmp = (u64)er32(RXSTMPL);
534 rxstmp |= (u64)er32(RXSTMPH) << 32;
535 e1000e_systim_to_hwtstamp(adapter, skb_hwtstamps(skb), rxstmp);
536
537 adapter->flags2 &= ~FLAG2_CHECK_RX_HWTSTAMP;
538}
539
bc7f75fa 540/**
ad68076e 541 * e1000_receive_skb - helper function to handle Rx indications
bc7f75fa 542 * @adapter: board private structure
b67e1913 543 * @staterr: descriptor extended error and status field as written by hardware
bc7f75fa
AK		 544 * @vlan: descriptor vlan field as written by hardware (no le/be conversion)
545 * @skb: pointer to sk_buff to be indicated to stack
546 **/
547static void e1000_receive_skb(struct e1000_adapter *adapter,
af667a29 548 struct net_device *netdev, struct sk_buff *skb,
b67e1913 549 u32 staterr, __le16 vlan)
bc7f75fa 550{
86d70e53 551 u16 tag = le16_to_cpu(vlan);
b67e1913
BA		 552
553 e1000e_rx_hwtstamp(adapter, staterr, skb);
554
bc7f75fa
AK		 555 skb->protocol = eth_type_trans(skb, netdev);
556
b67e1913 557 if (staterr & E1000_RXD_STAT_VP)
86d70e53
JK		 558 __vlan_hwaccel_put_tag(skb, tag);
559
560 napi_gro_receive(&adapter->napi, skb);
bc7f75fa
AK		 561}
562
563/**
af667a29 564 * e1000_rx_checksum - Receive Checksum Offload
afd12939
BA		 565 * @adapter: board private structure
566 * @status_err: receive descriptor status and error fields
567 * @csum: receive descriptor csum field
568 * @sk_buff: socket buffer with received data
bc7f75fa
AK		 569 **/
570static void e1000_rx_checksum(struct e1000_adapter *adapter, u32 status_err,
2e1706f2 571 struct sk_buff *skb)
bc7f75fa
AK		 572{
573 u16 status = (u16)status_err;
574 u8 errors = (u8)(status_err >> 24);
bc8acf2c
ED		 575
576 skb_checksum_none_assert(skb);
bc7f75fa 577
afd12939
BA		 578 /* Rx checksum disabled */
579 if (!(adapter->netdev->features & NETIF_F_RXCSUM))
580 return;
581
bc7f75fa
AK		 582 /* Ignore Checksum bit is set */
583 if (status & E1000_RXD_STAT_IXSM)
584 return;
afd12939 585
2e1706f2
BA		 586 /* TCP/UDP checksum error bit or IP checksum error bit is set */
587 if (errors & (E1000_RXD_ERR_TCPE | E1000_RXD_ERR_IPE)) {
bc7f75fa
AK		 588 /* let the stack verify checksum errors */
589 adapter->hw_csum_err++;
590 return;
591 }
592
593 /* TCP/UDP Checksum has not been calculated */
594 if (!(status & (E1000_RXD_STAT_TCPCS | E1000_RXD_STAT_UDPCS)))
595 return;
596
597 /* It must be a TCP or UDP packet with a valid checksum */
2e1706f2 598 skb->ip_summed = CHECKSUM_UNNECESSARY;
bc7f75fa
AK		 599 adapter->hw_csum_good++;
600}
601
55aa6985 602static void e1000e_update_rdt_wa(struct e1000_ring *rx_ring, unsigned int i)
c6e7f51e 603{
55aa6985 604 struct e1000_adapter *adapter = rx_ring->adapter;
c6e7f51e 605 struct e1000_hw *hw = &adapter->hw;
bdc125f7
BA		 606 s32 ret_val = __ew32_prepare(hw);
607
608 writel(i, rx_ring->tail);
c6e7f51e 609
bdc125f7 610 if (unlikely(!ret_val && (i != readl(rx_ring->tail)))) {
c6e7f51e
BA		 611 u32 rctl = er32(RCTL);
612 ew32(RCTL, rctl & ~E1000_RCTL_EN);
613 e_err("ME firmware caused invalid RDT - resetting\n");
614 schedule_work(&adapter->reset_task);
615 }
616}
617
55aa6985 618static void e1000e_update_tdt_wa(struct e1000_ring *tx_ring, unsigned int i)
c6e7f51e 619{
55aa6985 620 struct e1000_adapter *adapter = tx_ring->adapter;
c6e7f51e 621 struct e1000_hw *hw = &adapter->hw;
bdc125f7 622 s32 ret_val = __ew32_prepare(hw);
c6e7f51e 623
bdc125f7
BA		 624 writel(i, tx_ring->tail);
625
626 if (unlikely(!ret_val && (i != readl(tx_ring->tail)))) {
c6e7f51e
BA		 627 u32 tctl = er32(TCTL);
628 ew32(TCTL, tctl & ~E1000_TCTL_EN);
629 e_err("ME firmware caused invalid TDT - resetting\n");
630 schedule_work(&adapter->reset_task);
631 }
632}
633
bc7f75fa 634/**
5f450212 635 * e1000_alloc_rx_buffers - Replace used receive buffers
55aa6985 636 * @rx_ring: Rx descriptor ring
bc7f75fa 637 **/
55aa6985 638static void e1000_alloc_rx_buffers(struct e1000_ring *rx_ring,
c2fed996 639 int cleaned_count, gfp_t gfp)
bc7f75fa 640{
55aa6985 641 struct e1000_adapter *adapter = rx_ring->adapter;
bc7f75fa
AK		 642 struct net_device *netdev = adapter->netdev;
643 struct pci_dev *pdev = adapter->pdev;
5f450212 644 union e1000_rx_desc_extended *rx_desc;
bc7f75fa
AK		 645 struct e1000_buffer *buffer_info;
646 struct sk_buff *skb;
647 unsigned int i;
89d71a66 648 unsigned int bufsz = adapter->rx_buffer_len;
bc7f75fa
AK		 649
650 i = rx_ring->next_to_use;
651 buffer_info = &rx_ring->buffer_info[i];
652
653 while (cleaned_count--) {
654 skb = buffer_info->skb;
655 if (skb) {
656 skb_trim(skb, 0);
657 goto map_skb;
658 }
659
c2fed996 660 skb = __netdev_alloc_skb_ip_align(netdev, bufsz, gfp);
bc7f75fa
AK		 661 if (!skb) {
662 /* Better luck next round */
663 adapter->alloc_rx_buff_failed++;
664 break;
665 }
666
bc7f75fa
AK		 667 buffer_info->skb = skb;
668map_skb:
0be3f55f 669 buffer_info->dma = dma_map_single(&pdev->dev, skb->data,
bc7f75fa 670 adapter->rx_buffer_len,
0be3f55f
NN		 671 DMA_FROM_DEVICE);
672 if (dma_mapping_error(&pdev->dev, buffer_info->dma)) {
af667a29 673 dev_err(&pdev->dev, "Rx DMA map failed\n");
bc7f75fa
AK		 674 adapter->rx_dma_failed++;
675 break;
676 }
677
5f450212
BA		 678 rx_desc = E1000_RX_DESC_EXT(*rx_ring, i);
679 rx_desc->read.buffer_addr = cpu_to_le64(buffer_info->dma);
bc7f75fa 680
50849d79 681 if (unlikely(!(i & (E1000_RX_BUFFER_WRITE - 1)))) {
e921eb1a 682 /* Force memory writes to complete before letting h/w
50849d79
TH		 683 * know there are new descriptors to fetch. (Only
684 * applicable for weak-ordered memory model archs,
685 * such as IA-64).
686 */
687 wmb();
c6e7f51e 688 if (adapter->flags2 & FLAG2_PCIM2PCI_ARBITER_WA)
55aa6985 689 e1000e_update_rdt_wa(rx_ring, i);
c6e7f51e 690 else
c5083cf6 691 writel(i, rx_ring->tail);
50849d79 692 }
bc7f75fa
AK		 693 i++;
694 if (i == rx_ring->count)
695 i = 0;
696 buffer_info = &rx_ring->buffer_info[i];
697 }
698
50849d79 699 rx_ring->next_to_use = i;
bc7f75fa
AK		 700}
701
702/**
703 * e1000_alloc_rx_buffers_ps - Replace used receive buffers; packet split
55aa6985 704 * @rx_ring: Rx descriptor ring
bc7f75fa 705 **/
55aa6985 706static void e1000_alloc_rx_buffers_ps(struct e1000_ring *rx_ring,
c2fed996 707 int cleaned_count, gfp_t gfp)
bc7f75fa 708{
55aa6985 709 struct e1000_adapter *adapter = rx_ring->adapter;
bc7f75fa
AK		 710 struct net_device *netdev = adapter->netdev;
711 struct pci_dev *pdev = adapter->pdev;
712 union e1000_rx_desc_packet_split *rx_desc;
bc7f75fa
AK		 713 struct e1000_buffer *buffer_info;
714 struct e1000_ps_page *ps_page;
715 struct sk_buff *skb;
716 unsigned int i, j;
717
718 i = rx_ring->next_to_use;
719 buffer_info = &rx_ring->buffer_info[i];
720
721 while (cleaned_count--) {
722 rx_desc = E1000_RX_DESC_PS(*rx_ring, i);
723
724 for (j = 0; j < PS_PAGE_BUFFERS; j++) {
47f44e40
AK		 725 ps_page = &buffer_info->ps_pages[j];
726 if (j >= adapter->rx_ps_pages) {
727 /* all unused desc entries get hw null ptr */
af667a29
BA		 728 rx_desc->read.buffer_addr[j + 1] =
729 ~cpu_to_le64(0);
47f44e40
AK		 730 continue;
731 }
732 if (!ps_page->page) {
c2fed996 733 ps_page->page = alloc_page(gfp);
bc7f75fa 734 if (!ps_page->page) {
47f44e40
AK		 735 adapter->alloc_rx_buff_failed++;
736 goto no_buffers;
737 }
0be3f55f
NN		 738 ps_page->dma = dma_map_page(&pdev->dev,
739 ps_page->page,
740 0, PAGE_SIZE,
741 DMA_FROM_DEVICE);
742 if (dma_mapping_error(&pdev->dev,
743 ps_page->dma)) {
47f44e40 744 dev_err(&adapter->pdev->dev,
af667a29 745 "Rx DMA page map failed\n");
47f44e40
AK		 746 adapter->rx_dma_failed++;
747 goto no_buffers;
bc7f75fa 748 }
bc7f75fa 749 }
e921eb1a 750 /* Refresh the desc even if buffer_addrs
47f44e40
AK		 751 * didn't change because each write-back
752 * erases this info.
753 */
af667a29
BA		 754 rx_desc->read.buffer_addr[j + 1] =
755 cpu_to_le64(ps_page->dma);
bc7f75fa
AK		 756 }
757
c2fed996
JK		 758 skb = __netdev_alloc_skb_ip_align(netdev,
759 adapter->rx_ps_bsize0,
760 gfp);
bc7f75fa
AK		 761
762 if (!skb) {
763 adapter->alloc_rx_buff_failed++;
764 break;
765 }
766
bc7f75fa 767 buffer_info->skb = skb;
0be3f55f 768 buffer_info->dma = dma_map_single(&pdev->dev, skb->data,
bc7f75fa 769 adapter->rx_ps_bsize0,
0be3f55f
NN		 770 DMA_FROM_DEVICE);
771 if (dma_mapping_error(&pdev->dev, buffer_info->dma)) {
af667a29 772 dev_err(&pdev->dev, "Rx DMA map failed\n");
bc7f75fa
AK		 773 adapter->rx_dma_failed++;
774 /* cleanup skb */
775 dev_kfree_skb_any(skb);
776 buffer_info->skb = NULL;
777 break;
778 }
779
780 rx_desc->read.buffer_addr[0] = cpu_to_le64(buffer_info->dma);
781
50849d79 782 if (unlikely(!(i & (E1000_RX_BUFFER_WRITE - 1)))) {
e921eb1a 783 /* Force memory writes to complete before letting h/w
50849d79
TH		 784 * know there are new descriptors to fetch. (Only
785 * applicable for weak-ordered memory model archs,
786 * such as IA-64).
787 */
788 wmb();
c6e7f51e 789 if (adapter->flags2 & FLAG2_PCIM2PCI_ARBITER_WA)
55aa6985 790 e1000e_update_rdt_wa(rx_ring, i << 1);
c6e7f51e 791 else
c5083cf6 792 writel(i << 1, rx_ring->tail);
50849d79
TH		 793 }
794
bc7f75fa
AK		 795 i++;
796 if (i == rx_ring->count)
797 i = 0;
798 buffer_info = &rx_ring->buffer_info[i];
799 }
800
801no_buffers:
50849d79 802 rx_ring->next_to_use = i;
bc7f75fa
AK		 803}
804
97ac8cae
BA		 805/**
806 * e1000_alloc_jumbo_rx_buffers - Replace used jumbo receive buffers
55aa6985 807 * @rx_ring: Rx descriptor ring
97ac8cae
BA		 808 * @cleaned_count: number of buffers to allocate this pass
809 **/
810
55aa6985 811static void e1000_alloc_jumbo_rx_buffers(struct e1000_ring *rx_ring,
c2fed996 812 int cleaned_count, gfp_t gfp)
97ac8cae 813{
55aa6985 814 struct e1000_adapter *adapter = rx_ring->adapter;
97ac8cae
BA		 815 struct net_device *netdev = adapter->netdev;
816 struct pci_dev *pdev = adapter->pdev;
5f450212 817 union e1000_rx_desc_extended *rx_desc;
97ac8cae
BA		 818 struct e1000_buffer *buffer_info;
819 struct sk_buff *skb;
820 unsigned int i;
2a2293b9 821 unsigned int bufsz = 256 - 16; /* for skb_reserve */
97ac8cae
BA		 822
823 i = rx_ring->next_to_use;
824 buffer_info = &rx_ring->buffer_info[i];
825
826 while (cleaned_count--) {
827 skb = buffer_info->skb;
828 if (skb) {
829 skb_trim(skb, 0);
830 goto check_page;
831 }
832
c2fed996 833 skb = __netdev_alloc_skb_ip_align(netdev, bufsz, gfp);
97ac8cae
BA		 834 if (unlikely(!skb)) {
835 /* Better luck next round */
836 adapter->alloc_rx_buff_failed++;
837 break;
838 }
839
97ac8cae
BA		 840 buffer_info->skb = skb;
841check_page:
842 /* allocate a new page if necessary */
843 if (!buffer_info->page) {
c2fed996 844 buffer_info->page = alloc_page(gfp);
97ac8cae
BA		 845 if (unlikely(!buffer_info->page)) {
846 adapter->alloc_rx_buff_failed++;
847 break;
848 }
849 }
850
851 if (!buffer_info->dma)
0be3f55f 852 buffer_info->dma = dma_map_page(&pdev->dev,
f0ff4398
BA		 853 buffer_info->page, 0,
854 PAGE_SIZE,
0be3f55f 855 DMA_FROM_DEVICE);
97ac8cae 856
5f450212
BA		 857 rx_desc = E1000_RX_DESC_EXT(*rx_ring, i);
858 rx_desc->read.buffer_addr = cpu_to_le64(buffer_info->dma);
97ac8cae
BA		 859
860 if (unlikely(++i == rx_ring->count))
861 i = 0;
862 buffer_info = &rx_ring->buffer_info[i];
863 }
864
865 if (likely(rx_ring->next_to_use != i)) {
866 rx_ring->next_to_use = i;
867 if (unlikely(i-- == 0))
868 i = (rx_ring->count - 1);
869
870 /* Force memory writes to complete before letting h/w
871 * know there are new descriptors to fetch. (Only
872 * applicable for weak-ordered memory model archs,
e921eb1a
BA		 873 * such as IA-64).
874 */
97ac8cae 875 wmb();
c6e7f51e 876 if (adapter->flags2 & FLAG2_PCIM2PCI_ARBITER_WA)
55aa6985 877 e1000e_update_rdt_wa(rx_ring, i);
c6e7f51e 878 else
c5083cf6 879 writel(i, rx_ring->tail);
97ac8cae
BA		 880 }
881}
882
70495a50
BA		 883static inline void e1000_rx_hash(struct net_device *netdev, __le32 rss,
884 struct sk_buff *skb)
885{
886 if (netdev->features & NETIF_F_RXHASH)
887 skb->rxhash = le32_to_cpu(rss);
888}
889
bc7f75fa 890/**
55aa6985
BA		 891 * e1000_clean_rx_irq - Send received data up the network stack
892 * @rx_ring: Rx descriptor ring
bc7f75fa
AK		 893 *
894 * the return value indicates whether actual cleaning was done, there
895 * is no guarantee that everything was cleaned
896 **/
55aa6985
BA		 897static bool e1000_clean_rx_irq(struct e1000_ring *rx_ring, int *work_done,
898 int work_to_do)
bc7f75fa 899{
55aa6985 900 struct e1000_adapter *adapter = rx_ring->adapter;
bc7f75fa
AK		 901 struct net_device *netdev = adapter->netdev;
902 struct pci_dev *pdev = adapter->pdev;
3bb99fe2 903 struct e1000_hw *hw = &adapter->hw;
5f450212 904 union e1000_rx_desc_extended *rx_desc, *next_rxd;
bc7f75fa 905 struct e1000_buffer *buffer_info, *next_buffer;
5f450212 906 u32 length, staterr;
bc7f75fa
AK		 907 unsigned int i;
908 int cleaned_count = 0;
3db1cd5c 909 bool cleaned = false;
bc7f75fa
AK		 910 unsigned int total_rx_bytes = 0, total_rx_packets = 0;
911
912 i = rx_ring->next_to_clean;
5f450212
BA		 913 rx_desc = E1000_RX_DESC_EXT(*rx_ring, i);
914 staterr = le32_to_cpu(rx_desc->wb.upper.status_error);
bc7f75fa
AK		 915 buffer_info = &rx_ring->buffer_info[i];
916
5f450212 917 while (staterr & E1000_RXD_STAT_DD) {
bc7f75fa 918 struct sk_buff *skb;
bc7f75fa
AK		 919
920 if (*work_done >= work_to_do)
921 break;
922 (*work_done)++;
2d0bb1c1 923 rmb(); /* read descriptor and rx_buffer_info after status DD */
bc7f75fa 924
bc7f75fa
AK		 925 skb = buffer_info->skb;
926 buffer_info->skb = NULL;
927
928 prefetch(skb->data - NET_IP_ALIGN);
929
930 i++;
931 if (i == rx_ring->count)
932 i = 0;
5f450212 933 next_rxd = E1000_RX_DESC_EXT(*rx_ring, i);
bc7f75fa
AK		 934 prefetch(next_rxd);
935
936 next_buffer = &rx_ring->buffer_info[i];
937
3db1cd5c 938 cleaned = true;
bc7f75fa 939 cleaned_count++;
0be3f55f 940 dma_unmap_single(&pdev->dev,
bc7f75fa
AK		 941 buffer_info->dma,
942 adapter->rx_buffer_len,
0be3f55f 943 DMA_FROM_DEVICE);
bc7f75fa
AK		 944 buffer_info->dma = 0;
945
5f450212 946 length = le16_to_cpu(rx_desc->wb.upper.length);
bc7f75fa 947
e921eb1a 948 /* !EOP means multiple descriptors were used to store a single
b94b5028
JB		 949 * packet, if that's the case we need to toss it. In fact, we
950 * need to toss every packet with the EOP bit clear and the
951 * next frame that _does_ have the EOP bit set, as it is by
952 * definition only a frame fragment
953 */
5f450212 954 if (unlikely(!(staterr & E1000_RXD_STAT_EOP)))
b94b5028
JB		 955 adapter->flags2 |= FLAG2_IS_DISCARDING;
956
957 if (adapter->flags2 & FLAG2_IS_DISCARDING) {
bc7f75fa 958 /* All receives must fit into a single buffer */
3bb99fe2 959 e_dbg("Receive packet consumed multiple buffers\n");
bc7f75fa
AK		 960 /* recycle */
961 buffer_info->skb = skb;
5f450212 962 if (staterr & E1000_RXD_STAT_EOP)
b94b5028 963 adapter->flags2 &= ~FLAG2_IS_DISCARDING;
bc7f75fa
AK		 964 goto next_desc;
965 }
966
cf955e6c
BG		 967 if (unlikely((staterr & E1000_RXDEXT_ERR_FRAME_ERR_MASK) &&
968 !(netdev->features & NETIF_F_RXALL))) {
bc7f75fa
AK		 969 /* recycle */
970 buffer_info->skb = skb;
971 goto next_desc;
972 }
973
eb7c3adb 974 /* adjust length to remove Ethernet CRC */
0184039a
BG		 975 if (!(adapter->flags2 & FLAG2_CRC_STRIPPING)) {
976 /* If configured to store CRC, don't subtract FCS,
977 * but keep the FCS bytes out of the total_rx_bytes
978 * counter
979 */
980 if (netdev->features & NETIF_F_RXFCS)
981 total_rx_bytes -= 4;
982 else
983 length -= 4;
984 }
eb7c3adb 985
bc7f75fa
AK		 986 total_rx_bytes += length;
987 total_rx_packets++;
988
e921eb1a 989 /* code added for copybreak, this should improve
bc7f75fa 990 * performance for small packets with large amounts
ad68076e
BA		 991 * of reassembly being done in the stack
992 */
bc7f75fa
AK		 993 if (length < copybreak) {
994 struct sk_buff *new_skb =
89d71a66 995 netdev_alloc_skb_ip_align(netdev, length);
bc7f75fa 996 if (new_skb) {
808ff676
BA		 997 skb_copy_to_linear_data_offset(new_skb,
998 -NET_IP_ALIGN,
999 (skb->data -
1000 NET_IP_ALIGN),
1001 (length +
1002 NET_IP_ALIGN));
bc7f75fa
AK		 1003 /* save the skb in buffer_info as good */
1004 buffer_info->skb = skb;
1005 skb = new_skb;
1006 }
1007 /* else just continue with the old one */
1008 }
1009 /* end copybreak code */
1010 skb_put(skb, length);
1011
1012 /* Receive Checksum Offload */
2e1706f2 1013 e1000_rx_checksum(adapter, staterr, skb);
bc7f75fa 1014
70495a50
BA		 1015 e1000_rx_hash(netdev, rx_desc->wb.lower.hi_dword.rss, skb);
1016
5f450212
BA		 1017 e1000_receive_skb(adapter, netdev, skb, staterr,
1018 rx_desc->wb.upper.vlan);
bc7f75fa
AK		 1019
1020next_desc:
5f450212 1021 rx_desc->wb.upper.status_error &= cpu_to_le32(~0xFF);
bc7f75fa
AK		 1022
1023 /* return some buffers to hardware, one at a time is too slow */
1024 if (cleaned_count >= E1000_RX_BUFFER_WRITE) {
55aa6985 1025 adapter->alloc_rx_buf(rx_ring, cleaned_count,
c2fed996 1026 GFP_ATOMIC);
bc7f75fa
AK		 1027 cleaned_count = 0;
1028 }
1029
1030 /* use prefetched values */
1031 rx_desc = next_rxd;
1032 buffer_info = next_buffer;
5f450212
BA		 1033
1034 staterr = le32_to_cpu(rx_desc->wb.upper.status_error);
bc7f75fa
AK		 1035 }
1036 rx_ring->next_to_clean = i;
1037
1038 cleaned_count = e1000_desc_unused(rx_ring);
1039 if (cleaned_count)
55aa6985 1040 adapter->alloc_rx_buf(rx_ring, cleaned_count, GFP_ATOMIC);
bc7f75fa 1041
bc7f75fa 1042 adapter->total_rx_bytes += total_rx_bytes;
7c25769f 1043 adapter->total_rx_packets += total_rx_packets;
bc7f75fa
AK		 1044 return cleaned;
1045}
1046
55aa6985
BA		 1047static void e1000_put_txbuf(struct e1000_ring *tx_ring,
1048 struct e1000_buffer *buffer_info)
bc7f75fa 1049{
55aa6985
BA		 1050 struct e1000_adapter *adapter = tx_ring->adapter;
1051
03b1320d
AD		 1052 if (buffer_info->dma) {
1053 if (buffer_info->mapped_as_page)
0be3f55f
NN		 1054 dma_unmap_page(&adapter->pdev->dev, buffer_info->dma,
1055 buffer_info->length, DMA_TO_DEVICE);
03b1320d 1056 else
0be3f55f
NN		 1057 dma_unmap_single(&adapter->pdev->dev, buffer_info->dma,
1058 buffer_info->length, DMA_TO_DEVICE);
03b1320d
AD		 1059 buffer_info->dma = 0;
1060 }
bc7f75fa
AK		 1061 if (buffer_info->skb) {
1062 dev_kfree_skb_any(buffer_info->skb);
1063 buffer_info->skb = NULL;
1064 }
1b7719c4 1065 buffer_info->time_stamp = 0;
bc7f75fa
AK		 1066}
1067
41cec6f1 1068static void e1000_print_hw_hang(struct work_struct *work)
bc7f75fa 1069{
41cec6f1 1070 struct e1000_adapter *adapter = container_of(work,
f0ff4398
BA		 1071 struct e1000_adapter,
1072 print_hang_task);
09357b00 1073 struct net_device *netdev = adapter->netdev;
bc7f75fa
AK		 1074 struct e1000_ring *tx_ring = adapter->tx_ring;
1075 unsigned int i = tx_ring->next_to_clean;
1076 unsigned int eop = tx_ring->buffer_info[i].next_to_watch;
1077 struct e1000_tx_desc *eop_desc = E1000_TX_DESC(*tx_ring, eop);
41cec6f1
BA		 1078 struct e1000_hw *hw = &adapter->hw;
1079 u16 phy_status, phy_1000t_status, phy_ext_status;
1080 u16 pci_status;
1081
615b32af
JB		 1082 if (test_bit(__E1000_DOWN, &adapter->state))
1083 return;
1084
09357b00
JK		 1085 if (!adapter->tx_hang_recheck &&
1086 (adapter->flags2 & FLAG2_DMA_BURST)) {
e921eb1a 1087 /* May be block on write-back, flush and detect again
09357b00
JK		 1088 * flush pending descriptor writebacks to memory
1089 */
1090 ew32(TIDV, adapter->tx_int_delay | E1000_TIDV_FPD);
1091 /* execute the writes immediately */
1092 e1e_flush();
e921eb1a 1093 /* Due to rare timing issues, write to TIDV again to ensure
bf03085f
MV		 1094 * the write is successful
1095 */
1096 ew32(TIDV, adapter->tx_int_delay | E1000_TIDV_FPD);
1097 /* execute the writes immediately */
1098 e1e_flush();
09357b00
JK		 1099 adapter->tx_hang_recheck = true;
1100 return;
1101 }
1102 /* Real hang detected */
1103 adapter->tx_hang_recheck = false;
1104 netif_stop_queue(netdev);
1105
c2ade1a4
BA		 1106 e1e_rphy(hw, MII_BMSR, &phy_status);
1107 e1e_rphy(hw, MII_STAT1000, &phy_1000t_status);
1108 e1e_rphy(hw, MII_ESTATUS, &phy_ext_status);
bc7f75fa 1109
41cec6f1
BA		 1110 pci_read_config_word(adapter->pdev, PCI_STATUS, &pci_status);
1111
1112 /* detected Hardware unit hang */
1113 e_err("Detected Hardware Unit Hang:\n"
44defeb3
JK		 1114 " TDH <%x>\n"
1115 " TDT <%x>\n"
1116 " next_to_use <%x>\n"
1117 " next_to_clean <%x>\n"
1118 "buffer_info[next_to_clean]:\n"
1119 " time_stamp <%lx>\n"
1120 " next_to_watch <%x>\n"
1121 " jiffies <%lx>\n"
41cec6f1
BA		 1122 " next_to_watch.status <%x>\n"
1123 "MAC Status <%x>\n"
1124 "PHY Status <%x>\n"
1125 "PHY 1000BASE-T Status <%x>\n"
1126 "PHY Extended Status <%x>\n"
1127 "PCI Status <%x>\n",
c5083cf6
BA		 1128 readl(tx_ring->head),
1129 readl(tx_ring->tail),
44defeb3
JK		 1130 tx_ring->next_to_use,
1131 tx_ring->next_to_clean,
1132 tx_ring->buffer_info[eop].time_stamp,
1133 eop,
1134 jiffies,
41cec6f1
BA		 1135 eop_desc->upper.fields.status,
1136 er32(STATUS),
1137 phy_status,
1138 phy_1000t_status,
1139 phy_ext_status,
1140 pci_status);
7c0427ee
BA		 1141
1142 /* Suggest workaround for known h/w issue */
1143 if ((hw->mac.type == e1000_pchlan) && (er32(CTRL) & E1000_CTRL_TFCE))
1144 e_err("Try turning off Tx pause (flow control) via ethtool\n");
bc7f75fa
AK		 1145}
1146
b67e1913
BA		 1147/**
1148 * e1000e_tx_hwtstamp_work - check for Tx time stamp
1149 * @work: pointer to work struct
1150 *
1151 * This work function polls the TSYNCTXCTL valid bit to determine when a
1152 * timestamp has been taken for the current stored skb. The timestamp must
1153 * be for this skb because only one such packet is allowed in the queue.
1154 */
1155static void e1000e_tx_hwtstamp_work(struct work_struct *work)
1156{
1157 struct e1000_adapter *adapter = container_of(work, struct e1000_adapter,
1158 tx_hwtstamp_work);
1159 struct e1000_hw *hw = &adapter->hw;
1160
1161 if (!adapter->tx_hwtstamp_skb)
1162 return;
1163
1164 if (er32(TSYNCTXCTL) & E1000_TSYNCTXCTL_VALID) {
1165 struct skb_shared_hwtstamps shhwtstamps;
1166 u64 txstmp;
1167
1168 txstmp = er32(TXSTMPL);
1169 txstmp |= (u64)er32(TXSTMPH) << 32;
1170
1171 e1000e_systim_to_hwtstamp(adapter, &shhwtstamps, txstmp);
1172
1173 skb_tstamp_tx(adapter->tx_hwtstamp_skb, &shhwtstamps);
1174 dev_kfree_skb_any(adapter->tx_hwtstamp_skb);
1175 adapter->tx_hwtstamp_skb = NULL;
1176 } else {
1177 /* reschedule to check later */
1178 schedule_work(&adapter->tx_hwtstamp_work);
1179 }
1180}
1181
bc7f75fa
AK		 1182/**
1183 * e1000_clean_tx_irq - Reclaim resources after transmit completes
55aa6985 1184 * @tx_ring: Tx descriptor ring
bc7f75fa
AK		 1185 *
1186 * the return value indicates whether actual cleaning was done, there
1187 * is no guarantee that everything was cleaned
1188 **/
55aa6985 1189static bool e1000_clean_tx_irq(struct e1000_ring *tx_ring)
bc7f75fa 1190{
55aa6985 1191 struct e1000_adapter *adapter = tx_ring->adapter;
bc7f75fa
AK		 1192 struct net_device *netdev = adapter->netdev;
1193 struct e1000_hw *hw = &adapter->hw;
bc7f75fa
AK		 1194 struct e1000_tx_desc *tx_desc, *eop_desc;
1195 struct e1000_buffer *buffer_info;
1196 unsigned int i, eop;
1197 unsigned int count = 0;
bc7f75fa 1198 unsigned int total_tx_bytes = 0, total_tx_packets = 0;
3f0cfa3b 1199 unsigned int bytes_compl = 0, pkts_compl = 0;
bc7f75fa
AK		 1200
1201 i = tx_ring->next_to_clean;
1202 eop = tx_ring->buffer_info[i].next_to_watch;
1203 eop_desc = E1000_TX_DESC(*tx_ring, eop);
1204
12d04a3c
AD		 1205 while ((eop_desc->upper.data & cpu_to_le32(E1000_TXD_STAT_DD)) &&
1206 (count < tx_ring->count)) {
a86043c2 1207 bool cleaned = false;
2d0bb1c1 1208 rmb(); /* read buffer_info after eop_desc */
a86043c2 1209 for (; !cleaned; count++) {
bc7f75fa
AK		 1210 tx_desc = E1000_TX_DESC(*tx_ring, i);
1211 buffer_info = &tx_ring->buffer_info[i];
1212 cleaned = (i == eop);
1213
1214 if (cleaned) {
9ed318d5
TH		 1215 total_tx_packets += buffer_info->segs;
1216 total_tx_bytes += buffer_info->bytecount;
3f0cfa3b
TH		 1217 if (buffer_info->skb) {
1218 bytes_compl += buffer_info->skb->len;
1219 pkts_compl++;
1220 }
bc7f75fa
AK		 1221 }
1222
55aa6985 1223 e1000_put_txbuf(tx_ring, buffer_info);
bc7f75fa
AK		 1224 tx_desc->upper.data = 0;
1225
1226 i++;
1227 if (i == tx_ring->count)
1228 i = 0;
1229 }
1230
dac87619
TL		 1231 if (i == tx_ring->next_to_use)
1232 break;
bc7f75fa
AK		 1233 eop = tx_ring->buffer_info[i].next_to_watch;
1234 eop_desc = E1000_TX_DESC(*tx_ring, eop);
bc7f75fa
AK		 1235 }
1236
1237 tx_ring->next_to_clean = i;
1238
3f0cfa3b
TH		 1239 netdev_completed_queue(netdev, pkts_compl, bytes_compl);
1240
bc7f75fa 1241#define TX_WAKE_THRESHOLD 32
a86043c2
JB		 1242 if (count && netif_carrier_ok(netdev) &&
1243 e1000_desc_unused(tx_ring) >= TX_WAKE_THRESHOLD) {
bc7f75fa
AK		 1244 /* Make sure that anybody stopping the queue after this
1245 * sees the new next_to_clean.
1246 */
1247 smp_mb();
1248
1249 if (netif_queue_stopped(netdev) &&
1250 !(test_bit(__E1000_DOWN, &adapter->state))) {
1251 netif_wake_queue(netdev);
1252 ++adapter->restart_queue;
1253 }
1254 }
1255
1256 if (adapter->detect_tx_hung) {
e921eb1a 1257 /* Detect a transmit hang in hardware, this serializes the
41cec6f1
BA		 1258 * check with the clearing of time_stamp and movement of i
1259 */
3db1cd5c 1260 adapter->detect_tx_hung = false;
12d04a3c
AD		 1261 if (tx_ring->buffer_info[i].time_stamp &&
1262 time_after(jiffies, tx_ring->buffer_info[i].time_stamp
8e95a202 1263 + (adapter->tx_timeout_factor * HZ)) &&
09357b00 1264 !(er32(STATUS) & E1000_STATUS_TXOFF))
41cec6f1 1265 schedule_work(&adapter->print_hang_task);
09357b00
JK		 1266 else
1267 adapter->tx_hang_recheck = false;
bc7f75fa
AK		 1268 }
1269 adapter->total_tx_bytes += total_tx_bytes;
1270 adapter->total_tx_packets += total_tx_packets;
807540ba 1271 return count < tx_ring->count;
bc7f75fa
AK		 1272}
1273
bc7f75fa
AK		 1274/**
1275 * e1000_clean_rx_irq_ps - Send received data up the network stack; packet split
55aa6985 1276 * @rx_ring: Rx descriptor ring
bc7f75fa
AK		 1277 *
1278 * the return value indicates whether actual cleaning was done, there
1279 * is no guarantee that everything was cleaned
1280 **/
55aa6985
BA		 1281static bool e1000_clean_rx_irq_ps(struct e1000_ring *rx_ring, int *work_done,
1282 int work_to_do)
bc7f75fa 1283{
55aa6985 1284 struct e1000_adapter *adapter = rx_ring->adapter;
3bb99fe2 1285 struct e1000_hw *hw = &adapter->hw;
bc7f75fa
AK		 1286 union e1000_rx_desc_packet_split *rx_desc, *next_rxd;
1287 struct net_device *netdev = adapter->netdev;
1288 struct pci_dev *pdev = adapter->pdev;
bc7f75fa
AK		 1289 struct e1000_buffer *buffer_info, *next_buffer;
1290 struct e1000_ps_page *ps_page;
1291 struct sk_buff *skb;
1292 unsigned int i, j;
1293 u32 length, staterr;
1294 int cleaned_count = 0;
3db1cd5c 1295 bool cleaned = false;
bc7f75fa
AK		 1296 unsigned int total_rx_bytes = 0, total_rx_packets = 0;
1297
1298 i = rx_ring->next_to_clean;
1299 rx_desc = E1000_RX_DESC_PS(*rx_ring, i);
1300 staterr = le32_to_cpu(rx_desc->wb.middle.status_error);
1301 buffer_info = &rx_ring->buffer_info[i];
1302
1303 while (staterr & E1000_RXD_STAT_DD) {
1304 if (*work_done >= work_to_do)
1305 break;
1306 (*work_done)++;
1307 skb = buffer_info->skb;
2d0bb1c1 1308 rmb(); /* read descriptor and rx_buffer_info after status DD */
bc7f75fa
AK		 1309
1310 /* in the packet split case this is header only */
1311 prefetch(skb->data - NET_IP_ALIGN);
1312
1313 i++;
1314 if (i == rx_ring->count)
1315 i = 0;
1316 next_rxd = E1000_RX_DESC_PS(*rx_ring, i);
1317 prefetch(next_rxd);
1318
1319 next_buffer = &rx_ring->buffer_info[i];
1320
3db1cd5c 1321 cleaned = true;
bc7f75fa 1322 cleaned_count++;
0be3f55f 1323 dma_unmap_single(&pdev->dev, buffer_info->dma,
af667a29 1324 adapter->rx_ps_bsize0, DMA_FROM_DEVICE);
bc7f75fa
AK		 1325 buffer_info->dma = 0;
1326
af667a29 1327 /* see !EOP comment in other Rx routine */
b94b5028
JB		 1328 if (!(staterr & E1000_RXD_STAT_EOP))
1329 adapter->flags2 |= FLAG2_IS_DISCARDING;
1330
1331 if (adapter->flags2 & FLAG2_IS_DISCARDING) {
ef456f85 1332 e_dbg("Packet Split buffers didn't pick up the full packet\n");
bc7f75fa 1333 dev_kfree_skb_irq(skb);
b94b5028
JB		 1334 if (staterr & E1000_RXD_STAT_EOP)
1335 adapter->flags2 &= ~FLAG2_IS_DISCARDING;
bc7f75fa
AK		 1336 goto next_desc;
1337 }
1338
cf955e6c
BG		 1339 if (unlikely((staterr & E1000_RXDEXT_ERR_FRAME_ERR_MASK) &&
1340 !(netdev->features & NETIF_F_RXALL))) {
bc7f75fa
AK		 1341 dev_kfree_skb_irq(skb);
1342 goto next_desc;
1343 }
1344
1345 length = le16_to_cpu(rx_desc->wb.middle.length0);
1346
1347 if (!length) {
ef456f85 1348 e_dbg("Last part of the packet spanning multiple descriptors\n");
bc7f75fa
AK		 1349 dev_kfree_skb_irq(skb);
1350 goto next_desc;
1351 }
1352
1353 /* Good Receive */
1354 skb_put(skb, length);
1355
1356 {
e921eb1a 1357 /* this looks ugly, but it seems compiler issues make
0e15df49
BA		 1358 * it more efficient than reusing j
1359 */
1360 int l1 = le16_to_cpu(rx_desc->wb.upper.length[0]);
bc7f75fa 1361
e921eb1a 1362 /* page alloc/put takes too long and effects small
0e15df49
BA		 1363 * packet throughput, so unsplit small packets and
1364 * save the alloc/put only valid in softirq (napi)
1365 * context to call kmap_*
ad68076e 1366 */
0e15df49
BA		 1367 if (l1 && (l1 <= copybreak) &&
1368 ((length + l1) <= adapter->rx_ps_bsize0)) {
1369 u8 *vaddr;
1370
1371 ps_page = &buffer_info->ps_pages[0];
1372
e921eb1a 1373 /* there is no documentation about how to call
0e15df49
BA		 1374 * kmap_atomic, so we can't hold the mapping
1375 * very long
1376 */
1377 dma_sync_single_for_cpu(&pdev->dev,
1378 ps_page->dma,
1379 PAGE_SIZE,
1380 DMA_FROM_DEVICE);
9f393834 1381 vaddr = kmap_atomic(ps_page->page);
0e15df49 1382 memcpy(skb_tail_pointer(skb), vaddr, l1);
9f393834 1383 kunmap_atomic(vaddr);
0e15df49
BA		 1384 dma_sync_single_for_device(&pdev->dev,
1385 ps_page->dma,
1386 PAGE_SIZE,
1387 DMA_FROM_DEVICE);
1388
1389 /* remove the CRC */
0184039a
BG		 1390 if (!(adapter->flags2 & FLAG2_CRC_STRIPPING)) {
1391 if (!(netdev->features & NETIF_F_RXFCS))
1392 l1 -= 4;
1393 }
0e15df49
BA		 1394
1395 skb_put(skb, l1);
1396 goto copydone;
1397 } /* if */
bc7f75fa
AK		 1398 }
1399
1400 for (j = 0; j < PS_PAGE_BUFFERS; j++) {
1401 length = le16_to_cpu(rx_desc->wb.upper.length[j]);
1402 if (!length)
1403 break;
1404
47f44e40 1405 ps_page = &buffer_info->ps_pages[j];
0be3f55f
NN		 1406 dma_unmap_page(&pdev->dev, ps_page->dma, PAGE_SIZE,
1407 DMA_FROM_DEVICE);
bc7f75fa
AK		 1408 ps_page->dma = 0;
1409 skb_fill_page_desc(skb, j, ps_page->page, 0, length);
1410 ps_page->page = NULL;
1411 skb->len += length;
1412 skb->data_len += length;
98a045d7 1413 skb->truesize += PAGE_SIZE;
bc7f75fa
AK		 1414 }
1415
eb7c3adb
JK		 1416 /* strip the ethernet crc, problem is we're using pages now so
1417 * this whole operation can get a little cpu intensive
1418 */
0184039a
BG		 1419 if (!(adapter->flags2 & FLAG2_CRC_STRIPPING)) {
1420 if (!(netdev->features & NETIF_F_RXFCS))
1421 pskb_trim(skb, skb->len - 4);
1422 }
eb7c3adb 1423
bc7f75fa
AK		 1424copydone:
1425 total_rx_bytes += skb->len;
1426 total_rx_packets++;
1427
2e1706f2 1428 e1000_rx_checksum(adapter, staterr, skb);
bc7f75fa 1429
70495a50
BA		 1430 e1000_rx_hash(netdev, rx_desc->wb.lower.hi_dword.rss, skb);
1431
bc7f75fa 1432 if (rx_desc->wb.upper.header_status &
17e813ec 1433 cpu_to_le16(E1000_RXDPS_HDRSTAT_HDRSP))
bc7f75fa
AK		 1434 adapter->rx_hdr_split++;
1435
b67e1913
BA		 1436 e1000_receive_skb(adapter, netdev, skb, staterr,
1437 rx_desc->wb.middle.vlan);
bc7f75fa
AK		 1438
1439next_desc:
1440 rx_desc->wb.middle.status_error &= cpu_to_le32(~0xFF);
1441 buffer_info->skb = NULL;
1442
1443 /* return some buffers to hardware, one at a time is too slow */
1444 if (cleaned_count >= E1000_RX_BUFFER_WRITE) {
55aa6985 1445 adapter->alloc_rx_buf(rx_ring, cleaned_count,
c2fed996 1446 GFP_ATOMIC);
bc7f75fa
AK		 1447 cleaned_count = 0;
1448 }
1449
1450 /* use prefetched values */
1451 rx_desc = next_rxd;
1452 buffer_info = next_buffer;
1453
1454 staterr = le32_to_cpu(rx_desc->wb.middle.status_error);
1455 }
1456 rx_ring->next_to_clean = i;
1457
1458 cleaned_count = e1000_desc_unused(rx_ring);
1459 if (cleaned_count)
55aa6985 1460 adapter->alloc_rx_buf(rx_ring, cleaned_count, GFP_ATOMIC);
bc7f75fa 1461
bc7f75fa 1462 adapter->total_rx_bytes += total_rx_bytes;
7c25769f 1463 adapter->total_rx_packets += total_rx_packets;
bc7f75fa
AK		 1464 return cleaned;
1465}
1466
97ac8cae
BA		 1467/**
1468 * e1000_consume_page - helper function
1469 **/
1470static void e1000_consume_page(struct e1000_buffer *bi, struct sk_buff *skb,
66501f56 1471 u16 length)
97ac8cae
BA		 1472{
1473 bi->page = NULL;
1474 skb->len += length;
1475 skb->data_len += length;
98a045d7 1476 skb->truesize += PAGE_SIZE;
97ac8cae
BA		 1477}
1478
1479/**
1480 * e1000_clean_jumbo_rx_irq - Send received data up the network stack; legacy
1481 * @adapter: board private structure
1482 *
1483 * the return value indicates whether actual cleaning was done, there
1484 * is no guarantee that everything was cleaned
1485 **/
55aa6985
BA		 1486static bool e1000_clean_jumbo_rx_irq(struct e1000_ring *rx_ring, int *work_done,
1487 int work_to_do)
97ac8cae 1488{
55aa6985 1489 struct e1000_adapter *adapter = rx_ring->adapter;
97ac8cae
BA		 1490 struct net_device *netdev = adapter->netdev;
1491 struct pci_dev *pdev = adapter->pdev;
5f450212 1492 union e1000_rx_desc_extended *rx_desc, *next_rxd;
97ac8cae 1493 struct e1000_buffer *buffer_info, *next_buffer;
5f450212 1494 u32 length, staterr;
97ac8cae
BA		 1495 unsigned int i;
1496 int cleaned_count = 0;
1497 bool cleaned = false;
362e20ca 1498 unsigned int total_rx_bytes = 0, total_rx_packets = 0;
17e813ec 1499 struct skb_shared_info *shinfo;
97ac8cae
BA		 1500
1501 i = rx_ring->next_to_clean;
5f450212
BA		 1502 rx_desc = E1000_RX_DESC_EXT(*rx_ring, i);
1503 staterr = le32_to_cpu(rx_desc->wb.upper.status_error);
97ac8cae
BA		 1504 buffer_info = &rx_ring->buffer_info[i];
1505
5f450212 1506 while (staterr & E1000_RXD_STAT_DD) {
97ac8cae 1507 struct sk_buff *skb;
97ac8cae
BA		 1508
1509 if (*work_done >= work_to_do)
1510 break;
1511 (*work_done)++;
2d0bb1c1 1512 rmb(); /* read descriptor and rx_buffer_info after status DD */
97ac8cae 1513
97ac8cae
BA		 1514 skb = buffer_info->skb;
1515 buffer_info->skb = NULL;
1516
1517 ++i;
1518 if (i == rx_ring->count)
1519 i = 0;
5f450212 1520 next_rxd = E1000_RX_DESC_EXT(*rx_ring, i);
97ac8cae
BA		 1521 prefetch(next_rxd);
1522
1523 next_buffer = &rx_ring->buffer_info[i];
1524
1525 cleaned = true;
1526 cleaned_count++;
0be3f55f
NN		 1527 dma_unmap_page(&pdev->dev, buffer_info->dma, PAGE_SIZE,
1528 DMA_FROM_DEVICE);
97ac8cae
BA		 1529 buffer_info->dma = 0;
1530
5f450212 1531 length = le16_to_cpu(rx_desc->wb.upper.length);
97ac8cae
BA		 1532
1533 /* errors is only valid for DD + EOP descriptors */
5f450212 1534 if (unlikely((staterr & E1000_RXD_STAT_EOP) &&
cf955e6c
BG		 1535 ((staterr & E1000_RXDEXT_ERR_FRAME_ERR_MASK) &&
1536 !(netdev->features & NETIF_F_RXALL)))) {
5f450212
BA		 1537 /* recycle both page and skb */
1538 buffer_info->skb = skb;
1539 /* an error means any chain goes out the window too */
1540 if (rx_ring->rx_skb_top)
1541 dev_kfree_skb_irq(rx_ring->rx_skb_top);
1542 rx_ring->rx_skb_top = NULL;
1543 goto next_desc;
97ac8cae
BA		 1544 }
1545
f0f1a172 1546#define rxtop (rx_ring->rx_skb_top)
5f450212 1547 if (!(staterr & E1000_RXD_STAT_EOP)) {
97ac8cae
BA		 1548 /* this descriptor is only the beginning (or middle) */
1549 if (!rxtop) {
1550 /* this is the beginning of a chain */
1551 rxtop = skb;
1552 skb_fill_page_desc(rxtop, 0, buffer_info->page,
f0ff4398 1553 0, length);
97ac8cae
BA		 1554 } else {
1555 /* this is the middle of a chain */
17e813ec
BA		 1556 shinfo = skb_shinfo(rxtop);
1557 skb_fill_page_desc(rxtop, shinfo->nr_frags,
1558 buffer_info->page, 0,
1559 length);
97ac8cae
BA		 1560 /* re-use the skb, only consumed the page */
1561 buffer_info->skb = skb;
1562 }
1563 e1000_consume_page(buffer_info, rxtop, length);
1564 goto next_desc;
1565 } else {
1566 if (rxtop) {
1567 /* end of the chain */
17e813ec
BA		 1568 shinfo = skb_shinfo(rxtop);
1569 skb_fill_page_desc(rxtop, shinfo->nr_frags,
1570 buffer_info->page, 0,
1571 length);
97ac8cae 1572 /* re-use the current skb, we only consumed the
e921eb1a
BA		 1573 * page
1574 */
97ac8cae
BA		 1575 buffer_info->skb = skb;
1576 skb = rxtop;
1577 rxtop = NULL;
1578 e1000_consume_page(buffer_info, skb, length);
1579 } else {
1580 /* no chain, got EOP, this buf is the packet
e921eb1a
BA		 1581 * copybreak to save the put_page/alloc_page
1582 */
97ac8cae
BA		 1583 if (length <= copybreak &&
1584 skb_tailroom(skb) >= length) {
1585 u8 *vaddr;
4679026d 1586 vaddr = kmap_atomic(buffer_info->page);
97ac8cae
BA		 1587 memcpy(skb_tail_pointer(skb), vaddr,
1588 length);
4679026d 1589 kunmap_atomic(vaddr);
97ac8cae 1590 /* re-use the page, so don't erase
e921eb1a
BA		 1591 * buffer_info->page
1592 */
97ac8cae
BA		 1593 skb_put(skb, length);
1594 } else {
1595 skb_fill_page_desc(skb, 0,
f0ff4398
BA		 1596 buffer_info->page, 0,
1597 length);
97ac8cae 1598 e1000_consume_page(buffer_info, skb,
f0ff4398 1599 length);
97ac8cae
BA		 1600 }
1601 }
1602 }
1603
2e1706f2
BA		 1604 /* Receive Checksum Offload */
1605 e1000_rx_checksum(adapter, staterr, skb);
97ac8cae 1606
70495a50
BA		 1607 e1000_rx_hash(netdev, rx_desc->wb.lower.hi_dword.rss, skb);
1608
97ac8cae
BA		 1609 /* probably a little skewed due to removing CRC */
1610 total_rx_bytes += skb->len;
1611 total_rx_packets++;
1612
1613 /* eth type trans needs skb->data to point to something */
1614 if (!pskb_may_pull(skb, ETH_HLEN)) {
44defeb3 1615 e_err("pskb_may_pull failed.\n");
ef5ab89c 1616 dev_kfree_skb_irq(skb);
97ac8cae
BA		 1617 goto next_desc;
1618 }
1619
5f450212
BA		 1620 e1000_receive_skb(adapter, netdev, skb, staterr,
1621 rx_desc->wb.upper.vlan);
97ac8cae
BA		 1622
1623next_desc:
5f450212 1624 rx_desc->wb.upper.status_error &= cpu_to_le32(~0xFF);
97ac8cae
BA		 1625
1626 /* return some buffers to hardware, one at a time is too slow */
1627 if (unlikely(cleaned_count >= E1000_RX_BUFFER_WRITE)) {
55aa6985 1628 adapter->alloc_rx_buf(rx_ring, cleaned_count,
c2fed996 1629 GFP_ATOMIC);
97ac8cae
BA		 1630 cleaned_count = 0;
1631 }
1632
1633 /* use prefetched values */
1634 rx_desc = next_rxd;
1635 buffer_info = next_buffer;
5f450212
BA		 1636
1637 staterr = le32_to_cpu(rx_desc->wb.upper.status_error);
97ac8cae
BA		 1638 }
1639 rx_ring->next_to_clean = i;
1640
1641 cleaned_count = e1000_desc_unused(rx_ring);
1642 if (cleaned_count)
55aa6985 1643 adapter->alloc_rx_buf(rx_ring, cleaned_count, GFP_ATOMIC);
97ac8cae
BA		 1644
1645 adapter->total_rx_bytes += total_rx_bytes;
1646 adapter->total_rx_packets += total_rx_packets;
97ac8cae
BA		 1647 return cleaned;
1648}
1649
bc7f75fa
AK		 1650/**
1651 * e1000_clean_rx_ring - Free Rx Buffers per Queue
55aa6985 1652 * @rx_ring: Rx descriptor ring
bc7f75fa 1653 **/
55aa6985 1654static void e1000_clean_rx_ring(struct e1000_ring *rx_ring)
bc7f75fa 1655{
55aa6985 1656 struct e1000_adapter *adapter = rx_ring->adapter;
bc7f75fa
AK		 1657 struct e1000_buffer *buffer_info;
1658 struct e1000_ps_page *ps_page;
1659 struct pci_dev *pdev = adapter->pdev;
bc7f75fa
AK		 1660 unsigned int i, j;
1661
1662 /* Free all the Rx ring sk_buffs */
1663 for (i = 0; i < rx_ring->count; i++) {
1664 buffer_info = &rx_ring->buffer_info[i];
1665 if (buffer_info->dma) {
1666 if (adapter->clean_rx == e1000_clean_rx_irq)
0be3f55f 1667 dma_unmap_single(&pdev->dev, buffer_info->dma,
bc7f75fa 1668 adapter->rx_buffer_len,
0be3f55f 1669 DMA_FROM_DEVICE);
97ac8cae 1670 else if (adapter->clean_rx == e1000_clean_jumbo_rx_irq)
0be3f55f 1671 dma_unmap_page(&pdev->dev, buffer_info->dma,
f0ff4398 1672 PAGE_SIZE, DMA_FROM_DEVICE);
bc7f75fa 1673 else if (adapter->clean_rx == e1000_clean_rx_irq_ps)
0be3f55f 1674 dma_unmap_single(&pdev->dev, buffer_info->dma,
bc7f75fa 1675 adapter->rx_ps_bsize0,
0be3f55f 1676 DMA_FROM_DEVICE);
bc7f75fa
AK		 1677 buffer_info->dma = 0;
1678 }
1679
97ac8cae
BA		 1680 if (buffer_info->page) {
1681 put_page(buffer_info->page);
1682 buffer_info->page = NULL;
1683 }
1684
bc7f75fa
AK		 1685 if (buffer_info->skb) {
1686 dev_kfree_skb(buffer_info->skb);
1687 buffer_info->skb = NULL;
1688 }
1689
1690 for (j = 0; j < PS_PAGE_BUFFERS; j++) {
47f44e40 1691 ps_page = &buffer_info->ps_pages[j];
bc7f75fa
AK		 1692 if (!ps_page->page)
1693 break;
0be3f55f
NN		 1694 dma_unmap_page(&pdev->dev, ps_page->dma, PAGE_SIZE,
1695 DMA_FROM_DEVICE);
bc7f75fa
AK		 1696 ps_page->dma = 0;
1697 put_page(ps_page->page);
1698 ps_page->page = NULL;
1699 }
1700 }
1701
1702 /* there also may be some cached data from a chained receive */
1703 if (rx_ring->rx_skb_top) {
1704 dev_kfree_skb(rx_ring->rx_skb_top);
1705 rx_ring->rx_skb_top = NULL;
1706 }
1707
bc7f75fa
AK		 1708 /* Zero out the descriptor ring */
1709 memset(rx_ring->desc, 0, rx_ring->size);
1710
1711 rx_ring->next_to_clean = 0;
1712 rx_ring->next_to_use = 0;
b94b5028 1713 adapter->flags2 &= ~FLAG2_IS_DISCARDING;
bc7f75fa 1714
c5083cf6 1715 writel(0, rx_ring->head);
bdc125f7
BA		 1716 if (rx_ring->adapter->flags2 & FLAG2_PCIM2PCI_ARBITER_WA)
1717 e1000e_update_rdt_wa(rx_ring, 0);
1718 else
1719 writel(0, rx_ring->tail);
bc7f75fa
AK		 1720}
1721
a8f88ff5
JB		 1722static void e1000e_downshift_workaround(struct work_struct *work)
1723{
1724 struct e1000_adapter *adapter = container_of(work,
17e813ec
BA		 1725 struct e1000_adapter,
1726 downshift_task);
a8f88ff5 1727
615b32af
JB		 1728 if (test_bit(__E1000_DOWN, &adapter->state))
1729 return;
1730
a8f88ff5
JB		 1731 e1000e_gig_downshift_workaround_ich8lan(&adapter->hw);
1732}
1733
bc7f75fa
AK		 1734/**
1735 * e1000_intr_msi - Interrupt Handler
1736 * @irq: interrupt number
1737 * @data: pointer to a network interface device structure
1738 **/
8bb62869 1739static irqreturn_t e1000_intr_msi(int __always_unused irq, void *data)
bc7f75fa
AK		 1740{
1741 struct net_device *netdev = data;
1742 struct e1000_adapter *adapter = netdev_priv(netdev);
1743 struct e1000_hw *hw = &adapter->hw;
1744 u32 icr = er32(ICR);
1745
e921eb1a 1746 /* read ICR disables interrupts using IAM */
573cca8c 1747 if (icr & E1000_ICR_LSC) {
f92518dd 1748 hw->mac.get_link_status = true;
e921eb1a 1749 /* ICH8 workaround-- Call gig speed drop workaround on cable
ad68076e
BA		 1750 * disconnect (LSC) before accessing any PHY registers
1751 */
bc7f75fa
AK		 1752 if ((adapter->flags & FLAG_LSC_GIG_SPEED_DROP) &&
1753 (!(er32(STATUS) & E1000_STATUS_LU)))
a8f88ff5 1754 schedule_work(&adapter->downshift_task);
bc7f75fa 1755
e921eb1a 1756 /* 80003ES2LAN workaround-- For packet buffer work-around on
bc7f75fa 1757 * link down event; disable receives here in the ISR and reset
ad68076e
BA		 1758 * adapter in watchdog
1759 */
bc7f75fa
AK		 1760 if (netif_carrier_ok(netdev) &&
1761 adapter->flags & FLAG_RX_NEEDS_RESTART) {
1762 /* disable receives */
1763 u32 rctl = er32(RCTL);
1764 ew32(RCTL, rctl & ~E1000_RCTL_EN);
12d43f7d 1765 adapter->flags |= FLAG_RESTART_NOW;
bc7f75fa
AK		 1766 }
1767 /* guard against interrupt when we're going down */
1768 if (!test_bit(__E1000_DOWN, &adapter->state))
1769 mod_timer(&adapter->watchdog_timer, jiffies + 1);
1770 }
1771
94fb848b
BA		 1772 /* Reset on uncorrectable ECC error */
1773 if ((icr & E1000_ICR_ECCER) && (hw->mac.type == e1000_pch_lpt)) {
1774 u32 pbeccsts = er32(PBECCSTS);
1775
1776 adapter->corr_errors +=
1777 pbeccsts & E1000_PBECCSTS_CORR_ERR_CNT_MASK;
1778 adapter->uncorr_errors +=
1779 (pbeccsts & E1000_PBECCSTS_UNCORR_ERR_CNT_MASK) >>
1780 E1000_PBECCSTS_UNCORR_ERR_CNT_SHIFT;
1781
1782 /* Do the reset outside of interrupt context */
1783 schedule_work(&adapter->reset_task);
1784
1785 /* return immediately since reset is imminent */
1786 return IRQ_HANDLED;
1787 }
1788
288379f0 1789 if (napi_schedule_prep(&adapter->napi)) {
bc7f75fa
AK		 1790 adapter->total_tx_bytes = 0;
1791 adapter->total_tx_packets = 0;
1792 adapter->total_rx_bytes = 0;
1793 adapter->total_rx_packets = 0;
288379f0 1794 __napi_schedule(&adapter->napi);
bc7f75fa
AK		 1795 }
1796
1797 return IRQ_HANDLED;
1798}
1799
1800/**
1801 * e1000_intr - Interrupt Handler
1802 * @irq: interrupt number
1803 * @data: pointer to a network interface device structure
1804 **/
8bb62869 1805static irqreturn_t e1000_intr(int __always_unused irq, void *data)
bc7f75fa
AK		 1806{
1807 struct net_device *netdev = data;
1808 struct e1000_adapter *adapter = netdev_priv(netdev);
1809 struct e1000_hw *hw = &adapter->hw;
bc7f75fa 1810 u32 rctl, icr = er32(ICR);
4662e82b 1811
a68ea775 1812 if (!icr || test_bit(__E1000_DOWN, &adapter->state))
bc7f75fa
AK		 1813 return IRQ_NONE; /* Not our interrupt */
1814
e921eb1a 1815 /* IMS will not auto-mask if INT_ASSERTED is not set, and if it is
ad68076e
BA		 1816 * not set, then the adapter didn't send an interrupt
1817 */
bc7f75fa
AK		 1818 if (!(icr & E1000_ICR_INT_ASSERTED))
1819 return IRQ_NONE;
1820
e921eb1a 1821 /* Interrupt Auto-Mask...upon reading ICR,
ad68076e
BA		 1822 * interrupts are masked. No need for the
1823 * IMC write
1824 */
bc7f75fa 1825
573cca8c 1826 if (icr & E1000_ICR_LSC) {
f92518dd 1827 hw->mac.get_link_status = true;
e921eb1a 1828 /* ICH8 workaround-- Call gig speed drop workaround on cable
ad68076e
BA		 1829 * disconnect (LSC) before accessing any PHY registers
1830 */
bc7f75fa
AK		 1831 if ((adapter->flags & FLAG_LSC_GIG_SPEED_DROP) &&
1832 (!(er32(STATUS) & E1000_STATUS_LU)))
a8f88ff5 1833 schedule_work(&adapter->downshift_task);
bc7f75fa 1834
e921eb1a 1835 /* 80003ES2LAN workaround--
bc7f75fa
AK		 1836 * For packet buffer work-around on link down event;
1837 * disable receives here in the ISR and
1838 * reset adapter in watchdog
1839 */
1840 if (netif_carrier_ok(netdev) &&
1841 (adapter->flags & FLAG_RX_NEEDS_RESTART)) {
1842 /* disable receives */
1843 rctl = er32(RCTL);
1844 ew32(RCTL, rctl & ~E1000_RCTL_EN);
12d43f7d 1845 adapter->flags |= FLAG_RESTART_NOW;
bc7f75fa
AK		 1846 }
1847 /* guard against interrupt when we're going down */
1848 if (!test_bit(__E1000_DOWN, &adapter->state))
1849 mod_timer(&adapter->watchdog_timer, jiffies + 1);
1850 }
1851
94fb848b
BA		 1852 /* Reset on uncorrectable ECC error */
1853 if ((icr & E1000_ICR_ECCER) && (hw->mac.type == e1000_pch_lpt)) {
1854 u32 pbeccsts = er32(PBECCSTS);
1855
1856 adapter->corr_errors +=
1857 pbeccsts & E1000_PBECCSTS_CORR_ERR_CNT_MASK;
1858 adapter->uncorr_errors +=
1859 (pbeccsts & E1000_PBECCSTS_UNCORR_ERR_CNT_MASK) >>
1860 E1000_PBECCSTS_UNCORR_ERR_CNT_SHIFT;
1861
1862 /* Do the reset outside of interrupt context */
1863 schedule_work(&adapter->reset_task);
1864
1865 /* return immediately since reset is imminent */
1866 return IRQ_HANDLED;
1867 }
1868
288379f0 1869 if (napi_schedule_prep(&adapter->napi)) {
bc7f75fa
AK		 1870 adapter->total_tx_bytes = 0;
1871 adapter->total_tx_packets = 0;
1872 adapter->total_rx_bytes = 0;
1873 adapter->total_rx_packets = 0;
288379f0 1874 __napi_schedule(&adapter->napi);
bc7f75fa
AK		 1875 }
1876
1877 return IRQ_HANDLED;
1878}
1879
8bb62869 1880static irqreturn_t e1000_msix_other(int __always_unused irq, void *data)
4662e82b
BA		 1881{
1882 struct net_device *netdev = data;
1883 struct e1000_adapter *adapter = netdev_priv(netdev);
1884 struct e1000_hw *hw = &adapter->hw;
1885 u32 icr = er32(ICR);
1886
1887 if (!(icr & E1000_ICR_INT_ASSERTED)) {
a3c69fef
JB		 1888 if (!test_bit(__E1000_DOWN, &adapter->state))
1889 ew32(IMS, E1000_IMS_OTHER);
4662e82b
BA		 1890 return IRQ_NONE;
1891 }
1892
1893 if (icr & adapter->eiac_mask)
1894 ew32(ICS, (icr & adapter->eiac_mask));
1895
1896 if (icr & E1000_ICR_OTHER) {
1897 if (!(icr & E1000_ICR_LSC))
1898 goto no_link_interrupt;
f92518dd 1899 hw->mac.get_link_status = true;
4662e82b
BA		 1900 /* guard against interrupt when we're going down */
1901 if (!test_bit(__E1000_DOWN, &adapter->state))
1902 mod_timer(&adapter->watchdog_timer, jiffies + 1);
1903 }
1904
1905no_link_interrupt:
a3c69fef
JB		 1906 if (!test_bit(__E1000_DOWN, &adapter->state))
1907 ew32(IMS, E1000_IMS_LSC | E1000_IMS_OTHER);
4662e82b
BA		 1908
1909 return IRQ_HANDLED;
1910}
1911
8bb62869 1912static irqreturn_t e1000_intr_msix_tx(int __always_unused irq, void *data)
4662e82b
BA		 1913{
1914 struct net_device *netdev = data;
1915 struct e1000_adapter *adapter = netdev_priv(netdev);
1916 struct e1000_hw *hw = &adapter->hw;
1917 struct e1000_ring *tx_ring = adapter->tx_ring;
1918
1919
1920 adapter->total_tx_bytes = 0;
1921 adapter->total_tx_packets = 0;
1922
55aa6985 1923 if (!e1000_clean_tx_irq(tx_ring))
4662e82b
BA		 1924 /* Ring was not completely cleaned, so fire another interrupt */
1925 ew32(ICS, tx_ring->ims_val);
1926
1927 return IRQ_HANDLED;
1928}
1929
8bb62869 1930static irqreturn_t e1000_intr_msix_rx(int __always_unused irq, void *data)
4662e82b
BA		 1931{
1932 struct net_device *netdev = data;
1933 struct e1000_adapter *adapter = netdev_priv(netdev);
55aa6985 1934 struct e1000_ring *rx_ring = adapter->rx_ring;
4662e82b
BA		 1935
1936 /* Write the ITR value calculated at the end of the
1937 * previous interrupt.
1938 */
55aa6985
BA		 1939 if (rx_ring->set_itr) {
1940 writel(1000000000 / (rx_ring->itr_val * 256),
1941 rx_ring->itr_register);
1942 rx_ring->set_itr = 0;
4662e82b
BA		 1943 }
1944
288379f0 1945 if (napi_schedule_prep(&adapter->napi)) {
4662e82b
BA		 1946 adapter->total_rx_bytes = 0;
1947 adapter->total_rx_packets = 0;
288379f0 1948 __napi_schedule(&adapter->napi);
4662e82b
BA		 1949 }
1950 return IRQ_HANDLED;
1951}
1952
1953/**
1954 * e1000_configure_msix - Configure MSI-X hardware
1955 *
1956 * e1000_configure_msix sets up the hardware to properly
1957 * generate MSI-X interrupts.
1958 **/
1959static void e1000_configure_msix(struct e1000_adapter *adapter)
1960{
1961 struct e1000_hw *hw = &adapter->hw;
1962 struct e1000_ring *rx_ring = adapter->rx_ring;
1963 struct e1000_ring *tx_ring = adapter->tx_ring;
1964 int vector = 0;
1965 u32 ctrl_ext, ivar = 0;
1966
1967 adapter->eiac_mask = 0;
1968
1969 /* Workaround issue with spurious interrupts on 82574 in MSI-X mode */
1970 if (hw->mac.type == e1000_82574) {
1971 u32 rfctl = er32(RFCTL);
1972 rfctl |= E1000_RFCTL_ACK_DIS;
1973 ew32(RFCTL, rfctl);
1974 }
1975
1976#define E1000_IVAR_INT_ALLOC_VALID 0x8
1977 /* Configure Rx vector */
1978 rx_ring->ims_val = E1000_IMS_RXQ0;
1979 adapter->eiac_mask |= rx_ring->ims_val;
1980 if (rx_ring->itr_val)
1981 writel(1000000000 / (rx_ring->itr_val * 256),
c5083cf6 1982 rx_ring->itr_register);
4662e82b 1983 else
c5083cf6 1984 writel(1, rx_ring->itr_register);
4662e82b
BA		 1985 ivar = E1000_IVAR_INT_ALLOC_VALID | vector;
1986
1987 /* Configure Tx vector */
1988 tx_ring->ims_val = E1000_IMS_TXQ0;
1989 vector++;
1990 if (tx_ring->itr_val)
1991 writel(1000000000 / (tx_ring->itr_val * 256),
c5083cf6 1992 tx_ring->itr_register);
4662e82b 1993 else
c5083cf6 1994 writel(1, tx_ring->itr_register);
4662e82b
BA		 1995 adapter->eiac_mask |= tx_ring->ims_val;
1996 ivar |= ((E1000_IVAR_INT_ALLOC_VALID | vector) << 8);
1997
1998 /* set vector for Other Causes, e.g. link changes */
1999 vector++;
2000 ivar |= ((E1000_IVAR_INT_ALLOC_VALID | vector) << 16);
2001 if (rx_ring->itr_val)
2002 writel(1000000000 / (rx_ring->itr_val * 256),
2003 hw->hw_addr + E1000_EITR_82574(vector));
2004 else
2005 writel(1, hw->hw_addr + E1000_EITR_82574(vector));
2006
2007 /* Cause Tx interrupts on every write back */
2008 ivar |= (1 << 31);
2009
2010 ew32(IVAR, ivar);
2011
2012 /* enable MSI-X PBA support */
2013 ctrl_ext = er32(CTRL_EXT);
2014 ctrl_ext |= E1000_CTRL_EXT_PBA_CLR;
2015
2016 /* Auto-Mask Other interrupts upon ICR read */
4662e82b
BA		 2017 ew32(IAM, ~E1000_EIAC_MASK_82574 | E1000_IMS_OTHER);
2018 ctrl_ext |= E1000_CTRL_EXT_EIAME;
2019 ew32(CTRL_EXT, ctrl_ext);
2020 e1e_flush();
2021}
2022
2023void e1000e_reset_interrupt_capability(struct e1000_adapter *adapter)
2024{
2025 if (adapter->msix_entries) {
2026 pci_disable_msix(adapter->pdev);
2027 kfree(adapter->msix_entries);
2028 adapter->msix_entries = NULL;
2029 } else if (adapter->flags & FLAG_MSI_ENABLED) {
2030 pci_disable_msi(adapter->pdev);
2031 adapter->flags &= ~FLAG_MSI_ENABLED;
2032 }
4662e82b
BA		 2033}
2034
2035/**
2036 * e1000e_set_interrupt_capability - set MSI or MSI-X if supported
2037 *
2038 * Attempt to configure interrupts using the best available
2039 * capabilities of the hardware and kernel.
2040 **/
2041void e1000e_set_interrupt_capability(struct e1000_adapter *adapter)
2042{
2043 int err;
8e86acd7 2044 int i;
4662e82b
BA		 2045
2046 switch (adapter->int_mode) {
2047 case E1000E_INT_MODE_MSIX:
2048 if (adapter->flags & FLAG_HAS_MSIX) {
8e86acd7
JK		 2049 adapter->num_vectors = 3; /* RxQ0, TxQ0 and other */
2050 adapter->msix_entries = kcalloc(adapter->num_vectors,
17e813ec
BA		 2051 sizeof(struct
2052 msix_entry),
2053 GFP_KERNEL);
4662e82b 2054 if (adapter->msix_entries) {
8e86acd7 2055 for (i = 0; i < adapter->num_vectors; i++)
4662e82b
BA		 2056 adapter->msix_entries[i].entry = i;
2057
2058 err = pci_enable_msix(adapter->pdev,
2059 adapter->msix_entries,
8e86acd7 2060 adapter->num_vectors);
b1cdfead 2061 if (err == 0)
4662e82b
BA		 2062 return;
2063 }
2064 /* MSI-X failed, so fall through and try MSI */
ef456f85 2065 e_err("Failed to initialize MSI-X interrupts. Falling back to MSI interrupts.\n");
4662e82b
BA		 2066 e1000e_reset_interrupt_capability(adapter);
2067 }
2068 adapter->int_mode = E1000E_INT_MODE_MSI;
2069 /* Fall through */
2070 case E1000E_INT_MODE_MSI:
2071 if (!pci_enable_msi(adapter->pdev)) {
2072 adapter->flags |= FLAG_MSI_ENABLED;
2073 } else {
2074 adapter->int_mode = E1000E_INT_MODE_LEGACY;
ef456f85 2075 e_err("Failed to initialize MSI interrupts. Falling back to legacy interrupts.\n");
4662e82b
BA		 2076 }
2077 /* Fall through */
2078 case E1000E_INT_MODE_LEGACY:
2079 /* Don't do anything; this is the system default */
2080 break;
2081 }
8e86acd7
JK		 2082
2083 /* store the number of vectors being used */
2084 adapter->num_vectors = 1;
4662e82b
BA		 2085}
2086
2087/**
2088 * e1000_request_msix - Initialize MSI-X interrupts
2089 *
2090 * e1000_request_msix allocates MSI-X vectors and requests interrupts from the
2091 * kernel.
2092 **/
2093static int e1000_request_msix(struct e1000_adapter *adapter)
2094{
2095 struct net_device *netdev = adapter->netdev;
2096 int err = 0, vector = 0;
2097
2098 if (strlen(netdev->name) < (IFNAMSIZ - 5))
79f5e840
BA		 2099 snprintf(adapter->rx_ring->name,
2100 sizeof(adapter->rx_ring->name) - 1,
2101 "%s-rx-0", netdev->name);
4662e82b
BA		 2102 else
2103 memcpy(adapter->rx_ring->name, netdev->name, IFNAMSIZ);
2104 err = request_irq(adapter->msix_entries[vector].vector,
a0607fd3 2105 e1000_intr_msix_rx, 0, adapter->rx_ring->name,
4662e82b
BA		 2106 netdev);
2107 if (err)
5015e53a 2108 return err;
c5083cf6
BA		 2109 adapter->rx_ring->itr_register = adapter->hw.hw_addr +
2110 E1000_EITR_82574(vector);
4662e82b
BA		 2111 adapter->rx_ring->itr_val = adapter->itr;
2112 vector++;
2113
2114 if (strlen(netdev->name) < (IFNAMSIZ - 5))
79f5e840
BA		 2115 snprintf(adapter->tx_ring->name,
2116 sizeof(adapter->tx_ring->name) - 1,
2117 "%s-tx-0", netdev->name);
4662e82b
BA		 2118 else
2119 memcpy(adapter->tx_ring->name, netdev->name, IFNAMSIZ);
2120 err = request_irq(adapter->msix_entries[vector].vector,
a0607fd3 2121 e1000_intr_msix_tx, 0, adapter->tx_ring->name,
4662e82b
BA		 2122 netdev);
2123 if (err)
5015e53a 2124 return err;
c5083cf6
BA		 2125 adapter->tx_ring->itr_register = adapter->hw.hw_addr +
2126 E1000_EITR_82574(vector);
4662e82b
BA		 2127 adapter->tx_ring->itr_val = adapter->itr;
2128 vector++;
2129
2130 err = request_irq(adapter->msix_entries[vector].vector,
a0607fd3 2131 e1000_msix_other, 0, netdev->name, netdev);
4662e82b 2132 if (err)
5015e53a 2133 return err;
4662e82b
BA		 2134
2135 e1000_configure_msix(adapter);
5015e53a 2136
4662e82b 2137 return 0;
4662e82b
BA		 2138}
2139
f8d59f78
BA		 2140/**
2141 * e1000_request_irq - initialize interrupts
2142 *
2143 * Attempts to configure interrupts using the best available
2144 * capabilities of the hardware and kernel.
2145 **/
bc7f75fa
AK		 2146static int e1000_request_irq(struct e1000_adapter *adapter)
2147{
2148 struct net_device *netdev = adapter->netdev;
bc7f75fa
AK		 2149 int err;
2150
4662e82b
BA		 2151 if (adapter->msix_entries) {
2152 err = e1000_request_msix(adapter);
2153 if (!err)
2154 return err;
2155 /* fall back to MSI */
2156 e1000e_reset_interrupt_capability(adapter);
2157 adapter->int_mode = E1000E_INT_MODE_MSI;
2158 e1000e_set_interrupt_capability(adapter);
bc7f75fa 2159 }
4662e82b 2160 if (adapter->flags & FLAG_MSI_ENABLED) {
a0607fd3 2161 err = request_irq(adapter->pdev->irq, e1000_intr_msi, 0,
4662e82b
BA		 2162 netdev->name, netdev);
2163 if (!err)
2164 return err;
bc7f75fa 2165
4662e82b
BA		 2166 /* fall back to legacy interrupt */
2167 e1000e_reset_interrupt_capability(adapter);
2168 adapter->int_mode = E1000E_INT_MODE_LEGACY;
bc7f75fa
AK		 2169 }
2170
a0607fd3 2171 err = request_irq(adapter->pdev->irq, e1000_intr, IRQF_SHARED,
4662e82b
BA		 2172 netdev->name, netdev);
2173 if (err)
2174 e_err("Unable to allocate interrupt, Error: %d\n", err);
2175
bc7f75fa
AK		 2176 return err;
2177}
2178
2179static void e1000_free_irq(struct e1000_adapter *adapter)
2180{
2181 struct net_device *netdev = adapter->netdev;
2182
4662e82b
BA		 2183 if (adapter->msix_entries) {
2184 int vector = 0;
2185
2186 free_irq(adapter->msix_entries[vector].vector, netdev);
2187 vector++;
2188
2189 free_irq(adapter->msix_entries[vector].vector, netdev);
2190 vector++;
2191
2192 /* Other Causes interrupt vector */
2193 free_irq(adapter->msix_entries[vector].vector, netdev);
2194 return;
bc7f75fa 2195 }
4662e82b
BA		 2196
2197 free_irq(adapter->pdev->irq, netdev);
bc7f75fa
AK		 2198}
2199
2200/**
2201 * e1000_irq_disable - Mask off interrupt generation on the NIC
2202 **/
2203static void e1000_irq_disable(struct e1000_adapter *adapter)
2204{
2205 struct e1000_hw *hw = &adapter->hw;
2206
bc7f75fa 2207 ew32(IMC, ~0);
4662e82b
BA		 2208 if (adapter->msix_entries)
2209 ew32(EIAC_82574, 0);
bc7f75fa 2210 e1e_flush();
8e86acd7
JK		 2211
2212 if (adapter->msix_entries) {
2213 int i;
2214 for (i = 0; i < adapter->num_vectors; i++)
2215 synchronize_irq(adapter->msix_entries[i].vector);
2216 } else {
2217 synchronize_irq(adapter->pdev->irq);
2218 }
bc7f75fa
AK		 2219}
2220
2221/**
2222 * e1000_irq_enable - Enable default interrupt generation settings
2223 **/
2224static void e1000_irq_enable(struct e1000_adapter *adapter)
2225{
2226 struct e1000_hw *hw = &adapter->hw;
2227
4662e82b
BA		 2228 if (adapter->msix_entries) {
2229 ew32(EIAC_82574, adapter->eiac_mask & E1000_EIAC_MASK_82574);
2230 ew32(IMS, adapter->eiac_mask | E1000_IMS_OTHER | E1000_IMS_LSC);
94fb848b
BA		 2231 } else if (hw->mac.type == e1000_pch_lpt) {
2232 ew32(IMS, IMS_ENABLE_MASK | E1000_IMS_ECCER);
4662e82b
BA		 2233 } else {
2234 ew32(IMS, IMS_ENABLE_MASK);
2235 }
74ef9c39 2236 e1e_flush();
bc7f75fa
AK		 2237}
2238
2239/**
31dbe5b4 2240 * e1000e_get_hw_control - get control of the h/w from f/w
bc7f75fa
AK		 2241 * @adapter: address of board private structure
2242 *
31dbe5b4 2243 * e1000e_get_hw_control sets {CTRL_EXT|SWSM}:DRV_LOAD bit.
bc7f75fa
AK		 2244 * For ASF and Pass Through versions of f/w this means that
2245 * the driver is loaded. For AMT version (only with 82573)
2246 * of the f/w this means that the network i/f is open.
2247 **/
31dbe5b4 2248void e1000e_get_hw_control(struct e1000_adapter *adapter)
bc7f75fa
AK		 2249{
2250 struct e1000_hw *hw = &adapter->hw;
2251 u32 ctrl_ext;
2252 u32 swsm;
2253
2254 /* Let firmware know the driver has taken over */
2255 if (adapter->flags & FLAG_HAS_SWSM_ON_LOAD) {
2256 swsm = er32(SWSM);
2257 ew32(SWSM, swsm | E1000_SWSM_DRV_LOAD);
2258 } else if (adapter->flags & FLAG_HAS_CTRLEXT_ON_LOAD) {
2259 ctrl_ext = er32(CTRL_EXT);
ad68076e 2260 ew32(CTRL_EXT, ctrl_ext | E1000_CTRL_EXT_DRV_LOAD);
bc7f75fa
AK		 2261 }
2262}
2263
2264/**
31dbe5b4 2265 * e1000e_release_hw_control - release control of the h/w to f/w
bc7f75fa
AK		 2266 * @adapter: address of board private structure
2267 *
31dbe5b4 2268 * e1000e_release_hw_control resets {CTRL_EXT|SWSM}:DRV_LOAD bit.
bc7f75fa
AK		 2269 * For ASF and Pass Through versions of f/w this means that the
2270 * driver is no longer loaded. For AMT version (only with 82573) i
2271 * of the f/w this means that the network i/f is closed.
2272 *
2273 **/
31dbe5b4 2274void e1000e_release_hw_control(struct e1000_adapter *adapter)
bc7f75fa
AK		 2275{
2276 struct e1000_hw *hw = &adapter->hw;
2277 u32 ctrl_ext;
2278 u32 swsm;
2279
2280 /* Let firmware taken over control of h/w */
2281 if (adapter->flags & FLAG_HAS_SWSM_ON_LOAD) {
2282 swsm = er32(SWSM);
2283 ew32(SWSM, swsm & ~E1000_SWSM_DRV_LOAD);
2284 } else if (adapter->flags & FLAG_HAS_CTRLEXT_ON_LOAD) {
2285 ctrl_ext = er32(CTRL_EXT);
ad68076e 2286 ew32(CTRL_EXT, ctrl_ext & ~E1000_CTRL_EXT_DRV_LOAD);
bc7f75fa
AK		 2287 }
2288}
2289
bc7f75fa 2290/**
49ce9c2c 2291 * e1000_alloc_ring_dma - allocate memory for a ring structure
bc7f75fa
AK		 2292 **/
2293static int e1000_alloc_ring_dma(struct e1000_adapter *adapter,
2294 struct e1000_ring *ring)
2295{
2296 struct pci_dev *pdev = adapter->pdev;
2297
2298 ring->desc = dma_alloc_coherent(&pdev->dev, ring->size, &ring->dma,
2299 GFP_KERNEL);
2300 if (!ring->desc)
2301 return -ENOMEM;
2302
2303 return 0;
2304}
2305
2306/**
2307 * e1000e_setup_tx_resources - allocate Tx resources (Descriptors)
55aa6985 2308 * @tx_ring: Tx descriptor ring
bc7f75fa
AK		 2309 *
2310 * Return 0 on success, negative on failure
2311 **/
55aa6985 2312int e1000e_setup_tx_resources(struct e1000_ring *tx_ring)
bc7f75fa 2313{
55aa6985 2314 struct e1000_adapter *adapter = tx_ring->adapter;
bc7f75fa
AK		 2315 int err = -ENOMEM, size;
2316
2317 size = sizeof(struct e1000_buffer) * tx_ring->count;
89bf67f1 2318 tx_ring->buffer_info = vzalloc(size);
bc7f75fa
AK		 2319 if (!tx_ring->buffer_info)
2320 goto err;
bc7f75fa
AK		 2321
2322 /* round up to nearest 4K */
2323 tx_ring->size = tx_ring->count * sizeof(struct e1000_tx_desc);
2324 tx_ring->size = ALIGN(tx_ring->size, 4096);
2325
2326 err = e1000_alloc_ring_dma(adapter, tx_ring);
2327 if (err)
2328 goto err;
2329
2330 tx_ring->next_to_use = 0;
2331 tx_ring->next_to_clean = 0;
bc7f75fa
AK		 2332
2333 return 0;
2334err:
2335 vfree(tx_ring->buffer_info);
44defeb3 2336 e_err("Unable to allocate memory for the transmit descriptor ring\n");
bc7f75fa
AK		 2337 return err;
2338}
2339
2340/**
2341 * e1000e_setup_rx_resources - allocate Rx resources (Descriptors)
55aa6985 2342 * @rx_ring: Rx descriptor ring
bc7f75fa
AK		 2343 *
2344 * Returns 0 on success, negative on failure
2345 **/
55aa6985 2346int e1000e_setup_rx_resources(struct e1000_ring *rx_ring)
bc7f75fa 2347{
55aa6985 2348 struct e1000_adapter *adapter = rx_ring->adapter;
47f44e40
AK		 2349 struct e1000_buffer *buffer_info;
2350 int i, size, desc_len, err = -ENOMEM;
bc7f75fa
AK		 2351
2352 size = sizeof(struct e1000_buffer) * rx_ring->count;
89bf67f1 2353 rx_ring->buffer_info = vzalloc(size);
bc7f75fa
AK		 2354 if (!rx_ring->buffer_info)
2355 goto err;
bc7f75fa 2356
47f44e40
AK		 2357 for (i = 0; i < rx_ring->count; i++) {
2358 buffer_info = &rx_ring->buffer_info[i];
2359 buffer_info->ps_pages = kcalloc(PS_PAGE_BUFFERS,
2360 sizeof(struct e1000_ps_page),
2361 GFP_KERNEL);
2362 if (!buffer_info->ps_pages)
2363 goto err_pages;
2364 }
bc7f75fa
AK		 2365
2366 desc_len = sizeof(union e1000_rx_desc_packet_split);
2367
2368 /* Round up to nearest 4K */
2369 rx_ring->size = rx_ring->count * desc_len;
2370 rx_ring->size = ALIGN(rx_ring->size, 4096);
2371
2372 err = e1000_alloc_ring_dma(adapter, rx_ring);
2373 if (err)
47f44e40 2374 goto err_pages;
bc7f75fa
AK		 2375
2376 rx_ring->next_to_clean = 0;
2377 rx_ring->next_to_use = 0;
2378 rx_ring->rx_skb_top = NULL;
2379
2380 return 0;
47f44e40
AK		 2381
2382err_pages:
2383 for (i = 0; i < rx_ring->count; i++) {
2384 buffer_info = &rx_ring->buffer_info[i];
2385 kfree(buffer_info->ps_pages);
2386 }
bc7f75fa
AK		 2387err:
2388 vfree(rx_ring->buffer_info);
e9262447 2389 e_err("Unable to allocate memory for the receive descriptor ring\n");
bc7f75fa
AK		 2390 return err;
2391}
2392
2393/**
2394 * e1000_clean_tx_ring - Free Tx Buffers
55aa6985 2395 * @tx_ring: Tx descriptor ring
bc7f75fa 2396 **/
55aa6985 2397static void e1000_clean_tx_ring(struct e1000_ring *tx_ring)
bc7f75fa 2398{
55aa6985 2399 struct e1000_adapter *adapter = tx_ring->adapter;
bc7f75fa
AK		 2400 struct e1000_buffer *buffer_info;
2401 unsigned long size;
2402 unsigned int i;
2403
2404 for (i = 0; i < tx_ring->count; i++) {
2405 buffer_info = &tx_ring->buffer_info[i];
55aa6985 2406 e1000_put_txbuf(tx_ring, buffer_info);
bc7f75fa
AK		 2407 }
2408
3f0cfa3b 2409 netdev_reset_queue(adapter->netdev);
bc7f75fa
AK		 2410 size = sizeof(struct e1000_buffer) * tx_ring->count;
2411 memset(tx_ring->buffer_info, 0, size);
2412
2413 memset(tx_ring->desc, 0, tx_ring->size);
2414
2415 tx_ring->next_to_use = 0;
2416 tx_ring->next_to_clean = 0;
2417
c5083cf6 2418 writel(0, tx_ring->head);
bdc125f7
BA		 2419 if (tx_ring->adapter->flags2 & FLAG2_PCIM2PCI_ARBITER_WA)
2420 e1000e_update_tdt_wa(tx_ring, 0);
2421 else
2422 writel(0, tx_ring->tail);
bc7f75fa
AK		 2423}
2424
2425/**
2426 * e1000e_free_tx_resources - Free Tx Resources per Queue
55aa6985 2427 * @tx_ring: Tx descriptor ring
bc7f75fa
AK		 2428 *
2429 * Free all transmit software resources
2430 **/
55aa6985 2431void e1000e_free_tx_resources(struct e1000_ring *tx_ring)
bc7f75fa 2432{
55aa6985 2433 struct e1000_adapter *adapter = tx_ring->adapter;
bc7f75fa 2434 struct pci_dev *pdev = adapter->pdev;
bc7f75fa 2435
55aa6985 2436 e1000_clean_tx_ring(tx_ring);
bc7f75fa
AK		 2437
2438 vfree(tx_ring->buffer_info);
2439 tx_ring->buffer_info = NULL;
2440
2441 dma_free_coherent(&pdev->dev, tx_ring->size, tx_ring->desc,
2442 tx_ring->dma);
2443 tx_ring->desc = NULL;
2444}
2445
2446/**
2447 * e1000e_free_rx_resources - Free Rx Resources
55aa6985 2448 * @rx_ring: Rx descriptor ring
bc7f75fa
AK		 2449 *
2450 * Free all receive software resources
2451 **/
55aa6985 2452void e1000e_free_rx_resources(struct e1000_ring *rx_ring)
bc7f75fa 2453{
55aa6985 2454 struct e1000_adapter *adapter = rx_ring->adapter;
bc7f75fa 2455 struct pci_dev *pdev = adapter->pdev;
47f44e40 2456 int i;
bc7f75fa 2457
55aa6985 2458 e1000_clean_rx_ring(rx_ring);
bc7f75fa 2459
b1cdfead 2460 for (i = 0; i < rx_ring->count; i++)
47f44e40 2461 kfree(rx_ring->buffer_info[i].ps_pages);
47f44e40 2462
bc7f75fa
AK		 2463 vfree(rx_ring->buffer_info);
2464 rx_ring->buffer_info = NULL;
2465
bc7f75fa
AK		 2466 dma_free_coherent(&pdev->dev, rx_ring->size, rx_ring->desc,
2467 rx_ring->dma);
2468 rx_ring->desc = NULL;
2469}
2470
2471/**
2472 * e1000_update_itr - update the dynamic ITR value based on statistics
489815ce
AK		 2473 * @adapter: pointer to adapter
2474 * @itr_setting: current adapter->itr
2475 * @packets: the number of packets during this measurement interval
2476 * @bytes: the number of bytes during this measurement interval
2477 *
bc7f75fa
AK		 2478 * Stores a new ITR value based on packets and byte
2479 * counts during the last interrupt. The advantage of per interrupt
2480 * computation is faster updates and more accurate ITR for the current
2481 * traffic pattern. Constants in this function were computed
2482 * based on theoretical maximum wire speed and thresholds were set based
2483 * on testing data as well as attempting to minimize response time
4662e82b
BA		 2484 * while increasing bulk throughput. This functionality is controlled
2485 * by the InterruptThrottleRate module parameter.
bc7f75fa 2486 **/
8bb62869 2487static unsigned int e1000_update_itr(u16 itr_setting, int packets, int bytes)
bc7f75fa
AK		 2488{
2489 unsigned int retval = itr_setting;
2490
2491 if (packets == 0)
5015e53a 2492 return itr_setting;
bc7f75fa
AK		 2493
2494 switch (itr_setting) {
2495 case lowest_latency:
2496 /* handle TSO and jumbo frames */
362e20ca 2497 if (bytes / packets > 8000)
bc7f75fa 2498 retval = bulk_latency;
b1cdfead 2499 else if ((packets < 5) && (bytes > 512))
bc7f75fa 2500 retval = low_latency;
bc7f75fa
AK		 2501 break;
2502 case low_latency: /* 50 usec aka 20000 ints/s */
2503 if (bytes > 10000) {
2504 /* this if handles the TSO accounting */
362e20ca 2505 if (bytes / packets > 8000)
bc7f75fa 2506 retval = bulk_latency;
362e20ca 2507 else if ((packets < 10) || ((bytes / packets) > 1200))
bc7f75fa 2508 retval = bulk_latency;
b1cdfead 2509 else if ((packets > 35))
bc7f75fa 2510 retval = lowest_latency;
362e20ca 2511 } else if (bytes / packets > 2000) {
bc7f75fa
AK		 2512 retval = bulk_latency;
2513 } else if (packets <= 2 && bytes < 512) {
2514 retval = lowest_latency;
2515 }
2516 break;
2517 case bulk_latency: /* 250 usec aka 4000 ints/s */
2518 if (bytes > 25000) {
b1cdfead 2519 if (packets > 35)
bc7f75fa 2520 retval = low_latency;
bc7f75fa
AK		 2521 } else if (bytes < 6000) {
2522 retval = low_latency;
2523 }
2524 break;
2525 }
2526
bc7f75fa
AK		 2527 return retval;
2528}
2529
2530static void e1000_set_itr(struct e1000_adapter *adapter)
2531{
bc7f75fa
AK		 2532 u16 current_itr;
2533 u32 new_itr = adapter->itr;
2534
2535 /* for non-gigabit speeds, just fix the interrupt rate at 4000 */
2536 if (adapter->link_speed != SPEED_1000) {
2537 current_itr = 0;
2538 new_itr = 4000;
2539 goto set_itr_now;
2540 }
2541
828bac87
BA		 2542 if (adapter->flags2 & FLAG2_DISABLE_AIM) {
2543 new_itr = 0;
2544 goto set_itr_now;
2545 }
2546
8bb62869
BA		 2547 adapter->tx_itr = e1000_update_itr(adapter->tx_itr,
2548 adapter->total_tx_packets,
2549 adapter->total_tx_bytes);
bc7f75fa
AK		 2550 /* conservative mode (itr 3) eliminates the lowest_latency setting */
2551 if (adapter->itr_setting == 3 && adapter->tx_itr == lowest_latency)
2552 adapter->tx_itr = low_latency;
2553
8bb62869
BA		 2554 adapter->rx_itr = e1000_update_itr(adapter->rx_itr,
2555 adapter->total_rx_packets,
2556 adapter->total_rx_bytes);
bc7f75fa
AK		 2557 /* conservative mode (itr 3) eliminates the lowest_latency setting */
2558 if (adapter->itr_setting == 3 && adapter->rx_itr == lowest_latency)
2559 adapter->rx_itr = low_latency;
2560
2561 current_itr = max(adapter->rx_itr, adapter->tx_itr);
2562
2563 switch (current_itr) {
2564 /* counts and packets in update_itr are dependent on these numbers */
2565 case lowest_latency:
2566 new_itr = 70000;
2567 break;
2568 case low_latency:
2569 new_itr = 20000; /* aka hwitr = ~200 */
2570 break;
2571 case bulk_latency:
2572 new_itr = 4000;
2573 break;
2574 default:
2575 break;
2576 }
2577
2578set_itr_now:
2579 if (new_itr != adapter->itr) {
e921eb1a 2580 /* this attempts to bias the interrupt rate towards Bulk
bc7f75fa 2581 * by adding intermediate steps when interrupt rate is
ad68076e
BA		 2582 * increasing
2583 */
bc7f75fa 2584 new_itr = new_itr > adapter->itr ?
f0ff4398 2585 min(adapter->itr + (new_itr >> 2), new_itr) : new_itr;
bc7f75fa 2586 adapter->itr = new_itr;
4662e82b
BA		 2587 adapter->rx_ring->itr_val = new_itr;
2588 if (adapter->msix_entries)
2589 adapter->rx_ring->set_itr = 1;
2590 else
e3d14b08 2591 e1000e_write_itr(adapter, new_itr);
bc7f75fa
AK		 2592 }
2593}
2594
22a4cca2
MV		 2595/**
2596 * e1000e_write_itr - write the ITR value to the appropriate registers
2597 * @adapter: address of board private structure
2598 * @itr: new ITR value to program
2599 *
2600 * e1000e_write_itr determines if the adapter is in MSI-X mode
2601 * and, if so, writes the EITR registers with the ITR value.
2602 * Otherwise, it writes the ITR value into the ITR register.
2603 **/
2604void e1000e_write_itr(struct e1000_adapter *adapter, u32 itr)
2605{
2606 struct e1000_hw *hw = &adapter->hw;
2607 u32 new_itr = itr ? 1000000000 / (itr * 256) : 0;
2608
2609 if (adapter->msix_entries) {
2610 int vector;
2611
2612 for (vector = 0; vector < adapter->num_vectors; vector++)
2613 writel(new_itr, hw->hw_addr + E1000_EITR_82574(vector));
2614 } else {
2615 ew32(ITR, new_itr);
2616 }
2617}
2618
4662e82b
BA		 2619/**
2620 * e1000_alloc_queues - Allocate memory for all rings
2621 * @adapter: board private structure to initialize
2622 **/
9f9a12f8 2623static int e1000_alloc_queues(struct e1000_adapter *adapter)
4662e82b 2624{
55aa6985
BA		 2625 int size = sizeof(struct e1000_ring);
2626
2627 adapter->tx_ring = kzalloc(size, GFP_KERNEL);
4662e82b
BA		 2628 if (!adapter->tx_ring)
2629 goto err;
55aa6985
BA		 2630 adapter->tx_ring->count = adapter->tx_ring_count;
2631 adapter->tx_ring->adapter = adapter;
4662e82b 2632
55aa6985 2633 adapter->rx_ring = kzalloc(size, GFP_KERNEL);
4662e82b
BA		 2634 if (!adapter->rx_ring)
2635 goto err;
55aa6985
BA		 2636 adapter->rx_ring->count = adapter->rx_ring_count;
2637 adapter->rx_ring->adapter = adapter;
4662e82b
BA		 2638
2639 return 0;
2640err:
2641 e_err("Unable to allocate memory for queues\n");
2642 kfree(adapter->rx_ring);
2643 kfree(adapter->tx_ring);
2644 return -ENOMEM;
2645}
2646
bc7f75fa 2647/**
c58c8a78 2648 * e1000e_poll - NAPI Rx polling callback
ad68076e 2649 * @napi: struct associated with this polling callback
c58c8a78 2650 * @weight: number of packets driver is allowed to process this poll
bc7f75fa 2651 **/
c58c8a78 2652static int e1000e_poll(struct napi_struct *napi, int weight)
bc7f75fa 2653{
c58c8a78
BA		 2654 struct e1000_adapter *adapter = container_of(napi, struct e1000_adapter,
2655 napi);
4662e82b 2656 struct e1000_hw *hw = &adapter->hw;
bc7f75fa 2657 struct net_device *poll_dev = adapter->netdev;
679e8a0f 2658 int tx_cleaned = 1, work_done = 0;
bc7f75fa 2659
4cf1653a 2660 adapter = netdev_priv(poll_dev);
bc7f75fa 2661
c58c8a78
BA		 2662 if (!adapter->msix_entries ||
2663 (adapter->rx_ring->ims_val & adapter->tx_ring->ims_val))
2664 tx_cleaned = e1000_clean_tx_irq(adapter->tx_ring);
4662e82b 2665
c58c8a78 2666 adapter->clean_rx(adapter->rx_ring, &work_done, weight);
d2c7ddd6 2667
12d04a3c 2668 if (!tx_cleaned)
c58c8a78 2669 work_done = weight;
bc7f75fa 2670
c58c8a78
BA		 2671 /* If weight not fully consumed, exit the polling mode */
2672 if (work_done < weight) {
bc7f75fa
AK		 2673 if (adapter->itr_setting & 3)
2674 e1000_set_itr(adapter);
288379f0 2675 napi_complete(napi);
a3c69fef
JB		 2676 if (!test_bit(__E1000_DOWN, &adapter->state)) {
2677 if (adapter->msix_entries)
2678 ew32(IMS, adapter->rx_ring->ims_val);
2679 else
2680 e1000_irq_enable(adapter);
2681 }
bc7f75fa
AK		 2682 }
2683
2684 return work_done;
2685}
2686
8e586137 2687static int e1000_vlan_rx_add_vid(struct net_device *netdev, u16 vid)
bc7f75fa
AK		 2688{
2689 struct e1000_adapter *adapter = netdev_priv(netdev);
2690 struct e1000_hw *hw = &adapter->hw;
2691 u32 vfta, index;
2692
2693 /* don't update vlan cookie if already programmed */
2694 if ((adapter->hw.mng_cookie.status &
2695 E1000_MNG_DHCP_COOKIE_STATUS_VLAN) &&
2696 (vid == adapter->mng_vlan_id))
8e586137 2697 return 0;
caaddaf8 2698
bc7f75fa 2699 /* add VID to filter table */
caaddaf8
BA		 2700 if (adapter->flags & FLAG_HAS_HW_VLAN_FILTER) {
2701 index = (vid >> 5) & 0x7F;
2702 vfta = E1000_READ_REG_ARRAY(hw, E1000_VFTA, index);
2703 vfta |= (1 << (vid & 0x1F));
2704 hw->mac.ops.write_vfta(hw, index, vfta);
2705 }
86d70e53
JK		 2706
2707 set_bit(vid, adapter->active_vlans);
8e586137
JP		 2708
2709 return 0;
bc7f75fa
AK		 2710}
2711
8e586137 2712static int e1000_vlan_rx_kill_vid(struct net_device *netdev, u16 vid)
bc7f75fa
AK		 2713{
2714 struct e1000_adapter *adapter = netdev_priv(netdev);
2715 struct e1000_hw *hw = &adapter->hw;
2716 u32 vfta, index;
2717
bc7f75fa
AK		 2718 if ((adapter->hw.mng_cookie.status &
2719 E1000_MNG_DHCP_COOKIE_STATUS_VLAN) &&
2720 (vid == adapter->mng_vlan_id)) {
2721 /* release control to f/w */
31dbe5b4 2722 e1000e_release_hw_control(adapter);
8e586137 2723 return 0;
bc7f75fa
AK		 2724 }
2725
2726 /* remove VID from filter table */
caaddaf8
BA		 2727 if (adapter->flags & FLAG_HAS_HW_VLAN_FILTER) {
2728 index = (vid >> 5) & 0x7F;
2729 vfta = E1000_READ_REG_ARRAY(hw, E1000_VFTA, index);
2730 vfta &= ~(1 << (vid & 0x1F));
2731 hw->mac.ops.write_vfta(hw, index, vfta);
2732 }
86d70e53
JK		 2733
2734 clear_bit(vid, adapter->active_vlans);
8e586137
JP		 2735
2736 return 0;
bc7f75fa
AK		 2737}
2738
86d70e53
JK		 2739/**
2740 * e1000e_vlan_filter_disable - helper to disable hw VLAN filtering
2741 * @adapter: board private structure to initialize
2742 **/
2743static void e1000e_vlan_filter_disable(struct e1000_adapter *adapter)
bc7f75fa
AK		 2744{
2745 struct net_device *netdev = adapter->netdev;
86d70e53
JK		 2746 struct e1000_hw *hw = &adapter->hw;
2747 u32 rctl;
bc7f75fa 2748
86d70e53
JK		 2749 if (adapter->flags & FLAG_HAS_HW_VLAN_FILTER) {
2750 /* disable VLAN receive filtering */
2751 rctl = er32(RCTL);
2752 rctl &= ~(E1000_RCTL_VFE | E1000_RCTL_CFIEN);
2753 ew32(RCTL, rctl);
2754
2755 if (adapter->mng_vlan_id != (u16)E1000_MNG_VLAN_NONE) {
2756 e1000_vlan_rx_kill_vid(netdev, adapter->mng_vlan_id);
2757 adapter->mng_vlan_id = E1000_MNG_VLAN_NONE;
bc7f75fa 2758 }
bc7f75fa
AK		 2759 }
2760}
2761
86d70e53
JK		 2762/**
2763 * e1000e_vlan_filter_enable - helper to enable HW VLAN filtering
2764 * @adapter: board private structure to initialize
2765 **/
2766static void e1000e_vlan_filter_enable(struct e1000_adapter *adapter)
2767{
2768 struct e1000_hw *hw = &adapter->hw;
2769 u32 rctl;
2770
2771 if (adapter->flags & FLAG_HAS_HW_VLAN_FILTER) {
2772 /* enable VLAN receive filtering */
2773 rctl = er32(RCTL);
2774 rctl |= E1000_RCTL_VFE;
2775 rctl &= ~E1000_RCTL_CFIEN;
2776 ew32(RCTL, rctl);
2777 }
2778}
bc7f75fa 2779
86d70e53
JK		 2780/**
2781 * e1000e_vlan_strip_enable - helper to disable HW VLAN stripping
2782 * @adapter: board private structure to initialize
2783 **/
2784static void e1000e_vlan_strip_disable(struct e1000_adapter *adapter)
bc7f75fa 2785{
bc7f75fa 2786 struct e1000_hw *hw = &adapter->hw;
86d70e53 2787 u32 ctrl;
bc7f75fa 2788
86d70e53
JK		 2789 /* disable VLAN tag insert/strip */
2790 ctrl = er32(CTRL);
2791 ctrl &= ~E1000_CTRL_VME;
2792 ew32(CTRL, ctrl);
2793}
bc7f75fa 2794
86d70e53
JK		 2795/**
2796 * e1000e_vlan_strip_enable - helper to enable HW VLAN stripping
2797 * @adapter: board private structure to initialize
2798 **/
2799static void e1000e_vlan_strip_enable(struct e1000_adapter *adapter)
2800{
2801 struct e1000_hw *hw = &adapter->hw;
2802 u32 ctrl;
bc7f75fa 2803
86d70e53
JK		 2804 /* enable VLAN tag insert/strip */
2805 ctrl = er32(CTRL);
2806 ctrl |= E1000_CTRL_VME;
2807 ew32(CTRL, ctrl);
2808}
bc7f75fa 2809
86d70e53
JK		 2810static void e1000_update_mng_vlan(struct e1000_adapter *adapter)
2811{
2812 struct net_device *netdev = adapter->netdev;
2813 u16 vid = adapter->hw.mng_cookie.vlan_id;
2814 u16 old_vid = adapter->mng_vlan_id;
2815
2816 if (adapter->hw.mng_cookie.status &
2817 E1000_MNG_DHCP_COOKIE_STATUS_VLAN) {
2818 e1000_vlan_rx_add_vid(netdev, vid);
2819 adapter->mng_vlan_id = vid;
bc7f75fa
AK		 2820 }
2821
86d70e53
JK		 2822 if ((old_vid != (u16)E1000_MNG_VLAN_NONE) && (vid != old_vid))
2823 e1000_vlan_rx_kill_vid(netdev, old_vid);
bc7f75fa
AK		 2824}
2825
2826static void e1000_restore_vlan(struct e1000_adapter *adapter)
2827{
2828 u16 vid;
2829
86d70e53 2830 e1000_vlan_rx_add_vid(adapter->netdev, 0);
bc7f75fa 2831
86d70e53 2832 for_each_set_bit(vid, adapter->active_vlans, VLAN_N_VID)
f0ff4398 2833 e1000_vlan_rx_add_vid(adapter->netdev, vid);
bc7f75fa
AK		 2834}
2835
cd791618 2836static void e1000_init_manageability_pt(struct e1000_adapter *adapter)
bc7f75fa
AK		 2837{
2838 struct e1000_hw *hw = &adapter->hw;
cd791618 2839 u32 manc, manc2h, mdef, i, j;
bc7f75fa
AK		 2840
2841 if (!(adapter->flags & FLAG_MNG_PT_ENABLED))
2842 return;
2843
2844 manc = er32(MANC);
2845
e921eb1a 2846 /* enable receiving management packets to the host. this will probably
bc7f75fa 2847 * generate destination unreachable messages from the host OS, but
ad68076e
BA		 2848 * the packets will be handled on SMBUS
2849 */
bc7f75fa
AK		 2850 manc |= E1000_MANC_EN_MNG2HOST;
2851 manc2h = er32(MANC2H);
cd791618
BA		 2852
2853 switch (hw->mac.type) {
2854 default:
2855 manc2h |= (E1000_MANC2H_PORT_623 | E1000_MANC2H_PORT_664);
2856 break;
2857 case e1000_82574:
2858 case e1000_82583:
e921eb1a 2859 /* Check if IPMI pass-through decision filter already exists;
cd791618
BA		 2860 * if so, enable it.
2861 */
2862 for (i = 0, j = 0; i < 8; i++) {
2863 mdef = er32(MDEF(i));
2864
2865 /* Ignore filters with anything other than IPMI ports */
3b21b508 2866 if (mdef & ~(E1000_MDEF_PORT_623 | E1000_MDEF_PORT_664))
cd791618
BA		 2867 continue;
2868
2869 /* Enable this decision filter in MANC2H */
2870 if (mdef)
2871 manc2h |= (1 << i);
2872
2873 j |= mdef;
2874 }
2875
2876 if (j == (E1000_MDEF_PORT_623 | E1000_MDEF_PORT_664))
2877 break;
2878
2879 /* Create new decision filter in an empty filter */
2880 for (i = 0, j = 0; i < 8; i++)
2881 if (er32(MDEF(i)) == 0) {
2882 ew32(MDEF(i), (E1000_MDEF_PORT_623 |
2883 E1000_MDEF_PORT_664));
2884 manc2h |= (1 << 1);
2885 j++;
2886 break;
2887 }
2888
2889 if (!j)
2890 e_warn("Unable to create IPMI pass-through filter\n");
2891 break;
2892 }
2893
bc7f75fa
AK		 2894 ew32(MANC2H, manc2h);
2895 ew32(MANC, manc);
2896}
2897
2898/**
af667a29 2899 * e1000_configure_tx - Configure Transmit Unit after Reset
bc7f75fa
AK		 2900 * @adapter: board private structure
2901 *
2902 * Configure the Tx unit of the MAC after a reset.
2903 **/
2904static void e1000_configure_tx(struct e1000_adapter *adapter)
2905{
2906 struct e1000_hw *hw = &adapter->hw;
2907 struct e1000_ring *tx_ring = adapter->tx_ring;
2908 u64 tdba;
c550b121 2909 u32 tdlen, tarc;
bc7f75fa
AK		 2910
2911 /* Setup the HW Tx Head and Tail descriptor pointers */
2912 tdba = tx_ring->dma;
2913 tdlen = tx_ring->count * sizeof(struct e1000_tx_desc);
1e36052e
BA		 2914 ew32(TDBAL(0), (tdba & DMA_BIT_MASK(32)));
2915 ew32(TDBAH(0), (tdba >> 32));
2916 ew32(TDLEN(0), tdlen);
2917 ew32(TDH(0), 0);
2918 ew32(TDT(0), 0);
2919 tx_ring->head = adapter->hw.hw_addr + E1000_TDH(0);
2920 tx_ring->tail = adapter->hw.hw_addr + E1000_TDT(0);
bc7f75fa 2921
bc7f75fa
AK		 2922 /* Set the Tx Interrupt Delay register */
2923 ew32(TIDV, adapter->tx_int_delay);
ad68076e 2924 /* Tx irq moderation */
bc7f75fa
AK		 2925 ew32(TADV, adapter->tx_abs_int_delay);
2926
3a3b7586
JB		 2927 if (adapter->flags2 & FLAG2_DMA_BURST) {
2928 u32 txdctl = er32(TXDCTL(0));
2929 txdctl &= ~(E1000_TXDCTL_PTHRESH | E1000_TXDCTL_HTHRESH |
2930 E1000_TXDCTL_WTHRESH);
e921eb1a 2931 /* set up some performance related parameters to encourage the
3a3b7586
JB		 2932 * hardware to use the bus more efficiently in bursts, depends
2933 * on the tx_int_delay to be enabled,
8edc0e62 2934 * wthresh = 1 ==> burst write is disabled to avoid Tx stalls
3a3b7586
JB		 2935 * hthresh = 1 ==> prefetch when one or more available
2936 * pthresh = 0x1f ==> prefetch if internal cache 31 or less
2937 * BEWARE: this seems to work but should be considered first if
af667a29 2938 * there are Tx hangs or other Tx related bugs
3a3b7586
JB		 2939 */
2940 txdctl |= E1000_TXDCTL_DMA_BURST_ENABLE;
2941 ew32(TXDCTL(0), txdctl);
3a3b7586 2942 }
56032be7
BA		 2943 /* erratum work around: set txdctl the same for both queues */
2944 ew32(TXDCTL(1), er32(TXDCTL(0)));
3a3b7586 2945
bc7f75fa 2946 if (adapter->flags & FLAG_TARC_SPEED_MODE_BIT) {
e9ec2c0f 2947 tarc = er32(TARC(0));
e921eb1a 2948 /* set the speed mode bit, we'll clear it if we're not at
ad68076e
BA		 2949 * gigabit link later
2950 */
bc7f75fa
AK		 2951#define SPEED_MODE_BIT (1 << 21)
2952 tarc |= SPEED_MODE_BIT;
e9ec2c0f 2953 ew32(TARC(0), tarc);
bc7f75fa
AK		 2954 }
2955
2956 /* errata: program both queues to unweighted RR */
2957 if (adapter->flags & FLAG_TARC_SET_BIT_ZERO) {
e9ec2c0f 2958 tarc = er32(TARC(0));
bc7f75fa 2959 tarc |= 1;
e9ec2c0f
JK		 2960 ew32(TARC(0), tarc);
2961 tarc = er32(TARC(1));
bc7f75fa 2962 tarc |= 1;
e9ec2c0f 2963 ew32(TARC(1), tarc);
bc7f75fa
AK		 2964 }
2965
bc7f75fa
AK		 2966 /* Setup Transmit Descriptor Settings for eop descriptor */
2967 adapter->txd_cmd = E1000_TXD_CMD_EOP | E1000_TXD_CMD_IFCS;
2968
2969 /* only set IDE if we are delaying interrupts using the timers */
2970 if (adapter->tx_int_delay)
2971 adapter->txd_cmd |= E1000_TXD_CMD_IDE;
2972
2973 /* enable Report Status bit */
2974 adapter->txd_cmd |= E1000_TXD_CMD_RS;
2975
57cde763 2976 hw->mac.ops.config_collision_dist(hw);
bc7f75fa
AK		 2977}
2978
2979/**
2980 * e1000_setup_rctl - configure the receive control registers
2981 * @adapter: Board private structure
2982 **/
2983#define PAGE_USE_COUNT(S) (((S) >> PAGE_SHIFT) + \
2984 (((S) & (PAGE_SIZE - 1)) ? 1 : 0))
2985static void e1000_setup_rctl(struct e1000_adapter *adapter)
2986{
2987 struct e1000_hw *hw = &adapter->hw;
2988 u32 rctl, rfctl;
bc7f75fa
AK		 2989 u32 pages = 0;
2990
2fbe4526
BA		 2991 /* Workaround Si errata on PCHx - configure jumbo frame flow */
2992 if (hw->mac.type >= e1000_pch2lan) {
a1ce6473
BA		 2993 s32 ret_val;
2994
2995 if (adapter->netdev->mtu > ETH_DATA_LEN)
2996 ret_val = e1000_lv_jumbo_workaround_ich8lan(hw, true);
2997 else
2998 ret_val = e1000_lv_jumbo_workaround_ich8lan(hw, false);
dd93f95e
BA		 2999
3000 if (ret_val)
3001 e_dbg("failed to enable jumbo frame workaround mode\n");
a1ce6473
BA		 3002 }
3003
bc7f75fa
AK		 3004 /* Program MC offset vector base */
3005 rctl = er32(RCTL);
3006 rctl &= ~(3 << E1000_RCTL_MO_SHIFT);
3007 rctl |= E1000_RCTL_EN | E1000_RCTL_BAM |
f0ff4398
BA		 3008 E1000_RCTL_LBM_NO | E1000_RCTL_RDMTS_HALF |
3009 (adapter->hw.mac.mc_filter_type << E1000_RCTL_MO_SHIFT);
bc7f75fa
AK		 3010
3011 /* Do not Store bad packets */
3012 rctl &= ~E1000_RCTL_SBP;
3013
3014 /* Enable Long Packet receive */
3015 if (adapter->netdev->mtu <= ETH_DATA_LEN)
3016 rctl &= ~E1000_RCTL_LPE;
3017 else
3018 rctl |= E1000_RCTL_LPE;
3019
eb7c3adb
JK		 3020 /* Some systems expect that the CRC is included in SMBUS traffic. The
3021 * hardware strips the CRC before sending to both SMBUS (BMC) and to
3022 * host memory when this is enabled
3023 */
3024 if (adapter->flags2 & FLAG2_CRC_STRIPPING)
3025 rctl |= E1000_RCTL_SECRC;
5918bd88 3026
a4f58f54
BA		 3027 /* Workaround Si errata on 82577 PHY - configure IPG for jumbos */
3028 if ((hw->phy.type == e1000_phy_82577) && (rctl & E1000_RCTL_LPE)) {
3029 u16 phy_data;
3030
3031 e1e_rphy(hw, PHY_REG(770, 26), &phy_data);
3032 phy_data &= 0xfff8;
3033 phy_data |= (1 << 2);
3034 e1e_wphy(hw, PHY_REG(770, 26), phy_data);
3035
3036 e1e_rphy(hw, 22, &phy_data);
3037 phy_data &= 0x0fff;
3038 phy_data |= (1 << 14);
3039 e1e_wphy(hw, 0x10, 0x2823);
3040 e1e_wphy(hw, 0x11, 0x0003);
3041 e1e_wphy(hw, 22, phy_data);
3042 }
3043
bc7f75fa
AK		 3044 /* Setup buffer sizes */
3045 rctl &= ~E1000_RCTL_SZ_4096;
3046 rctl |= E1000_RCTL_BSEX;
3047 switch (adapter->rx_buffer_len) {
bc7f75fa
AK		 3048 case 2048:
3049 default:
3050 rctl |= E1000_RCTL_SZ_2048;
3051 rctl &= ~E1000_RCTL_BSEX;
3052 break;
3053 case 4096:
3054 rctl |= E1000_RCTL_SZ_4096;
3055 break;
3056 case 8192:
3057 rctl |= E1000_RCTL_SZ_8192;
3058 break;
3059 case 16384:
3060 rctl |= E1000_RCTL_SZ_16384;
3061 break;
3062 }
3063
5f450212
BA		 3064 /* Enable Extended Status in all Receive Descriptors */
3065 rfctl = er32(RFCTL);
3066 rfctl |= E1000_RFCTL_EXTEN;
f6bd5577 3067 ew32(RFCTL, rfctl);
5f450212 3068
e921eb1a 3069 /* 82571 and greater support packet-split where the protocol
bc7f75fa
AK		 3070 * header is placed in skb->data and the packet data is
3071 * placed in pages hanging off of skb_shinfo(skb)->nr_frags.
3072 * In the case of a non-split, skb->data is linearly filled,
3073 * followed by the page buffers. Therefore, skb->data is
3074 * sized to hold the largest protocol header.
3075 *
3076 * allocations using alloc_page take too long for regular MTU
3077 * so only enable packet split for jumbo frames
3078 *
3079 * Using pages when the page size is greater than 16k wastes
3080 * a lot of memory, since we allocate 3 pages at all times
3081 * per packet.
3082 */
bc7f75fa 3083 pages = PAGE_USE_COUNT(adapter->netdev->mtu);
79d4e908 3084 if ((pages <= 3) && (PAGE_SIZE <= 16384) && (rctl & E1000_RCTL_LPE))
bc7f75fa 3085 adapter->rx_ps_pages = pages;
97ac8cae
BA		 3086 else
3087 adapter->rx_ps_pages = 0;
bc7f75fa
AK		 3088
3089 if (adapter->rx_ps_pages) {
90da0669
BA		 3090 u32 psrctl = 0;
3091
140a7480
AK		 3092 /* Enable Packet split descriptors */
3093 rctl |= E1000_RCTL_DTYP_PS;
bc7f75fa
AK		 3094
3095 psrctl |= adapter->rx_ps_bsize0 >>
3096 E1000_PSRCTL_BSIZE0_SHIFT;
3097
3098 switch (adapter->rx_ps_pages) {
3099 case 3:
3100 psrctl |= PAGE_SIZE <<
3101 E1000_PSRCTL_BSIZE3_SHIFT;
3102 case 2:
3103 psrctl |= PAGE_SIZE <<
3104 E1000_PSRCTL_BSIZE2_SHIFT;
3105 case 1:
3106 psrctl |= PAGE_SIZE >>
3107 E1000_PSRCTL_BSIZE1_SHIFT;
3108 break;
3109 }
3110
3111 ew32(PSRCTL, psrctl);
3112 }
3113
cf955e6c
BG		 3114 /* This is useful for sniffing bad packets. */
3115 if (adapter->netdev->features & NETIF_F_RXALL) {
3116 /* UPE and MPE will be handled by normal PROMISC logic
e921eb1a
BA		 3117 * in e1000e_set_rx_mode
3118 */
cf955e6c
BG		 3119 rctl |= (E1000_RCTL_SBP | /* Receive bad packets */
3120 E1000_RCTL_BAM | /* RX All Bcast Pkts */
3121 E1000_RCTL_PMCF); /* RX All MAC Ctrl Pkts */
3122
3123 rctl &= ~(E1000_RCTL_VFE | /* Disable VLAN filter */
3124 E1000_RCTL_DPF | /* Allow filtered pause */
3125 E1000_RCTL_CFIEN); /* Dis VLAN CFIEN Filter */
3126 /* Do not mess with E1000_CTRL_VME, it affects transmit as well,
3127 * and that breaks VLANs.
3128 */
3129 }
3130
bc7f75fa 3131 ew32(RCTL, rctl);
318a94d6 3132 /* just started the receive unit, no need to restart */
12d43f7d 3133 adapter->flags &= ~FLAG_RESTART_NOW;
bc7f75fa
AK		 3134}
3135
3136/**
3137 * e1000_configure_rx - Configure Receive Unit after Reset
3138 * @adapter: board private structure
3139 *
3140 * Configure the Rx unit of the MAC after a reset.
3141 **/
3142static void e1000_configure_rx(struct e1000_adapter *adapter)
3143{
3144 struct e1000_hw *hw = &adapter->hw;
3145 struct e1000_ring *rx_ring = adapter->rx_ring;
3146 u64 rdba;
3147 u32 rdlen, rctl, rxcsum, ctrl_ext;
3148
3149 if (adapter->rx_ps_pages) {
3150 /* this is a 32 byte descriptor */
3151 rdlen = rx_ring->count *
af667a29 3152 sizeof(union e1000_rx_desc_packet_split);
bc7f75fa
AK		 3153 adapter->clean_rx = e1000_clean_rx_irq_ps;
3154 adapter->alloc_rx_buf = e1000_alloc_rx_buffers_ps;
97ac8cae 3155 } else if (adapter->netdev->mtu > ETH_FRAME_LEN + ETH_FCS_LEN) {
5f450212 3156 rdlen = rx_ring->count * sizeof(union e1000_rx_desc_extended);
97ac8cae
BA		 3157 adapter->clean_rx = e1000_clean_jumbo_rx_irq;
3158 adapter->alloc_rx_buf = e1000_alloc_jumbo_rx_buffers;
bc7f75fa 3159 } else {
5f450212 3160 rdlen = rx_ring->count * sizeof(union e1000_rx_desc_extended);
bc7f75fa
AK		 3161 adapter->clean_rx = e1000_clean_rx_irq;
3162 adapter->alloc_rx_buf = e1000_alloc_rx_buffers;
3163 }
3164
3165 /* disable receives while setting up the descriptors */
3166 rctl = er32(RCTL);
7f99ae63
BA		 3167 if (!(adapter->flags2 & FLAG2_NO_DISABLE_RX))
3168 ew32(RCTL, rctl & ~E1000_RCTL_EN);
bc7f75fa 3169 e1e_flush();
1bba4386 3170 usleep_range(10000, 20000);
bc7f75fa 3171
3a3b7586 3172 if (adapter->flags2 & FLAG2_DMA_BURST) {
e921eb1a 3173 /* set the writeback threshold (only takes effect if the RDTR
3a3b7586 3174 * is set). set GRAN=1 and write back up to 0x4 worth, and
af667a29 3175 * enable prefetching of 0x20 Rx descriptors
3a3b7586
JB		 3176 * granularity = 01
3177 * wthresh = 04,
3178 * hthresh = 04,
3179 * pthresh = 0x20
3180 */
3181 ew32(RXDCTL(0), E1000_RXDCTL_DMA_BURST_ENABLE);
3182 ew32(RXDCTL(1), E1000_RXDCTL_DMA_BURST_ENABLE);
3183
e921eb1a 3184 /* override the delay timers for enabling bursting, only if
3a3b7586
JB		 3185 * the value was not set by the user via module options
3186 */
3187 if (adapter->rx_int_delay == DEFAULT_RDTR)
3188 adapter->rx_int_delay = BURST_RDTR;
3189 if (adapter->rx_abs_int_delay == DEFAULT_RADV)
3190 adapter->rx_abs_int_delay = BURST_RADV;
3191 }
3192
bc7f75fa
AK		 3193 /* set the Receive Delay Timer Register */
3194 ew32(RDTR, adapter->rx_int_delay);
3195
3196 /* irq moderation */
3197 ew32(RADV, adapter->rx_abs_int_delay);
828bac87 3198 if ((adapter->itr_setting != 0) && (adapter->itr != 0))
22a4cca2 3199 e1000e_write_itr(adapter, adapter->itr);
bc7f75fa
AK		 3200
3201 ctrl_ext = er32(CTRL_EXT);
bc7f75fa
AK		 3202 /* Auto-Mask interrupts upon ICR access */
3203 ctrl_ext |= E1000_CTRL_EXT_IAME;
3204 ew32(IAM, 0xffffffff);
3205 ew32(CTRL_EXT, ctrl_ext);
3206 e1e_flush();
3207
e921eb1a 3208 /* Setup the HW Rx Head and Tail Descriptor Pointers and
ad68076e
BA		 3209 * the Base and Length of the Rx Descriptor Ring
3210 */
bc7f75fa 3211 rdba = rx_ring->dma;
1e36052e
BA		 3212 ew32(RDBAL(0), (rdba & DMA_BIT_MASK(32)));
3213 ew32(RDBAH(0), (rdba >> 32));
3214 ew32(RDLEN(0), rdlen);
3215 ew32(RDH(0), 0);
3216 ew32(RDT(0), 0);
3217 rx_ring->head = adapter->hw.hw_addr + E1000_RDH(0);
3218 rx_ring->tail = adapter->hw.hw_addr + E1000_RDT(0);
bc7f75fa
AK		 3219
3220 /* Enable Receive Checksum Offload for TCP and UDP */
3221 rxcsum = er32(RXCSUM);
2e1706f2 3222 if (adapter->netdev->features & NETIF_F_RXCSUM)
bc7f75fa 3223 rxcsum |= E1000_RXCSUM_TUOFL;
2e1706f2 3224 else
bc7f75fa 3225 rxcsum &= ~E1000_RXCSUM_TUOFL;
bc7f75fa
AK		 3226 ew32(RXCSUM, rxcsum);
3227
3e35d991
BA		 3228 /* With jumbo frames, excessive C-state transition latencies result
3229 * in dropped transactions.
3230 */
3231 if (adapter->netdev->mtu > ETH_DATA_LEN) {
3232 u32 lat =
3233 ((er32(PBA) & E1000_PBA_RXA_MASK) * 1024 -
3234 adapter->max_frame_size) * 8 / 1000;
3235
3236 if (adapter->flags & FLAG_IS_ICH) {
53ec5498
BA		 3237 u32 rxdctl = er32(RXDCTL(0));
3238 ew32(RXDCTL(0), rxdctl | 0x3);
53ec5498 3239 }
3e35d991
BA		 3240
3241 pm_qos_update_request(&adapter->netdev->pm_qos_req, lat);
3242 } else {
3243 pm_qos_update_request(&adapter->netdev->pm_qos_req,
3244 PM_QOS_DEFAULT_VALUE);
97ac8cae 3245 }
bc7f75fa
AK		 3246
3247 /* Enable Receives */
3248 ew32(RCTL, rctl);
3249}
3250
3251/**
ef9b965a
JB		 3252 * e1000e_write_mc_addr_list - write multicast addresses to MTA
3253 * @netdev: network interface device structure
bc7f75fa 3254 *
ef9b965a
JB		 3255 * Writes multicast address list to the MTA hash table.
3256 * Returns: -ENOMEM on failure
3257 * 0 on no addresses written
3258 * X on writing X addresses to MTA
3259 */
3260static int e1000e_write_mc_addr_list(struct net_device *netdev)
3261{
3262 struct e1000_adapter *adapter = netdev_priv(netdev);
3263 struct e1000_hw *hw = &adapter->hw;
3264 struct netdev_hw_addr *ha;
3265 u8 *mta_list;
3266 int i;
3267
3268 if (netdev_mc_empty(netdev)) {
3269 /* nothing to program, so clear mc list */
3270 hw->mac.ops.update_mc_addr_list(hw, NULL, 0);
3271 return 0;
3272 }
3273
3274 mta_list = kzalloc(netdev_mc_count(netdev) * ETH_ALEN, GFP_ATOMIC);
3275 if (!mta_list)
3276 return -ENOMEM;
3277
3278 /* update_mc_addr_list expects a packed array of only addresses. */
3279 i = 0;
3280 netdev_for_each_mc_addr(ha, netdev)
f0ff4398 3281 memcpy(mta_list + (i++ * ETH_ALEN), ha->addr, ETH_ALEN);
ef9b965a
JB		 3282
3283 hw->mac.ops.update_mc_addr_list(hw, mta_list, i);
3284 kfree(mta_list);
3285
3286 return netdev_mc_count(netdev);
3287}
3288
3289/**
3290 * e1000e_write_uc_addr_list - write unicast addresses to RAR table
3291 * @netdev: network interface device structure
bc7f75fa 3292 *
ef9b965a
JB		 3293 * Writes unicast address list to the RAR table.
3294 * Returns: -ENOMEM on failure/insufficient address space
3295 * 0 on no addresses written
3296 * X on writing X addresses to the RAR table
bc7f75fa 3297 **/
ef9b965a 3298static int e1000e_write_uc_addr_list(struct net_device *netdev)
bc7f75fa 3299{
ef9b965a
JB		 3300 struct e1000_adapter *adapter = netdev_priv(netdev);
3301 struct e1000_hw *hw = &adapter->hw;
3302 unsigned int rar_entries = hw->mac.rar_entry_count;
3303 int count = 0;
3304
3305 /* save a rar entry for our hardware address */
3306 rar_entries--;
3307
3308 /* save a rar entry for the LAA workaround */
3309 if (adapter->flags & FLAG_RESET_OVERWRITES_LAA)
3310 rar_entries--;
3311
3312 /* return ENOMEM indicating insufficient memory for addresses */
3313 if (netdev_uc_count(netdev) > rar_entries)
3314 return -ENOMEM;
3315
3316 if (!netdev_uc_empty(netdev) && rar_entries) {
3317 struct netdev_hw_addr *ha;
3318
e921eb1a 3319 /* write the addresses in reverse order to avoid write
ef9b965a
JB		 3320 * combining
3321 */
3322 netdev_for_each_uc_addr(ha, netdev) {
3323 if (!rar_entries)
3324 break;
69e1e019 3325 hw->mac.ops.rar_set(hw, ha->addr, rar_entries--);
ef9b965a
JB		 3326 count++;
3327 }
3328 }
3329
3330 /* zero out the remaining RAR entries not used above */
3331 for (; rar_entries > 0; rar_entries--) {
3332 ew32(RAH(rar_entries), 0);
3333 ew32(RAL(rar_entries), 0);
3334 }
3335 e1e_flush();
3336
3337 return count;
bc7f75fa
AK		 3338}
3339
3340/**
ef9b965a 3341 * e1000e_set_rx_mode - secondary unicast, Multicast and Promiscuous mode set
bc7f75fa
AK		 3342 * @netdev: network interface device structure
3343 *
ef9b965a
JB		 3344 * The ndo_set_rx_mode entry point is called whenever the unicast or multicast
3345 * address list or the network interface flags are updated. This routine is
3346 * responsible for configuring the hardware for proper unicast, multicast,
bc7f75fa
AK		 3347 * promiscuous mode, and all-multi behavior.
3348 **/
ef9b965a 3349static void e1000e_set_rx_mode(struct net_device *netdev)
bc7f75fa
AK		 3350{
3351 struct e1000_adapter *adapter = netdev_priv(netdev);
3352 struct e1000_hw *hw = &adapter->hw;
bc7f75fa 3353 u32 rctl;
bc7f75fa
AK		 3354
3355 /* Check for Promiscuous and All Multicast modes */
bc7f75fa
AK		 3356 rctl = er32(RCTL);
3357
ef9b965a
JB		 3358 /* clear the affected bits */
3359 rctl &= ~(E1000_RCTL_UPE | E1000_RCTL_MPE);
3360
bc7f75fa
AK		 3361 if (netdev->flags & IFF_PROMISC) {
3362 rctl |= (E1000_RCTL_UPE | E1000_RCTL_MPE);
86d70e53
JK		 3363 /* Do not hardware filter VLANs in promisc mode */
3364 e1000e_vlan_filter_disable(adapter);
bc7f75fa 3365 } else {
ef9b965a 3366 int count;
3d3a1676 3367
746b9f02
PM		 3368 if (netdev->flags & IFF_ALLMULTI) {
3369 rctl |= E1000_RCTL_MPE;
746b9f02 3370 } else {
e921eb1a 3371 /* Write addresses to the MTA, if the attempt fails
ef9b965a
JB		 3372 * then we should just turn on promiscuous mode so
3373 * that we can at least receive multicast traffic
3374 */
3375 count = e1000e_write_mc_addr_list(netdev);
3376 if (count < 0)
3377 rctl |= E1000_RCTL_MPE;
746b9f02 3378 }
86d70e53 3379 e1000e_vlan_filter_enable(adapter);
e921eb1a 3380 /* Write addresses to available RAR registers, if there is not
ef9b965a
JB		 3381 * sufficient space to store all the addresses then enable
3382 * unicast promiscuous mode
bc7f75fa 3383 */
ef9b965a
JB		 3384 count = e1000e_write_uc_addr_list(netdev);
3385 if (count < 0)
3386 rctl |= E1000_RCTL_UPE;
bc7f75fa 3387 }
86d70e53 3388
ef9b965a
JB		 3389 ew32(RCTL, rctl);
3390
86d70e53
JK		 3391 if (netdev->features & NETIF_F_HW_VLAN_RX)
3392 e1000e_vlan_strip_enable(adapter);
3393 else
3394 e1000e_vlan_strip_disable(adapter);
bc7f75fa
AK		 3395}
3396
70495a50
BA		 3397static void e1000e_setup_rss_hash(struct e1000_adapter *adapter)
3398{
3399 struct e1000_hw *hw = &adapter->hw;
3400 u32 mrqc, rxcsum;
3401 int i;
3402 static const u32 rsskey[10] = {
3403 0xda565a6d, 0xc20e5b25, 0x3d256741, 0xb08fa343, 0xcb2bcad0,
3404 0xb4307bae, 0xa32dcb77, 0x0cf23080, 0x3bb7426a, 0xfa01acbe
3405 };
3406
3407 /* Fill out hash function seed */
3408 for (i = 0; i < 10; i++)
3409 ew32(RSSRK(i), rsskey[i]);
3410
3411 /* Direct all traffic to queue 0 */
3412 for (i = 0; i < 32; i++)
3413 ew32(RETA(i), 0);
3414
e921eb1a 3415 /* Disable raw packet checksumming so that RSS hash is placed in
70495a50
BA		 3416 * descriptor on writeback.
3417 */
3418 rxcsum = er32(RXCSUM);
3419 rxcsum |= E1000_RXCSUM_PCSD;
3420
3421 ew32(RXCSUM, rxcsum);
3422
3423 mrqc = (E1000_MRQC_RSS_FIELD_IPV4 |
3424 E1000_MRQC_RSS_FIELD_IPV4_TCP |
3425 E1000_MRQC_RSS_FIELD_IPV6 |
3426 E1000_MRQC_RSS_FIELD_IPV6_TCP |
3427 E1000_MRQC_RSS_FIELD_IPV6_TCP_EX);
3428
3429 ew32(MRQC, mrqc);
3430}
3431
b67e1913
BA		 3432/**
3433 * e1000e_get_base_timinca - get default SYSTIM time increment attributes
3434 * @adapter: board private structure
3435 * @timinca: pointer to returned time increment attributes
3436 *
3437 * Get attributes for incrementing the System Time Register SYSTIML/H at
3438 * the default base frequency, and set the cyclecounter shift value.
3439 **/
d89777bf 3440s32 e1000e_get_base_timinca(struct e1000_adapter *adapter, u32 *timinca)
b67e1913
BA		 3441{
3442 struct e1000_hw *hw = &adapter->hw;
3443 u32 incvalue, incperiod, shift;
3444
3445 /* Make sure clock is enabled on I217 before checking the frequency */
3446 if ((hw->mac.type == e1000_pch_lpt) &&
3447 !(er32(TSYNCTXCTL) & E1000_TSYNCTXCTL_ENABLED) &&
3448 !(er32(TSYNCRXCTL) & E1000_TSYNCRXCTL_ENABLED)) {
3449 u32 fextnvm7 = er32(FEXTNVM7);
3450
3451 if (!(fextnvm7 & (1 << 0))) {
3452 ew32(FEXTNVM7, fextnvm7 | (1 << 0));
3453 e1e_flush();
3454 }
3455 }
3456
3457 switch (hw->mac.type) {
3458 case e1000_pch2lan:
3459 case e1000_pch_lpt:
3460 /* On I217, the clock frequency is 25MHz or 96MHz as
3461 * indicated by the System Clock Frequency Indication
3462 */
3463 if ((hw->mac.type != e1000_pch_lpt) ||
3464 (er32(TSYNCRXCTL) & E1000_TSYNCRXCTL_SYSCFI)) {
3465 /* Stable 96MHz frequency */
3466 incperiod = INCPERIOD_96MHz;
3467 incvalue = INCVALUE_96MHz;
3468 shift = INCVALUE_SHIFT_96MHz;
3469 adapter->cc.shift = shift + INCPERIOD_SHIFT_96MHz;
3470 break;
3471 }
3472 /* fall-through */
3473 case e1000_82574:
3474 case e1000_82583:
3475 /* Stable 25MHz frequency */
3476 incperiod = INCPERIOD_25MHz;
3477 incvalue = INCVALUE_25MHz;
3478 shift = INCVALUE_SHIFT_25MHz;
3479 adapter->cc.shift = shift;
3480 break;
3481 default:
3482 return -EINVAL;
3483 }
3484
3485 *timinca = ((incperiod << E1000_TIMINCA_INCPERIOD_SHIFT) |
3486 ((incvalue << shift) & E1000_TIMINCA_INCVALUE_MASK));
3487
3488 return 0;
3489}
3490
3491/**
3492 * e1000e_config_hwtstamp - configure the hwtstamp registers and enable/disable
3493 * @adapter: board private structure
3494 *
3495 * Outgoing time stamping can be enabled and disabled. Play nice and
3496 * disable it when requested, although it shouldn't cause any overhead
3497 * when no packet needs it. At most one packet in the queue may be
3498 * marked for time stamping, otherwise it would be impossible to tell
3499 * for sure to which packet the hardware time stamp belongs.
3500 *
3501 * Incoming time stamping has to be configured via the hardware filters.
3502 * Not all combinations are supported, in particular event type has to be
3503 * specified. Matching the kind of event packet is not supported, with the
3504 * exception of "all V2 events regardless of level 2 or 4".
3505 **/
3506static int e1000e_config_hwtstamp(struct e1000_adapter *adapter)
3507{
3508 struct e1000_hw *hw = &adapter->hw;
3509 struct hwtstamp_config *config = &adapter->hwtstamp_config;
3510 u32 tsync_tx_ctl = E1000_TSYNCTXCTL_ENABLED;
3511 u32 tsync_rx_ctl = E1000_TSYNCRXCTL_ENABLED;
d89777bf
BA		 3512 u32 rxmtrl = 0;
3513 u16 rxudp = 0;
3514 bool is_l4 = false;
3515 bool is_l2 = false;
b67e1913
BA		 3516 u32 regval;
3517 s32 ret_val;
3518
3519 if (!(adapter->flags & FLAG_HAS_HW_TIMESTAMP))
3520 return -EINVAL;
3521
3522 /* flags reserved for future extensions - must be zero */
3523 if (config->flags)
3524 return -EINVAL;
3525
3526 switch (config->tx_type) {
3527 case HWTSTAMP_TX_OFF:
3528 tsync_tx_ctl = 0;
3529 break;
3530 case HWTSTAMP_TX_ON:
3531 break;
3532 default:
3533 return -ERANGE;
3534 }
3535
3536 switch (config->rx_filter) {
3537 case HWTSTAMP_FILTER_NONE:
3538 tsync_rx_ctl = 0;
3539 break;
d89777bf
BA		 3540 case HWTSTAMP_FILTER_PTP_V1_L4_SYNC:
3541 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_L4_V1;
3542 rxmtrl = E1000_RXMTRL_PTP_V1_SYNC_MESSAGE;
3543 is_l4 = true;
3544 break;
3545 case HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ:
3546 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_L4_V1;
3547 rxmtrl = E1000_RXMTRL_PTP_V1_DELAY_REQ_MESSAGE;
3548 is_l4 = true;
3549 break;
3550 case HWTSTAMP_FILTER_PTP_V2_L2_SYNC:
3551 /* Also time stamps V2 L2 Path Delay Request/Response */
3552 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_L2_V2;
3553 rxmtrl = E1000_RXMTRL_PTP_V2_SYNC_MESSAGE;
3554 is_l2 = true;
3555 break;
3556 case HWTSTAMP_FILTER_PTP_V2_L2_DELAY_REQ:
3557 /* Also time stamps V2 L2 Path Delay Request/Response. */
3558 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_L2_V2;
3559 rxmtrl = E1000_RXMTRL_PTP_V2_DELAY_REQ_MESSAGE;
3560 is_l2 = true;
3561 break;
3562 case HWTSTAMP_FILTER_PTP_V2_L4_SYNC:
3563 /* Hardware cannot filter just V2 L4 Sync messages;
3564 * fall-through to V2 (both L2 and L4) Sync.
3565 */
3566 case HWTSTAMP_FILTER_PTP_V2_SYNC:
3567 /* Also time stamps V2 Path Delay Request/Response. */
3568 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_L2_L4_V2;
3569 rxmtrl = E1000_RXMTRL_PTP_V2_SYNC_MESSAGE;
3570 is_l2 = true;
3571 is_l4 = true;
3572 break;
3573 case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ:
3574 /* Hardware cannot filter just V2 L4 Delay Request messages;
3575 * fall-through to V2 (both L2 and L4) Delay Request.
3576 */
3577 case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ:
3578 /* Also time stamps V2 Path Delay Request/Response. */
3579 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_L2_L4_V2;
3580 rxmtrl = E1000_RXMTRL_PTP_V2_DELAY_REQ_MESSAGE;
3581 is_l2 = true;
3582 is_l4 = true;
3583 break;
3584 case HWTSTAMP_FILTER_PTP_V2_L4_EVENT:
3585 case HWTSTAMP_FILTER_PTP_V2_L2_EVENT:
3586 /* Hardware cannot filter just V2 L4 or L2 Event messages;
3587 * fall-through to all V2 (both L2 and L4) Events.
3588 */
3589 case HWTSTAMP_FILTER_PTP_V2_EVENT:
3590 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_EVENT_V2;
3591 config->rx_filter = HWTSTAMP_FILTER_PTP_V2_EVENT;
3592 is_l2 = true;
3593 is_l4 = true;
3594 break;
3595 case HWTSTAMP_FILTER_PTP_V1_L4_EVENT:
3596 /* For V1, the hardware can only filter Sync messages or
3597 * Delay Request messages but not both so fall-through to
3598 * time stamp all packets.
3599 */
b67e1913 3600 case HWTSTAMP_FILTER_ALL:
d89777bf
BA		 3601 is_l2 = true;
3602 is_l4 = true;
b67e1913
BA		 3603 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_ALL;
3604 config->rx_filter = HWTSTAMP_FILTER_ALL;
3605 break;
3606 default:
3607 return -ERANGE;
3608 }
3609
3610 /* enable/disable Tx h/w time stamping */
3611 regval = er32(TSYNCTXCTL);
3612 regval &= ~E1000_TSYNCTXCTL_ENABLED;
3613 regval |= tsync_tx_ctl;
3614 ew32(TSYNCTXCTL, regval);
3615 if ((er32(TSYNCTXCTL) & E1000_TSYNCTXCTL_ENABLED) !=
3616 (regval & E1000_TSYNCTXCTL_ENABLED)) {
3617 e_err("Timesync Tx Control register not set as expected\n");
3618 return -EAGAIN;
3619 }
3620
3621 /* enable/disable Rx h/w time stamping */
3622 regval = er32(TSYNCRXCTL);
3623 regval &= ~(E1000_TSYNCRXCTL_ENABLED | E1000_TSYNCRXCTL_TYPE_MASK);
3624 regval |= tsync_rx_ctl;
3625 ew32(TSYNCRXCTL, regval);
3626 if ((er32(TSYNCRXCTL) & (E1000_TSYNCRXCTL_ENABLED |
3627 E1000_TSYNCRXCTL_TYPE_MASK)) !=
3628 (regval & (E1000_TSYNCRXCTL_ENABLED |
3629 E1000_TSYNCRXCTL_TYPE_MASK))) {
3630 e_err("Timesync Rx Control register not set as expected\n");
3631 return -EAGAIN;
3632 }
3633
d89777bf
BA		 3634 /* L2: define ethertype filter for time stamped packets */
3635 if (is_l2)
3636 rxmtrl |= ETH_P_1588;
3637
3638 /* define which PTP packets get time stamped */
3639 ew32(RXMTRL, rxmtrl);
3640
3641 /* Filter by destination port */
3642 if (is_l4) {
3643 rxudp = PTP_EV_PORT;
3644 cpu_to_be16s(&rxudp);
3645 }
3646 ew32(RXUDP, rxudp);
3647
3648 e1e_flush();
3649
b67e1913 3650 /* Clear TSYNCRXCTL_VALID & TSYNCTXCTL_VALID bit */
70806a7f
BA		 3651 er32(RXSTMPH);
3652 er32(TXSTMPH);
b67e1913
BA		 3653
3654 /* Get and set the System Time Register SYSTIM base frequency */
3655 ret_val = e1000e_get_base_timinca(adapter, &regval);
3656 if (ret_val)
3657 return ret_val;
3658 ew32(TIMINCA, regval);
3659
3660 /* reset the ns time counter */
3661 timecounter_init(&adapter->tc, &adapter->cc,
3662 ktime_to_ns(ktime_get_real()));
3663
3664 return 0;
3665}
3666
bc7f75fa 3667/**
ad68076e 3668 * e1000_configure - configure the hardware for Rx and Tx
bc7f75fa
AK		 3669 * @adapter: private board structure
3670 **/
3671static void e1000_configure(struct e1000_adapter *adapter)
3672{
55aa6985
BA		 3673 struct e1000_ring *rx_ring = adapter->rx_ring;
3674
ef9b965a 3675 e1000e_set_rx_mode(adapter->netdev);
bc7f75fa
AK		 3676
3677 e1000_restore_vlan(adapter);
cd791618 3678 e1000_init_manageability_pt(adapter);
bc7f75fa
AK		 3679
3680 e1000_configure_tx(adapter);
70495a50
BA		 3681
3682 if (adapter->netdev->features & NETIF_F_RXHASH)
3683 e1000e_setup_rss_hash(adapter);
bc7f75fa
AK		 3684 e1000_setup_rctl(adapter);
3685 e1000_configure_rx(adapter);
55aa6985 3686 adapter->alloc_rx_buf(rx_ring, e1000_desc_unused(rx_ring), GFP_KERNEL);
bc7f75fa
AK		 3687}
3688
3689/**
3690 * e1000e_power_up_phy - restore link in case the phy was powered down
3691 * @adapter: address of board private structure
3692 *
3693 * The phy may be powered down to save power and turn off link when the
3694 * driver is unloaded and wake on lan is not enabled (among others)
3695 * *** this routine MUST be followed by a call to e1000e_reset ***
3696 **/
3697void e1000e_power_up_phy(struct e1000_adapter *adapter)
3698{
17f208de
BA		 3699 if (adapter->hw.phy.ops.power_up)
3700 adapter->hw.phy.ops.power_up(&adapter->hw);
bc7f75fa
AK		 3701
3702 adapter->hw.mac.ops.setup_link(&adapter->hw);
3703}
3704
3705/**
3706 * e1000_power_down_phy - Power down the PHY
3707 *
17f208de
BA		 3708 * Power down the PHY so no link is implied when interface is down.
3709 * The PHY cannot be powered down if management or WoL is active.
bc7f75fa
AK		 3710 */
3711static void e1000_power_down_phy(struct e1000_adapter *adapter)
3712{
bc7f75fa 3713 /* WoL is enabled */
23b66e2b 3714 if (adapter->wol)
bc7f75fa
AK		 3715 return;
3716
17f208de
BA		 3717 if (adapter->hw.phy.ops.power_down)
3718 adapter->hw.phy.ops.power_down(&adapter->hw);
bc7f75fa
AK		 3719}
3720
3721/**
3722 * e1000e_reset - bring the hardware into a known good state
3723 *
3724 * This function boots the hardware and enables some settings that
3725 * require a configuration cycle of the hardware - those cannot be
3726 * set/changed during runtime. After reset the device needs to be
ad68076e 3727 * properly configured for Rx, Tx etc.
bc7f75fa
AK		 3728 */
3729void e1000e_reset(struct e1000_adapter *adapter)
3730{
3731 struct e1000_mac_info *mac = &adapter->hw.mac;
318a94d6 3732 struct e1000_fc_info *fc = &adapter->hw.fc;
bc7f75fa
AK		 3733 struct e1000_hw *hw = &adapter->hw;
3734 u32 tx_space, min_tx_space, min_rx_space;
318a94d6 3735 u32 pba = adapter->pba;
bc7f75fa
AK		 3736 u16 hwm;
3737
ad68076e 3738 /* reset Packet Buffer Allocation to default */
318a94d6 3739 ew32(PBA, pba);
df762464 3740
318a94d6 3741 if (adapter->max_frame_size > ETH_FRAME_LEN + ETH_FCS_LEN) {
e921eb1a 3742 /* To maintain wire speed transmits, the Tx FIFO should be
bc7f75fa
AK		 3743 * large enough to accommodate two full transmit packets,
3744 * rounded up to the next 1KB and expressed in KB. Likewise,
3745 * the Rx FIFO should be large enough to accommodate at least
3746 * one full receive packet and is similarly rounded up and
ad68076e
BA		 3747 * expressed in KB.
3748 */
df762464 3749 pba = er32(PBA);
bc7f75fa 3750 /* upper 16 bits has Tx packet buffer allocation size in KB */
df762464 3751 tx_space = pba >> 16;
bc7f75fa 3752 /* lower 16 bits has Rx packet buffer allocation size in KB */
df762464 3753 pba &= 0xffff;
e921eb1a 3754 /* the Tx fifo also stores 16 bytes of information about the Tx
ad68076e 3755 * but don't include ethernet FCS because hardware appends it
318a94d6
JK		 3756 */
3757 min_tx_space = (adapter->max_frame_size +
bc7f75fa
AK		 3758 sizeof(struct e1000_tx_desc) -
3759 ETH_FCS_LEN) * 2;
3760 min_tx_space = ALIGN(min_tx_space, 1024);
3761 min_tx_space >>= 10;
3762 /* software strips receive CRC, so leave room for it */
318a94d6 3763 min_rx_space = adapter->max_frame_size;
bc7f75fa
AK		 3764 min_rx_space = ALIGN(min_rx_space, 1024);
3765 min_rx_space >>= 10;
3766
e921eb1a 3767 /* If current Tx allocation is less than the min Tx FIFO size,
bc7f75fa 3768 * and the min Tx FIFO size is less than the current Rx FIFO
ad68076e
BA		 3769 * allocation, take space away from current Rx allocation
3770 */
df762464
AK		 3771 if ((tx_space < min_tx_space) &&
3772 ((min_tx_space - tx_space) < pba)) {
3773 pba -= min_tx_space - tx_space;
bc7f75fa 3774
e921eb1a 3775 /* if short on Rx space, Rx wins and must trump Tx
419e551c 3776 * adjustment
ad68076e 3777 */
79d4e908 3778 if (pba < min_rx_space)
df762464 3779 pba = min_rx_space;
bc7f75fa 3780 }
df762464
AK		 3781
3782 ew32(PBA, pba);
bc7f75fa
AK		 3783 }
3784
e921eb1a 3785 /* flow control settings
ad68076e 3786 *
38eb394e 3787 * The high water mark must be low enough to fit one full frame
bc7f75fa
AK		 3788 * (or the size used for early receive) above it in the Rx FIFO.
3789 * Set it to the lower of:
3790 * - 90% of the Rx FIFO size, and
38eb394e 3791 * - the full Rx FIFO size minus one full frame
ad68076e 3792 */
d3738bb8
BA		 3793 if (adapter->flags & FLAG_DISABLE_FC_PAUSE_TIME)
3794 fc->pause_time = 0xFFFF;
3795 else
3796 fc->pause_time = E1000_FC_PAUSE_TIME;
b20caa80 3797 fc->send_xon = true;
d3738bb8
BA		 3798 fc->current_mode = fc->requested_mode;
3799
3800 switch (hw->mac.type) {
79d4e908
BA		 3801 case e1000_ich9lan:
3802 case e1000_ich10lan:
3803 if (adapter->netdev->mtu > ETH_DATA_LEN) {
3804 pba = 14;
3805 ew32(PBA, pba);
3806 fc->high_water = 0x2800;
3807 fc->low_water = fc->high_water - 8;
3808 break;
3809 }
3810 /* fall-through */
d3738bb8 3811 default:
79d4e908
BA		 3812 hwm = min(((pba << 10) * 9 / 10),
3813 ((pba << 10) - adapter->max_frame_size));
d3738bb8
BA		 3814
3815 fc->high_water = hwm & E1000_FCRTH_RTH; /* 8-byte granularity */
3816 fc->low_water = fc->high_water - 8;
3817 break;
3818 case e1000_pchlan:
e921eb1a 3819 /* Workaround PCH LOM adapter hangs with certain network
38eb394e
BA		 3820 * loads. If hangs persist, try disabling Tx flow control.
3821 */
3822 if (adapter->netdev->mtu > ETH_DATA_LEN) {
3823 fc->high_water = 0x3500;
3824 fc->low_water = 0x1500;
3825 } else {
3826 fc->high_water = 0x5000;
3827 fc->low_water = 0x3000;
3828 }
a305595b 3829 fc->refresh_time = 0x1000;
d3738bb8
BA		 3830 break;
3831 case e1000_pch2lan:
2fbe4526 3832 case e1000_pch_lpt:
d3738bb8 3833 fc->refresh_time = 0x0400;
347b5201
BA		 3834
3835 if (adapter->netdev->mtu <= ETH_DATA_LEN) {
3836 fc->high_water = 0x05C20;
3837 fc->low_water = 0x05048;
3838 fc->pause_time = 0x0650;
3839 break;
828bac87 3840 }
347b5201
BA		 3841
3842 fc->high_water = ((pba << 10) * 9 / 10) & E1000_FCRTH_RTH;
3843 fc->low_water = ((pba << 10) * 8 / 10) & E1000_FCRTL_RTL;
d3738bb8 3844 break;
38eb394e 3845 }
bc7f75fa 3846
e921eb1a 3847 /* Alignment of Tx data is on an arbitrary byte boundary with the
d821a4c4
BA		 3848 * maximum size per Tx descriptor limited only to the transmit
3849 * allocation of the packet buffer minus 96 bytes with an upper
3850 * limit of 24KB due to receive synchronization limitations.
3851 */
3852 adapter->tx_fifo_limit = min_t(u32, ((er32(PBA) >> 16) << 10) - 96,
3853 24 << 10);
3854
e921eb1a 3855 /* Disable Adaptive Interrupt Moderation if 2 full packets cannot
79d4e908 3856 * fit in receive buffer.
828bac87
BA		 3857 */
3858 if (adapter->itr_setting & 0x3) {
79d4e908 3859 if ((adapter->max_frame_size * 2) > (pba << 10)) {
828bac87
BA		 3860 if (!(adapter->flags2 & FLAG2_DISABLE_AIM)) {
3861 dev_info(&adapter->pdev->dev,
17e813ec 3862 "Interrupt Throttle Rate off\n");
828bac87 3863 adapter->flags2 |= FLAG2_DISABLE_AIM;
22a4cca2 3864 e1000e_write_itr(adapter, 0);
828bac87
BA		 3865 }
3866 } else if (adapter->flags2 & FLAG2_DISABLE_AIM) {
3867 dev_info(&adapter->pdev->dev,
17e813ec 3868 "Interrupt Throttle Rate on\n");
828bac87
BA		 3869 adapter->flags2 &= ~FLAG2_DISABLE_AIM;
3870 adapter->itr = 20000;
22a4cca2 3871 e1000e_write_itr(adapter, adapter->itr);
828bac87
BA		 3872 }
3873 }
3874
bc7f75fa
AK		 3875 /* Allow time for pending master requests to run */
3876 mac->ops.reset_hw(hw);
97ac8cae 3877
e921eb1a 3878 /* For parts with AMT enabled, let the firmware know
97ac8cae
BA		 3879 * that the network interface is in control
3880 */
c43bc57e 3881 if (adapter->flags & FLAG_HAS_AMT)
31dbe5b4 3882 e1000e_get_hw_control(adapter);
97ac8cae 3883
bc7f75fa
AK		 3884 ew32(WUC, 0);
3885
3886 if (mac->ops.init_hw(hw))
44defeb3 3887 e_err("Hardware Error\n");
bc7f75fa
AK		 3888
3889 e1000_update_mng_vlan(adapter);
3890
3891 /* Enable h/w to recognize an 802.1Q VLAN Ethernet packet */
3892 ew32(VET, ETH_P_8021Q);
3893
3894 e1000e_reset_adaptive(hw);
31dbe5b4 3895
b67e1913
BA		 3896 /* initialize systim and reset the ns time counter */
3897 e1000e_config_hwtstamp(adapter);
3898
31dbe5b4
BA		 3899 if (!netif_running(adapter->netdev) &&
3900 !test_bit(__E1000_TESTING, &adapter->state)) {
3901 e1000_power_down_phy(adapter);
3902 return;
3903 }
3904
bc7f75fa
AK		 3905 e1000_get_phy_info(hw);
3906
918d7197
BA		 3907 if ((adapter->flags & FLAG_HAS_SMART_POWER_DOWN) &&
3908 !(adapter->flags & FLAG_SMART_POWER_DOWN)) {
bc7f75fa 3909 u16 phy_data = 0;
e921eb1a 3910 /* speed up time to link by disabling smart power down, ignore
bc7f75fa 3911 * the return value of this function because there is nothing
ad68076e
BA		 3912 * different we would do if it failed
3913 */
bc7f75fa
AK		 3914 e1e_rphy(hw, IGP02E1000_PHY_POWER_MGMT, &phy_data);
3915 phy_data &= ~IGP02E1000_PM_SPD;
3916 e1e_wphy(hw, IGP02E1000_PHY_POWER_MGMT, phy_data);
3917 }
bc7f75fa
AK		 3918}
3919
3920int e1000e_up(struct e1000_adapter *adapter)
3921{
3922 struct e1000_hw *hw = &adapter->hw;
3923
3924 /* hardware has been reset, we need to reload some things */
3925 e1000_configure(adapter);
3926
3927 clear_bit(__E1000_DOWN, &adapter->state);
3928
4662e82b
BA		 3929 if (adapter->msix_entries)
3930 e1000_configure_msix(adapter);
bc7f75fa
AK		 3931 e1000_irq_enable(adapter);
3932
400484fa 3933 netif_start_queue(adapter->netdev);
4cb9be7a 3934
bc7f75fa 3935 /* fire a link change interrupt to start the watchdog */
52a9b231
BA		 3936 if (adapter->msix_entries)
3937 ew32(ICS, E1000_ICS_LSC | E1000_ICR_OTHER);
3938 else
3939 ew32(ICS, E1000_ICS_LSC);
3940
bc7f75fa
AK		 3941 return 0;
3942}
3943
713b3c9e
JB		 3944static void e1000e_flush_descriptors(struct e1000_adapter *adapter)
3945{
3946 struct e1000_hw *hw = &adapter->hw;
3947
3948 if (!(adapter->flags2 & FLAG2_DMA_BURST))
3949 return;
3950
3951 /* flush pending descriptor writebacks to memory */
3952 ew32(TIDV, adapter->tx_int_delay | E1000_TIDV_FPD);
3953 ew32(RDTR, adapter->rx_int_delay | E1000_RDTR_FPD);
3954
3955 /* execute the writes immediately */
3956 e1e_flush();
bf03085f 3957
e921eb1a 3958 /* due to rare timing issues, write to TIDV/RDTR again to ensure the
bf03085f
MV		 3959 * write is successful
3960 */
3961 ew32(TIDV, adapter->tx_int_delay | E1000_TIDV_FPD);
3962 ew32(RDTR, adapter->rx_int_delay | E1000_RDTR_FPD);
713b3c9e
JB		 3963
3964 /* execute the writes immediately */
3965 e1e_flush();
3966}
3967
67fd4fcb
JK		 3968static void e1000e_update_stats(struct e1000_adapter *adapter);
3969
bc7f75fa
AK		 3970void e1000e_down(struct e1000_adapter *adapter)
3971{
3972 struct net_device *netdev = adapter->netdev;
3973 struct e1000_hw *hw = &adapter->hw;
3974 u32 tctl, rctl;
3975
e921eb1a 3976 /* signal that we're down so the interrupt handler does not
ad68076e
BA		 3977 * reschedule our watchdog timer
3978 */
bc7f75fa
AK		 3979 set_bit(__E1000_DOWN, &adapter->state);
3980
3981 /* disable receives in the hardware */
3982 rctl = er32(RCTL);
7f99ae63
BA		 3983 if (!(adapter->flags2 & FLAG2_NO_DISABLE_RX))
3984 ew32(RCTL, rctl & ~E1000_RCTL_EN);
bc7f75fa
AK		 3985 /* flush and sleep below */
3986
4cb9be7a 3987 netif_stop_queue(netdev);
bc7f75fa
AK		 3988
3989 /* disable transmits in the hardware */
3990 tctl = er32(TCTL);
3991 tctl &= ~E1000_TCTL_EN;
3992 ew32(TCTL, tctl);
7f99ae63 3993
bc7f75fa
AK		 3994 /* flush both disables and wait for them to finish */
3995 e1e_flush();
1bba4386 3996 usleep_range(10000, 20000);
bc7f75fa 3997
bc7f75fa
AK		 3998 e1000_irq_disable(adapter);
3999
4000 del_timer_sync(&adapter->watchdog_timer);
4001 del_timer_sync(&adapter->phy_info_timer);
4002
bc7f75fa 4003 netif_carrier_off(netdev);
67fd4fcb
JK		 4004
4005 spin_lock(&adapter->stats64_lock);
4006 e1000e_update_stats(adapter);
4007 spin_unlock(&adapter->stats64_lock);
4008
400484fa 4009 e1000e_flush_descriptors(adapter);
55aa6985
BA		 4010 e1000_clean_tx_ring(adapter->tx_ring);
4011 e1000_clean_rx_ring(adapter->rx_ring);
400484fa 4012
bc7f75fa
AK		 4013 adapter->link_speed = 0;
4014 adapter->link_duplex = 0;
4015
52cc3086
JK		 4016 if (!pci_channel_offline(adapter->pdev))
4017 e1000e_reset(adapter);
713b3c9e 4018
e921eb1a 4019 /* TODO: for power management, we could drop the link and
bc7f75fa
AK		 4020 * pci_disable_device here.
4021 */
4022}
4023
4024void e1000e_reinit_locked(struct e1000_adapter *adapter)
4025{
4026 might_sleep();
4027 while (test_and_set_bit(__E1000_RESETTING, &adapter->state))
1bba4386 4028 usleep_range(1000, 2000);
bc7f75fa
AK		 4029 e1000e_down(adapter);
4030 e1000e_up(adapter);
4031 clear_bit(__E1000_RESETTING, &adapter->state);
4032}
4033
b67e1913
BA		 4034/**
4035 * e1000e_cyclecounter_read - read raw cycle counter (used by time counter)
4036 * @cc: cyclecounter structure
4037 **/
4038static cycle_t e1000e_cyclecounter_read(const struct cyclecounter *cc)
4039{
4040 struct e1000_adapter *adapter = container_of(cc, struct e1000_adapter,
4041 cc);
4042 struct e1000_hw *hw = &adapter->hw;
4043 cycle_t systim;
4044
4045 /* latch SYSTIMH on read of SYSTIML */
4046 systim = (cycle_t)er32(SYSTIML);
4047 systim |= (cycle_t)er32(SYSTIMH) << 32;
4048
4049 return systim;
4050}
4051
bc7f75fa
AK		 4052/**
4053 * e1000_sw_init - Initialize general software structures (struct e1000_adapter)
4054 * @adapter: board private structure to initialize
4055 *
4056 * e1000_sw_init initializes the Adapter private data structure.
4057 * Fields are initialized based on PCI device information and
4058 * OS network device settings (MTU size).
4059 **/
9f9a12f8 4060static int e1000_sw_init(struct e1000_adapter *adapter)
bc7f75fa 4061{
bc7f75fa
AK		 4062 struct net_device *netdev = adapter->netdev;
4063
4064 adapter->rx_buffer_len = ETH_FRAME_LEN + VLAN_HLEN + ETH_FCS_LEN;
4065 adapter->rx_ps_bsize0 = 128;
318a94d6
JK		 4066 adapter->max_frame_size = netdev->mtu + ETH_HLEN + ETH_FCS_LEN;
4067 adapter->min_frame_size = ETH_ZLEN + ETH_FCS_LEN;
55aa6985
BA		 4068 adapter->tx_ring_count = E1000_DEFAULT_TXD;
4069 adapter->rx_ring_count = E1000_DEFAULT_RXD;
bc7f75fa 4070
67fd4fcb
JK		 4071 spin_lock_init(&adapter->stats64_lock);
4072
4662e82b 4073 e1000e_set_interrupt_capability(adapter);
bc7f75fa 4074
4662e82b
BA		 4075 if (e1000_alloc_queues(adapter))
4076 return -ENOMEM;
bc7f75fa 4077
b67e1913
BA		 4078 /* Setup hardware time stamping cyclecounter */
4079 if (adapter->flags & FLAG_HAS_HW_TIMESTAMP) {
4080 adapter->cc.read = e1000e_cyclecounter_read;
4081 adapter->cc.mask = CLOCKSOURCE_MASK(64);
4082 adapter->cc.mult = 1;
4083 /* cc.shift set in e1000e_get_base_tininca() */
4084
4085 spin_lock_init(&adapter->systim_lock);
4086 INIT_WORK(&adapter->tx_hwtstamp_work, e1000e_tx_hwtstamp_work);
4087 }
4088
bc7f75fa 4089 /* Explicitly disable IRQ since the NIC can be in any state. */
bc7f75fa
AK		 4090 e1000_irq_disable(adapter);
4091
bc7f75fa
AK		 4092 set_bit(__E1000_DOWN, &adapter->state);
4093 return 0;
bc7f75fa
AK		 4094}
4095
f8d59f78
BA		 4096/**
4097 * e1000_intr_msi_test - Interrupt Handler
4098 * @irq: interrupt number
4099 * @data: pointer to a network interface device structure
4100 **/
8bb62869 4101static irqreturn_t e1000_intr_msi_test(int __always_unused irq, void *data)
f8d59f78
BA		 4102{
4103 struct net_device *netdev = data;
4104 struct e1000_adapter *adapter = netdev_priv(netdev);
4105 struct e1000_hw *hw = &adapter->hw;
4106 u32 icr = er32(ICR);
4107
3bb99fe2 4108 e_dbg("icr is %08X\n", icr);
f8d59f78
BA		 4109 if (icr & E1000_ICR_RXSEQ) {
4110 adapter->flags &= ~FLAG_MSI_TEST_FAILED;
e921eb1a 4111 /* Force memory writes to complete before acknowledging the
bc76329d
BA		 4112 * interrupt is handled.
4113 */
f8d59f78
BA		 4114 wmb();
4115 }
4116
4117 return IRQ_HANDLED;
4118}
4119
4120/**
4121 * e1000_test_msi_interrupt - Returns 0 for successful test
4122 * @adapter: board private struct
4123 *
4124 * code flow taken from tg3.c
4125 **/
4126static int e1000_test_msi_interrupt(struct e1000_adapter *adapter)
4127{
4128 struct net_device *netdev = adapter->netdev;
4129 struct e1000_hw *hw = &adapter->hw;
4130 int err;
4131
4132 /* poll_enable hasn't been called yet, so don't need disable */
4133 /* clear any pending events */
4134 er32(ICR);
4135
4136 /* free the real vector and request a test handler */
4137 e1000_free_irq(adapter);
4662e82b 4138 e1000e_reset_interrupt_capability(adapter);
f8d59f78
BA		 4139
4140 /* Assume that the test fails, if it succeeds then the test
e921eb1a
BA		 4141 * MSI irq handler will unset this flag
4142 */
f8d59f78
BA		 4143 adapter->flags |= FLAG_MSI_TEST_FAILED;
4144
4145 err = pci_enable_msi(adapter->pdev);
4146 if (err)
4147 goto msi_test_failed;
4148
a0607fd3 4149 err = request_irq(adapter->pdev->irq, e1000_intr_msi_test, 0,
f8d59f78
BA		 4150 netdev->name, netdev);
4151 if (err) {
4152 pci_disable_msi(adapter->pdev);
4153 goto msi_test_failed;
4154 }
4155
e921eb1a 4156 /* Force memory writes to complete before enabling and firing an
bc76329d
BA		 4157 * interrupt.
4158 */
f8d59f78
BA		 4159 wmb();
4160
4161 e1000_irq_enable(adapter);
4162
4163 /* fire an unusual interrupt on the test handler */
4164 ew32(ICS, E1000_ICS_RXSEQ);
4165 e1e_flush();
569a3aff 4166 msleep(100);
f8d59f78
BA		 4167
4168 e1000_irq_disable(adapter);
4169
bc76329d 4170 rmb(); /* read flags after interrupt has been fired */
f8d59f78
BA		 4171
4172 if (adapter->flags & FLAG_MSI_TEST_FAILED) {
4662e82b 4173 adapter->int_mode = E1000E_INT_MODE_LEGACY;
068e8a30 4174 e_info("MSI interrupt test failed, using legacy interrupt.\n");
24b706b2 4175 } else {
068e8a30 4176 e_dbg("MSI interrupt test succeeded!\n");
24b706b2 4177 }
f8d59f78
BA		 4178
4179 free_irq(adapter->pdev->irq, netdev);
4180 pci_disable_msi(adapter->pdev);
4181
f8d59f78 4182msi_test_failed:
4662e82b 4183 e1000e_set_interrupt_capability(adapter);
068e8a30 4184 return e1000_request_irq(adapter);
f8d59f78
BA		 4185}
4186
4187/**
4188 * e1000_test_msi - Returns 0 if MSI test succeeds or INTx mode is restored
4189 * @adapter: board private struct
4190 *
4191 * code flow taken from tg3.c, called with e1000 interrupts disabled.
4192 **/
4193static int e1000_test_msi(struct e1000_adapter *adapter)
4194{
4195 int err;
4196 u16 pci_cmd;
4197
4198 if (!(adapter->flags & FLAG_MSI_ENABLED))
4199 return 0;
4200
4201 /* disable SERR in case the MSI write causes a master abort */
4202 pci_read_config_word(adapter->pdev, PCI_COMMAND, &pci_cmd);
36f2407f
DN		 4203 if (pci_cmd & PCI_COMMAND_SERR)
4204 pci_write_config_word(adapter->pdev, PCI_COMMAND,
4205 pci_cmd & ~PCI_COMMAND_SERR);
f8d59f78
BA		 4206
4207 err = e1000_test_msi_interrupt(adapter);
4208
36f2407f
DN		 4209 /* re-enable SERR */
4210 if (pci_cmd & PCI_COMMAND_SERR) {
4211 pci_read_config_word(adapter->pdev, PCI_COMMAND, &pci_cmd);
4212 pci_cmd |= PCI_COMMAND_SERR;
4213 pci_write_config_word(adapter->pdev, PCI_COMMAND, pci_cmd);
4214 }
f8d59f78 4215
f8d59f78
BA		 4216 return err;
4217}
4218
bc7f75fa
AK		 4219/**
4220 * e1000_open - Called when a network interface is made active
4221 * @netdev: network interface device structure
4222 *
4223 * Returns 0 on success, negative value on failure
4224 *
4225 * The open entry point is called when a network interface is made
4226 * active by the system (IFF_UP). At this point all resources needed
4227 * for transmit and receive operations are allocated, the interrupt
4228 * handler is registered with the OS, the watchdog timer is started,
4229 * and the stack is notified that the interface is ready.
4230 **/
4231static int e1000_open(struct net_device *netdev)
4232{
4233 struct e1000_adapter *adapter = netdev_priv(netdev);
4234 struct e1000_hw *hw = &adapter->hw;
23606cf5 4235 struct pci_dev *pdev = adapter->pdev;
bc7f75fa
AK		 4236 int err;
4237
4238 /* disallow open during test */
4239 if (test_bit(__E1000_TESTING, &adapter->state))
4240 return -EBUSY;
4241
23606cf5
RW		 4242 pm_runtime_get_sync(&pdev->dev);
4243
9c563d20
JB		 4244 netif_carrier_off(netdev);
4245
bc7f75fa 4246 /* allocate transmit descriptors */
55aa6985 4247 err = e1000e_setup_tx_resources(adapter->tx_ring);
bc7f75fa
AK		 4248 if (err)
4249 goto err_setup_tx;
4250
4251 /* allocate receive descriptors */
55aa6985 4252 err = e1000e_setup_rx_resources(adapter->rx_ring);
bc7f75fa
AK		 4253 if (err)
4254 goto err_setup_rx;
4255
e921eb1a 4256 /* If AMT is enabled, let the firmware know that the network
11b08be8
BA		 4257 * interface is now open and reset the part to a known state.
4258 */
4259 if (adapter->flags & FLAG_HAS_AMT) {
31dbe5b4 4260 e1000e_get_hw_control(adapter);
11b08be8
BA		 4261 e1000e_reset(adapter);
4262 }
4263
bc7f75fa
AK		 4264 e1000e_power_up_phy(adapter);
4265
4266 adapter->mng_vlan_id = E1000_MNG_VLAN_NONE;
4267 if ((adapter->hw.mng_cookie.status &
4268 E1000_MNG_DHCP_COOKIE_STATUS_VLAN))
4269 e1000_update_mng_vlan(adapter);
4270
79d4e908 4271 /* DMA latency requirement to workaround jumbo issue */
3e35d991
BA		 4272 pm_qos_add_request(&adapter->netdev->pm_qos_req, PM_QOS_CPU_DMA_LATENCY,
4273 PM_QOS_DEFAULT_VALUE);
c128ec29 4274
e921eb1a 4275 /* before we allocate an interrupt, we must be ready to handle it.
bc7f75fa
AK		 4276 * Setting DEBUG_SHIRQ in the kernel makes it fire an interrupt
4277 * as soon as we call pci_request_irq, so we have to setup our
ad68076e
BA		 4278 * clean_rx handler before we do so.
4279 */
bc7f75fa
AK		 4280 e1000_configure(adapter);
4281
4282 err = e1000_request_irq(adapter);
4283 if (err)
4284 goto err_req_irq;
4285
e921eb1a 4286 /* Work around PCIe errata with MSI interrupts causing some chipsets to
f8d59f78
BA		 4287 * ignore e1000e MSI messages, which means we need to test our MSI
4288 * interrupt now
4289 */
4662e82b 4290 if (adapter->int_mode != E1000E_INT_MODE_LEGACY) {
f8d59f78
BA		 4291 err = e1000_test_msi(adapter);
4292 if (err) {
4293 e_err("Interrupt allocation failed\n");
4294 goto err_req_irq;
4295 }
4296 }
4297
bc7f75fa
AK		 4298 /* From here on the code is the same as e1000e_up() */
4299 clear_bit(__E1000_DOWN, &adapter->state);
4300
4301 napi_enable(&adapter->napi);
4302
4303 e1000_irq_enable(adapter);
4304
09357b00 4305 adapter->tx_hang_recheck = false;
4cb9be7a 4306 netif_start_queue(netdev);
d55b53ff 4307
23606cf5
RW		 4308 adapter->idle_check = true;
4309 pm_runtime_put(&pdev->dev);
4310
bc7f75fa 4311 /* fire a link status change interrupt to start the watchdog */
52a9b231
BA		 4312 if (adapter->msix_entries)
4313 ew32(ICS, E1000_ICS_LSC | E1000_ICR_OTHER);
4314 else
4315 ew32(ICS, E1000_ICS_LSC);
bc7f75fa
AK		 4316
4317 return 0;
4318
4319err_req_irq:
31dbe5b4 4320 e1000e_release_hw_control(adapter);
bc7f75fa 4321 e1000_power_down_phy(adapter);
55aa6985 4322 e1000e_free_rx_resources(adapter->rx_ring);
bc7f75fa 4323err_setup_rx:
55aa6985 4324 e1000e_free_tx_resources(adapter->tx_ring);
bc7f75fa
AK		 4325err_setup_tx:
4326 e1000e_reset(adapter);
23606cf5 4327 pm_runtime_put_sync(&pdev->dev);
bc7f75fa
AK		 4328
4329 return err;
4330}
4331
4332/**
4333 * e1000_close - Disables a network interface
4334 * @netdev: network interface device structure
4335 *
4336 * Returns 0, this is not allowed to fail
4337 *
4338 * The close entry point is called when an interface is de-activated
4339 * by the OS. The hardware is still under the drivers control, but
4340 * needs to be disabled. A global MAC reset is issued to stop the
4341 * hardware, and all transmit and receive resources are freed.
4342 **/
4343static int e1000_close(struct net_device *netdev)
4344{
4345 struct e1000_adapter *adapter = netdev_priv(netdev);
23606cf5 4346 struct pci_dev *pdev = adapter->pdev;
bb9e44d0
BA		 4347 int count = E1000_CHECK_RESET_COUNT;
4348
4349 while (test_bit(__E1000_RESETTING, &adapter->state) && count--)
4350 usleep_range(10000, 20000);
bc7f75fa
AK		 4351
4352 WARN_ON(test_bit(__E1000_RESETTING, &adapter->state));
23606cf5
RW		 4353
4354 pm_runtime_get_sync(&pdev->dev);
4355
5f4a780d
BA		 4356 napi_disable(&adapter->napi);
4357
23606cf5
RW		 4358 if (!test_bit(__E1000_DOWN, &adapter->state)) {
4359 e1000e_down(adapter);
4360 e1000_free_irq(adapter);
4361 }
bc7f75fa 4362 e1000_power_down_phy(adapter);
bc7f75fa 4363
55aa6985
BA		 4364 e1000e_free_tx_resources(adapter->tx_ring);
4365 e1000e_free_rx_resources(adapter->rx_ring);
bc7f75fa 4366
e921eb1a 4367 /* kill manageability vlan ID if supported, but not if a vlan with
ad68076e
BA		 4368 * the same ID is registered on the host OS (let 8021q kill it)
4369 */
86d70e53
JK		 4370 if (adapter->hw.mng_cookie.status &
4371 E1000_MNG_DHCP_COOKIE_STATUS_VLAN)
bc7f75fa
AK		 4372 e1000_vlan_rx_kill_vid(netdev, adapter->mng_vlan_id);
4373
e921eb1a 4374 /* If AMT is enabled, let the firmware know that the network
ad68076e
BA		 4375 * interface is now closed
4376 */
31dbe5b4
BA		 4377 if ((adapter->flags & FLAG_HAS_AMT) &&
4378 !test_bit(__E1000_TESTING, &adapter->state))
4379 e1000e_release_hw_control(adapter);
bc7f75fa 4380
3e35d991 4381 pm_qos_remove_request(&adapter->netdev->pm_qos_req);
c128ec29 4382
23606cf5
RW		 4383 pm_runtime_put_sync(&pdev->dev);
4384
bc7f75fa
AK		 4385 return 0;
4386}
4387/**
4388 * e1000_set_mac - Change the Ethernet Address of the NIC
4389 * @netdev: network interface device structure
4390 * @p: pointer to an address structure
4391 *
4392 * Returns 0 on success, negative on failure
4393 **/
4394static int e1000_set_mac(struct net_device *netdev, void *p)
4395{
4396 struct e1000_adapter *adapter = netdev_priv(netdev);
69e1e019 4397 struct e1000_hw *hw = &adapter->hw;
bc7f75fa
AK		 4398 struct sockaddr *addr = p;
4399
4400 if (!is_valid_ether_addr(addr->sa_data))
4401 return -EADDRNOTAVAIL;
4402
4403 memcpy(netdev->dev_addr, addr->sa_data, netdev->addr_len);
4404 memcpy(adapter->hw.mac.addr, addr->sa_data, netdev->addr_len);
4405
69e1e019 4406 hw->mac.ops.rar_set(&adapter->hw, adapter->hw.mac.addr, 0);
bc7f75fa
AK		 4407
4408 if (adapter->flags & FLAG_RESET_OVERWRITES_LAA) {
4409 /* activate the work around */
4410 e1000e_set_laa_state_82571(&adapter->hw, 1);
4411
e921eb1a 4412 /* Hold a copy of the LAA in RAR[14] This is done so that
bc7f75fa
AK		 4413 * between the time RAR[0] gets clobbered and the time it
4414 * gets fixed (in e1000_watchdog), the actual LAA is in one
4415 * of the RARs and no incoming packets directed to this port
4416 * are dropped. Eventually the LAA will be in RAR[0] and
ad68076e
BA		 4417 * RAR[14]
4418 */
69e1e019
BA		 4419 hw->mac.ops.rar_set(&adapter->hw, adapter->hw.mac.addr,
4420 adapter->hw.mac.rar_entry_count - 1);
bc7f75fa
AK		 4421 }
4422
4423 return 0;
4424}
4425
a8f88ff5
JB		 4426/**
4427 * e1000e_update_phy_task - work thread to update phy
4428 * @work: pointer to our work struct
4429 *
4430 * this worker thread exists because we must acquire a
4431 * semaphore to read the phy, which we could msleep while
4432 * waiting for it, and we can't msleep in a timer.
4433 **/
4434static void e1000e_update_phy_task(struct work_struct *work)
4435{
4436 struct e1000_adapter *adapter = container_of(work,
17e813ec
BA		 4437 struct e1000_adapter,
4438 update_phy_task);
615b32af
JB		 4439
4440 if (test_bit(__E1000_DOWN, &adapter->state))
4441 return;
4442
a8f88ff5
JB		 4443 e1000_get_phy_info(&adapter->hw);
4444}
4445
e921eb1a
BA		 4446/**
4447 * e1000_update_phy_info - timre call-back to update PHY info
4448 * @data: pointer to adapter cast into an unsigned long
4449 *
ad68076e
BA		 4450 * Need to wait a few seconds after link up to get diagnostic information from
4451 * the phy
e921eb1a 4452 **/
bc7f75fa
AK		 4453static void e1000_update_phy_info(unsigned long data)
4454{
4455 struct e1000_adapter *adapter = (struct e1000_adapter *) data;
615b32af
JB		 4456
4457 if (test_bit(__E1000_DOWN, &adapter->state))
4458 return;
4459
a8f88ff5 4460 schedule_work(&adapter->update_phy_task);
bc7f75fa
AK		 4461}
4462
8c7bbb92
BA		 4463/**
4464 * e1000e_update_phy_stats - Update the PHY statistics counters
4465 * @adapter: board private structure
2b6b168d
BA		 4466 *
4467 * Read/clear the upper 16-bit PHY registers and read/accumulate lower
8c7bbb92
BA		 4468 **/
4469static void e1000e_update_phy_stats(struct e1000_adapter *adapter)
4470{
4471 struct e1000_hw *hw = &adapter->hw;
4472 s32 ret_val;
4473 u16 phy_data;
4474
4475 ret_val = hw->phy.ops.acquire(hw);
4476 if (ret_val)
4477 return;
4478
e921eb1a 4479 /* A page set is expensive so check if already on desired page.
8c7bbb92
BA		 4480 * If not, set to the page with the PHY status registers.
4481 */
2b6b168d 4482 hw->phy.addr = 1;
8c7bbb92
BA		 4483 ret_val = e1000e_read_phy_reg_mdic(hw, IGP01E1000_PHY_PAGE_SELECT,
4484 &phy_data);
4485 if (ret_val)
4486 goto release;
2b6b168d
BA		 4487 if (phy_data != (HV_STATS_PAGE << IGP_PAGE_SHIFT)) {
4488 ret_val = hw->phy.ops.set_page(hw,
4489 HV_STATS_PAGE << IGP_PAGE_SHIFT);
8c7bbb92
BA		 4490 if (ret_val)
4491 goto release;
4492 }
4493
8c7bbb92 4494 /* Single Collision Count */
2b6b168d
BA		 4495 hw->phy.ops.read_reg_page(hw, HV_SCC_UPPER, &phy_data);
4496 ret_val = hw->phy.ops.read_reg_page(hw, HV_SCC_LOWER, &phy_data);
8c7bbb92
BA		 4497 if (!ret_val)
4498 adapter->stats.scc += phy_data;
4499
4500 /* Excessive Collision Count */
2b6b168d
BA		 4501 hw->phy.ops.read_reg_page(hw, HV_ECOL_UPPER, &phy_data);
4502 ret_val = hw->phy.ops.read_reg_page(hw, HV_ECOL_LOWER, &phy_data);
8c7bbb92
BA		 4503 if (!ret_val)
4504 adapter->stats.ecol += phy_data;
4505
4506 /* Multiple Collision Count */
2b6b168d
BA		 4507 hw->phy.ops.read_reg_page(hw, HV_MCC_UPPER, &phy_data);
4508 ret_val = hw->phy.ops.read_reg_page(hw, HV_MCC_LOWER, &phy_data);
8c7bbb92
BA		 4509 if (!ret_val)
4510 adapter->stats.mcc += phy_data;
4511
4512 /* Late Collision Count */
2b6b168d
BA		 4513 hw->phy.ops.read_reg_page(hw, HV_LATECOL_UPPER, &phy_data);
4514 ret_val = hw->phy.ops.read_reg_page(hw, HV_LATECOL_LOWER, &phy_data);
8c7bbb92
BA		 4515 if (!ret_val)
4516 adapter->stats.latecol += phy_data;
4517
4518 /* Collision Count - also used for adaptive IFS */
2b6b168d
BA		 4519 hw->phy.ops.read_reg_page(hw, HV_COLC_UPPER, &phy_data);
4520 ret_val = hw->phy.ops.read_reg_page(hw, HV_COLC_LOWER, &phy_data);
8c7bbb92
BA		 4521 if (!ret_val)
4522 hw->mac.collision_delta = phy_data;
4523
4524 /* Defer Count */
2b6b168d
BA		 4525 hw->phy.ops.read_reg_page(hw, HV_DC_UPPER, &phy_data);
4526 ret_val = hw->phy.ops.read_reg_page(hw, HV_DC_LOWER, &phy_data);
8c7bbb92
BA		 4527 if (!ret_val)
4528 adapter->stats.dc += phy_data;
4529
4530 /* Transmit with no CRS */
2b6b168d
BA		 4531 hw->phy.ops.read_reg_page(hw, HV_TNCRS_UPPER, &phy_data);
4532 ret_val = hw->phy.ops.read_reg_page(hw, HV_TNCRS_LOWER, &phy_data);
8c7bbb92
BA		 4533 if (!ret_val)
4534 adapter->stats.tncrs += phy_data;
4535
4536release:
4537 hw->phy.ops.release(hw);
4538}
4539
bc7f75fa
AK		 4540/**
4541 * e1000e_update_stats - Update the board statistics counters
4542 * @adapter: board private structure
4543 **/
67fd4fcb 4544static void e1000e_update_stats(struct e1000_adapter *adapter)
bc7f75fa 4545{
7274c20f 4546 struct net_device *netdev = adapter->netdev;
bc7f75fa
AK		 4547 struct e1000_hw *hw = &adapter->hw;
4548 struct pci_dev *pdev = adapter->pdev;
bc7f75fa 4549
e921eb1a 4550 /* Prevent stats update while adapter is being reset, or if the pci
bc7f75fa
AK		 4551 * connection is down.
4552 */
4553 if (adapter->link_speed == 0)
4554 return;
4555 if (pci_channel_offline(pdev))
4556 return;
4557
bc7f75fa
AK		 4558 adapter->stats.crcerrs += er32(CRCERRS);
4559 adapter->stats.gprc += er32(GPRC);
7c25769f
BA		 4560 adapter->stats.gorc += er32(GORCL);
4561 er32(GORCH); /* Clear gorc */
bc7f75fa
AK		 4562 adapter->stats.bprc += er32(BPRC);
4563 adapter->stats.mprc += er32(MPRC);
4564 adapter->stats.roc += er32(ROC);
4565
bc7f75fa 4566 adapter->stats.mpc += er32(MPC);
8c7bbb92
BA		 4567
4568 /* Half-duplex statistics */
4569 if (adapter->link_duplex == HALF_DUPLEX) {
4570 if (adapter->flags2 & FLAG2_HAS_PHY_STATS) {
4571 e1000e_update_phy_stats(adapter);
4572 } else {
4573 adapter->stats.scc += er32(SCC);
4574 adapter->stats.ecol += er32(ECOL);
4575 adapter->stats.mcc += er32(MCC);
4576 adapter->stats.latecol += er32(LATECOL);
4577 adapter->stats.dc += er32(DC);
4578
4579 hw->mac.collision_delta = er32(COLC);
4580
4581 if ((hw->mac.type != e1000_82574) &&
4582 (hw->mac.type != e1000_82583))
4583 adapter->stats.tncrs += er32(TNCRS);
4584 }
4585 adapter->stats.colc += hw->mac.collision_delta;
a4f58f54 4586 }
8c7bbb92 4587
bc7f75fa
AK		 4588 adapter->stats.xonrxc += er32(XONRXC);
4589 adapter->stats.xontxc += er32(XONTXC);
4590 adapter->stats.xoffrxc += er32(XOFFRXC);
4591 adapter->stats.xofftxc += er32(XOFFTXC);
bc7f75fa 4592 adapter->stats.gptc += er32(GPTC);
7c25769f
BA		 4593 adapter->stats.gotc += er32(GOTCL);
4594 er32(GOTCH); /* Clear gotc */
bc7f75fa
AK		 4595 adapter->stats.rnbc += er32(RNBC);
4596 adapter->stats.ruc += er32(RUC);
bc7f75fa
AK		 4597
4598 adapter->stats.mptc += er32(MPTC);
4599 adapter->stats.bptc += er32(BPTC);
4600
4601 /* used for adaptive IFS */
4602
4603 hw->mac.tx_packet_delta = er32(TPT);
4604 adapter->stats.tpt += hw->mac.tx_packet_delta;
bc7f75fa
AK		 4605
4606 adapter->stats.algnerrc += er32(ALGNERRC);
4607 adapter->stats.rxerrc += er32(RXERRC);
bc7f75fa
AK		 4608 adapter->stats.cexterr += er32(CEXTERR);
4609 adapter->stats.tsctc += er32(TSCTC);
4610 adapter->stats.tsctfc += er32(TSCTFC);
4611
bc7f75fa 4612 /* Fill out the OS statistics structure */
7274c20f
AK		 4613 netdev->stats.multicast = adapter->stats.mprc;
4614 netdev->stats.collisions = adapter->stats.colc;
bc7f75fa
AK		 4615
4616 /* Rx Errors */
4617
e921eb1a 4618 /* RLEC on some newer hardware can be incorrect so build
ad68076e
BA		 4619 * our own version based on RUC and ROC
4620 */
7274c20f 4621 netdev->stats.rx_errors = adapter->stats.rxerrc +
f0ff4398
BA		 4622 adapter->stats.crcerrs + adapter->stats.algnerrc +
4623 adapter->stats.ruc + adapter->stats.roc + adapter->stats.cexterr;
7274c20f 4624 netdev->stats.rx_length_errors = adapter->stats.ruc +
f0ff4398 4625 adapter->stats.roc;
7274c20f
AK		 4626 netdev->stats.rx_crc_errors = adapter->stats.crcerrs;
4627 netdev->stats.rx_frame_errors = adapter->stats.algnerrc;
4628 netdev->stats.rx_missed_errors = adapter->stats.mpc;
bc7f75fa
AK		 4629
4630 /* Tx Errors */
f0ff4398 4631 netdev->stats.tx_errors = adapter->stats.ecol + adapter->stats.latecol;
7274c20f
AK		 4632 netdev->stats.tx_aborted_errors = adapter->stats.ecol;
4633 netdev->stats.tx_window_errors = adapter->stats.latecol;
4634 netdev->stats.tx_carrier_errors = adapter->stats.tncrs;
bc7f75fa
AK		 4635
4636 /* Tx Dropped needs to be maintained elsewhere */
4637
bc7f75fa
AK		 4638 /* Management Stats */
4639 adapter->stats.mgptc += er32(MGTPTC);
4640 adapter->stats.mgprc += er32(MGTPRC);
4641 adapter->stats.mgpdc += er32(MGTPDC);
94fb848b
BA		 4642
4643 /* Correctable ECC Errors */
4644 if (hw->mac.type == e1000_pch_lpt) {
4645 u32 pbeccsts = er32(PBECCSTS);
4646 adapter->corr_errors +=
4647 pbeccsts & E1000_PBECCSTS_CORR_ERR_CNT_MASK;
4648 adapter->uncorr_errors +=
4649 (pbeccsts & E1000_PBECCSTS_UNCORR_ERR_CNT_MASK) >>
4650 E1000_PBECCSTS_UNCORR_ERR_CNT_SHIFT;
4651 }
bc7f75fa
AK		 4652}
4653
7c25769f
BA		 4654/**
4655 * e1000_phy_read_status - Update the PHY register status snapshot
4656 * @adapter: board private structure
4657 **/
4658static void e1000_phy_read_status(struct e1000_adapter *adapter)
4659{
4660 struct e1000_hw *hw = &adapter->hw;
4661 struct e1000_phy_regs *phy = &adapter->phy_regs;
7c25769f
BA		 4662
4663 if ((er32(STATUS) & E1000_STATUS_LU) &&
4664 (adapter->hw.phy.media_type == e1000_media_type_copper)) {
90da0669
BA		 4665 int ret_val;
4666
c2ade1a4
BA		 4667 ret_val = e1e_rphy(hw, MII_BMCR, &phy->bmcr);
4668 ret_val |= e1e_rphy(hw, MII_BMSR, &phy->bmsr);
4669 ret_val |= e1e_rphy(hw, MII_ADVERTISE, &phy->advertise);
4670 ret_val |= e1e_rphy(hw, MII_LPA, &phy->lpa);
4671 ret_val |= e1e_rphy(hw, MII_EXPANSION, &phy->expansion);
4672 ret_val |= e1e_rphy(hw, MII_CTRL1000, &phy->ctrl1000);
4673 ret_val |= e1e_rphy(hw, MII_STAT1000, &phy->stat1000);
4674 ret_val |= e1e_rphy(hw, MII_ESTATUS, &phy->estatus);
7c25769f 4675 if (ret_val)
44defeb3 4676 e_warn("Error reading PHY register\n");
7c25769f 4677 } else {
e921eb1a 4678 /* Do not read PHY registers if link is not up
7c25769f
BA		 4679 * Set values to typical power-on defaults
4680 */
4681 phy->bmcr = (BMCR_SPEED1000 | BMCR_ANENABLE | BMCR_FULLDPLX);
4682 phy->bmsr = (BMSR_100FULL | BMSR_100HALF | BMSR_10FULL |
4683 BMSR_10HALF | BMSR_ESTATEN | BMSR_ANEGCAPABLE |
4684 BMSR_ERCAP);
4685 phy->advertise = (ADVERTISE_PAUSE_ASYM | ADVERTISE_PAUSE_CAP |
4686 ADVERTISE_ALL | ADVERTISE_CSMA);
4687 phy->lpa = 0;
4688 phy->expansion = EXPANSION_ENABLENPAGE;
4689 phy->ctrl1000 = ADVERTISE_1000FULL;
4690 phy->stat1000 = 0;
4691 phy->estatus = (ESTATUS_1000_TFULL | ESTATUS_1000_THALF);
4692 }
7c25769f
BA		 4693}
4694
bc7f75fa
AK		 4695static void e1000_print_link_info(struct e1000_adapter *adapter)
4696{
bc7f75fa
AK		 4697 struct e1000_hw *hw = &adapter->hw;
4698 u32 ctrl = er32(CTRL);
4699
8f12fe86 4700 /* Link status message must follow this format for user tools */
7dbc1672
BA		 4701 pr_info("%s NIC Link is Up %d Mbps %s Duplex, Flow Control: %s\n",
4702 adapter->netdev->name, adapter->link_speed,
ef456f85
JK		 4703 adapter->link_duplex == FULL_DUPLEX ? "Full" : "Half",
4704 (ctrl & E1000_CTRL_TFCE) && (ctrl & E1000_CTRL_RFCE) ? "Rx/Tx" :
4705 (ctrl & E1000_CTRL_RFCE) ? "Rx" :
4706 (ctrl & E1000_CTRL_TFCE) ? "Tx" : "None");
bc7f75fa
AK		 4707}
4708
0c6bdb30 4709static bool e1000e_has_link(struct e1000_adapter *adapter)
318a94d6
JK		 4710{
4711 struct e1000_hw *hw = &adapter->hw;
3db1cd5c 4712 bool link_active = false;
318a94d6
JK		 4713 s32 ret_val = 0;
4714
e921eb1a 4715 /* get_link_status is set on LSC (link status) interrupt or
318a94d6
JK		 4716 * Rx sequence error interrupt. get_link_status will stay
4717 * false until the check_for_link establishes link
4718 * for copper adapters ONLY
4719 */
4720 switch (hw->phy.media_type) {
4721 case e1000_media_type_copper:
4722 if (hw->mac.get_link_status) {
4723 ret_val = hw->mac.ops.check_for_link(hw);
4724 link_active = !hw->mac.get_link_status;
4725 } else {
3db1cd5c 4726 link_active = true;
318a94d6
JK		 4727 }
4728 break;
4729 case e1000_media_type_fiber:
4730 ret_val = hw->mac.ops.check_for_link(hw);
4731 link_active = !!(er32(STATUS) & E1000_STATUS_LU);
4732 break;
4733 case e1000_media_type_internal_serdes:
4734 ret_val = hw->mac.ops.check_for_link(hw);
4735 link_active = adapter->hw.mac.serdes_has_link;
4736 break;
4737 default:
4738 case e1000_media_type_unknown:
4739 break;
4740 }
4741
4742 if ((ret_val == E1000_ERR_PHY) && (hw->phy.type == e1000_phy_igp_3) &&
4743 (er32(CTRL) & E1000_PHY_CTRL_GBE_DISABLE)) {
4744 /* See e1000_kmrn_lock_loss_workaround_ich8lan() */
44defeb3 4745 e_info("Gigabit has been disabled, downgrading speed\n");
318a94d6
JK		 4746 }
4747
4748 return link_active;
4749}
4750
4751static void e1000e_enable_receives(struct e1000_adapter *adapter)
4752{
4753 /* make sure the receive unit is started */
4754 if ((adapter->flags & FLAG_RX_NEEDS_RESTART) &&
12d43f7d 4755 (adapter->flags & FLAG_RESTART_NOW)) {
318a94d6
JK		 4756 struct e1000_hw *hw = &adapter->hw;
4757 u32 rctl = er32(RCTL);
4758 ew32(RCTL, rctl | E1000_RCTL_EN);
12d43f7d 4759 adapter->flags &= ~FLAG_RESTART_NOW;
318a94d6
JK		 4760 }
4761}
4762
ff10e13c
CW		 4763static void e1000e_check_82574_phy_workaround(struct e1000_adapter *adapter)
4764{
4765 struct e1000_hw *hw = &adapter->hw;
4766
e921eb1a 4767 /* With 82574 controllers, PHY needs to be checked periodically
ff10e13c
CW		 4768 * for hung state and reset, if two calls return true
4769 */
4770 if (e1000_check_phy_82574(hw))
4771 adapter->phy_hang_count++;
4772 else
4773 adapter->phy_hang_count = 0;
4774
4775 if (adapter->phy_hang_count > 1) {
4776 adapter->phy_hang_count = 0;
4777 schedule_work(&adapter->reset_task);
4778 }
4779}
4780
bc7f75fa
AK		 4781/**
4782 * e1000_watchdog - Timer Call-back
4783 * @data: pointer to adapter cast into an unsigned long
4784 **/
4785static void e1000_watchdog(unsigned long data)
4786{
4787 struct e1000_adapter *adapter = (struct e1000_adapter *) data;
4788
4789 /* Do the rest outside of interrupt context */
4790 schedule_work(&adapter->watchdog_task);
4791
4792 /* TODO: make this use queue_delayed_work() */
4793}
4794
4795static void e1000_watchdog_task(struct work_struct *work)
4796{
4797 struct e1000_adapter *adapter = container_of(work,
17e813ec
BA		 4798 struct e1000_adapter,
4799 watchdog_task);
bc7f75fa
AK		 4800 struct net_device *netdev = adapter->netdev;
4801 struct e1000_mac_info *mac = &adapter->hw.mac;
75eb0fad 4802 struct e1000_phy_info *phy = &adapter->hw.phy;
bc7f75fa
AK		 4803 struct e1000_ring *tx_ring = adapter->tx_ring;
4804 struct e1000_hw *hw = &adapter->hw;
4805 u32 link, tctl;
bc7f75fa 4806
615b32af
JB		 4807 if (test_bit(__E1000_DOWN, &adapter->state))
4808 return;
4809
b405e8df 4810 link = e1000e_has_link(adapter);
318a94d6 4811 if ((netif_carrier_ok(netdev)) && link) {
23606cf5
RW		 4812 /* Cancel scheduled suspend requests. */
4813 pm_runtime_resume(netdev->dev.parent);
4814
318a94d6 4815 e1000e_enable_receives(adapter);
bc7f75fa 4816 goto link_up;
bc7f75fa
AK		 4817 }
4818
4819 if ((e1000e_enable_tx_pkt_filtering(hw)) &&
4820 (adapter->mng_vlan_id != adapter->hw.mng_cookie.vlan_id))
4821 e1000_update_mng_vlan(adapter);
4822
bc7f75fa
AK		 4823 if (link) {
4824 if (!netif_carrier_ok(netdev)) {
3db1cd5c 4825 bool txb2b = true;
23606cf5
RW		 4826
4827 /* Cancel scheduled suspend requests. */
4828 pm_runtime_resume(netdev->dev.parent);
4829
318a94d6 4830 /* update snapshot of PHY registers on LSC */
7c25769f 4831 e1000_phy_read_status(adapter);
bc7f75fa 4832 mac->ops.get_link_up_info(&adapter->hw,
17e813ec
BA		 4833 &adapter->link_speed,
4834 &adapter->link_duplex);
bc7f75fa 4835 e1000_print_link_info(adapter);
e792cd91
KS		 4836
4837 /* check if SmartSpeed worked */
4838 e1000e_check_downshift(hw);
4839 if (phy->speed_downgraded)
4840 netdev_warn(netdev,
4841 "Link Speed was downgraded by SmartSpeed\n");
4842
e921eb1a 4843 /* On supported PHYs, check for duplex mismatch only
f4187b56
BA		 4844 * if link has autonegotiated at 10/100 half
4845 */
4846 if ((hw->phy.type == e1000_phy_igp_3 ||
4847 hw->phy.type == e1000_phy_bm) &&
4848 (hw->mac.autoneg == true) &&
4849 (adapter->link_speed == SPEED_10 ||
4850 adapter->link_speed == SPEED_100) &&
4851 (adapter->link_duplex == HALF_DUPLEX)) {
4852 u16 autoneg_exp;
4853
c2ade1a4 4854 e1e_rphy(hw, MII_EXPANSION, &autoneg_exp);
f4187b56 4855
c2ade1a4 4856 if (!(autoneg_exp & EXPANSION_NWAY))
ef456f85 4857 e_info("Autonegotiated half duplex but link partner cannot autoneg. Try forcing full duplex if link gets many collisions.\n");
f4187b56
BA		 4858 }
4859
f49c57e1 4860 /* adjust timeout factor according to speed/duplex */
bc7f75fa
AK		 4861 adapter->tx_timeout_factor = 1;
4862 switch (adapter->link_speed) {
4863 case SPEED_10:
3db1cd5c 4864 txb2b = false;
10f1b492 4865 adapter->tx_timeout_factor = 16;
bc7f75fa
AK		 4866 break;
4867 case SPEED_100:
3db1cd5c 4868 txb2b = false;
4c86e0b9 4869 adapter->tx_timeout_factor = 10;
bc7f75fa
AK		 4870 break;
4871 }
4872
e921eb1a 4873 /* workaround: re-program speed mode bit after
ad68076e
BA		 4874 * link-up event
4875 */
bc7f75fa
AK		 4876 if ((adapter->flags & FLAG_TARC_SPEED_MODE_BIT) &&
4877 !txb2b) {
4878 u32 tarc0;
e9ec2c0f 4879 tarc0 = er32(TARC(0));
bc7f75fa 4880 tarc0 &= ~SPEED_MODE_BIT;
e9ec2c0f 4881 ew32(TARC(0), tarc0);
bc7f75fa
AK		 4882 }
4883
e921eb1a 4884 /* disable TSO for pcie and 10/100 speeds, to avoid
ad68076e
BA		 4885 * some hardware issues
4886 */
bc7f75fa
AK		 4887 if (!(adapter->flags & FLAG_TSO_FORCE)) {
4888 switch (adapter->link_speed) {
4889 case SPEED_10:
4890 case SPEED_100:
44defeb3 4891 e_info("10/100 speed: disabling TSO\n");
bc7f75fa
AK		 4892 netdev->features &= ~NETIF_F_TSO;
4893 netdev->features &= ~NETIF_F_TSO6;
4894 break;
4895 case SPEED_1000:
4896 netdev->features |= NETIF_F_TSO;
4897 netdev->features |= NETIF_F_TSO6;
4898 break;
4899 default:
4900 /* oops */
4901 break;
4902 }
4903 }
4904
e921eb1a 4905 /* enable transmits in the hardware, need to do this
ad68076e
BA		 4906 * after setting TARC(0)
4907 */
bc7f75fa
AK		 4908 tctl = er32(TCTL);
4909 tctl |= E1000_TCTL_EN;
4910 ew32(TCTL, tctl);
4911
e921eb1a 4912 /* Perform any post-link-up configuration before
75eb0fad
BA		 4913 * reporting link up.
4914 */
4915 if (phy->ops.cfg_on_link_up)
4916 phy->ops.cfg_on_link_up(hw);
4917
bc7f75fa 4918 netif_carrier_on(netdev);
bc7f75fa
AK		 4919
4920 if (!test_bit(__E1000_DOWN, &adapter->state))
4921 mod_timer(&adapter->phy_info_timer,
4922 round_jiffies(jiffies + 2 * HZ));
bc7f75fa
AK		 4923 }
4924 } else {
4925 if (netif_carrier_ok(netdev)) {
4926 adapter->link_speed = 0;
4927 adapter->link_duplex = 0;
8f12fe86 4928 /* Link status message must follow this format */
7dbc1672 4929 pr_info("%s NIC Link is Down\n", adapter->netdev->name);
bc7f75fa 4930 netif_carrier_off(netdev);
bc7f75fa
AK		 4931 if (!test_bit(__E1000_DOWN, &adapter->state))
4932 mod_timer(&adapter->phy_info_timer,
4933 round_jiffies(jiffies + 2 * HZ));
4934
12d43f7d
BA		 4935 /* The link is lost so the controller stops DMA.
4936 * If there is queued Tx work that cannot be done
4937 * or if on an 8000ES2LAN which requires a Rx packet
4938 * buffer work-around on link down event, reset the
4939 * controller to flush the Tx/Rx packet buffers.
4940 * (Do the reset outside of interrupt context).
4941 */
4942 if ((adapter->flags & FLAG_RX_NEEDS_RESTART) ||
4943 (e1000_desc_unused(tx_ring) + 1 < tx_ring->count))
4944 adapter->flags |= FLAG_RESTART_NOW;
23606cf5
RW		 4945 else
4946 pm_schedule_suspend(netdev->dev.parent,
17e813ec 4947 LINK_TIMEOUT);
bc7f75fa
AK		 4948 }
4949 }
4950
4951link_up:
67fd4fcb 4952 spin_lock(&adapter->stats64_lock);
bc7f75fa
AK		 4953 e1000e_update_stats(adapter);
4954
4955 mac->tx_packet_delta = adapter->stats.tpt - adapter->tpt_old;
4956 adapter->tpt_old = adapter->stats.tpt;
4957 mac->collision_delta = adapter->stats.colc - adapter->colc_old;
4958 adapter->colc_old = adapter->stats.colc;
4959
7c25769f
BA		 4960 adapter->gorc = adapter->stats.gorc - adapter->gorc_old;
4961 adapter->gorc_old = adapter->stats.gorc;
4962 adapter->gotc = adapter->stats.gotc - adapter->gotc_old;
4963 adapter->gotc_old = adapter->stats.gotc;
2084b114 4964 spin_unlock(&adapter->stats64_lock);
bc7f75fa 4965
12d43f7d 4966 if (adapter->flags & FLAG_RESTART_NOW) {
90da0669
BA		 4967 schedule_work(&adapter->reset_task);
4968 /* return immediately since reset is imminent */
4969 return;
bc7f75fa
AK		 4970 }
4971
12d43f7d
BA		 4972 e1000e_update_adaptive(&adapter->hw);
4973
eab2abf5
JB		 4974 /* Simple mode for Interrupt Throttle Rate (ITR) */
4975 if (adapter->itr_setting == 4) {
e921eb1a 4976 /* Symmetric Tx/Rx gets a reduced ITR=2000;
eab2abf5
JB		 4977 * Total asymmetrical Tx or Rx gets ITR=8000;
4978 * everyone else is between 2000-8000.
4979 */
4980 u32 goc = (adapter->gotc + adapter->gorc) / 10000;
4981 u32 dif = (adapter->gotc > adapter->gorc ?
17e813ec
BA		 4982 adapter->gotc - adapter->gorc :
4983 adapter->gorc - adapter->gotc) / 10000;
eab2abf5
JB		 4984 u32 itr = goc > 0 ? (dif * 6000 / goc + 2000) : 8000;
4985
22a4cca2 4986 e1000e_write_itr(adapter, itr);
eab2abf5
JB		 4987 }
4988
ad68076e 4989 /* Cause software interrupt to ensure Rx ring is cleaned */
4662e82b
BA		 4990 if (adapter->msix_entries)
4991 ew32(ICS, adapter->rx_ring->ims_val);
4992 else
4993 ew32(ICS, E1000_ICS_RXDMT0);
bc7f75fa 4994
713b3c9e
JB		 4995 /* flush pending descriptors to memory before detecting Tx hang */
4996 e1000e_flush_descriptors(adapter);
4997
bc7f75fa 4998 /* Force detection of hung controller every watchdog period */
3db1cd5c 4999 adapter->detect_tx_hung = true;
bc7f75fa 5000
e921eb1a 5001 /* With 82571 controllers, LAA may be overwritten due to controller
ad68076e
BA		 5002 * reset from the other port. Set the appropriate LAA in RAR[0]
5003 */
bc7f75fa 5004 if (e1000e_get_laa_state_82571(hw))
69e1e019 5005 hw->mac.ops.rar_set(hw, adapter->hw.mac.addr, 0);
bc7f75fa 5006
ff10e13c
CW		 5007 if (adapter->flags2 & FLAG2_CHECK_PHY_HANG)
5008 e1000e_check_82574_phy_workaround(adapter);
5009
b67e1913
BA		 5010 /* Clear valid timestamp stuck in RXSTMPL/H due to a Rx error */
5011 if (adapter->hwtstamp_config.rx_filter != HWTSTAMP_FILTER_NONE) {
5012 if ((adapter->flags2 & FLAG2_CHECK_RX_HWTSTAMP) &&
5013 (er32(TSYNCRXCTL) & E1000_TSYNCRXCTL_VALID)) {
5014 er32(RXSTMPH);
5015 adapter->rx_hwtstamp_cleared++;
5016 } else {
5017 adapter->flags2 |= FLAG2_CHECK_RX_HWTSTAMP;
5018 }
5019 }
5020
bc7f75fa
AK		 5021 /* Reset the timer */
5022 if (!test_bit(__E1000_DOWN, &adapter->state))
5023 mod_timer(&adapter->watchdog_timer,
5024 round_jiffies(jiffies + 2 * HZ));
5025}
5026
5027#define E1000_TX_FLAGS_CSUM 0x00000001
5028#define E1000_TX_FLAGS_VLAN 0x00000002
5029#define E1000_TX_FLAGS_TSO 0x00000004
5030#define E1000_TX_FLAGS_IPV4 0x00000008
943146de 5031#define E1000_TX_FLAGS_NO_FCS 0x00000010
b67e1913 5032#define E1000_TX_FLAGS_HWTSTAMP 0x00000020
bc7f75fa
AK		 5033#define E1000_TX_FLAGS_VLAN_MASK 0xffff0000
5034#define E1000_TX_FLAGS_VLAN_SHIFT 16
5035
55aa6985 5036static int e1000_tso(struct e1000_ring *tx_ring, struct sk_buff *skb)
bc7f75fa 5037{
bc7f75fa
AK		 5038 struct e1000_context_desc *context_desc;
5039 struct e1000_buffer *buffer_info;
5040 unsigned int i;
5041 u32 cmd_length = 0;
70443ae9 5042 u16 ipcse = 0, mss;
bc7f75fa 5043 u8 ipcss, ipcso, tucss, tucso, hdr_len;
bc7f75fa 5044
3d5e33c9
BA		 5045 if (!skb_is_gso(skb))
5046 return 0;
bc7f75fa 5047
3d5e33c9 5048 if (skb_header_cloned(skb)) {
90da0669
BA		 5049 int err = pskb_expand_head(skb, 0, 0, GFP_ATOMIC);
5050
3d5e33c9
BA		 5051 if (err)
5052 return err;
bc7f75fa
AK		 5053 }
5054
3d5e33c9
BA		 5055 hdr_len = skb_transport_offset(skb) + tcp_hdrlen(skb);
5056 mss = skb_shinfo(skb)->gso_size;
5057 if (skb->protocol == htons(ETH_P_IP)) {
5058 struct iphdr *iph = ip_hdr(skb);
5059 iph->tot_len = 0;
5060 iph->check = 0;
5061 tcp_hdr(skb)->check = ~csum_tcpudp_magic(iph->saddr, iph->daddr,
f0ff4398 5062 0, IPPROTO_TCP, 0);
3d5e33c9
BA		 5063 cmd_length = E1000_TXD_CMD_IP;
5064 ipcse = skb_transport_offset(skb) - 1;
8e1e8a47 5065 } else if (skb_is_gso_v6(skb)) {
3d5e33c9
BA		 5066 ipv6_hdr(skb)->payload_len = 0;
5067 tcp_hdr(skb)->check = ~csum_ipv6_magic(&ipv6_hdr(skb)->saddr,
f0ff4398
BA		 5068 &ipv6_hdr(skb)->daddr,
5069 0, IPPROTO_TCP, 0);
3d5e33c9
BA		 5070 ipcse = 0;
5071 }
5072 ipcss = skb_network_offset(skb);
5073 ipcso = (void *)&(ip_hdr(skb)->check) - (void *)skb->data;
5074 tucss = skb_transport_offset(skb);
5075 tucso = (void *)&(tcp_hdr(skb)->check) - (void *)skb->data;
3d5e33c9
BA		 5076
5077 cmd_length |= (E1000_TXD_CMD_DEXT | E1000_TXD_CMD_TSE |
f0ff4398 5078 E1000_TXD_CMD_TCP | (skb->len - (hdr_len)));
3d5e33c9
BA		 5079
5080 i = tx_ring->next_to_use;
5081 context_desc = E1000_CONTEXT_DESC(*tx_ring, i);
5082 buffer_info = &tx_ring->buffer_info[i];
5083
5084 context_desc->lower_setup.ip_fields.ipcss = ipcss;
5085 context_desc->lower_setup.ip_fields.ipcso = ipcso;
5086 context_desc->lower_setup.ip_fields.ipcse = cpu_to_le16(ipcse);
5087 context_desc->upper_setup.tcp_fields.tucss = tucss;
5088 context_desc->upper_setup.tcp_fields.tucso = tucso;
70443ae9 5089 context_desc->upper_setup.tcp_fields.tucse = 0;
3d5e33c9
BA		 5090 context_desc->tcp_seg_setup.fields.mss = cpu_to_le16(mss);
5091 context_desc->tcp_seg_setup.fields.hdr_len = hdr_len;
5092 context_desc->cmd_and_length = cpu_to_le32(cmd_length);
5093
5094 buffer_info->time_stamp = jiffies;
5095 buffer_info->next_to_watch = i;
5096
5097 i++;
5098 if (i == tx_ring->count)
5099 i = 0;
5100 tx_ring->next_to_use = i;
5101
5102 return 1;
bc7f75fa
AK		 5103}
5104
55aa6985 5105static bool e1000_tx_csum(struct e1000_ring *tx_ring, struct sk_buff *skb)
bc7f75fa 5106{
55aa6985 5107 struct e1000_adapter *adapter = tx_ring->adapter;
bc7f75fa
AK		 5108 struct e1000_context_desc *context_desc;
5109 struct e1000_buffer *buffer_info;
5110 unsigned int i;
5111 u8 css;
af807c82 5112 u32 cmd_len = E1000_TXD_CMD_DEXT;
5f66f208 5113 __be16 protocol;
bc7f75fa 5114
af807c82
DG		 5115 if (skb->ip_summed != CHECKSUM_PARTIAL)
5116 return 0;
bc7f75fa 5117
5f66f208
AJ		 5118 if (skb->protocol == cpu_to_be16(ETH_P_8021Q))
5119 protocol = vlan_eth_hdr(skb)->h_vlan_encapsulated_proto;
5120 else
5121 protocol = skb->protocol;
5122
3f518390 5123 switch (protocol) {
09640e63 5124 case cpu_to_be16(ETH_P_IP):
af807c82
DG		 5125 if (ip_hdr(skb)->protocol == IPPROTO_TCP)
5126 cmd_len |= E1000_TXD_CMD_TCP;
5127 break;
09640e63 5128 case cpu_to_be16(ETH_P_IPV6):
af807c82
DG		 5129 /* XXX not handling all IPV6 headers */
5130 if (ipv6_hdr(skb)->nexthdr == IPPROTO_TCP)
5131 cmd_len |= E1000_TXD_CMD_TCP;
5132 break;
5133 default:
5134 if (unlikely(net_ratelimit()))
5f66f208
AJ		 5135 e_warn("checksum_partial proto=%x!\n",
5136 be16_to_cpu(protocol));
af807c82 5137 break;
bc7f75fa
AK		 5138 }
5139
0d0b1672 5140 css = skb_checksum_start_offset(skb);
af807c82
DG		 5141
5142 i = tx_ring->next_to_use;
5143 buffer_info = &tx_ring->buffer_info[i];
5144 context_desc = E1000_CONTEXT_DESC(*tx_ring, i);
5145
5146 context_desc->lower_setup.ip_config = 0;
5147 context_desc->upper_setup.tcp_fields.tucss = css;
f0ff4398 5148 context_desc->upper_setup.tcp_fields.tucso = css + skb->csum_offset;
af807c82
DG		 5149 context_desc->upper_setup.tcp_fields.tucse = 0;
5150 context_desc->tcp_seg_setup.data = 0;
5151 context_desc->cmd_and_length = cpu_to_le32(cmd_len);
5152
5153 buffer_info->time_stamp = jiffies;
5154 buffer_info->next_to_watch = i;
5155
5156 i++;
5157 if (i == tx_ring->count)
5158 i = 0;
5159 tx_ring->next_to_use = i;
5160
5161 return 1;
bc7f75fa
AK		 5162}
5163
55aa6985
BA		 5164static int e1000_tx_map(struct e1000_ring *tx_ring, struct sk_buff *skb,
5165 unsigned int first, unsigned int max_per_txd,
d821a4c4 5166 unsigned int nr_frags)
bc7f75fa 5167{
55aa6985 5168 struct e1000_adapter *adapter = tx_ring->adapter;
03b1320d 5169 struct pci_dev *pdev = adapter->pdev;
1b7719c4 5170 struct e1000_buffer *buffer_info;
8ddc951c 5171 unsigned int len = skb_headlen(skb);
03b1320d 5172 unsigned int offset = 0, size, count = 0, i;
9ed318d5 5173 unsigned int f, bytecount, segs;
bc7f75fa
AK		 5174
5175 i = tx_ring->next_to_use;
5176
5177 while (len) {
1b7719c4 5178 buffer_info = &tx_ring->buffer_info[i];
bc7f75fa
AK		 5179 size = min(len, max_per_txd);
5180
bc7f75fa 5181 buffer_info->length = size;
bc7f75fa 5182 buffer_info->time_stamp = jiffies;
bc7f75fa 5183 buffer_info->next_to_watch = i;
0be3f55f
NN		 5184 buffer_info->dma = dma_map_single(&pdev->dev,
5185 skb->data + offset,
af667a29 5186 size, DMA_TO_DEVICE);
03b1320d 5187 buffer_info->mapped_as_page = false;
0be3f55f 5188 if (dma_mapping_error(&pdev->dev, buffer_info->dma))
03b1320d 5189 goto dma_error;
bc7f75fa
AK		 5190
5191 len -= size;
5192 offset += size;
03b1320d 5193 count++;
1b7719c4
AD		 5194
5195 if (len) {
5196 i++;
5197 if (i == tx_ring->count)
5198 i = 0;
5199 }
bc7f75fa
AK		 5200 }
5201
5202 for (f = 0; f < nr_frags; f++) {
9e903e08 5203 const struct skb_frag_struct *frag;
bc7f75fa
AK		 5204
5205 frag = &skb_shinfo(skb)->frags[f];
9e903e08 5206 len = skb_frag_size(frag);
877749bf 5207 offset = 0;
bc7f75fa
AK		 5208
5209 while (len) {
1b7719c4
AD		 5210 i++;
5211 if (i == tx_ring->count)
5212 i = 0;
5213
bc7f75fa
AK		 5214 buffer_info = &tx_ring->buffer_info[i];
5215 size = min(len, max_per_txd);
bc7f75fa
AK		 5216
5217 buffer_info->length = size;
5218 buffer_info->time_stamp = jiffies;
bc7f75fa 5219 buffer_info->next_to_watch = i;
877749bf 5220 buffer_info->dma = skb_frag_dma_map(&pdev->dev, frag,
17e813ec
BA		 5221 offset, size,
5222 DMA_TO_DEVICE);
03b1320d 5223 buffer_info->mapped_as_page = true;
0be3f55f 5224 if (dma_mapping_error(&pdev->dev, buffer_info->dma))
03b1320d 5225 goto dma_error;
bc7f75fa
AK		 5226
5227 len -= size;
5228 offset += size;
5229 count++;
bc7f75fa
AK		 5230 }
5231 }
5232
af667a29 5233 segs = skb_shinfo(skb)->gso_segs ? : 1;
9ed318d5
TH		 5234 /* multiply data chunks by size of headers */
5235 bytecount = ((segs - 1) * skb_headlen(skb)) + skb->len;
5236
bc7f75fa 5237 tx_ring->buffer_info[i].skb = skb;
9ed318d5
TH		 5238 tx_ring->buffer_info[i].segs = segs;
5239 tx_ring->buffer_info[i].bytecount = bytecount;
bc7f75fa
AK		 5240 tx_ring->buffer_info[first].next_to_watch = i;
5241
5242 return count;
03b1320d
AD		 5243
5244dma_error:
af667a29 5245 dev_err(&pdev->dev, "Tx DMA map failed\n");
03b1320d 5246 buffer_info->dma = 0;
c1fa347f 5247 if (count)
03b1320d 5248 count--;
c1fa347f
RK		 5249
5250 while (count--) {
af667a29 5251 if (i == 0)
03b1320d 5252 i += tx_ring->count;
c1fa347f 5253 i--;
03b1320d 5254 buffer_info = &tx_ring->buffer_info[i];
55aa6985 5255 e1000_put_txbuf(tx_ring, buffer_info);
03b1320d
AD		 5256 }
5257
5258 return 0;
bc7f75fa
AK		 5259}
5260
55aa6985 5261static void e1000_tx_queue(struct e1000_ring *tx_ring, int tx_flags, int count)
bc7f75fa 5262{
55aa6985 5263 struct e1000_adapter *adapter = tx_ring->adapter;
bc7f75fa
AK		 5264 struct e1000_tx_desc *tx_desc = NULL;
5265 struct e1000_buffer *buffer_info;
5266 u32 txd_upper = 0, txd_lower = E1000_TXD_CMD_IFCS;
5267 unsigned int i;
5268
5269 if (tx_flags & E1000_TX_FLAGS_TSO) {
5270 txd_lower |= E1000_TXD_CMD_DEXT | E1000_TXD_DTYP_D |
f0ff4398 5271 E1000_TXD_CMD_TSE;
bc7f75fa
AK		 5272 txd_upper |= E1000_TXD_POPTS_TXSM << 8;
5273
5274 if (tx_flags & E1000_TX_FLAGS_IPV4)
5275 txd_upper |= E1000_TXD_POPTS_IXSM << 8;
5276 }
5277
5278 if (tx_flags & E1000_TX_FLAGS_CSUM) {
5279 txd_lower |= E1000_TXD_CMD_DEXT | E1000_TXD_DTYP_D;
5280 txd_upper |= E1000_TXD_POPTS_TXSM << 8;
5281 }
5282
5283 if (tx_flags & E1000_TX_FLAGS_VLAN) {
5284 txd_lower |= E1000_TXD_CMD_VLE;
5285 txd_upper |= (tx_flags & E1000_TX_FLAGS_VLAN_MASK);
5286 }
5287
943146de
BG		 5288 if (unlikely(tx_flags & E1000_TX_FLAGS_NO_FCS))
5289 txd_lower &= ~(E1000_TXD_CMD_IFCS);
5290
b67e1913
BA		 5291 if (unlikely(tx_flags & E1000_TX_FLAGS_HWTSTAMP)) {
5292 txd_lower |= E1000_TXD_CMD_DEXT | E1000_TXD_DTYP_D;
5293 txd_upper |= E1000_TXD_EXTCMD_TSTAMP;
5294 }
5295
bc7f75fa
AK		 5296 i = tx_ring->next_to_use;
5297
36b973df 5298 do {
bc7f75fa
AK		 5299 buffer_info = &tx_ring->buffer_info[i];
5300 tx_desc = E1000_TX_DESC(*tx_ring, i);
5301 tx_desc->buffer_addr = cpu_to_le64(buffer_info->dma);
f0ff4398
BA		 5302 tx_desc->lower.data = cpu_to_le32(txd_lower |
5303 buffer_info->length);
bc7f75fa
AK		 5304 tx_desc->upper.data = cpu_to_le32(txd_upper);
5305
5306 i++;
5307 if (i == tx_ring->count)
5308 i = 0;
36b973df 5309 } while (--count > 0);
bc7f75fa
AK		 5310
5311 tx_desc->lower.data |= cpu_to_le32(adapter->txd_cmd);
5312
943146de
BG		 5313 /* txd_cmd re-enables FCS, so we'll re-disable it here as desired. */
5314 if (unlikely(tx_flags & E1000_TX_FLAGS_NO_FCS))
5315 tx_desc->lower.data &= ~(cpu_to_le32(E1000_TXD_CMD_IFCS));
5316
e921eb1a 5317 /* Force memory writes to complete before letting h/w
bc7f75fa
AK		 5318 * know there are new descriptors to fetch. (Only
5319 * applicable for weak-ordered memory model archs,
ad68076e
BA		 5320 * such as IA-64).
5321 */
bc7f75fa
AK		 5322 wmb();
5323
5324 tx_ring->next_to_use = i;
c6e7f51e
BA		 5325
5326 if (adapter->flags2 & FLAG2_PCIM2PCI_ARBITER_WA)
55aa6985 5327 e1000e_update_tdt_wa(tx_ring, i);
c6e7f51e 5328 else
c5083cf6 5329 writel(i, tx_ring->tail);
c6e7f51e 5330
e921eb1a 5331 /* we need this if more than one processor can write to our tail
ad68076e
BA		 5332 * at a time, it synchronizes IO on IA64/Altix systems
5333 */
bc7f75fa
AK		 5334 mmiowb();
5335}
5336
5337#define MINIMUM_DHCP_PACKET_SIZE 282
5338static int e1000_transfer_dhcp_info(struct e1000_adapter *adapter,
5339 struct sk_buff *skb)
5340{
5341 struct e1000_hw *hw = &adapter->hw;
5342 u16 length, offset;
5343
d60923c4
BA		 5344 if (vlan_tx_tag_present(skb) &&
5345 !((vlan_tx_tag_get(skb) == adapter->hw.mng_cookie.vlan_id) &&
5346 (adapter->hw.mng_cookie.status &
5347 E1000_MNG_DHCP_COOKIE_STATUS_VLAN)))
5348 return 0;
bc7f75fa
AK		 5349
5350 if (skb->len <= MINIMUM_DHCP_PACKET_SIZE)
5351 return 0;
5352
5353 if (((struct ethhdr *) skb->data)->h_proto != htons(ETH_P_IP))
5354 return 0;
5355
5356 {
362e20ca 5357 const struct iphdr *ip = (struct iphdr *)((u8 *)skb->data + 14);
bc7f75fa
AK		 5358 struct udphdr *udp;
5359
5360 if (ip->protocol != IPPROTO_UDP)
5361 return 0;
5362
5363 udp = (struct udphdr *)((u8 *)ip + (ip->ihl << 2));
5364 if (ntohs(udp->dest) != 67)
5365 return 0;
5366
5367 offset = (u8 *)udp + 8 - skb->data;
5368 length = skb->len - offset;
5369 return e1000e_mng_write_dhcp_info(hw, (u8 *)udp + 8, length);
5370 }
5371
5372 return 0;
5373}
5374
55aa6985 5375static int __e1000_maybe_stop_tx(struct e1000_ring *tx_ring, int size)
bc7f75fa 5376{
55aa6985 5377 struct e1000_adapter *adapter = tx_ring->adapter;
bc7f75fa 5378
55aa6985 5379 netif_stop_queue(adapter->netdev);
e921eb1a 5380 /* Herbert's original patch had:
bc7f75fa 5381 * smp_mb__after_netif_stop_queue();
ad68076e
BA		 5382 * but since that doesn't exist yet, just open code it.
5383 */
bc7f75fa
AK		 5384 smp_mb();
5385
e921eb1a 5386 /* We need to check again in a case another CPU has just
ad68076e
BA		 5387 * made room available.
5388 */
55aa6985 5389 if (e1000_desc_unused(tx_ring) < size)
bc7f75fa
AK		 5390 return -EBUSY;
5391
5392 /* A reprieve! */
55aa6985 5393 netif_start_queue(adapter->netdev);
bc7f75fa
AK		 5394 ++adapter->restart_queue;
5395 return 0;
5396}
5397
55aa6985 5398static int e1000_maybe_stop_tx(struct e1000_ring *tx_ring, int size)
bc7f75fa 5399{
d821a4c4
BA		 5400 BUG_ON(size > tx_ring->count);
5401
55aa6985 5402 if (e1000_desc_unused(tx_ring) >= size)
bc7f75fa 5403 return 0;
55aa6985 5404 return __e1000_maybe_stop_tx(tx_ring, size);
bc7f75fa
AK		 5405}
5406
3b29a56d
SH		 5407static netdev_tx_t e1000_xmit_frame(struct sk_buff *skb,
5408 struct net_device *netdev)
bc7f75fa
AK		 5409{
5410 struct e1000_adapter *adapter = netdev_priv(netdev);
5411 struct e1000_ring *tx_ring = adapter->tx_ring;
5412 unsigned int first;
bc7f75fa 5413 unsigned int tx_flags = 0;
e743d313 5414 unsigned int len = skb_headlen(skb);
4e6c709c
AK		 5415 unsigned int nr_frags;
5416 unsigned int mss;
bc7f75fa
AK		 5417 int count = 0;
5418 int tso;
5419 unsigned int f;
bc7f75fa
AK		 5420
5421 if (test_bit(__E1000_DOWN, &adapter->state)) {
5422 dev_kfree_skb_any(skb);
5423 return NETDEV_TX_OK;
5424 }
5425
5426 if (skb->len <= 0) {
5427 dev_kfree_skb_any(skb);
5428 return NETDEV_TX_OK;
5429 }
5430
e921eb1a 5431 /* The minimum packet size with TCTL.PSP set is 17 bytes so
6e97c170
TD		 5432 * pad skb in order to meet this minimum size requirement
5433 */
5434 if (unlikely(skb->len < 17)) {
5435 if (skb_pad(skb, 17 - skb->len))
5436 return NETDEV_TX_OK;
5437 skb->len = 17;
5438 skb_set_tail_pointer(skb, 17);
5439 }
5440
bc7f75fa 5441 mss = skb_shinfo(skb)->gso_size;
bc7f75fa
AK		 5442 if (mss) {
5443 u8 hdr_len;
bc7f75fa 5444
e921eb1a 5445 /* TSO Workaround for 82571/2/3 Controllers -- if skb->data
ad68076e
BA		 5446 * points to just header, pull a few bytes of payload from
5447 * frags into skb->data
5448 */
bc7f75fa 5449 hdr_len = skb_transport_offset(skb) + tcp_hdrlen(skb);
e921eb1a 5450 /* we do this workaround for ES2LAN, but it is un-necessary,
ad68076e
BA		 5451 * avoiding it could save a lot of cycles
5452 */
4e6c709c 5453 if (skb->data_len && (hdr_len == len)) {
bc7f75fa
AK		 5454 unsigned int pull_size;
5455
a2a5b323 5456 pull_size = min_t(unsigned int, 4, skb->data_len);
bc7f75fa 5457 if (!__pskb_pull_tail(skb, pull_size)) {
44defeb3 5458 e_err("__pskb_pull_tail failed.\n");
bc7f75fa
AK		 5459 dev_kfree_skb_any(skb);
5460 return NETDEV_TX_OK;
5461 }
e743d313 5462 len = skb_headlen(skb);
bc7f75fa
AK		 5463 }
5464 }
5465
5466 /* reserve a descriptor for the offload context */
5467 if ((mss) || (skb->ip_summed == CHECKSUM_PARTIAL))
5468 count++;
5469 count++;
5470
d821a4c4 5471 count += DIV_ROUND_UP(len, adapter->tx_fifo_limit);
bc7f75fa
AK		 5472
5473 nr_frags = skb_shinfo(skb)->nr_frags;
5474 for (f = 0; f < nr_frags; f++)
d821a4c4
BA		 5475 count += DIV_ROUND_UP(skb_frag_size(&skb_shinfo(skb)->frags[f]),
5476 adapter->tx_fifo_limit);
bc7f75fa
AK		 5477
5478 if (adapter->hw.mac.tx_pkt_filtering)
5479 e1000_transfer_dhcp_info(adapter, skb);
5480
e921eb1a 5481 /* need: count + 2 desc gap to keep tail from touching
ad68076e
BA		 5482 * head, otherwise try next time
5483 */
55aa6985 5484 if (e1000_maybe_stop_tx(tx_ring, count + 2))
bc7f75fa 5485 return NETDEV_TX_BUSY;
bc7f75fa 5486
eab6d18d 5487 if (vlan_tx_tag_present(skb)) {
bc7f75fa
AK		 5488 tx_flags |= E1000_TX_FLAGS_VLAN;
5489 tx_flags |= (vlan_tx_tag_get(skb) << E1000_TX_FLAGS_VLAN_SHIFT);
5490 }
5491
5492 first = tx_ring->next_to_use;
5493
55aa6985 5494 tso = e1000_tso(tx_ring, skb);
bc7f75fa
AK		 5495 if (tso < 0) {
5496 dev_kfree_skb_any(skb);
bc7f75fa
AK		 5497 return NETDEV_TX_OK;
5498 }
5499
5500 if (tso)
5501 tx_flags |= E1000_TX_FLAGS_TSO;
55aa6985 5502 else if (e1000_tx_csum(tx_ring, skb))
bc7f75fa
AK		 5503 tx_flags |= E1000_TX_FLAGS_CSUM;
5504
e921eb1a 5505 /* Old method was to assume IPv4 packet by default if TSO was enabled.
bc7f75fa 5506 * 82571 hardware supports TSO capabilities for IPv6 as well...
ad68076e
BA		 5507 * no longer assume, we must.
5508 */
bc7f75fa
AK		 5509 if (skb->protocol == htons(ETH_P_IP))
5510 tx_flags |= E1000_TX_FLAGS_IPV4;
5511
943146de
BG		 5512 if (unlikely(skb->no_fcs))
5513 tx_flags |= E1000_TX_FLAGS_NO_FCS;
5514
25985edc 5515 /* if count is 0 then mapping error has occurred */
d821a4c4
BA		 5516 count = e1000_tx_map(tx_ring, skb, first, adapter->tx_fifo_limit,
5517 nr_frags);
1b7719c4 5518 if (count) {
b67e1913
BA		 5519 if (unlikely((skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) &&
5520 !adapter->tx_hwtstamp_skb)) {
5521 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;
5522 tx_flags |= E1000_TX_FLAGS_HWTSTAMP;
5523 adapter->tx_hwtstamp_skb = skb_get(skb);
5524 schedule_work(&adapter->tx_hwtstamp_work);
5525 } else {
5526 skb_tx_timestamp(skb);
5527 }
80be3129 5528
3f0cfa3b 5529 netdev_sent_queue(netdev, skb->len);
55aa6985 5530 e1000_tx_queue(tx_ring, tx_flags, count);
1b7719c4 5531 /* Make sure there is space in the ring for the next send. */
d821a4c4
BA		 5532 e1000_maybe_stop_tx(tx_ring,
5533 (MAX_SKB_FRAGS *
5534 DIV_ROUND_UP(PAGE_SIZE,
5535 adapter->tx_fifo_limit) + 2));
1b7719c4 5536 } else {
bc7f75fa 5537 dev_kfree_skb_any(skb);
1b7719c4
AD		 5538 tx_ring->buffer_info[first].time_stamp = 0;
5539 tx_ring->next_to_use = first;
bc7f75fa
AK		 5540 }
5541
bc7f75fa
AK		 5542 return NETDEV_TX_OK;
5543}
5544
5545/**
5546 * e1000_tx_timeout - Respond to a Tx Hang
5547 * @netdev: network interface device structure
5548 **/
5549static void e1000_tx_timeout(struct net_device *netdev)
5550{
5551 struct e1000_adapter *adapter = netdev_priv(netdev);
5552
5553 /* Do the reset outside of interrupt context */
5554 adapter->tx_timeout_count++;
5555 schedule_work(&adapter->reset_task);
5556}
5557
5558static void e1000_reset_task(struct work_struct *work)
5559{
5560 struct e1000_adapter *adapter;
5561 adapter = container_of(work, struct e1000_adapter, reset_task);
5562
615b32af
JB		 5563 /* don't run the task if already down */
5564 if (test_bit(__E1000_DOWN, &adapter->state))
5565 return;
5566
12d43f7d 5567 if (!(adapter->flags & FLAG_RESTART_NOW)) {
affa9dfb 5568 e1000e_dump(adapter);
12d43f7d 5569 e_err("Reset adapter unexpectedly\n");
affa9dfb 5570 }
bc7f75fa
AK		 5571 e1000e_reinit_locked(adapter);
5572}
5573
5574/**
67fd4fcb 5575 * e1000_get_stats64 - Get System Network Statistics
bc7f75fa 5576 * @netdev: network interface device structure
67fd4fcb 5577 * @stats: rtnl_link_stats64 pointer
bc7f75fa
AK		 5578 *
5579 * Returns the address of the device statistics structure.
bc7f75fa 5580 **/
67fd4fcb 5581struct rtnl_link_stats64 *e1000e_get_stats64(struct net_device *netdev,
66501f56 5582 struct rtnl_link_stats64 *stats)
bc7f75fa 5583{
67fd4fcb
JK		 5584 struct e1000_adapter *adapter = netdev_priv(netdev);
5585
5586 memset(stats, 0, sizeof(struct rtnl_link_stats64));
5587 spin_lock(&adapter->stats64_lock);
5588 e1000e_update_stats(adapter);
5589 /* Fill out the OS statistics structure */
5590 stats->rx_bytes = adapter->stats.gorc;
5591 stats->rx_packets = adapter->stats.gprc;
5592 stats->tx_bytes = adapter->stats.gotc;
5593 stats->tx_packets = adapter->stats.gptc;
5594 stats->multicast = adapter->stats.mprc;
5595 stats->collisions = adapter->stats.colc;
5596
5597 /* Rx Errors */
5598
e921eb1a 5599 /* RLEC on some newer hardware can be incorrect so build
67fd4fcb
JK		 5600 * our own version based on RUC and ROC
5601 */
5602 stats->rx_errors = adapter->stats.rxerrc +
f0ff4398
BA		 5603 adapter->stats.crcerrs + adapter->stats.algnerrc +
5604 adapter->stats.ruc + adapter->stats.roc + adapter->stats.cexterr;
5605 stats->rx_length_errors = adapter->stats.ruc + adapter->stats.roc;
67fd4fcb
JK		 5606 stats->rx_crc_errors = adapter->stats.crcerrs;
5607 stats->rx_frame_errors = adapter->stats.algnerrc;
5608 stats->rx_missed_errors = adapter->stats.mpc;
5609
5610 /* Tx Errors */
f0ff4398 5611 stats->tx_errors = adapter->stats.ecol + adapter->stats.latecol;
67fd4fcb
JK		 5612 stats->tx_aborted_errors = adapter->stats.ecol;
5613 stats->tx_window_errors = adapter->stats.latecol;
5614 stats->tx_carrier_errors = adapter->stats.tncrs;
5615
5616 /* Tx Dropped needs to be maintained elsewhere */
5617
5618 spin_unlock(&adapter->stats64_lock);
5619 return stats;
bc7f75fa
AK		 5620}
5621
5622/**
5623 * e1000_change_mtu - Change the Maximum Transfer Unit
5624 * @netdev: network interface device structure
5625 * @new_mtu: new value for maximum frame size
5626 *
5627 * Returns 0 on success, negative on failure
5628 **/
5629static int e1000_change_mtu(struct net_device *netdev, int new_mtu)
5630{
5631 struct e1000_adapter *adapter = netdev_priv(netdev);
5632 int max_frame = new_mtu + ETH_HLEN + ETH_FCS_LEN;
5633
2adc55c9 5634 /* Jumbo frame support */
2e1706f2
BA		 5635 if ((max_frame > ETH_FRAME_LEN + ETH_FCS_LEN) &&
5636 !(adapter->flags & FLAG_HAS_JUMBO_FRAMES)) {
5637 e_err("Jumbo Frames not supported.\n");
5638 return -EINVAL;
bc7f75fa
AK		 5639 }
5640
2adc55c9
BA		 5641 /* Supported frame sizes */
5642 if ((new_mtu < ETH_ZLEN + ETH_FCS_LEN + VLAN_HLEN) ||
5643 (max_frame > adapter->max_hw_frame_size)) {
5644 e_err("Unsupported MTU setting\n");
bc7f75fa
AK		 5645 return -EINVAL;
5646 }
5647
2fbe4526
BA		 5648 /* Jumbo frame workaround on 82579 and newer requires CRC be stripped */
5649 if ((adapter->hw.mac.type >= e1000_pch2lan) &&
a1ce6473
BA		 5650 !(adapter->flags2 & FLAG2_CRC_STRIPPING) &&
5651 (new_mtu > ETH_DATA_LEN)) {
2fbe4526 5652 e_err("Jumbo Frames not supported on this device when CRC stripping is disabled.\n");
a1ce6473
BA		 5653 return -EINVAL;
5654 }
5655
bc7f75fa 5656 while (test_and_set_bit(__E1000_RESETTING, &adapter->state))
1bba4386 5657 usleep_range(1000, 2000);
610c9928 5658 /* e1000e_down -> e1000e_reset dependent on max_frame_size & mtu */
318a94d6 5659 adapter->max_frame_size = max_frame;
610c9928
BA		 5660 e_info("changing MTU from %d to %d\n", netdev->mtu, new_mtu);
5661 netdev->mtu = new_mtu;
bc7f75fa
AK		 5662 if (netif_running(netdev))
5663 e1000e_down(adapter);
5664
e921eb1a 5665 /* NOTE: netdev_alloc_skb reserves 16 bytes, and typically NET_IP_ALIGN
bc7f75fa
AK		 5666 * means we reserve 2 more, this pushes us to allocate from the next
5667 * larger slab size.
ad68076e 5668 * i.e. RXBUFFER_2048 --> size-4096 slab
97ac8cae
BA		 5669 * However with the new *_jumbo_rx* routines, jumbo receives will use
5670 * fragmented skbs
ad68076e 5671 */
bc7f75fa 5672
9926146b 5673 if (max_frame <= 2048)
bc7f75fa
AK		 5674 adapter->rx_buffer_len = 2048;
5675 else
5676 adapter->rx_buffer_len = 4096;
5677
5678 /* adjust allocation if LPE protects us, and we aren't using SBP */
5679 if ((max_frame == ETH_FRAME_LEN + ETH_FCS_LEN) ||
17e813ec 5680 (max_frame == ETH_FRAME_LEN + VLAN_HLEN + ETH_FCS_LEN))
bc7f75fa 5681 adapter->rx_buffer_len = ETH_FRAME_LEN + VLAN_HLEN
17e813ec 5682 + ETH_FCS_LEN;
bc7f75fa 5683
bc7f75fa
AK		 5684 if (netif_running(netdev))
5685 e1000e_up(adapter);
5686 else
5687 e1000e_reset(adapter);
5688
5689 clear_bit(__E1000_RESETTING, &adapter->state);
5690
5691 return 0;
5692}
5693
5694static int e1000_mii_ioctl(struct net_device *netdev, struct ifreq *ifr,
5695 int cmd)
5696{
5697 struct e1000_adapter *adapter = netdev_priv(netdev);
5698 struct mii_ioctl_data *data = if_mii(ifr);
bc7f75fa 5699
318a94d6 5700 if (adapter->hw.phy.media_type != e1000_media_type_copper)
bc7f75fa
AK		 5701 return -EOPNOTSUPP;
5702
5703 switch (cmd) {
5704 case SIOCGMIIPHY:
5705 data->phy_id = adapter->hw.phy.addr;
5706 break;
5707 case SIOCGMIIREG:
b16a002e
BA		 5708 e1000_phy_read_status(adapter);
5709
7c25769f
BA		 5710 switch (data->reg_num & 0x1F) {
5711 case MII_BMCR:
5712 data->val_out = adapter->phy_regs.bmcr;
5713 break;
5714 case MII_BMSR:
5715 data->val_out = adapter->phy_regs.bmsr;
5716 break;
5717 case MII_PHYSID1:
5718 data->val_out = (adapter->hw.phy.id >> 16);
5719 break;
5720 case MII_PHYSID2:
5721 data->val_out = (adapter->hw.phy.id & 0xFFFF);
5722 break;
5723 case MII_ADVERTISE:
5724 data->val_out = adapter->phy_regs.advertise;
5725 break;
5726 case MII_LPA:
5727 data->val_out = adapter->phy_regs.lpa;
5728 break;
5729 case MII_EXPANSION:
5730 data->val_out = adapter->phy_regs.expansion;
5731 break;
5732 case MII_CTRL1000:
5733 data->val_out = adapter->phy_regs.ctrl1000;
5734 break;
5735 case MII_STAT1000:
5736 data->val_out = adapter->phy_regs.stat1000;
5737 break;
5738 case MII_ESTATUS:
5739 data->val_out = adapter->phy_regs.estatus;
5740 break;
5741 default:
bc7f75fa
AK		 5742 return -EIO;
5743 }
bc7f75fa
AK		 5744 break;
5745 case SIOCSMIIREG:
5746 default:
5747 return -EOPNOTSUPP;
5748 }
5749 return 0;
5750}
5751
b67e1913
BA		 5752/**
5753 * e1000e_hwtstamp_ioctl - control hardware time stamping
5754 * @netdev: network interface device structure
5755 * @ifreq: interface request
5756 *
5757 * Outgoing time stamping can be enabled and disabled. Play nice and
5758 * disable it when requested, although it shouldn't cause any overhead
5759 * when no packet needs it. At most one packet in the queue may be
5760 * marked for time stamping, otherwise it would be impossible to tell
5761 * for sure to which packet the hardware time stamp belongs.
5762 *
5763 * Incoming time stamping has to be configured via the hardware filters.
5764 * Not all combinations are supported, in particular event type has to be
5765 * specified. Matching the kind of event packet is not supported, with the
5766 * exception of "all V2 events regardless of level 2 or 4".
5767 **/
5768static int e1000e_hwtstamp_ioctl(struct net_device *netdev, struct ifreq *ifr)
5769{
5770 struct e1000_adapter *adapter = netdev_priv(netdev);
5771 struct hwtstamp_config config;
5772 int ret_val;
5773
5774 if (copy_from_user(&config, ifr->ifr_data, sizeof(config)))
5775 return -EFAULT;
5776
5777 adapter->hwtstamp_config = config;
5778
5779 ret_val = e1000e_config_hwtstamp(adapter);
5780 if (ret_val)
5781 return ret_val;
5782
5783 config = adapter->hwtstamp_config;
5784
d89777bf
BA		 5785 switch (config.rx_filter) {
5786 case HWTSTAMP_FILTER_PTP_V2_L4_SYNC:
5787 case HWTSTAMP_FILTER_PTP_V2_L2_SYNC:
5788 case HWTSTAMP_FILTER_PTP_V2_SYNC:
5789 case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ:
5790 case HWTSTAMP_FILTER_PTP_V2_L2_DELAY_REQ:
5791 case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ:
5792 /* With V2 type filters which specify a Sync or Delay Request,
5793 * Path Delay Request/Response messages are also time stamped
5794 * by hardware so notify the caller the requested packets plus
5795 * some others are time stamped.
5796 */
5797 config.rx_filter = HWTSTAMP_FILTER_SOME;
5798 break;
5799 default:
5800 break;
5801 }
5802
b67e1913
BA		 5803 return copy_to_user(ifr->ifr_data, &config,
5804 sizeof(config)) ? -EFAULT : 0;
5805}
5806
bc7f75fa
AK		 5807static int e1000_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd)
5808{
5809 switch (cmd) {
5810 case SIOCGMIIPHY:
5811 case SIOCGMIIREG:
5812 case SIOCSMIIREG:
5813 return e1000_mii_ioctl(netdev, ifr, cmd);
b67e1913
BA		 5814 case SIOCSHWTSTAMP:
5815 return e1000e_hwtstamp_ioctl(netdev, ifr);
bc7f75fa
AK		 5816 default:
5817 return -EOPNOTSUPP;
5818 }
5819}
5820
a4f58f54
BA		 5821static int e1000_init_phy_wakeup(struct e1000_adapter *adapter, u32 wufc)
5822{
5823 struct e1000_hw *hw = &adapter->hw;
5824 u32 i, mac_reg;
2b6b168d 5825 u16 phy_reg, wuc_enable;
70806a7f 5826 int retval;
a4f58f54
BA		 5827
5828 /* copy MAC RARs to PHY RARs */
d3738bb8 5829 e1000_copy_rx_addrs_to_phy_ich8lan(hw);
a4f58f54 5830
2b6b168d
BA		 5831 retval = hw->phy.ops.acquire(hw);
5832 if (retval) {
5833 e_err("Could not acquire PHY\n");
5834 return retval;
5835 }
5836
5837 /* Enable access to wakeup registers on and set page to BM_WUC_PAGE */
5838 retval = e1000_enable_phy_wakeup_reg_access_bm(hw, &wuc_enable);
5839 if (retval)
75ce1532 5840 goto release;
2b6b168d
BA		 5841
5842 /* copy MAC MTA to PHY MTA - only needed for pchlan */
a4f58f54
BA		 5843 for (i = 0; i < adapter->hw.mac.mta_reg_count; i++) {
5844 mac_reg = E1000_READ_REG_ARRAY(hw, E1000_MTA, i);
2b6b168d
BA		 5845 hw->phy.ops.write_reg_page(hw, BM_MTA(i),
5846 (u16)(mac_reg & 0xFFFF));
5847 hw->phy.ops.write_reg_page(hw, BM_MTA(i) + 1,
5848 (u16)((mac_reg >> 16) & 0xFFFF));
a4f58f54
BA		 5849 }
5850
5851 /* configure PHY Rx Control register */
2b6b168d 5852 hw->phy.ops.read_reg_page(&adapter->hw, BM_RCTL, &phy_reg);
a4f58f54
BA		 5853 mac_reg = er32(RCTL);
5854 if (mac_reg & E1000_RCTL_UPE)
5855 phy_reg |= BM_RCTL_UPE;
5856 if (mac_reg & E1000_RCTL_MPE)
5857 phy_reg |= BM_RCTL_MPE;
5858 phy_reg &= ~(BM_RCTL_MO_MASK);
5859 if (mac_reg & E1000_RCTL_MO_3)
5860 phy_reg |= (((mac_reg & E1000_RCTL_MO_3) >> E1000_RCTL_MO_SHIFT)
17e813ec 5861 << BM_RCTL_MO_SHIFT);
a4f58f54
BA		 5862 if (mac_reg & E1000_RCTL_BAM)
5863 phy_reg |= BM_RCTL_BAM;
5864 if (mac_reg & E1000_RCTL_PMCF)
5865 phy_reg |= BM_RCTL_PMCF;
5866 mac_reg = er32(CTRL);
5867 if (mac_reg & E1000_CTRL_RFCE)
5868 phy_reg |= BM_RCTL_RFCE;
2b6b168d 5869 hw->phy.ops.write_reg_page(&adapter->hw, BM_RCTL, phy_reg);
a4f58f54
BA		 5870
5871 /* enable PHY wakeup in MAC register */
5872 ew32(WUFC, wufc);
5873 ew32(WUC, E1000_WUC_PHY_WAKE | E1000_WUC_PME_EN);
5874
5875 /* configure and enable PHY wakeup in PHY registers */
2b6b168d
BA		 5876 hw->phy.ops.write_reg_page(&adapter->hw, BM_WUFC, wufc);
5877 hw->phy.ops.write_reg_page(&adapter->hw, BM_WUC, E1000_WUC_PME_EN);
a4f58f54
BA		 5878
5879 /* activate PHY wakeup */
2b6b168d
BA		 5880 wuc_enable |= BM_WUC_ENABLE_BIT | BM_WUC_HOST_WU_BIT;
5881 retval = e1000_disable_phy_wakeup_reg_access_bm(hw, &wuc_enable);
a4f58f54
BA		 5882 if (retval)
5883 e_err("Could not set PHY Host Wakeup bit\n");
75ce1532 5884release:
94d8186a 5885 hw->phy.ops.release(hw);
a4f58f54
BA		 5886
5887 return retval;
5888}
5889
23606cf5
RW		 5890static int __e1000_shutdown(struct pci_dev *pdev, bool *enable_wake,
5891 bool runtime)
bc7f75fa
AK		 5892{
5893 struct net_device *netdev = pci_get_drvdata(pdev);
5894 struct e1000_adapter *adapter = netdev_priv(netdev);
5895 struct e1000_hw *hw = &adapter->hw;
5896 u32 ctrl, ctrl_ext, rctl, status;
23606cf5
RW		 5897 /* Runtime suspend should only enable wakeup for link changes */
5898 u32 wufc = runtime ? E1000_WUFC_LNKC : adapter->wol;
bc7f75fa
AK		 5899 int retval = 0;
5900
5901 netif_device_detach(netdev);
5902
5903 if (netif_running(netdev)) {
bb9e44d0
BA		 5904 int count = E1000_CHECK_RESET_COUNT;
5905
5906 while (test_bit(__E1000_RESETTING, &adapter->state) && count--)
5907 usleep_range(10000, 20000);
5908
bc7f75fa
AK		 5909 WARN_ON(test_bit(__E1000_RESETTING, &adapter->state));
5910 e1000e_down(adapter);
5911 e1000_free_irq(adapter);
5912 }
4662e82b 5913 e1000e_reset_interrupt_capability(adapter);
bc7f75fa
AK		 5914
5915 retval = pci_save_state(pdev);
5916 if (retval)
5917 return retval;
5918
5919 status = er32(STATUS);
5920 if (status & E1000_STATUS_LU)
5921 wufc &= ~E1000_WUFC_LNKC;
5922
5923 if (wufc) {
5924 e1000_setup_rctl(adapter);
ef9b965a 5925 e1000e_set_rx_mode(netdev);
bc7f75fa
AK		 5926
5927 /* turn on all-multi mode if wake on multicast is enabled */
5928 if (wufc & E1000_WUFC_MC) {
5929 rctl = er32(RCTL);
5930 rctl |= E1000_RCTL_MPE;
5931 ew32(RCTL, rctl);
5932 }
5933
5934 ctrl = er32(CTRL);
5935 /* advertise wake from D3Cold */
5936 #define E1000_CTRL_ADVD3WUC 0x00100000
5937 /* phy power management enable */
5938 #define E1000_CTRL_EN_PHY_PWR_MGMT 0x00200000
a4f58f54
BA		 5939 ctrl |= E1000_CTRL_ADVD3WUC;
5940 if (!(adapter->flags2 & FLAG2_HAS_PHY_WAKEUP))
5941 ctrl |= E1000_CTRL_EN_PHY_PWR_MGMT;
bc7f75fa
AK		 5942 ew32(CTRL, ctrl);
5943
318a94d6
JK		 5944 if (adapter->hw.phy.media_type == e1000_media_type_fiber ||
5945 adapter->hw.phy.media_type ==
5946 e1000_media_type_internal_serdes) {
bc7f75fa
AK		 5947 /* keep the laser running in D3 */
5948 ctrl_ext = er32(CTRL_EXT);
93a23f48 5949 ctrl_ext |= E1000_CTRL_EXT_SDP3_DATA;
bc7f75fa
AK		 5950 ew32(CTRL_EXT, ctrl_ext);
5951 }
5952
97ac8cae 5953 if (adapter->flags & FLAG_IS_ICH)
99730e4c 5954 e1000_suspend_workarounds_ich8lan(&adapter->hw);
97ac8cae 5955
bc7f75fa
AK		 5956 /* Allow time for pending master requests to run */
5957 e1000e_disable_pcie_master(&adapter->hw);
5958
82776a4b 5959 if (adapter->flags2 & FLAG2_HAS_PHY_WAKEUP) {
a4f58f54
BA		 5960 /* enable wakeup by the PHY */
5961 retval = e1000_init_phy_wakeup(adapter, wufc);
5962 if (retval)
5963 return retval;
5964 } else {
5965 /* enable wakeup by the MAC */
5966 ew32(WUFC, wufc);
5967 ew32(WUC, E1000_WUC_PME_EN);
5968 }
bc7f75fa
AK		 5969 } else {
5970 ew32(WUC, 0);
5971 ew32(WUFC, 0);
bc7f75fa
AK		 5972 }
5973
4f9de721
RW		 5974 *enable_wake = !!wufc;
5975
bc7f75fa 5976 /* make sure adapter isn't asleep if manageability is enabled */
82776a4b
BA		 5977 if ((adapter->flags & FLAG_MNG_PT_ENABLED) ||
5978 (hw->mac.ops.check_mng_mode(hw)))
4f9de721 5979 *enable_wake = true;
bc7f75fa
AK		 5980
5981 if (adapter->hw.phy.type == e1000_phy_igp_3)
5982 e1000e_igp3_phy_powerdown_workaround_ich8lan(&adapter->hw);
5983
e921eb1a 5984 /* Release control of h/w to f/w. If f/w is AMT enabled, this
ad68076e
BA		 5985 * would have already happened in close and is redundant.
5986 */
31dbe5b4 5987 e1000e_release_hw_control(adapter);
bc7f75fa
AK		 5988
5989 pci_disable_device(pdev);
5990
4f9de721
RW		 5991 return 0;
5992}
5993
5994static void e1000_power_off(struct pci_dev *pdev, bool sleep, bool wake)
5995{
5996 if (sleep && wake) {
5997 pci_prepare_to_sleep(pdev);
5998 return;
5999 }
6000
6001 pci_wake_from_d3(pdev, wake);
6002 pci_set_power_state(pdev, PCI_D3hot);
6003}
6004
f0ff4398 6005static void e1000_complete_shutdown(struct pci_dev *pdev, bool sleep, bool wake)
4f9de721
RW		 6006{
6007 struct net_device *netdev = pci_get_drvdata(pdev);
6008 struct e1000_adapter *adapter = netdev_priv(netdev);
6009
e921eb1a 6010 /* The pci-e switch on some quad port adapters will report a
005cbdfc
AD		 6011 * correctable error when the MAC transitions from D0 to D3. To
6012 * prevent this we need to mask off the correctable errors on the
6013 * downstream port of the pci-e switch.
6014 */
6015 if (adapter->flags & FLAG_IS_QUAD_PORT) {
6016 struct pci_dev *us_dev = pdev->bus->self;
005cbdfc
AD		 6017 u16 devctl;
6018
f8c0fcac
JL		 6019 pcie_capability_read_word(us_dev, PCI_EXP_DEVCTL, &devctl);
6020 pcie_capability_write_word(us_dev, PCI_EXP_DEVCTL,
6021 (devctl & ~PCI_EXP_DEVCTL_CERE));
005cbdfc 6022
4f9de721 6023 e1000_power_off(pdev, sleep, wake);
005cbdfc 6024
f8c0fcac 6025 pcie_capability_write_word(us_dev, PCI_EXP_DEVCTL, devctl);
005cbdfc 6026 } else {
4f9de721 6027 e1000_power_off(pdev, sleep, wake);
005cbdfc 6028 }
bc7f75fa
AK		 6029}
6030
6f461f6c
BA		 6031#ifdef CONFIG_PCIEASPM
6032static void __e1000e_disable_aspm(struct pci_dev *pdev, u16 state)
6033{
9f728f53 6034 pci_disable_link_state_locked(pdev, state);
6f461f6c
BA		 6035}
6036#else
6037static void __e1000e_disable_aspm(struct pci_dev *pdev, u16 state)
1eae4eb2 6038{
ffe0b2ff
BH		 6039 u16 aspm_ctl = 0;
6040
6041 if (state & PCIE_LINK_STATE_L0S)
6042 aspm_ctl |= PCI_EXP_LNKCTL_ASPM_L0S;
6043 if (state & PCIE_LINK_STATE_L1)
6044 aspm_ctl |= PCI_EXP_LNKCTL_ASPM_L1;
6045
e921eb1a 6046 /* Both device and parent should have the same ASPM setting.
6f461f6c 6047 * Disable ASPM in downstream component first and then upstream.
1eae4eb2 6048 */
ffe0b2ff 6049 pcie_capability_clear_word(pdev, PCI_EXP_LNKCTL, aspm_ctl);
0c75ba22 6050
f8c0fcac
JL		 6051 if (pdev->bus->self)
6052 pcie_capability_clear_word(pdev->bus->self, PCI_EXP_LNKCTL,
ffe0b2ff 6053 aspm_ctl);
6f461f6c
BA		 6054}
6055#endif
78cd29d5 6056static void e1000e_disable_aspm(struct pci_dev *pdev, u16 state)
6f461f6c
BA		 6057{
6058 dev_info(&pdev->dev, "Disabling ASPM %s %s\n",
6059 (state & PCIE_LINK_STATE_L0S) ? "L0s" : "",
6060 (state & PCIE_LINK_STATE_L1) ? "L1" : "");
6061
6062 __e1000e_disable_aspm(pdev, state);
1eae4eb2
AK		 6063}
6064
aa338601 6065#ifdef CONFIG_PM
23606cf5 6066static bool e1000e_pm_ready(struct e1000_adapter *adapter)
4f9de721 6067{
23606cf5 6068 return !!adapter->tx_ring->buffer_info;
4f9de721
RW		 6069}
6070
23606cf5 6071static int __e1000_resume(struct pci_dev *pdev)
bc7f75fa
AK		 6072{
6073 struct net_device *netdev = pci_get_drvdata(pdev);
6074 struct e1000_adapter *adapter = netdev_priv(netdev);
6075 struct e1000_hw *hw = &adapter->hw;
78cd29d5 6076 u16 aspm_disable_flag = 0;
bc7f75fa
AK		 6077 u32 err;
6078
78cd29d5
BA		 6079 if (adapter->flags2 & FLAG2_DISABLE_ASPM_L0S)
6080 aspm_disable_flag = PCIE_LINK_STATE_L0S;
6081 if (adapter->flags2 & FLAG2_DISABLE_ASPM_L1)
6082 aspm_disable_flag |= PCIE_LINK_STATE_L1;
6083 if (aspm_disable_flag)
6084 e1000e_disable_aspm(pdev, aspm_disable_flag);
6085
bc7f75fa
AK		 6086 pci_set_power_state(pdev, PCI_D0);
6087 pci_restore_state(pdev);
28b8f04a 6088 pci_save_state(pdev);
6e4f6f6b 6089
4662e82b 6090 e1000e_set_interrupt_capability(adapter);
bc7f75fa
AK		 6091 if (netif_running(netdev)) {
6092 err = e1000_request_irq(adapter);
6093 if (err)
6094 return err;
6095 }
6096
2fbe4526 6097 if (hw->mac.type >= e1000_pch2lan)
99730e4c
BA		 6098 e1000_resume_workarounds_pchlan(&adapter->hw);
6099
bc7f75fa 6100 e1000e_power_up_phy(adapter);
a4f58f54
BA		 6101
6102 /* report the system wakeup cause from S3/S4 */
6103 if (adapter->flags2 & FLAG2_HAS_PHY_WAKEUP) {
6104 u16 phy_data;
6105
6106 e1e_rphy(&adapter->hw, BM_WUS, &phy_data);
6107 if (phy_data) {
6108 e_info("PHY Wakeup cause - %s\n",
17e813ec
BA		 6109 phy_data & E1000_WUS_EX ? "Unicast Packet" :
6110 phy_data & E1000_WUS_MC ? "Multicast Packet" :
6111 phy_data & E1000_WUS_BC ? "Broadcast Packet" :
6112 phy_data & E1000_WUS_MAG ? "Magic Packet" :
6113 phy_data & E1000_WUS_LNKC ?
6114 "Link Status Change" : "other");
a4f58f54
BA		 6115 }
6116 e1e_wphy(&adapter->hw, BM_WUS, ~0);
6117 } else {
6118 u32 wus = er32(WUS);
6119 if (wus) {
6120 e_info("MAC Wakeup cause - %s\n",
17e813ec
BA		 6121 wus & E1000_WUS_EX ? "Unicast Packet" :
6122 wus & E1000_WUS_MC ? "Multicast Packet" :
6123 wus & E1000_WUS_BC ? "Broadcast Packet" :
6124 wus & E1000_WUS_MAG ? "Magic Packet" :
6125 wus & E1000_WUS_LNKC ? "Link Status Change" :
6126 "other");
a4f58f54
BA		 6127 }
6128 ew32(WUS, ~0);
6129 }
6130
bc7f75fa 6131 e1000e_reset(adapter);
bc7f75fa 6132
cd791618 6133 e1000_init_manageability_pt(adapter);
bc7f75fa
AK		 6134
6135 if (netif_running(netdev))
6136 e1000e_up(adapter);
6137
6138 netif_device_attach(netdev);
6139
e921eb1a 6140 /* If the controller has AMT, do not set DRV_LOAD until the interface
bc7f75fa 6141 * is up. For all other cases, let the f/w know that the h/w is now
ad68076e
BA		 6142 * under the control of the driver.
6143 */
c43bc57e 6144 if (!(adapter->flags & FLAG_HAS_AMT))
31dbe5b4 6145 e1000e_get_hw_control(adapter);
bc7f75fa
AK		 6146
6147 return 0;
6148}
23606cf5 6149
a0340162
RW		 6150#ifdef CONFIG_PM_SLEEP
6151static int e1000_suspend(struct device *dev)
6152{
6153 struct pci_dev *pdev = to_pci_dev(dev);
6154 int retval;
6155 bool wake;
6156
6157 retval = __e1000_shutdown(pdev, &wake, false);
6158 if (!retval)
6159 e1000_complete_shutdown(pdev, true, wake);
6160
6161 return retval;
6162}
6163
23606cf5
RW		 6164static int e1000_resume(struct device *dev)
6165{
6166 struct pci_dev *pdev = to_pci_dev(dev);
6167 struct net_device *netdev = pci_get_drvdata(pdev);
6168 struct e1000_adapter *adapter = netdev_priv(netdev);
6169
6170 if (e1000e_pm_ready(adapter))
6171 adapter->idle_check = true;
6172
6173 return __e1000_resume(pdev);
6174}
a0340162
RW		 6175#endif /* CONFIG_PM_SLEEP */
6176
6177#ifdef CONFIG_PM_RUNTIME
6178static int e1000_runtime_suspend(struct device *dev)
6179{
6180 struct pci_dev *pdev = to_pci_dev(dev);
6181 struct net_device *netdev = pci_get_drvdata(pdev);
6182 struct e1000_adapter *adapter = netdev_priv(netdev);
6183
6184 if (e1000e_pm_ready(adapter)) {
6185 bool wake;
6186
6187 __e1000_shutdown(pdev, &wake, true);
6188 }
6189
6190 return 0;
6191}
6192
6193static int e1000_idle(struct device *dev)
6194{
6195 struct pci_dev *pdev = to_pci_dev(dev);
6196 struct net_device *netdev = pci_get_drvdata(pdev);
6197 struct e1000_adapter *adapter = netdev_priv(netdev);
6198
6199 if (!e1000e_pm_ready(adapter))
6200 return 0;
6201
6202 if (adapter->idle_check) {
6203 adapter->idle_check = false;
6204 if (!e1000e_has_link(adapter))
6205 pm_schedule_suspend(dev, MSEC_PER_SEC);
6206 }
6207
6208 return -EBUSY;
6209}
23606cf5
RW		 6210
6211static int e1000_runtime_resume(struct device *dev)
6212{
6213 struct pci_dev *pdev = to_pci_dev(dev);
6214 struct net_device *netdev = pci_get_drvdata(pdev);
6215 struct e1000_adapter *adapter = netdev_priv(netdev);
6216
6217 if (!e1000e_pm_ready(adapter))
6218 return 0;
6219
6220 adapter->idle_check = !dev->power.runtime_auto;
6221 return __e1000_resume(pdev);
6222}
a0340162 6223#endif /* CONFIG_PM_RUNTIME */
aa338601 6224#endif /* CONFIG_PM */
bc7f75fa
AK		 6225
6226static void e1000_shutdown(struct pci_dev *pdev)
6227{
4f9de721
RW		 6228 bool wake = false;
6229
23606cf5 6230 __e1000_shutdown(pdev, &wake, false);
4f9de721
RW		 6231
6232 if (system_state == SYSTEM_POWER_OFF)
6233 e1000_complete_shutdown(pdev, false, wake);
bc7f75fa
AK		 6234}
6235
6236#ifdef CONFIG_NET_POLL_CONTROLLER
147b2c8c 6237
8bb62869 6238static irqreturn_t e1000_intr_msix(int __always_unused irq, void *data)
147b2c8c
DD		 6239{
6240 struct net_device *netdev = data;
6241 struct e1000_adapter *adapter = netdev_priv(netdev);
147b2c8c
DD		 6242
6243 if (adapter->msix_entries) {
90da0669
BA		 6244 int vector, msix_irq;
6245
147b2c8c
DD		 6246 vector = 0;
6247 msix_irq = adapter->msix_entries[vector].vector;
6248 disable_irq(msix_irq);
6249 e1000_intr_msix_rx(msix_irq, netdev);
6250 enable_irq(msix_irq);
6251
6252 vector++;
6253 msix_irq = adapter->msix_entries[vector].vector;
6254 disable_irq(msix_irq);
6255 e1000_intr_msix_tx(msix_irq, netdev);
6256 enable_irq(msix_irq);
6257
6258 vector++;
6259 msix_irq = adapter->msix_entries[vector].vector;
6260 disable_irq(msix_irq);
6261 e1000_msix_other(msix_irq, netdev);
6262 enable_irq(msix_irq);
6263 }
6264
6265 return IRQ_HANDLED;
6266}
6267
e921eb1a
BA		 6268/**
6269 * e1000_netpoll
6270 * @netdev: network interface device structure
6271 *
bc7f75fa
AK		 6272 * Polling 'interrupt' - used by things like netconsole to send skbs
6273 * without having to re-enable interrupts. It's not called while
6274 * the interrupt routine is executing.
6275 */
6276static void e1000_netpoll(struct net_device *netdev)
6277{
6278 struct e1000_adapter *adapter = netdev_priv(netdev);
6279
147b2c8c
DD		 6280 switch (adapter->int_mode) {
6281 case E1000E_INT_MODE_MSIX:
6282 e1000_intr_msix(adapter->pdev->irq, netdev);
6283 break;
6284 case E1000E_INT_MODE_MSI:
6285 disable_irq(adapter->pdev->irq);
6286 e1000_intr_msi(adapter->pdev->irq, netdev);
6287 enable_irq(adapter->pdev->irq);
6288 break;
6289 default: /* E1000E_INT_MODE_LEGACY */
6290 disable_irq(adapter->pdev->irq);
6291 e1000_intr(adapter->pdev->irq, netdev);
6292 enable_irq(adapter->pdev->irq);
6293 break;
6294 }
bc7f75fa
AK		 6295}
6296#endif
6297
6298/**
6299 * e1000_io_error_detected - called when PCI error is detected
6300 * @pdev: Pointer to PCI device
6301 * @state: The current pci connection state
6302 *
6303 * This function is called after a PCI bus error affecting
6304 * this device has been detected.
6305 */
6306static pci_ers_result_t e1000_io_error_detected(struct pci_dev *pdev,
6307 pci_channel_state_t state)
6308{
6309 struct net_device *netdev = pci_get_drvdata(pdev);
6310 struct e1000_adapter *adapter = netdev_priv(netdev);
6311
6312 netif_device_detach(netdev);
6313
c93b5a76
MM		 6314 if (state == pci_channel_io_perm_failure)
6315 return PCI_ERS_RESULT_DISCONNECT;
6316
bc7f75fa
AK		 6317 if (netif_running(netdev))
6318 e1000e_down(adapter);
6319 pci_disable_device(pdev);
6320
6321 /* Request a slot slot reset. */
6322 return PCI_ERS_RESULT_NEED_RESET;
6323}
6324
6325/**
6326 * e1000_io_slot_reset - called after the pci bus has been reset.
6327 * @pdev: Pointer to PCI device
6328 *
6329 * Restart the card from scratch, as if from a cold-boot. Implementation
6330 * resembles the first-half of the e1000_resume routine.
6331 */
6332static pci_ers_result_t e1000_io_slot_reset(struct pci_dev *pdev)
6333{
6334 struct net_device *netdev = pci_get_drvdata(pdev);
6335 struct e1000_adapter *adapter = netdev_priv(netdev);
6336 struct e1000_hw *hw = &adapter->hw;
78cd29d5 6337 u16 aspm_disable_flag = 0;
6e4f6f6b 6338 int err;
111b9dc5 6339 pci_ers_result_t result;
bc7f75fa 6340
78cd29d5
BA		 6341 if (adapter->flags2 & FLAG2_DISABLE_ASPM_L0S)
6342 aspm_disable_flag = PCIE_LINK_STATE_L0S;
6f461f6c 6343 if (adapter->flags2 & FLAG2_DISABLE_ASPM_L1)
78cd29d5
BA		 6344 aspm_disable_flag |= PCIE_LINK_STATE_L1;
6345 if (aspm_disable_flag)
6346 e1000e_disable_aspm(pdev, aspm_disable_flag);
6347
f0f422e5 6348 err = pci_enable_device_mem(pdev);
6e4f6f6b 6349 if (err) {
bc7f75fa
AK		 6350 dev_err(&pdev->dev,
6351 "Cannot re-enable PCI device after reset.\n");
111b9dc5
JB		 6352 result = PCI_ERS_RESULT_DISCONNECT;
6353 } else {
6354 pci_set_master(pdev);
23606cf5 6355 pdev->state_saved = true;
111b9dc5 6356 pci_restore_state(pdev);
bc7f75fa 6357
111b9dc5
JB		 6358 pci_enable_wake(pdev, PCI_D3hot, 0);
6359 pci_enable_wake(pdev, PCI_D3cold, 0);
bc7f75fa 6360
111b9dc5
JB		 6361 e1000e_reset(adapter);
6362 ew32(WUS, ~0);
6363 result = PCI_ERS_RESULT_RECOVERED;
6364 }
bc7f75fa 6365
111b9dc5
JB		 6366 pci_cleanup_aer_uncorrect_error_status(pdev);
6367
6368 return result;
bc7f75fa
AK		 6369}
6370
6371/**
6372 * e1000_io_resume - called when traffic can start flowing again.
6373 * @pdev: Pointer to PCI device
6374 *
6375 * This callback is called when the error recovery driver tells us that
6376 * its OK to resume normal operation. Implementation resembles the
6377 * second-half of the e1000_resume routine.
6378 */
6379static void e1000_io_resume(struct pci_dev *pdev)
6380{
6381 struct net_device *netdev = pci_get_drvdata(pdev);
6382 struct e1000_adapter *adapter = netdev_priv(netdev);
6383
cd791618 6384 e1000_init_manageability_pt(adapter);
bc7f75fa
AK		 6385
6386 if (netif_running(netdev)) {
6387 if (e1000e_up(adapter)) {
6388 dev_err(&pdev->dev,
6389 "can't bring device back up after reset\n");
6390 return;
6391 }
6392 }
6393
6394 netif_device_attach(netdev);
6395
e921eb1a 6396 /* If the controller has AMT, do not set DRV_LOAD until the interface
bc7f75fa 6397 * is up. For all other cases, let the f/w know that the h/w is now
ad68076e
BA		 6398 * under the control of the driver.
6399 */
c43bc57e 6400 if (!(adapter->flags & FLAG_HAS_AMT))
31dbe5b4 6401 e1000e_get_hw_control(adapter);
bc7f75fa
AK		 6402}
6403
6404static void e1000_print_device_info(struct e1000_adapter *adapter)
6405{
6406 struct e1000_hw *hw = &adapter->hw;
6407 struct net_device *netdev = adapter->netdev;
073287c0
BA		 6408 u32 ret_val;
6409 u8 pba_str[E1000_PBANUM_LENGTH];
bc7f75fa
AK		 6410
6411 /* print bus type/speed/width info */
a5cc7642 6412 e_info("(PCI Express:2.5GT/s:%s) %pM\n",
44defeb3
JK		 6413 /* bus width */
6414 ((hw->bus.width == e1000_bus_width_pcie_x4) ? "Width x4" :
f0ff4398 6415 "Width x1"),
44defeb3 6416 /* MAC address */
7c510e4b 6417 netdev->dev_addr);
44defeb3
JK		 6418 e_info("Intel(R) PRO/%s Network Connection\n",
6419 (hw->phy.type == e1000_phy_ife) ? "10/100" : "1000");
073287c0
BA		 6420 ret_val = e1000_read_pba_string_generic(hw, pba_str,
6421 E1000_PBANUM_LENGTH);
6422 if (ret_val)
f2315bf1 6423 strlcpy((char *)pba_str, "Unknown", sizeof(pba_str));
073287c0
BA		 6424 e_info("MAC: %d, PHY: %d, PBA No: %s\n",
6425 hw->mac.type, hw->phy.type, pba_str);
bc7f75fa
AK		 6426}
6427
10aa4c04
AK		 6428static void e1000_eeprom_checks(struct e1000_adapter *adapter)
6429{
6430 struct e1000_hw *hw = &adapter->hw;
6431 int ret_val;
6432 u16 buf = 0;
6433
6434 if (hw->mac.type != e1000_82573)
6435 return;
6436
6437 ret_val = e1000_read_nvm(hw, NVM_INIT_CONTROL2_REG, 1, &buf);
e885d762
BA		 6438 le16_to_cpus(&buf);
6439 if (!ret_val && (!(buf & (1 << 0)))) {
10aa4c04 6440 /* Deep Smart Power Down (DSPD) */
6c2a9efa
FP		 6441 dev_warn(&adapter->pdev->dev,
6442 "Warning: detected DSPD enabled in EEPROM\n");
10aa4c04 6443 }
10aa4c04
AK		 6444}
6445
c8f44aff 6446static int e1000_set_features(struct net_device *netdev,
70495a50 6447 netdev_features_t features)
dc221294
BA		 6448{
6449 struct e1000_adapter *adapter = netdev_priv(netdev);
c8f44aff 6450 netdev_features_t changed = features ^ netdev->features;
dc221294
BA		 6451
6452 if (changed & (NETIF_F_TSO | NETIF_F_TSO6))
6453 adapter->flags |= FLAG_TSO_FORCE;
6454
6455 if (!(changed & (NETIF_F_HW_VLAN_RX | NETIF_F_HW_VLAN_TX |
cf955e6c
BG		 6456 NETIF_F_RXCSUM | NETIF_F_RXHASH | NETIF_F_RXFCS |
6457 NETIF_F_RXALL)))
dc221294
BA		 6458 return 0;
6459
0184039a
BG		 6460 if (changed & NETIF_F_RXFCS) {
6461 if (features & NETIF_F_RXFCS) {
6462 adapter->flags2 &= ~FLAG2_CRC_STRIPPING;
6463 } else {
6464 /* We need to take it back to defaults, which might mean
6465 * stripping is still disabled at the adapter level.
6466 */
6467 if (adapter->flags2 & FLAG2_DFLT_CRC_STRIPPING)
6468 adapter->flags2 |= FLAG2_CRC_STRIPPING;
6469 else
6470 adapter->flags2 &= ~FLAG2_CRC_STRIPPING;
6471 }
6472 }
6473
70495a50
BA		 6474 netdev->features = features;
6475
dc221294
BA		 6476 if (netif_running(netdev))
6477 e1000e_reinit_locked(adapter);
6478 else
6479 e1000e_reset(adapter);
6480
6481 return 0;
6482}
6483
651c2466
SH		 6484static const struct net_device_ops e1000e_netdev_ops = {
6485 .ndo_open = e1000_open,
6486 .ndo_stop = e1000_close,
00829823 6487 .ndo_start_xmit = e1000_xmit_frame,
67fd4fcb 6488 .ndo_get_stats64 = e1000e_get_stats64,
ef9b965a 6489 .ndo_set_rx_mode = e1000e_set_rx_mode,
651c2466
SH		 6490 .ndo_set_mac_address = e1000_set_mac,
6491 .ndo_change_mtu = e1000_change_mtu,
6492 .ndo_do_ioctl = e1000_ioctl,
6493 .ndo_tx_timeout = e1000_tx_timeout,
6494 .ndo_validate_addr = eth_validate_addr,
6495
651c2466
SH		 6496 .ndo_vlan_rx_add_vid = e1000_vlan_rx_add_vid,
6497 .ndo_vlan_rx_kill_vid = e1000_vlan_rx_kill_vid,
6498#ifdef CONFIG_NET_POLL_CONTROLLER
6499 .ndo_poll_controller = e1000_netpoll,
6500#endif
dc221294 6501 .ndo_set_features = e1000_set_features,
651c2466
SH		 6502};
6503
bc7f75fa
AK		 6504/**
6505 * e1000_probe - Device Initialization Routine
6506 * @pdev: PCI device information struct
6507 * @ent: entry in e1000_pci_tbl
6508 *
6509 * Returns 0 on success, negative on failure
6510 *
6511 * e1000_probe initializes an adapter identified by a pci_dev structure.
6512 * The OS initialization, configuring of the adapter private structure,
6513 * and a hardware reset occur.
6514 **/
1dd06ae8 6515static int e1000_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
bc7f75fa
AK		 6516{
6517 struct net_device *netdev;
6518 struct e1000_adapter *adapter;
6519 struct e1000_hw *hw;
6520 const struct e1000_info *ei = e1000_info_tbl[ent->driver_data];
f47e81fc
BB		 6521 resource_size_t mmio_start, mmio_len;
6522 resource_size_t flash_start, flash_len;
bc7f75fa 6523 static int cards_found;
78cd29d5 6524 u16 aspm_disable_flag = 0;
17e813ec 6525 int bars, i, err, pci_using_dac;
bc7f75fa
AK		 6526 u16 eeprom_data = 0;
6527 u16 eeprom_apme_mask = E1000_EEPROM_APME;
6528
78cd29d5
BA		 6529 if (ei->flags2 & FLAG2_DISABLE_ASPM_L0S)
6530 aspm_disable_flag = PCIE_LINK_STATE_L0S;
6f461f6c 6531 if (ei->flags2 & FLAG2_DISABLE_ASPM_L1)
78cd29d5
BA		 6532 aspm_disable_flag |= PCIE_LINK_STATE_L1;
6533 if (aspm_disable_flag)
6534 e1000e_disable_aspm(pdev, aspm_disable_flag);
6e4f6f6b 6535
f0f422e5 6536 err = pci_enable_device_mem(pdev);
bc7f75fa
AK		 6537 if (err)
6538 return err;
6539
6540 pci_using_dac = 0;
0be3f55f 6541 err = dma_set_mask(&pdev->dev, DMA_BIT_MASK(64));
bc7f75fa 6542 if (!err) {
0be3f55f 6543 err = dma_set_coherent_mask(&pdev->dev, DMA_BIT_MASK(64));
bc7f75fa
AK		 6544 if (!err)
6545 pci_using_dac = 1;
6546 } else {
0be3f55f 6547 err = dma_set_mask(&pdev->dev, DMA_BIT_MASK(32));
bc7f75fa 6548 if (err) {
0be3f55f
NN		 6549 err = dma_set_coherent_mask(&pdev->dev,
6550 DMA_BIT_MASK(32));
bc7f75fa 6551 if (err) {
f0ff4398
BA		 6552 dev_err(&pdev->dev,
6553 "No usable DMA configuration, aborting\n");
bc7f75fa
AK		 6554 goto err_dma;
6555 }
6556 }
6557 }
6558
17e813ec
BA		 6559 bars = pci_select_bars(pdev, IORESOURCE_MEM);
6560 err = pci_request_selected_regions_exclusive(pdev, bars,
6561 e1000e_driver_name);
bc7f75fa
AK		 6562 if (err)
6563 goto err_pci_reg;
6564
68eac460 6565 /* AER (Advanced Error Reporting) hooks */
19d5afd4 6566 pci_enable_pcie_error_reporting(pdev);
68eac460 6567
bc7f75fa 6568 pci_set_master(pdev);
438b365a
BA		 6569 /* PCI config space info */
6570 err = pci_save_state(pdev);
6571 if (err)
6572 goto err_alloc_etherdev;
bc7f75fa
AK		 6573
6574 err = -ENOMEM;
6575 netdev = alloc_etherdev(sizeof(struct e1000_adapter));
6576 if (!netdev)
6577 goto err_alloc_etherdev;
6578
bc7f75fa
AK		 6579 SET_NETDEV_DEV(netdev, &pdev->dev);
6580
f85e4dfa
TH		 6581 netdev->irq = pdev->irq;
6582
bc7f75fa
AK		 6583 pci_set_drvdata(pdev, netdev);
6584 adapter = netdev_priv(netdev);
6585 hw = &adapter->hw;
6586 adapter->netdev = netdev;
6587 adapter->pdev = pdev;
6588 adapter->ei = ei;
6589 adapter->pba = ei->pba;
6590 adapter->flags = ei->flags;
eb7c3adb 6591 adapter->flags2 = ei->flags2;
bc7f75fa
AK		 6592 adapter->hw.adapter = adapter;
6593 adapter->hw.mac.type = ei->mac;
2adc55c9 6594 adapter->max_hw_frame_size = ei->max_hw_frame_size;
b3f4d599 6595 adapter->msg_enable = netif_msg_init(debug, DEFAULT_MSG_ENABLE);
bc7f75fa
AK		 6596
6597 mmio_start = pci_resource_start(pdev, 0);
6598 mmio_len = pci_resource_len(pdev, 0);
6599
6600 err = -EIO;
6601 adapter->hw.hw_addr = ioremap(mmio_start, mmio_len);
6602 if (!adapter->hw.hw_addr)
6603 goto err_ioremap;
6604
6605 if ((adapter->flags & FLAG_HAS_FLASH) &&
6606 (pci_resource_flags(pdev, 1) & IORESOURCE_MEM)) {
6607 flash_start = pci_resource_start(pdev, 1);
6608 flash_len = pci_resource_len(pdev, 1);
6609 adapter->hw.flash_address = ioremap(flash_start, flash_len);
6610 if (!adapter->hw.flash_address)
6611 goto err_flashmap;
6612 }
6613
6614 /* construct the net_device struct */
651c2466 6615 netdev->netdev_ops = &e1000e_netdev_ops;
bc7f75fa 6616 e1000e_set_ethtool_ops(netdev);
bc7f75fa 6617 netdev->watchdog_timeo = 5 * HZ;
c58c8a78 6618 netif_napi_add(netdev, &adapter->napi, e1000e_poll, 64);
f2315bf1 6619 strlcpy(netdev->name, pci_name(pdev), sizeof(netdev->name));
bc7f75fa
AK		 6620
6621 netdev->mem_start = mmio_start;
6622 netdev->mem_end = mmio_start + mmio_len;
6623
6624 adapter->bd_number = cards_found++;
6625
4662e82b
BA		 6626 e1000e_check_options(adapter);
6627
bc7f75fa
AK		 6628 /* setup adapter struct */
6629 err = e1000_sw_init(adapter);
6630 if (err)
6631 goto err_sw_init;
6632
bc7f75fa
AK		 6633 memcpy(&hw->mac.ops, ei->mac_ops, sizeof(hw->mac.ops));
6634 memcpy(&hw->nvm.ops, ei->nvm_ops, sizeof(hw->nvm.ops));
6635 memcpy(&hw->phy.ops, ei->phy_ops, sizeof(hw->phy.ops));
6636
69e3fd8c 6637 err = ei->get_variants(adapter);
bc7f75fa
AK		 6638 if (err)
6639 goto err_hw_init;
6640
4a770358
BA		 6641 if ((adapter->flags & FLAG_IS_ICH) &&
6642 (adapter->flags & FLAG_READ_ONLY_NVM))
6643 e1000e_write_protect_nvm_ich8lan(&adapter->hw);
6644
bc7f75fa
AK		 6645 hw->mac.ops.get_bus_info(&adapter->hw);
6646
318a94d6 6647 adapter->hw.phy.autoneg_wait_to_complete = 0;
bc7f75fa
AK		 6648
6649 /* Copper options */
318a94d6 6650 if (adapter->hw.phy.media_type == e1000_media_type_copper) {
bc7f75fa
AK		 6651 adapter->hw.phy.mdix = AUTO_ALL_MODES;
6652 adapter->hw.phy.disable_polarity_correction = 0;
6653 adapter->hw.phy.ms_type = e1000_ms_hw_default;
6654 }
6655
470a5420 6656 if (hw->phy.ops.check_reset_block && hw->phy.ops.check_reset_block(hw))
185095fb
BA		 6657 dev_info(&pdev->dev,
6658 "PHY reset is blocked due to SOL/IDER session.\n");
bc7f75fa 6659
dc221294
BA		 6660 /* Set initial default active device features */
6661 netdev->features = (NETIF_F_SG |
6662 NETIF_F_HW_VLAN_RX |
6663 NETIF_F_HW_VLAN_TX |
6664 NETIF_F_TSO |
6665 NETIF_F_TSO6 |
70495a50 6666 NETIF_F_RXHASH |
dc221294
BA		 6667 NETIF_F_RXCSUM |
6668 NETIF_F_HW_CSUM);
6669
6670 /* Set user-changeable features (subset of all device features) */
6671 netdev->hw_features = netdev->features;
0184039a 6672 netdev->hw_features |= NETIF_F_RXFCS;
943146de 6673 netdev->priv_flags |= IFF_SUPP_NOFCS;
cf955e6c 6674 netdev->hw_features |= NETIF_F_RXALL;
bc7f75fa
AK		 6675
6676 if (adapter->flags & FLAG_HAS_HW_VLAN_FILTER)
6677 netdev->features |= NETIF_F_HW_VLAN_FILTER;
6678
dc221294
BA		 6679 netdev->vlan_features |= (NETIF_F_SG |
6680 NETIF_F_TSO |
6681 NETIF_F_TSO6 |
6682 NETIF_F_HW_CSUM);
a5136e23 6683
ef9b965a
JB		 6684 netdev->priv_flags |= IFF_UNICAST_FLT;
6685
7b872a55 6686 if (pci_using_dac) {
bc7f75fa 6687 netdev->features |= NETIF_F_HIGHDMA;
7b872a55
YZ		 6688 netdev->vlan_features |= NETIF_F_HIGHDMA;
6689 }
bc7f75fa 6690
bc7f75fa
AK		 6691 if (e1000e_enable_mng_pass_thru(&adapter->hw))
6692 adapter->flags |= FLAG_MNG_PT_ENABLED;
6693
e921eb1a 6694 /* before reading the NVM, reset the controller to
ad68076e
BA		 6695 * put the device in a known good starting state
6696 */
bc7f75fa
AK		 6697 adapter->hw.mac.ops.reset_hw(&adapter->hw);
6698
e921eb1a 6699 /* systems with ASPM and others may see the checksum fail on the first
bc7f75fa
AK		 6700 * attempt. Let's give it a few tries
6701 */
6702 for (i = 0;; i++) {
6703 if (e1000_validate_nvm_checksum(&adapter->hw) >= 0)
6704 break;
6705 if (i == 2) {
185095fb 6706 dev_err(&pdev->dev, "The NVM Checksum Is Not Valid\n");
bc7f75fa
AK		 6707 err = -EIO;
6708 goto err_eeprom;
6709 }
6710 }
6711
10aa4c04
AK		 6712 e1000_eeprom_checks(adapter);
6713
608f8a0d 6714 /* copy the MAC address */
bc7f75fa 6715 if (e1000e_read_mac_addr(&adapter->hw))
185095fb
BA		 6716 dev_err(&pdev->dev,
6717 "NVM Read Error while reading MAC address\n");
bc7f75fa
AK		 6718
6719 memcpy(netdev->dev_addr, adapter->hw.mac.addr, netdev->addr_len);
bc7f75fa 6720
aaeb6cdf 6721 if (!is_valid_ether_addr(netdev->dev_addr)) {
185095fb 6722 dev_err(&pdev->dev, "Invalid MAC Address: %pM\n",
aaeb6cdf 6723 netdev->dev_addr);
bc7f75fa
AK		 6724 err = -EIO;
6725 goto err_eeprom;
6726 }
6727
6728 init_timer(&adapter->watchdog_timer);
c061b18d 6729 adapter->watchdog_timer.function = e1000_watchdog;
bc7f75fa
AK		 6730 adapter->watchdog_timer.data = (unsigned long) adapter;
6731
6732 init_timer(&adapter->phy_info_timer);
c061b18d 6733 adapter->phy_info_timer.function = e1000_update_phy_info;
bc7f75fa
AK		 6734 adapter->phy_info_timer.data = (unsigned long) adapter;
6735
6736 INIT_WORK(&adapter->reset_task, e1000_reset_task);
6737 INIT_WORK(&adapter->watchdog_task, e1000_watchdog_task);
a8f88ff5
JB		 6738 INIT_WORK(&adapter->downshift_task, e1000e_downshift_workaround);
6739 INIT_WORK(&adapter->update_phy_task, e1000e_update_phy_task);
41cec6f1 6740 INIT_WORK(&adapter->print_hang_task, e1000_print_hw_hang);
bc7f75fa 6741
bc7f75fa
AK		 6742 /* Initialize link parameters. User can change them with ethtool */
6743 adapter->hw.mac.autoneg = 1;
3db1cd5c 6744 adapter->fc_autoneg = true;
5c48ef3e
BA		 6745 adapter->hw.fc.requested_mode = e1000_fc_default;
6746 adapter->hw.fc.current_mode = e1000_fc_default;
bc7f75fa
AK		 6747 adapter->hw.phy.autoneg_advertised = 0x2f;
6748
6749 /* ring size defaults */
d821a4c4
BA		 6750 adapter->rx_ring->count = E1000_DEFAULT_RXD;
6751 adapter->tx_ring->count = E1000_DEFAULT_TXD;
bc7f75fa 6752
e921eb1a 6753 /* Initial Wake on LAN setting - If APM wake is enabled in
bc7f75fa
AK		 6754 * the EEPROM, enable the ACPI Magic Packet filter
6755 */
6756 if (adapter->flags & FLAG_APME_IN_WUC) {
6757 /* APME bit in EEPROM is mapped to WUC.APME */
6758 eeprom_data = er32(WUC);
6759 eeprom_apme_mask = E1000_WUC_APME;
4def99bb
BA		 6760 if ((hw->mac.type > e1000_ich10lan) &&
6761 (eeprom_data & E1000_WUC_PHY_WAKE))
a4f58f54 6762 adapter->flags2 |= FLAG2_HAS_PHY_WAKEUP;
bc7f75fa
AK		 6763 } else if (adapter->flags & FLAG_APME_IN_CTRL3) {
6764 if (adapter->flags & FLAG_APME_CHECK_PORT_B &&
6765 (adapter->hw.bus.func == 1))
3d3a1676
BA		 6766 e1000_read_nvm(&adapter->hw, NVM_INIT_CONTROL3_PORT_B,
6767 1, &eeprom_data);
bc7f75fa 6768 else
3d3a1676
BA		 6769 e1000_read_nvm(&adapter->hw, NVM_INIT_CONTROL3_PORT_A,
6770 1, &eeprom_data);
bc7f75fa
AK		 6771 }
6772
6773 /* fetch WoL from EEPROM */
6774 if (eeprom_data & eeprom_apme_mask)
6775 adapter->eeprom_wol |= E1000_WUFC_MAG;
6776
e921eb1a 6777 /* now that we have the eeprom settings, apply the special cases
bc7f75fa
AK		 6778 * where the eeprom may be wrong or the board simply won't support
6779 * wake on lan on a particular port
6780 */
6781 if (!(adapter->flags & FLAG_HAS_WOL))
6782 adapter->eeprom_wol = 0;
6783
6784 /* initialize the wol settings based on the eeprom settings */
6785 adapter->wol = adapter->eeprom_wol;
6ff68026 6786 device_set_wakeup_enable(&adapter->pdev->dev, adapter->wol);
bc7f75fa 6787
84527590
BA		 6788 /* save off EEPROM version number */
6789 e1000_read_nvm(&adapter->hw, 5, 1, &adapter->eeprom_vers);
6790
bc7f75fa
AK		 6791 /* reset the hardware with the new settings */
6792 e1000e_reset(adapter);
6793
e921eb1a 6794 /* If the controller has AMT, do not set DRV_LOAD until the interface
bc7f75fa 6795 * is up. For all other cases, let the f/w know that the h/w is now
ad68076e
BA		 6796 * under the control of the driver.
6797 */
c43bc57e 6798 if (!(adapter->flags & FLAG_HAS_AMT))
31dbe5b4 6799 e1000e_get_hw_control(adapter);
bc7f75fa 6800
f2315bf1 6801 strlcpy(netdev->name, "eth%d", sizeof(netdev->name));
bc7f75fa
AK		 6802 err = register_netdev(netdev);
6803 if (err)
6804 goto err_register;
6805
9c563d20
JB		 6806 /* carrier off reporting is important to ethtool even BEFORE open */
6807 netif_carrier_off(netdev);
6808
d89777bf
BA		 6809 /* init PTP hardware clock */
6810 e1000e_ptp_init(adapter);
6811
bc7f75fa
AK		 6812 e1000_print_device_info(adapter);
6813
f3ec4f87
AS		 6814 if (pci_dev_run_wake(pdev))
6815 pm_runtime_put_noidle(&pdev->dev);
23606cf5 6816
bc7f75fa
AK		 6817 return 0;
6818
6819err_register:
c43bc57e 6820 if (!(adapter->flags & FLAG_HAS_AMT))
31dbe5b4 6821 e1000e_release_hw_control(adapter);
bc7f75fa 6822err_eeprom:
470a5420 6823 if (hw->phy.ops.check_reset_block && !hw->phy.ops.check_reset_block(hw))
bc7f75fa 6824 e1000_phy_hw_reset(&adapter->hw);
c43bc57e 6825err_hw_init:
bc7f75fa
AK		 6826 kfree(adapter->tx_ring);
6827 kfree(adapter->rx_ring);
6828err_sw_init:
c43bc57e
JB		 6829 if (adapter->hw.flash_address)
6830 iounmap(adapter->hw.flash_address);
e82f54ba 6831 e1000e_reset_interrupt_capability(adapter);
c43bc57e 6832err_flashmap:
bc7f75fa
AK		 6833 iounmap(adapter->hw.hw_addr);
6834err_ioremap:
6835 free_netdev(netdev);
6836err_alloc_etherdev:
f0f422e5 6837 pci_release_selected_regions(pdev,
f0ff4398 6838 pci_select_bars(pdev, IORESOURCE_MEM));
bc7f75fa
AK		 6839err_pci_reg:
6840err_dma:
6841 pci_disable_device(pdev);
6842 return err;
6843}
6844
6845/**
6846 * e1000_remove - Device Removal Routine
6847 * @pdev: PCI device information struct
6848 *
6849 * e1000_remove is called by the PCI subsystem to alert the driver
6850 * that it should release a PCI device. The could be caused by a
6851 * Hot-Plug event, or because the driver is going to be removed from
6852 * memory.
6853 **/
9f9a12f8 6854static void e1000_remove(struct pci_dev *pdev)
bc7f75fa
AK		 6855{
6856 struct net_device *netdev = pci_get_drvdata(pdev);
6857 struct e1000_adapter *adapter = netdev_priv(netdev);
23606cf5
RW		 6858 bool down = test_bit(__E1000_DOWN, &adapter->state);
6859
d89777bf
BA		 6860 e1000e_ptp_remove(adapter);
6861
e921eb1a 6862 /* The timers may be rescheduled, so explicitly disable them
23f333a2 6863 * from being rescheduled.
ad68076e 6864 */
23606cf5
RW		 6865 if (!down)
6866 set_bit(__E1000_DOWN, &adapter->state);
bc7f75fa
AK		 6867 del_timer_sync(&adapter->watchdog_timer);
6868 del_timer_sync(&adapter->phy_info_timer);
6869
41cec6f1
BA		 6870 cancel_work_sync(&adapter->reset_task);
6871 cancel_work_sync(&adapter->watchdog_task);
6872 cancel_work_sync(&adapter->downshift_task);
6873 cancel_work_sync(&adapter->update_phy_task);
6874 cancel_work_sync(&adapter->print_hang_task);
bc7f75fa 6875
b67e1913
BA		 6876 if (adapter->flags & FLAG_HAS_HW_TIMESTAMP) {
6877 cancel_work_sync(&adapter->tx_hwtstamp_work);
6878 if (adapter->tx_hwtstamp_skb) {
6879 dev_kfree_skb_any(adapter->tx_hwtstamp_skb);
6880 adapter->tx_hwtstamp_skb = NULL;
6881 }
6882 }
6883
17f208de
BA		 6884 if (!(netdev->flags & IFF_UP))
6885 e1000_power_down_phy(adapter);
6886
23606cf5
RW		 6887 /* Don't lie to e1000_close() down the road. */
6888 if (!down)
6889 clear_bit(__E1000_DOWN, &adapter->state);
17f208de
BA		 6890 unregister_netdev(netdev);
6891
f3ec4f87
AS		 6892 if (pci_dev_run_wake(pdev))
6893 pm_runtime_get_noresume(&pdev->dev);
23606cf5 6894
e921eb1a 6895 /* Release control of h/w to f/w. If f/w is AMT enabled, this
ad68076e
BA		 6896 * would have already happened in close and is redundant.
6897 */
31dbe5b4 6898 e1000e_release_hw_control(adapter);
bc7f75fa 6899
4662e82b 6900 e1000e_reset_interrupt_capability(adapter);
bc7f75fa
AK		 6901 kfree(adapter->tx_ring);
6902 kfree(adapter->rx_ring);
6903
6904 iounmap(adapter->hw.hw_addr);
6905 if (adapter->hw.flash_address)
6906 iounmap(adapter->hw.flash_address);
f0f422e5 6907 pci_release_selected_regions(pdev,
f0ff4398 6908 pci_select_bars(pdev, IORESOURCE_MEM));
bc7f75fa
AK		 6909
6910 free_netdev(netdev);
6911
111b9dc5 6912 /* AER disable */
19d5afd4 6913 pci_disable_pcie_error_reporting(pdev);
111b9dc5 6914
bc7f75fa
AK		 6915 pci_disable_device(pdev);
6916}
6917
6918/* PCI Error Recovery (ERS) */
3646f0e5 6919static const struct pci_error_handlers e1000_err_handler = {
bc7f75fa
AK		 6920 .error_detected = e1000_io_error_detected,
6921 .slot_reset = e1000_io_slot_reset,
6922 .resume = e1000_io_resume,
6923};
6924
a3aa1884 6925static DEFINE_PCI_DEVICE_TABLE(e1000_pci_tbl) = {
bc7f75fa
AK		 6926 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82571EB_COPPER), board_82571 },
6927 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82571EB_FIBER), board_82571 },
6928 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82571EB_QUAD_COPPER), board_82571 },
c29c3ba5
BA		 6929 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82571EB_QUAD_COPPER_LP),
6930 board_82571 },
bc7f75fa
AK		 6931 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82571EB_QUAD_FIBER), board_82571 },
6932 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82571EB_SERDES), board_82571 },
040babf9
AK		 6933 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82571EB_SERDES_DUAL), board_82571 },
6934 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82571EB_SERDES_QUAD), board_82571 },
6935 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82571PT_QUAD_COPPER), board_82571 },
ad68076e 6936
bc7f75fa
AK		 6937 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82572EI), board_82572 },
6938 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82572EI_COPPER), board_82572 },
6939 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82572EI_FIBER), board_82572 },
6940 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82572EI_SERDES), board_82572 },
ad68076e 6941
bc7f75fa
AK		 6942 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82573E), board_82573 },
6943 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82573E_IAMT), board_82573 },
6944 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82573L), board_82573 },
ad68076e 6945
4662e82b 6946 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82574L), board_82574 },
bef28b11 6947 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82574LA), board_82574 },
8c81c9c3 6948 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82583V), board_82583 },
4662e82b 6949
bc7f75fa
AK		 6950 { PCI_VDEVICE(INTEL, E1000_DEV_ID_80003ES2LAN_COPPER_DPT),
6951 board_80003es2lan },
6952 { PCI_VDEVICE(INTEL, E1000_DEV_ID_80003ES2LAN_COPPER_SPT),
6953 board_80003es2lan },
6954 { PCI_VDEVICE(INTEL, E1000_DEV_ID_80003ES2LAN_SERDES_DPT),
6955 board_80003es2lan },
6956 { PCI_VDEVICE(INTEL, E1000_DEV_ID_80003ES2LAN_SERDES_SPT),
6957 board_80003es2lan },
ad68076e 6958
bc7f75fa
AK		 6959 { PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH8_IFE), board_ich8lan },
6960 { PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH8_IFE_G), board_ich8lan },
6961 { PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH8_IFE_GT), board_ich8lan },
6962 { PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH8_IGP_AMT), board_ich8lan },
6963 { PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH8_IGP_C), board_ich8lan },
6964 { PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH8_IGP_M), board_ich8lan },
6965 { PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH8_IGP_M_AMT), board_ich8lan },
9e135a2e 6966 { PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH8_82567V_3), board_ich8lan },
ad68076e 6967
bc7f75fa
AK		 6968 { PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH9_IFE), board_ich9lan },
6969 { PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH9_IFE_G), board_ich9lan },
6970 { PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH9_IFE_GT), board_ich9lan },
6971 { PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH9_IGP_AMT), board_ich9lan },
6972 { PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH9_IGP_C), board_ich9lan },
2f15f9d6 6973 { PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH9_BM), board_ich9lan },
97ac8cae
BA		 6974 { PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH9_IGP_M), board_ich9lan },
6975 { PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH9_IGP_M_AMT), board_ich9lan },
6976 { PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH9_IGP_M_V), board_ich9lan },
6977
6978 { PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH10_R_BM_LM), board_ich9lan },
6979 { PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH10_R_BM_LF), board_ich9lan },
6980 { PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH10_R_BM_V), board_ich9lan },
bc7f75fa 6981
f4187b56
BA		 6982 { PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH10_D_BM_LM), board_ich10lan },
6983 { PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH10_D_BM_LF), board_ich10lan },
10df0b91 6984 { PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH10_D_BM_V), board_ich10lan },
f4187b56 6985
a4f58f54
BA		 6986 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_M_HV_LM), board_pchlan },
6987 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_M_HV_LC), board_pchlan },
6988 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_D_HV_DM), board_pchlan },
6989 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_D_HV_DC), board_pchlan },
6990
d3738bb8
BA		 6991 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH2_LV_LM), board_pch2lan },
6992 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH2_LV_V), board_pch2lan },
6993
2fbe4526
BA		 6994 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_LPT_I217_LM), board_pch_lpt },
6995 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_LPT_I217_V), board_pch_lpt },
16e310ae
BA		 6996 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_LPTLP_I218_LM), board_pch_lpt },
6997 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_LPTLP_I218_V), board_pch_lpt },
2fbe4526 6998
f36bb6ca 6999 { 0, 0, 0, 0, 0, 0, 0 } /* terminate list */
bc7f75fa
AK		 7000};
7001MODULE_DEVICE_TABLE(pci, e1000_pci_tbl);
7002
aa338601 7003#ifdef CONFIG_PM
23606cf5 7004static const struct dev_pm_ops e1000_pm_ops = {
a0340162 7005 SET_SYSTEM_SLEEP_PM_OPS(e1000_suspend, e1000_resume)
17e813ec
BA		 7006 SET_RUNTIME_PM_OPS(e1000_runtime_suspend, e1000_runtime_resume,
7007 e1000_idle)
23606cf5 7008};
e50208a0 7009#endif
23606cf5 7010
bc7f75fa
AK		 7011/* PCI Device API Driver */
7012static struct pci_driver e1000_driver = {
7013 .name = e1000e_driver_name,
7014 .id_table = e1000_pci_tbl,
7015 .probe = e1000_probe,
9f9a12f8 7016 .remove = e1000_remove,
aa338601 7017#ifdef CONFIG_PM
f36bb6ca
BA		 7018 .driver = {
7019 .pm = &e1000_pm_ops,
7020 },
bc7f75fa
AK		 7021#endif
7022 .shutdown = e1000_shutdown,
7023 .err_handler = &e1000_err_handler
7024};
7025
7026/**
7027 * e1000_init_module - Driver Registration Routine
7028 *
7029 * e1000_init_module is the first routine called when the driver is
7030 * loaded. All it does is register with the PCI subsystem.
7031 **/
7032static int __init e1000_init_module(void)
7033{
7034 int ret;
8544b9f7
BA		 7035 pr_info("Intel(R) PRO/1000 Network Driver - %s\n",
7036 e1000e_driver_version);
bf67044b 7037 pr_info("Copyright(c) 1999 - 2013 Intel Corporation.\n");
bc7f75fa 7038 ret = pci_register_driver(&e1000_driver);
53ec5498 7039
bc7f75fa
AK		 7040 return ret;
7041}
7042module_init(e1000_init_module);
7043
7044/**
7045 * e1000_exit_module - Driver Exit Cleanup Routine
7046 *
7047 * e1000_exit_module is called just before the driver is removed
7048 * from memory.
7049 **/
7050static void __exit e1000_exit_module(void)
7051{
7052 pci_unregister_driver(&e1000_driver);
7053}
7054module_exit(e1000_exit_module);
7055
7056
7057MODULE_AUTHOR("Intel Corporation, <linux.nics@intel.com>");
7058MODULE_DESCRIPTION("Intel(R) PRO/1000 Network Driver");
7059MODULE_LICENSE("GPL");
7060MODULE_VERSION(DRV_VERSION);
7061
06c24b91 7062/* netdev.c */
Linux kernel - Deliverables
This page took 2.232899 seconds and 5 git commands to generate.