netdev.c 182 KB
Newer Older
1
2
3
/*******************************************************************************

  Intel PRO/1000 Linux driver
Bruce Allan's avatar
Bruce Allan committed
4
  Copyright(c) 1999 - 2012 Intel Corporation.
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28

  This program is free software; you can redistribute it and/or modify it
  under the terms and conditions of the GNU General Public License,
  version 2, as published by the Free Software Foundation.

  This program is distributed in the hope it will be useful, but WITHOUT
  ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  more details.

  You should have received a copy of the GNU General Public License along with
  this program; if not, write to the Free Software Foundation, Inc.,
  51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.

  The full GNU General Public License is included in this distribution in
  the file called "COPYING".

  Contact Information:
  Linux NICS <linux.nics@intel.com>
  e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
  Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497

*******************************************************************************/

29
30
#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt

31
32
33
34
35
36
37
38
#include <linux/module.h>
#include <linux/types.h>
#include <linux/init.h>
#include <linux/pci.h>
#include <linux/vmalloc.h>
#include <linux/pagemap.h>
#include <linux/delay.h>
#include <linux/netdevice.h>
39
#include <linux/interrupt.h>
40
41
#include <linux/tcp.h>
#include <linux/ipv6.h>
42
#include <linux/slab.h>
43
44
45
46
47
48
49
#include <net/checksum.h>
#include <net/ip6_checksum.h>
#include <linux/mii.h>
#include <linux/ethtool.h>
#include <linux/if_vlan.h>
#include <linux/cpu.h>
#include <linux/smp.h>
50
#include <linux/pm_qos.h>
51
#include <linux/pm_runtime.h>
Jesse Brandeburg's avatar
Jesse Brandeburg committed
52
#include <linux/aer.h>
53
#include <linux/prefetch.h>
54
55
56

#include "e1000.h"

Bruce Allan's avatar
Bruce Allan committed
57
#define DRV_EXTRAVERSION "-k"
58

59
#define DRV_VERSION "2.1.4" DRV_EXTRAVERSION
60
61
62
char e1000e_driver_name[] = "e1000e";
const char e1000e_driver_version[] = DRV_VERSION;

63
64
65
66
67
#define DEFAULT_MSG_ENABLE (NETIF_MSG_DRV|NETIF_MSG_PROBE|NETIF_MSG_LINK)
static int debug = -1;
module_param(debug, int, 0);
MODULE_PARM_DESC(debug, "Debug level (0=none,...,16=all)");

68
69
static void e1000e_disable_aspm(struct pci_dev *pdev, u16 state);

70
71
72
73
static const struct e1000_info *e1000_info_tbl[] = {
	[board_82571]		= &e1000_82571_info,
	[board_82572]		= &e1000_82572_info,
	[board_82573]		= &e1000_82573_info,
74
	[board_82574]		= &e1000_82574_info,
75
	[board_82583]		= &e1000_82583_info,
76
77
78
	[board_80003es2lan]	= &e1000_es2_info,
	[board_ich8lan]		= &e1000_ich8_info,
	[board_ich9lan]		= &e1000_ich9_info,
79
	[board_ich10lan]	= &e1000_ich10_info,
80
	[board_pchlan]		= &e1000_pch_info,
81
	[board_pch2lan]		= &e1000_pch2_info,
Bruce Allan's avatar
Bruce Allan committed
82
	[board_pch_lpt]		= &e1000_pch_lpt_info,
83
84
};

85
86
87
88
89
struct e1000_reg_info {
	u32 ofs;
	char *name;
};

90
91
92
93
94
95
96
97
98
99
100
#define E1000_RDFH	0x02410	/* Rx Data FIFO Head - RW */
#define E1000_RDFT	0x02418	/* Rx Data FIFO Tail - RW */
#define E1000_RDFHS	0x02420	/* Rx Data FIFO Head Saved - RW */
#define E1000_RDFTS	0x02428	/* Rx Data FIFO Tail Saved - RW */
#define E1000_RDFPC	0x02430	/* Rx Data FIFO Packet Count - RW */

#define E1000_TDFH	0x03410	/* Tx Data FIFO Head - RW */
#define E1000_TDFT	0x03418	/* Tx Data FIFO Tail - RW */
#define E1000_TDFHS	0x03420	/* Tx Data FIFO Head Saved - RW */
#define E1000_TDFTS	0x03428	/* Tx Data FIFO Tail Saved - RW */
#define E1000_TDFPC	0x03430	/* Tx Data FIFO Packet Count - RW */
101
102
103
104
105
106
107
108
109
110
111

static const struct e1000_reg_info e1000_reg_info_tbl[] = {

	/* General Registers */
	{E1000_CTRL, "CTRL"},
	{E1000_STATUS, "STATUS"},
	{E1000_CTRL_EXT, "CTRL_EXT"},

	/* Interrupt Registers */
	{E1000_ICR, "ICR"},

112
	/* Rx Registers */
113
	{E1000_RCTL, "RCTL"},
114
115
116
	{E1000_RDLEN(0), "RDLEN"},
	{E1000_RDH(0), "RDH"},
	{E1000_RDT(0), "RDT"},
117
118
119
	{E1000_RDTR, "RDTR"},
	{E1000_RXDCTL(0), "RXDCTL"},
	{E1000_ERT, "ERT"},
120
121
	{E1000_RDBAL(0), "RDBAL"},
	{E1000_RDBAH(0), "RDBAH"},
122
123
124
125
126
127
	{E1000_RDFH, "RDFH"},
	{E1000_RDFT, "RDFT"},
	{E1000_RDFHS, "RDFHS"},
	{E1000_RDFTS, "RDFTS"},
	{E1000_RDFPC, "RDFPC"},

128
	/* Tx Registers */
129
	{E1000_TCTL, "TCTL"},
130
131
132
133
134
	{E1000_TDBAL(0), "TDBAL"},
	{E1000_TDBAH(0), "TDBAH"},
	{E1000_TDLEN(0), "TDLEN"},
	{E1000_TDH(0), "TDH"},
	{E1000_TDT(0), "TDT"},
135
136
137
138
139
140
141
142
143
144
145
	{E1000_TIDV, "TIDV"},
	{E1000_TXDCTL(0), "TXDCTL"},
	{E1000_TADV, "TADV"},
	{E1000_TARC(0), "TARC"},
	{E1000_TDFH, "TDFH"},
	{E1000_TDFT, "TDFT"},
	{E1000_TDFHS, "TDFHS"},
	{E1000_TDFTS, "TDFTS"},
	{E1000_TDFPC, "TDFPC"},

	/* List Terminator */
146
	{0, NULL}
147
148
};

149
/**
150
 * e1000_regdump - register printout routine
151
152
153
 * @hw: pointer to the HW structure
 * @reginfo: pointer to the register info table
 **/
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
static void e1000_regdump(struct e1000_hw *hw, struct e1000_reg_info *reginfo)
{
	int n = 0;
	char rname[16];
	u32 regs[8];

	switch (reginfo->ofs) {
	case E1000_RXDCTL(0):
		for (n = 0; n < 2; n++)
			regs[n] = __er32(hw, E1000_RXDCTL(n));
		break;
	case E1000_TXDCTL(0):
		for (n = 0; n < 2; n++)
			regs[n] = __er32(hw, E1000_TXDCTL(n));
		break;
	case E1000_TARC(0):
		for (n = 0; n < 2; n++)
			regs[n] = __er32(hw, E1000_TARC(n));
		break;
	default:
174
175
		pr_info("%-15s %08x\n",
			reginfo->name, __er32(hw, reginfo->ofs));
176
177
178
179
		return;
	}

	snprintf(rname, 16, "%s%s", reginfo->name, "[0-1]");
180
	pr_info("%-15s %08x %08x\n", rname, regs[0], regs[1]);
181
182
}

183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
static void e1000e_dump_ps_pages(struct e1000_adapter *adapter,
				 struct e1000_buffer *bi)
{
	int i;
	struct e1000_ps_page *ps_page;

	for (i = 0; i < adapter->rx_ps_pages; i++) {
		ps_page = &bi->ps_pages[i];

		if (ps_page->page) {
			pr_info("packet dump for ps_page %d:\n", i);
			print_hex_dump(KERN_INFO, "", DUMP_PREFIX_ADDRESS,
				       16, 1, page_address(ps_page->page),
				       PAGE_SIZE, true);
		}
	}
}

201
/**
202
 * e1000e_dump - Print registers, Tx-ring and Rx-ring
203
204
 * @adapter: board private structure
 **/
205
206
207
208
209
210
211
static void e1000e_dump(struct e1000_adapter *adapter)
{
	struct net_device *netdev = adapter->netdev;
	struct e1000_hw *hw = &adapter->hw;
	struct e1000_reg_info *reginfo;
	struct e1000_ring *tx_ring = adapter->tx_ring;
	struct e1000_tx_desc *tx_desc;
212
	struct my_u0 {
213
214
		__le64 a;
		__le64 b;
215
	} *u0;
216
217
218
	struct e1000_buffer *buffer_info;
	struct e1000_ring *rx_ring = adapter->rx_ring;
	union e1000_rx_desc_packet_split *rx_desc_ps;
219
	union e1000_rx_desc_extended *rx_desc;
220
	struct my_u1 {
221
222
223
224
		__le64 a;
		__le64 b;
		__le64 c;
		__le64 d;
225
	} *u1;
226
227
228
229
230
231
232
233
234
	u32 staterr;
	int i = 0;

	if (!netif_msg_hw(adapter))
		return;

	/* Print netdevice Info */
	if (netdev) {
		dev_info(&adapter->pdev->dev, "Net device Info\n");
235
236
237
238
		pr_info("Device Name     state            trans_start      last_rx\n");
		pr_info("%-15s %016lX %016lX %016lX\n",
			netdev->name, netdev->state, netdev->trans_start,
			netdev->last_rx);
239
240
241
242
	}

	/* Print Registers */
	dev_info(&adapter->pdev->dev, "Register Dump\n");
243
	pr_info(" Register Name   Value\n");
244
245
246
247
248
	for (reginfo = (struct e1000_reg_info *)e1000_reg_info_tbl;
	     reginfo->name; reginfo++) {
		e1000_regdump(hw, reginfo);
	}

249
	/* Print Tx Ring Summary */
250
	if (!netdev || !netif_running(netdev))
251
		return;
252

253
	dev_info(&adapter->pdev->dev, "Tx Ring Summary\n");
254
	pr_info("Queue [NTU] [NTC] [bi(ntc)->dma  ] leng ntw timestamp\n");
255
	buffer_info = &tx_ring->buffer_info[tx_ring->next_to_clean];
256
257
258
259
260
261
	pr_info(" %5d %5X %5X %016llX %04X %3X %016llX\n",
		0, tx_ring->next_to_use, tx_ring->next_to_clean,
		(unsigned long long)buffer_info->dma,
		buffer_info->length,
		buffer_info->next_to_watch,
		(unsigned long long)buffer_info->time_stamp);
262

263
	/* Print Tx Ring */
264
265
266
	if (!netif_msg_tx_done(adapter))
		goto rx_ring_summary;

267
	dev_info(&adapter->pdev->dev, "Tx Ring Dump\n");
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295

	/* Transmit Descriptor Formats - DEXT[29] is 0 (Legacy) or 1 (Extended)
	 *
	 * Legacy Transmit Descriptor
	 *   +--------------------------------------------------------------+
	 * 0 |         Buffer Address [63:0] (Reserved on Write Back)       |
	 *   +--------------------------------------------------------------+
	 * 8 | Special  |    CSS     | Status |  CMD    |  CSO   |  Length  |
	 *   +--------------------------------------------------------------+
	 *   63       48 47        36 35    32 31     24 23    16 15        0
	 *
	 * Extended Context Descriptor (DTYP=0x0) for TSO or checksum offload
	 *   63      48 47    40 39       32 31             16 15    8 7      0
	 *   +----------------------------------------------------------------+
	 * 0 |  TUCSE  | TUCS0  |   TUCSS   |     IPCSE       | IPCS0 | IPCSS |
	 *   +----------------------------------------------------------------+
	 * 8 |   MSS   | HDRLEN | RSV | STA | TUCMD | DTYP |      PAYLEN      |
	 *   +----------------------------------------------------------------+
	 *   63      48 47    40 39 36 35 32 31   24 23  20 19                0
	 *
	 * Extended Data Descriptor (DTYP=0x1)
	 *   +----------------------------------------------------------------+
	 * 0 |                     Buffer Address [63:0]                      |
	 *   +----------------------------------------------------------------+
	 * 8 | VLAN tag |  POPTS  | Rsvd | Status | Command | DTYP |  DTALEN  |
	 *   +----------------------------------------------------------------+
	 *   63       48 47     40 39  36 35    32 31     24 23  20 19        0
	 */
296
297
298
	pr_info("Tl[desc]     [address 63:0  ] [SpeCssSCmCsLen] [bi->dma       ] leng  ntw timestamp        bi->skb <-- Legacy format\n");
	pr_info("Tc[desc]     [Ce CoCsIpceCoS] [MssHlRSCm0Plen] [bi->dma       ] leng  ntw timestamp        bi->skb <-- Ext Context format\n");
	pr_info("Td[desc]     [address 63:0  ] [VlaPoRSCm1Dlen] [bi->dma       ] leng  ntw timestamp        bi->skb <-- Ext Data format\n");
299
	for (i = 0; tx_ring->desc && (i < tx_ring->count); i++) {
300
		const char *next_desc;
301
302
303
304
		tx_desc = E1000_TX_DESC(*tx_ring, i);
		buffer_info = &tx_ring->buffer_info[i];
		u0 = (struct my_u0 *)tx_desc;
		if (i == tx_ring->next_to_use && i == tx_ring->next_to_clean)
305
			next_desc = " NTC/U";
306
		else if (i == tx_ring->next_to_use)
307
			next_desc = " NTU";
308
		else if (i == tx_ring->next_to_clean)
309
			next_desc = " NTC";
310
		else
311
312
313
314
315
316
317
318
319
320
321
			next_desc = "";
		pr_info("T%c[0x%03X]    %016llX %016llX %016llX %04X  %3X %016llX %p%s\n",
			(!(le64_to_cpu(u0->b) & (1 << 29)) ? 'l' :
			 ((le64_to_cpu(u0->b) & (1 << 20)) ? 'd' : 'c')),
			i,
			(unsigned long long)le64_to_cpu(u0->a),
			(unsigned long long)le64_to_cpu(u0->b),
			(unsigned long long)buffer_info->dma,
			buffer_info->length, buffer_info->next_to_watch,
			(unsigned long long)buffer_info->time_stamp,
			buffer_info->skb, next_desc);
322

323
		if (netif_msg_pktdata(adapter) && buffer_info->skb)
324
			print_hex_dump(KERN_INFO, "", DUMP_PREFIX_ADDRESS,
325
326
				       16, 1, buffer_info->skb->data,
				       buffer_info->skb->len, true);
327
328
	}

329
	/* Print Rx Ring Summary */
330
rx_ring_summary:
331
	dev_info(&adapter->pdev->dev, "Rx Ring Summary\n");
332
333
334
	pr_info("Queue [NTU] [NTC]\n");
	pr_info(" %5d %5X %5X\n",
		0, rx_ring->next_to_use, rx_ring->next_to_clean);
335

336
	/* Print Rx Ring */
337
	if (!netif_msg_rx_status(adapter))
338
		return;
339

340
	dev_info(&adapter->pdev->dev, "Rx Ring Dump\n");
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
	switch (adapter->rx_ps_pages) {
	case 1:
	case 2:
	case 3:
		/* [Extended] Packet Split Receive Descriptor Format
		 *
		 *    +-----------------------------------------------------+
		 *  0 |                Buffer Address 0 [63:0]              |
		 *    +-----------------------------------------------------+
		 *  8 |                Buffer Address 1 [63:0]              |
		 *    +-----------------------------------------------------+
		 * 16 |                Buffer Address 2 [63:0]              |
		 *    +-----------------------------------------------------+
		 * 24 |                Buffer Address 3 [63:0]              |
		 *    +-----------------------------------------------------+
		 */
357
		pr_info("R  [desc]      [buffer 0 63:0 ] [buffer 1 63:0 ] [buffer 2 63:0 ] [buffer 3 63:0 ] [bi->dma       ] [bi->skb] <-- Ext Pkt Split format\n");
358
359
360
361
362
363
364
365
366
367
368
		/* [Extended] Receive Descriptor (Write-Back) Format
		 *
		 *   63       48 47    32 31     13 12    8 7    4 3        0
		 *   +------------------------------------------------------+
		 * 0 | Packet   | IP     |  Rsvd   | MRQ   | Rsvd | MRQ RSS |
		 *   | Checksum | Ident  |         | Queue |      |  Type   |
		 *   +------------------------------------------------------+
		 * 8 | VLAN Tag | Length | Extended Error | Extended Status |
		 *   +------------------------------------------------------+
		 *   63       48 47    32 31            20 19               0
		 */
369
		pr_info("RWB[desc]      [ck ipid mrqhsh] [vl   l0 ee  es] [ l3  l2  l1 hs] [reserved      ] ---------------- [bi->skb] <-- Ext Rx Write-Back format\n");
370
		for (i = 0; i < rx_ring->count; i++) {
371
			const char *next_desc;
372
373
374
375
			buffer_info = &rx_ring->buffer_info[i];
			rx_desc_ps = E1000_RX_DESC_PS(*rx_ring, i);
			u1 = (struct my_u1 *)rx_desc_ps;
			staterr =
376
			    le32_to_cpu(rx_desc_ps->wb.middle.status_error);
377
378
379
380
381
382
383
384

			if (i == rx_ring->next_to_use)
				next_desc = " NTU";
			else if (i == rx_ring->next_to_clean)
				next_desc = " NTC";
			else
				next_desc = "";

385
386
			if (staterr & E1000_RXD_STAT_DD) {
				/* Descriptor Done */
387
388
389
390
391
392
393
				pr_info("%s[0x%03X]     %016llX %016llX %016llX %016llX ---------------- %p%s\n",
					"RWB", i,
					(unsigned long long)le64_to_cpu(u1->a),
					(unsigned long long)le64_to_cpu(u1->b),
					(unsigned long long)le64_to_cpu(u1->c),
					(unsigned long long)le64_to_cpu(u1->d),
					buffer_info->skb, next_desc);
394
			} else {
395
396
397
398
399
400
401
402
				pr_info("%s[0x%03X]     %016llX %016llX %016llX %016llX %016llX %p%s\n",
					"R  ", i,
					(unsigned long long)le64_to_cpu(u1->a),
					(unsigned long long)le64_to_cpu(u1->b),
					(unsigned long long)le64_to_cpu(u1->c),
					(unsigned long long)le64_to_cpu(u1->d),
					(unsigned long long)buffer_info->dma,
					buffer_info->skb, next_desc);
403
404

				if (netif_msg_pktdata(adapter))
405
406
					e1000e_dump_ps_pages(adapter,
							     buffer_info);
407
408
409
410
411
			}
		}
		break;
	default:
	case 0:
412
		/* Extended Receive Descriptor (Read) Format
413
		 *
414
415
416
417
418
		 *   +-----------------------------------------------------+
		 * 0 |                Buffer Address [63:0]                |
		 *   +-----------------------------------------------------+
		 * 8 |                      Reserved                       |
		 *   +-----------------------------------------------------+
419
		 */
420
		pr_info("R  [desc]      [buf addr 63:0 ] [reserved 63:0 ] [bi->dma       ] [bi->skb] <-- Ext (Read) format\n");
421
422
423
424
425
426
427
428
429
430
431
432
433
		/* Extended Receive Descriptor (Write-Back) Format
		 *
		 *   63       48 47    32 31    24 23            4 3        0
		 *   +------------------------------------------------------+
		 *   |     RSS Hash      |        |               |         |
		 * 0 +-------------------+  Rsvd  |   Reserved    | MRQ RSS |
		 *   | Packet   | IP     |        |               |  Type   |
		 *   | Checksum | Ident  |        |               |         |
		 *   +------------------------------------------------------+
		 * 8 | VLAN Tag | Length | Extended Error | Extended Status |
		 *   +------------------------------------------------------+
		 *   63       48 47    32 31            20 19               0
		 */
434
		pr_info("RWB[desc]      [cs ipid    mrq] [vt   ln xe  xs] [bi->skb] <-- Ext (Write-Back) format\n");
435
436

		for (i = 0; i < rx_ring->count; i++) {
437
438
			const char *next_desc;

439
			buffer_info = &rx_ring->buffer_info[i];
440
441
442
			rx_desc = E1000_RX_DESC_EXT(*rx_ring, i);
			u1 = (struct my_u1 *)rx_desc;
			staterr = le32_to_cpu(rx_desc->wb.upper.status_error);
443
444
445
446
447
448
449
450

			if (i == rx_ring->next_to_use)
				next_desc = " NTU";
			else if (i == rx_ring->next_to_clean)
				next_desc = " NTC";
			else
				next_desc = "";

451
452
			if (staterr & E1000_RXD_STAT_DD) {
				/* Descriptor Done */
453
454
455
456
457
				pr_info("%s[0x%03X]     %016llX %016llX ---------------- %p%s\n",
					"RWB", i,
					(unsigned long long)le64_to_cpu(u1->a),
					(unsigned long long)le64_to_cpu(u1->b),
					buffer_info->skb, next_desc);
458
			} else {
459
460
461
462
463
464
				pr_info("%s[0x%03X]     %016llX %016llX %016llX %p%s\n",
					"R  ", i,
					(unsigned long long)le64_to_cpu(u1->a),
					(unsigned long long)le64_to_cpu(u1->b),
					(unsigned long long)buffer_info->dma,
					buffer_info->skb, next_desc);
465

466
467
				if (netif_msg_pktdata(adapter) &&
				    buffer_info->skb)
468
469
470
					print_hex_dump(KERN_INFO, "",
						       DUMP_PREFIX_ADDRESS, 16,
						       1,
471
						       buffer_info->skb->data,
472
473
474
						       adapter->rx_buffer_len,
						       true);
			}
475
476
477
478
		}
	}
}

479
480
481
482
483
484
485
486
487
488
489
490
/**
 * e1000_desc_unused - calculate if we have unused descriptors
 **/
static int e1000_desc_unused(struct e1000_ring *ring)
{
	if (ring->next_to_clean > ring->next_to_use)
		return ring->next_to_clean - ring->next_to_use - 1;

	return ring->count + ring->next_to_clean - ring->next_to_use - 1;
}

/**
491
 * e1000_receive_skb - helper function to handle Rx indications
492
493
494
495
496
497
 * @adapter: board private structure
 * @status: descriptor status field as written by hardware
 * @vlan: descriptor vlan field as written by hardware (no le/be conversion)
 * @skb: pointer to sk_buff to be indicated to stack
 **/
static void e1000_receive_skb(struct e1000_adapter *adapter,
498
			      struct net_device *netdev, struct sk_buff *skb,
Al Viro's avatar
Al Viro committed
499
			      u8 status, __le16 vlan)
500
{
501
	u16 tag = le16_to_cpu(vlan);
502
503
	skb->protocol = eth_type_trans(skb, netdev);

504
505
506
507
	if (status & E1000_RXD_STAT_VP)
		__vlan_hwaccel_put_tag(skb, tag);

	napi_gro_receive(&adapter->napi, skb);
508
509
510
}

/**
511
 * e1000_rx_checksum - Receive Checksum Offload
512
513
514
515
 * @adapter: board private structure
 * @status_err: receive descriptor status and error fields
 * @csum: receive descriptor csum field
 * @sk_buff: socket buffer with received data
516
517
 **/
static void e1000_rx_checksum(struct e1000_adapter *adapter, u32 status_err,
518
			      struct sk_buff *skb)
519
520
521
{
	u16 status = (u16)status_err;
	u8 errors = (u8)(status_err >> 24);
522
523

	skb_checksum_none_assert(skb);
524

525
526
527
528
	/* Rx checksum disabled */
	if (!(adapter->netdev->features & NETIF_F_RXCSUM))
		return;

529
530
531
	/* Ignore Checksum bit is set */
	if (status & E1000_RXD_STAT_IXSM)
		return;
532

533
534
	/* TCP/UDP checksum error bit or IP checksum error bit is set */
	if (errors & (E1000_RXD_ERR_TCPE | E1000_RXD_ERR_IPE)) {
535
536
537
538
539
540
541
542
543
544
		/* let the stack verify checksum errors */
		adapter->hw_csum_err++;
		return;
	}

	/* TCP/UDP Checksum has not been calculated */
	if (!(status & (E1000_RXD_STAT_TCPCS | E1000_RXD_STAT_UDPCS)))
		return;

	/* It must be a TCP or UDP packet with a valid checksum */
545
	skb->ip_summed = CHECKSUM_UNNECESSARY;
546
547
548
	adapter->hw_csum_good++;
}

549
static void e1000e_update_rdt_wa(struct e1000_ring *rx_ring, unsigned int i)
550
{
551
	struct e1000_adapter *adapter = rx_ring->adapter;
552
	struct e1000_hw *hw = &adapter->hw;
553
554
555
	s32 ret_val = __ew32_prepare(hw);

	writel(i, rx_ring->tail);
556

557
	if (unlikely(!ret_val && (i != readl(rx_ring->tail)))) {
558
559
560
561
562
563
564
		u32 rctl = er32(RCTL);
		ew32(RCTL, rctl & ~E1000_RCTL_EN);
		e_err("ME firmware caused invalid RDT - resetting\n");
		schedule_work(&adapter->reset_task);
	}
}

565
static void e1000e_update_tdt_wa(struct e1000_ring *tx_ring, unsigned int i)
566
{
567
	struct e1000_adapter *adapter = tx_ring->adapter;
568
	struct e1000_hw *hw = &adapter->hw;
569
	s32 ret_val = __ew32_prepare(hw);
570

571
572
573
	writel(i, tx_ring->tail);

	if (unlikely(!ret_val && (i != readl(tx_ring->tail)))) {
574
575
576
577
578
579
580
		u32 tctl = er32(TCTL);
		ew32(TCTL, tctl & ~E1000_TCTL_EN);
		e_err("ME firmware caused invalid TDT - resetting\n");
		schedule_work(&adapter->reset_task);
	}
}

581
/**
582
 * e1000_alloc_rx_buffers - Replace used receive buffers
583
 * @rx_ring: Rx descriptor ring
584
 **/
585
static void e1000_alloc_rx_buffers(struct e1000_ring *rx_ring,
586
				   int cleaned_count, gfp_t gfp)
587
{
588
	struct e1000_adapter *adapter = rx_ring->adapter;
589
590
	struct net_device *netdev = adapter->netdev;
	struct pci_dev *pdev = adapter->pdev;
591
	union e1000_rx_desc_extended *rx_desc;
592
593
594
	struct e1000_buffer *buffer_info;
	struct sk_buff *skb;
	unsigned int i;
595
	unsigned int bufsz = adapter->rx_buffer_len;
596
597
598
599
600
601
602
603
604
605
606

	i = rx_ring->next_to_use;
	buffer_info = &rx_ring->buffer_info[i];

	while (cleaned_count--) {
		skb = buffer_info->skb;
		if (skb) {
			skb_trim(skb, 0);
			goto map_skb;
		}

607
		skb = __netdev_alloc_skb_ip_align(netdev, bufsz, gfp);
608
609
610
611
612
613
614
615
		if (!skb) {
			/* Better luck next round */
			adapter->alloc_rx_buff_failed++;
			break;
		}

		buffer_info->skb = skb;
map_skb:
616
		buffer_info->dma = dma_map_single(&pdev->dev, skb->data,
617
						  adapter->rx_buffer_len,
618
619
						  DMA_FROM_DEVICE);
		if (dma_mapping_error(&pdev->dev, buffer_info->dma)) {
620
			dev_err(&pdev->dev, "Rx DMA map failed\n");
621
622
623
624
			adapter->rx_dma_failed++;
			break;
		}

625
626
		rx_desc = E1000_RX_DESC_EXT(*rx_ring, i);
		rx_desc->read.buffer_addr = cpu_to_le64(buffer_info->dma);
627

628
		if (unlikely(!(i & (E1000_RX_BUFFER_WRITE - 1)))) {
629
			/* Force memory writes to complete before letting h/w
630
631
632
633
634
			 * know there are new descriptors to fetch.  (Only
			 * applicable for weak-ordered memory model archs,
			 * such as IA-64).
			 */
			wmb();
635
			if (adapter->flags2 & FLAG2_PCIM2PCI_ARBITER_WA)
636
				e1000e_update_rdt_wa(rx_ring, i);
637
			else
638
				writel(i, rx_ring->tail);
639
		}
640
641
642
643
644
645
		i++;
		if (i == rx_ring->count)
			i = 0;
		buffer_info = &rx_ring->buffer_info[i];
	}

646
	rx_ring->next_to_use = i;
647
648
649
650
}

/**
 * e1000_alloc_rx_buffers_ps - Replace used receive buffers; packet split
651
 * @rx_ring: Rx descriptor ring
652
 **/
653
static void e1000_alloc_rx_buffers_ps(struct e1000_ring *rx_ring,
654
				      int cleaned_count, gfp_t gfp)
655
{
656
	struct e1000_adapter *adapter = rx_ring->adapter;
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
	struct net_device *netdev = adapter->netdev;
	struct pci_dev *pdev = adapter->pdev;
	union e1000_rx_desc_packet_split *rx_desc;
	struct e1000_buffer *buffer_info;
	struct e1000_ps_page *ps_page;
	struct sk_buff *skb;
	unsigned int i, j;

	i = rx_ring->next_to_use;
	buffer_info = &rx_ring->buffer_info[i];

	while (cleaned_count--) {
		rx_desc = E1000_RX_DESC_PS(*rx_ring, i);

		for (j = 0; j < PS_PAGE_BUFFERS; j++) {
672
673
674
			ps_page = &buffer_info->ps_pages[j];
			if (j >= adapter->rx_ps_pages) {
				/* all unused desc entries get hw null ptr */
675
676
				rx_desc->read.buffer_addr[j + 1] =
				    ~cpu_to_le64(0);
677
678
679
				continue;
			}
			if (!ps_page->page) {
680
				ps_page->page = alloc_page(gfp);
681
				if (!ps_page->page) {
682
683
684
					adapter->alloc_rx_buff_failed++;
					goto no_buffers;
				}
685
686
687
688
689
690
				ps_page->dma = dma_map_page(&pdev->dev,
							    ps_page->page,
							    0, PAGE_SIZE,
							    DMA_FROM_DEVICE);
				if (dma_mapping_error(&pdev->dev,
						      ps_page->dma)) {
691
					dev_err(&adapter->pdev->dev,
692
						"Rx DMA page map failed\n");
693
694
					adapter->rx_dma_failed++;
					goto no_buffers;
695
696
				}
			}
697
			/* Refresh the desc even if buffer_addrs
698
699
700
			 * didn't change because each write-back
			 * erases this info.
			 */
701
702
			rx_desc->read.buffer_addr[j + 1] =
			    cpu_to_le64(ps_page->dma);
703
704
		}

705
706
707
		skb = __netdev_alloc_skb_ip_align(netdev,
						  adapter->rx_ps_bsize0,
						  gfp);
708
709
710
711
712
713
714

		if (!skb) {
			adapter->alloc_rx_buff_failed++;
			break;
		}

		buffer_info->skb = skb;
715
		buffer_info->dma = dma_map_single(&pdev->dev, skb->data,
716
						  adapter->rx_ps_bsize0,
717
718
						  DMA_FROM_DEVICE);
		if (dma_mapping_error(&pdev->dev, buffer_info->dma)) {
719
			dev_err(&pdev->dev, "Rx DMA map failed\n");
720
721
722
723
724
725
726
727
728
			adapter->rx_dma_failed++;
			/* cleanup skb */
			dev_kfree_skb_any(skb);
			buffer_info->skb = NULL;
			break;
		}

		rx_desc->read.buffer_addr[0] = cpu_to_le64(buffer_info->dma);

729
		if (unlikely(!(i & (E1000_RX_BUFFER_WRITE - 1)))) {
730
			/* Force memory writes to complete before letting h/w
731
732
733
734
735
			 * know there are new descriptors to fetch.  (Only
			 * applicable for weak-ordered memory model archs,
			 * such as IA-64).
			 */
			wmb();
736
			if (adapter->flags2 & FLAG2_PCIM2PCI_ARBITER_WA)
737
				e1000e_update_rdt_wa(rx_ring, i << 1);
738
			else
739
				writel(i << 1, rx_ring->tail);
740
741
		}

742
743
744
745
746
747
748
		i++;
		if (i == rx_ring->count)
			i = 0;
		buffer_info = &rx_ring->buffer_info[i];
	}

no_buffers:
749
	rx_ring->next_to_use = i;
750
751
}

752
753
/**
 * e1000_alloc_jumbo_rx_buffers - Replace used jumbo receive buffers
754
 * @rx_ring: Rx descriptor ring
755
756
757
 * @cleaned_count: number of buffers to allocate this pass
 **/

758
static void e1000_alloc_jumbo_rx_buffers(struct e1000_ring *rx_ring,
759
					 int cleaned_count, gfp_t gfp)
760
{
761
	struct e1000_adapter *adapter = rx_ring->adapter;
762
763
	struct net_device *netdev = adapter->netdev;
	struct pci_dev *pdev = adapter->pdev;
764
	union e1000_rx_desc_extended *rx_desc;
765
766
767
	struct e1000_buffer *buffer_info;
	struct sk_buff *skb;
	unsigned int i;
768
	unsigned int bufsz = 256 - 16;	/* for skb_reserve */
769
770
771
772
773
774
775
776
777
778
779

	i = rx_ring->next_to_use;
	buffer_info = &rx_ring->buffer_info[i];

	while (cleaned_count--) {
		skb = buffer_info->skb;
		if (skb) {
			skb_trim(skb, 0);
			goto check_page;
		}

780
		skb = __netdev_alloc_skb_ip_align(netdev, bufsz, gfp);
781
782
783
784
785
786
787
788
789
790
		if (unlikely(!skb)) {
			/* Better luck next round */
			adapter->alloc_rx_buff_failed++;
			break;
		}

		buffer_info->skb = skb;
check_page:
		/* allocate a new page if necessary */
		if (!buffer_info->page) {
791
			buffer_info->page = alloc_page(gfp);
792
793
794
795
796
797
798
			if (unlikely(!buffer_info->page)) {
				adapter->alloc_rx_buff_failed++;
				break;
			}
		}

		if (!buffer_info->dma)
799
			buffer_info->dma = dma_map_page(&pdev->dev,
800
801
			                                buffer_info->page, 0,
			                                PAGE_SIZE,
802
							DMA_FROM_DEVICE);
803

804
805
		rx_desc = E1000_RX_DESC_EXT(*rx_ring, i);
		rx_desc->read.buffer_addr = cpu_to_le64(buffer_info->dma);
806
807
808
809
810
811
812
813
814
815
816
817
818
819

		if (unlikely(++i == rx_ring->count))
			i = 0;
		buffer_info = &rx_ring->buffer_info[i];
	}

	if (likely(rx_ring->next_to_use != i)) {
		rx_ring->next_to_use = i;
		if (unlikely(i-- == 0))
			i = (rx_ring->count - 1);

		/* Force memory writes to complete before letting h/w
		 * know there are new descriptors to fetch.  (Only
		 * applicable for weak-ordered memory model archs,
820
821
		 * such as IA-64).
		 */
822
		wmb();
823
		if (adapter->flags2 & FLAG2_PCIM2PCI_ARBITER_WA)
824
			e1000e_update_rdt_wa(rx_ring, i);
825
		else
826
			writel(i, rx_ring->tail);
827
828
829
	}
}

830
831
832
833
834
835
836
static inline void e1000_rx_hash(struct net_device *netdev, __le32 rss,
				 struct sk_buff *skb)
{
	if (netdev->features & NETIF_F_RXHASH)
		skb->rxhash = le32_to_cpu(rss);
}

837
/**
838
839
 * e1000_clean_rx_irq - Send received data up the network stack
 * @rx_ring: Rx descriptor ring
840
841
842
843
 *
 * the return value indicates whether actual cleaning was done, there
 * is no guarantee that everything was cleaned
 **/
844
845
static bool e1000_clean_rx_irq(struct e1000_ring *rx_ring, int *work_done,
			       int work_to_do)
846
{
847
	struct e1000_adapter *adapter = rx_ring->adapter;
848
849
	struct net_device *netdev = adapter->netdev;
	struct pci_dev *pdev = adapter->pdev;
850
	struct e1000_hw *hw = &adapter->hw;
851
	union e1000_rx_desc_extended *rx_desc, *next_rxd;
852
	struct e1000_buffer *buffer_info, *next_buffer;
853
	u32 length, staterr;
854
855
	unsigned int i;
	int cleaned_count = 0;
856
	bool cleaned = false;
857
858
859
	unsigned int total_rx_bytes = 0, total_rx_packets = 0;

	i = rx_ring->next_to_clean;
860
861
	rx_desc = E1000_RX_DESC_EXT(*rx_ring, i);
	staterr = le32_to_cpu(rx_desc->wb.upper.status_error);
862
863
	buffer_info = &rx_ring->buffer_info[i];

864
	while (staterr & E1000_RXD_STAT_DD) {
865
866
867
868
869
		struct sk_buff *skb;

		if (*work_done >= work_to_do)
			break;
		(*work_done)++;
870
		rmb();	/* read descriptor and rx_buffer_info after status DD */
871
872
873
874
875
876
877
878
879

		skb = buffer_info->skb;
		buffer_info->skb = NULL;

		prefetch(skb->data - NET_IP_ALIGN);

		i++;
		if (i == rx_ring->count)
			i = 0;
880
		next_rxd = E1000_RX_DESC_EXT(*rx_ring, i);
881
882
883
884
		prefetch(next_rxd);

		next_buffer = &rx_ring->buffer_info[i];

885
		cleaned = true;
886
		cleaned_count++;
887
		dma_unmap_single(&pdev->dev,
888
889
				 buffer_info->dma,
				 adapter->rx_buffer_len,
890
				 DMA_FROM_DEVICE);
891
892
		buffer_info->dma = 0;

893
		length = le16_to_cpu(rx_desc->wb.upper.length);
894

895
		/* !EOP means multiple descriptors were used to store a single
896
897
898
899
900
		 * packet, if that's the case we need to toss it.  In fact, we
		 * need to toss every packet with the EOP bit clear and the
		 * next frame that _does_ have the EOP bit set, as it is by
		 * definition only a frame fragment
		 */
901
		if (unlikely(!(staterr & E1000_RXD_STAT_EOP)))
902
903
904
			adapter->flags2 |= FLAG2_IS_DISCARDING;

		if (adapter->flags2 & FLAG2_IS_DISCARDING) {
905
			/* All receives must fit into a single buffer */
906
			e_dbg("Receive packet consumed multiple buffers\n");
907
908
			/* recycle */
			buffer_info->skb = skb;
909
			if (staterr & E1000_RXD_STAT_EOP)
910
				adapter->flags2 &= ~FLAG2_IS_DISCARDING;
911
912
913
			goto next_desc;
		}

914
915
		if (unlikely((staterr & E1000_RXDEXT_ERR_FRAME_ERR_MASK) &&
			     !(netdev->features & NETIF_F_RXALL))) {
916
917
918
919
920
			/* recycle */
			buffer_info->skb = skb;
			goto next_desc;
		}

Jeff Kirsher's avatar
Jeff Kirsher committed
921
		/* adjust length to remove Ethernet CRC */
922
923
924
925
926
927
928
929
930
931
		if (!(adapter->flags2 & FLAG2_CRC_STRIPPING)) {
			/* If configured to store CRC, don't subtract FCS,
			 * but keep the FCS bytes out of the total_rx_bytes
			 * counter
			 */
			if (netdev->features & NETIF_F_RXFCS)
				total_rx_bytes -= 4;
			else
				length -= 4;
		}
Jeff Kirsher's avatar
Jeff Kirsher committed
932

933
934
935
		total_rx_bytes += length;
		total_rx_packets++;

936
		/* code added for copybreak, this should improve
937
		 * performance for small packets with large amounts
938
939
		 * of reassembly being done in the stack
		 */
940
941
		if (length < copybreak) {
			struct sk_buff *new_skb =
942
			    netdev_alloc_skb_ip_align(netdev, length);
943
			if (new_skb) {
944
945
946
947
948
949
				skb_copy_to_linear_data_offset(new_skb,
							       -NET_IP_ALIGN,
							       (skb->data -
								NET_IP_ALIGN),
							       (length +
								NET_IP_ALIGN));
950
951
952
953
954
955
956
957
958
959
				/* save the skb in buffer_info as good */
				buffer_info->skb = skb;
				skb = new_skb;
			}
			/* else just continue with the old one */
		}
		/* end copybreak code */
		skb_put(skb, length);

		/* Receive Checksum Offload */
960
		e1000_rx_checksum(adapter, staterr, skb);
961

962
963
		e1000_rx_hash(netdev, rx_desc->wb.lower.hi_dword.rss, skb);

964
965
		e1000_receive_skb(adapter, netdev, skb, staterr,
				  rx_desc->wb.upper.vlan);
966
967

next_desc:
968
		rx_desc->wb.upper.status_error &= cpu_to_le32(~0xFF);
969
970
971

		/* return some buffers to hardware, one at a time is too slow */
		if (cleaned_count >= E1000_RX_BUFFER_WRITE) {
972
			adapter->alloc_rx_buf(rx_ring, cleaned_count,
973
					      GFP_ATOMIC);
974
975
976
977
978
979
			cleaned_count = 0;
		}

		/* use prefetched values */
		rx_desc = next_rxd;
		buffer_info = next_buffer;
980
981

		staterr = le32_to_cpu(rx_desc->wb.upper.status_error);
982
983
984
985
986
	}
	rx_ring->next_to_clean = i;

	cleaned_count = e1000_desc_unused(rx_ring);
	if (cleaned_count)
987
		adapter->alloc_rx_buf(rx_ring, cleaned_count, GFP_ATOMIC);
988
989

	adapter->total_rx_bytes += total_rx_bytes;
990
	adapter->total_rx_packets += total_rx_packets;
991
992
993
	return cleaned;
}

994
995
static void e1000_put_txbuf(struct e1000_ring *tx_ring,
			    struct e1000_buffer *buffer_info)
996
{
997
998
	struct e1000_adapter *adapter = tx_ring->adapter;

999
1000
	if (buffer_info->dma) {
		if (buffer_info->mapped_as_page)
1001
1002
			dma_unmap_page(&adapter->pdev->dev, buffer_info->dma,
				       buffer_info->length, DMA_TO_DEVICE);
1003
		else
1004
1005
			dma_unmap_single(&adapter->pdev->dev, buffer_info->dma,
					 buffer_info->length, DMA_TO_DEVICE);
1006
1007
		buffer_info->dma = 0;
	}
1008
1009
1010
1011
	if (buffer_info->skb) {
		dev_kfree_skb_any(buffer_info->skb);
		buffer_info->skb = NULL;
	}
1012
	buffer_info->time_stamp = 0;
1013
1014
}

1015
static void e1000_print_hw_hang(struct work_struct *work)
1016
{
1017
1018
1019
	struct e1000_adapter *adapter = container_of(work,
	                                             struct e1000_adapter,
	                                             print_hang_task);
1020
	struct net_device *netdev = adapter->netdev;
1021
1022
1023
1024
	struct e1000_ring *tx_ring = adapter->tx_ring;
	unsigned int i = tx_ring->next_to_clean;
	unsigned int eop = tx_ring->buffer_info[i].next_to_watch;
	struct e1000_tx_desc *eop_desc = E1000_TX_DESC(*tx_ring, eop);
1025
1026
1027
1028
	struct e1000_hw *hw = &adapter->hw;
	u16 phy_status, phy_1000t_status, phy_ext_status;
	u16 pci_status;

1029
1030
1031
	if (test_bit(__E1000_DOWN, &adapter->state))
		return;

1032
1033
	if (!adapter->tx_hang_recheck &&
	    (adapter->flags2 & FLAG2_DMA_BURST)) {
1034
		/* May be block on write-back, flush and detect again
1035
1036
1037
1038
1039
		 * flush pending descriptor writebacks to memory
		 */
		ew32(TIDV, adapter->tx_int_delay | E1000_TIDV_FPD);
		/* execute the writes immediately */
		e1e_flush();
1040
		/* Due to rare timing issues, write to TIDV again to ensure
1041
1042
1043
1044
1045
		 * the write is successful
		 */
		ew32(TIDV, adapter->tx_int_delay | E1000_TIDV_FPD);
		/* execute the writes immediately */
		e1e_flush();
1046
1047
1048
1049
1050
1051
1052
		adapter->tx_hang_recheck = true;
		return;
	}
	/* Real hang detected */
	adapter->tx_hang_recheck = false;
	netif_stop_queue(netdev);

1053
1054
1055
	e1e_rphy(hw, PHY_STATUS, &phy_status);
	e1e_rphy(hw, PHY_1000T_STATUS, &phy_1000t_status);
	e1e_rphy(hw, PHY_EXT_STATUS, &phy_ext_status);
1056

1057
1058
1059
1060
	pci_read_config_word(adapter->pdev, PCI_STATUS, &pci_status);

	/* detected Hardware unit hang */
	e_err("Detected Hardware Unit Hang:\n"
1061
1062
1063
1064
1065
1066
1067
1068
	      "  TDH                  <%x>\n"
	      "  TDT                  <%x>\n"
	      "  next_to_use          <%x>\n"
	      "  next_to_clean        <%x>\n"
	      "buffer_info[next_to_clean]:\n"
	      "  time_stamp           <%lx>\n"
	      "  next_to_watch        <%x>\n"
	      "  jiffies              <%lx>\n"
1069
1070
1071
1072
1073
1074
	      "  next_to_watch.status <%x>\n"
	      "MAC Status             <%x>\n"
	      "PHY Status             <%x>\n"
	      "PHY 1000BASE-T Status  <%x>\n"
	      "PHY Extended Status    <%x>\n"
	      "PCI Status             <%x>\n",
1075
1076
	      readl(tx_ring->head),
	      readl(tx_ring->tail),
1077
1078
1079
1080
1081
	      tx_ring->next_to_use,
	      tx_ring->next_to_clean,
	      tx_ring->buffer_info[eop].time_stamp,
	      eop,
	      jiffies,
1082
1083
1084
1085
1086
1087
	      eop_desc->upper.fields.status,
	      er32(STATUS),
	      phy_status,
	      phy_1000t_status,
	      phy_ext_status,
	      pci_status);
1088
1089
1090
1091

	/* Suggest workaround for known h/w issue */
	if ((hw->mac.type == e1000_pchlan) && (er32(CTRL) & E1000_CTRL_TFCE))
		e_err("Try turning off Tx pause (flow control) via ethtool\n");
1092
1093
1094
1095
}

/**
 * e1000_clean_tx_irq - Reclaim resources after transmit completes
1096
 * @tx_ring: Tx descriptor ring
1097
1098
1099
1100
 *
 * the return value indicates whether actual cleaning was done, there
 * is no guarantee that everything was cleaned
 **/
1101
static bool e1000_clean_tx_irq(struct e1000_ring *tx_ring)
1102
{
1103
	struct e1000_adapter *adapter = tx_ring->adapter;
1104
1105
1106
1107
1108
1109
1110
	struct net_device *netdev = adapter->netdev;
	struct e1000_hw *hw = &adapter->hw;
	struct e1000_tx_desc *tx_desc, *eop_desc;
	struct e1000_buffer *buffer_info;
	unsigned int i, eop;
	unsigned int count = 0;
	unsigned int total_tx_bytes = 0, total_tx_packets = 0;
1111
	unsigned int bytes_compl = 0, pkts_compl = 0;
1112
1113
1114
1115
1116

	i = tx_ring->next_to_clean;
	eop = tx_ring->buffer_info[i].next_to_watch;
	eop_desc = E1000_TX_DESC(*tx_ring, eop);

1117
1118
	while ((eop_desc->upper.data & cpu_to_le32(E1000_TXD_STAT_DD)) &&
	       (count < tx_ring->count)) {
1119
		bool cleaned = false;
Jeff Kirsher's avatar