netdev.c 193 KB
Newer Older
1
2
3
/*******************************************************************************

  Intel PRO/1000 Linux driver
Bruce Allan's avatar
Bruce Allan committed
4
  Copyright(c) 1999 - 2012 Intel Corporation.
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28

  This program is free software; you can redistribute it and/or modify it
  under the terms and conditions of the GNU General Public License,
  version 2, as published by the Free Software Foundation.

  This program is distributed in the hope it will be useful, but WITHOUT
  ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  more details.

  You should have received a copy of the GNU General Public License along with
  this program; if not, write to the Free Software Foundation, Inc.,
  51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.

  The full GNU General Public License is included in this distribution in
  the file called "COPYING".

  Contact Information:
  Linux NICS <linux.nics@intel.com>
  e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
  Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497

*******************************************************************************/

29
30
#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt

31
32
33
34
35
36
37
38
#include <linux/module.h>
#include <linux/types.h>
#include <linux/init.h>
#include <linux/pci.h>
#include <linux/vmalloc.h>
#include <linux/pagemap.h>
#include <linux/delay.h>
#include <linux/netdevice.h>
39
#include <linux/interrupt.h>
40
41
#include <linux/tcp.h>
#include <linux/ipv6.h>
42
#include <linux/slab.h>
43
44
45
46
47
48
49
#include <net/checksum.h>
#include <net/ip6_checksum.h>
#include <linux/mii.h>
#include <linux/ethtool.h>
#include <linux/if_vlan.h>
#include <linux/cpu.h>
#include <linux/smp.h>
50
#include <linux/pm_qos.h>
51
#include <linux/pm_runtime.h>
Jesse Brandeburg's avatar
Jesse Brandeburg committed
52
#include <linux/aer.h>
53
#include <linux/prefetch.h>
54
55
56

#include "e1000.h"

Bruce Allan's avatar
Bruce Allan committed
57
#define DRV_EXTRAVERSION "-k"
58

59
#define DRV_VERSION "2.1.4" DRV_EXTRAVERSION
60
61
62
char e1000e_driver_name[] = "e1000e";
const char e1000e_driver_version[] = DRV_VERSION;

63
64
65
66
67
#define DEFAULT_MSG_ENABLE (NETIF_MSG_DRV|NETIF_MSG_PROBE|NETIF_MSG_LINK)
static int debug = -1;
module_param(debug, int, 0);
MODULE_PARM_DESC(debug, "Debug level (0=none,...,16=all)");

68
69
static void e1000e_disable_aspm(struct pci_dev *pdev, u16 state);

70
71
72
73
static const struct e1000_info *e1000_info_tbl[] = {
	[board_82571]		= &e1000_82571_info,
	[board_82572]		= &e1000_82572_info,
	[board_82573]		= &e1000_82573_info,
74
	[board_82574]		= &e1000_82574_info,
75
	[board_82583]		= &e1000_82583_info,
76
77
78
	[board_80003es2lan]	= &e1000_es2_info,
	[board_ich8lan]		= &e1000_ich8_info,
	[board_ich9lan]		= &e1000_ich9_info,
79
	[board_ich10lan]	= &e1000_ich10_info,
80
	[board_pchlan]		= &e1000_pch_info,
81
	[board_pch2lan]		= &e1000_pch2_info,
Bruce Allan's avatar
Bruce Allan committed
82
	[board_pch_lpt]		= &e1000_pch_lpt_info,
83
84
};

85
86
87
88
89
struct e1000_reg_info {
	u32 ofs;
	char *name;
};

90
91
92
93
94
95
96
97
98
99
100
#define E1000_RDFH	0x02410	/* Rx Data FIFO Head - RW */
#define E1000_RDFT	0x02418	/* Rx Data FIFO Tail - RW */
#define E1000_RDFHS	0x02420	/* Rx Data FIFO Head Saved - RW */
#define E1000_RDFTS	0x02428	/* Rx Data FIFO Tail Saved - RW */
#define E1000_RDFPC	0x02430	/* Rx Data FIFO Packet Count - RW */

#define E1000_TDFH	0x03410	/* Tx Data FIFO Head - RW */
#define E1000_TDFT	0x03418	/* Tx Data FIFO Tail - RW */
#define E1000_TDFHS	0x03420	/* Tx Data FIFO Head Saved - RW */
#define E1000_TDFTS	0x03428	/* Tx Data FIFO Tail Saved - RW */
#define E1000_TDFPC	0x03430	/* Tx Data FIFO Packet Count - RW */
101
102
103
104
105
106
107
108
109
110
111

static const struct e1000_reg_info e1000_reg_info_tbl[] = {

	/* General Registers */
	{E1000_CTRL, "CTRL"},
	{E1000_STATUS, "STATUS"},
	{E1000_CTRL_EXT, "CTRL_EXT"},

	/* Interrupt Registers */
	{E1000_ICR, "ICR"},

112
	/* Rx Registers */
113
	{E1000_RCTL, "RCTL"},
114
115
116
	{E1000_RDLEN(0), "RDLEN"},
	{E1000_RDH(0), "RDH"},
	{E1000_RDT(0), "RDT"},
117
118
119
	{E1000_RDTR, "RDTR"},
	{E1000_RXDCTL(0), "RXDCTL"},
	{E1000_ERT, "ERT"},
120
121
	{E1000_RDBAL(0), "RDBAL"},
	{E1000_RDBAH(0), "RDBAH"},
122
123
124
125
126
127
	{E1000_RDFH, "RDFH"},
	{E1000_RDFT, "RDFT"},
	{E1000_RDFHS, "RDFHS"},
	{E1000_RDFTS, "RDFTS"},
	{E1000_RDFPC, "RDFPC"},

128
	/* Tx Registers */
129
	{E1000_TCTL, "TCTL"},
130
131
132
133
134
	{E1000_TDBAL(0), "TDBAL"},
	{E1000_TDBAH(0), "TDBAH"},
	{E1000_TDLEN(0), "TDLEN"},
	{E1000_TDH(0), "TDH"},
	{E1000_TDT(0), "TDT"},
135
136
137
138
139
140
141
142
143
144
145
	{E1000_TIDV, "TIDV"},
	{E1000_TXDCTL(0), "TXDCTL"},
	{E1000_TADV, "TADV"},
	{E1000_TARC(0), "TARC"},
	{E1000_TDFH, "TDFH"},
	{E1000_TDFT, "TDFT"},
	{E1000_TDFHS, "TDFHS"},
	{E1000_TDFTS, "TDFTS"},
	{E1000_TDFPC, "TDFPC"},

	/* List Terminator */
146
	{0, NULL}
147
148
};

149
/**
150
 * e1000_regdump - register printout routine
151
152
153
 * @hw: pointer to the HW structure
 * @reginfo: pointer to the register info table
 **/
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
static void e1000_regdump(struct e1000_hw *hw, struct e1000_reg_info *reginfo)
{
	int n = 0;
	char rname[16];
	u32 regs[8];

	switch (reginfo->ofs) {
	case E1000_RXDCTL(0):
		for (n = 0; n < 2; n++)
			regs[n] = __er32(hw, E1000_RXDCTL(n));
		break;
	case E1000_TXDCTL(0):
		for (n = 0; n < 2; n++)
			regs[n] = __er32(hw, E1000_TXDCTL(n));
		break;
	case E1000_TARC(0):
		for (n = 0; n < 2; n++)
			regs[n] = __er32(hw, E1000_TARC(n));
		break;
	default:
174
175
		pr_info("%-15s %08x\n",
			reginfo->name, __er32(hw, reginfo->ofs));
176
177
178
179
		return;
	}

	snprintf(rname, 16, "%s%s", reginfo->name, "[0-1]");
180
	pr_info("%-15s %08x %08x\n", rname, regs[0], regs[1]);
181
182
}

183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
static void e1000e_dump_ps_pages(struct e1000_adapter *adapter,
				 struct e1000_buffer *bi)
{
	int i;
	struct e1000_ps_page *ps_page;

	for (i = 0; i < adapter->rx_ps_pages; i++) {
		ps_page = &bi->ps_pages[i];

		if (ps_page->page) {
			pr_info("packet dump for ps_page %d:\n", i);
			print_hex_dump(KERN_INFO, "", DUMP_PREFIX_ADDRESS,
				       16, 1, page_address(ps_page->page),
				       PAGE_SIZE, true);
		}
	}
}

201
/**
202
 * e1000e_dump - Print registers, Tx-ring and Rx-ring
203
204
 * @adapter: board private structure
 **/
205
206
207
208
209
210
211
static void e1000e_dump(struct e1000_adapter *adapter)
{
	struct net_device *netdev = adapter->netdev;
	struct e1000_hw *hw = &adapter->hw;
	struct e1000_reg_info *reginfo;
	struct e1000_ring *tx_ring = adapter->tx_ring;
	struct e1000_tx_desc *tx_desc;
212
	struct my_u0 {
213
214
		__le64 a;
		__le64 b;
215
	} *u0;
216
217
218
	struct e1000_buffer *buffer_info;
	struct e1000_ring *rx_ring = adapter->rx_ring;
	union e1000_rx_desc_packet_split *rx_desc_ps;
219
	union e1000_rx_desc_extended *rx_desc;
220
	struct my_u1 {
221
222
223
224
		__le64 a;
		__le64 b;
		__le64 c;
		__le64 d;
225
	} *u1;
226
227
228
229
230
231
232
233
234
	u32 staterr;
	int i = 0;

	if (!netif_msg_hw(adapter))
		return;

	/* Print netdevice Info */
	if (netdev) {
		dev_info(&adapter->pdev->dev, "Net device Info\n");
235
236
237
238
		pr_info("Device Name     state            trans_start      last_rx\n");
		pr_info("%-15s %016lX %016lX %016lX\n",
			netdev->name, netdev->state, netdev->trans_start,
			netdev->last_rx);
239
240
241
242
	}

	/* Print Registers */
	dev_info(&adapter->pdev->dev, "Register Dump\n");
243
	pr_info(" Register Name   Value\n");
244
245
246
247
248
	for (reginfo = (struct e1000_reg_info *)e1000_reg_info_tbl;
	     reginfo->name; reginfo++) {
		e1000_regdump(hw, reginfo);
	}

249
	/* Print Tx Ring Summary */
250
	if (!netdev || !netif_running(netdev))
251
		return;
252

253
	dev_info(&adapter->pdev->dev, "Tx Ring Summary\n");
254
	pr_info("Queue [NTU] [NTC] [bi(ntc)->dma  ] leng ntw timestamp\n");
255
	buffer_info = &tx_ring->buffer_info[tx_ring->next_to_clean];
256
257
258
259
260
261
	pr_info(" %5d %5X %5X %016llX %04X %3X %016llX\n",
		0, tx_ring->next_to_use, tx_ring->next_to_clean,
		(unsigned long long)buffer_info->dma,
		buffer_info->length,
		buffer_info->next_to_watch,
		(unsigned long long)buffer_info->time_stamp);
262

263
	/* Print Tx Ring */
264
265
266
	if (!netif_msg_tx_done(adapter))
		goto rx_ring_summary;

267
	dev_info(&adapter->pdev->dev, "Tx Ring Dump\n");
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295

	/* Transmit Descriptor Formats - DEXT[29] is 0 (Legacy) or 1 (Extended)
	 *
	 * Legacy Transmit Descriptor
	 *   +--------------------------------------------------------------+
	 * 0 |         Buffer Address [63:0] (Reserved on Write Back)       |
	 *   +--------------------------------------------------------------+
	 * 8 | Special  |    CSS     | Status |  CMD    |  CSO   |  Length  |
	 *   +--------------------------------------------------------------+
	 *   63       48 47        36 35    32 31     24 23    16 15        0
	 *
	 * Extended Context Descriptor (DTYP=0x0) for TSO or checksum offload
	 *   63      48 47    40 39       32 31             16 15    8 7      0
	 *   +----------------------------------------------------------------+
	 * 0 |  TUCSE  | TUCS0  |   TUCSS   |     IPCSE       | IPCS0 | IPCSS |
	 *   +----------------------------------------------------------------+
	 * 8 |   MSS   | HDRLEN | RSV | STA | TUCMD | DTYP |      PAYLEN      |
	 *   +----------------------------------------------------------------+
	 *   63      48 47    40 39 36 35 32 31   24 23  20 19                0
	 *
	 * Extended Data Descriptor (DTYP=0x1)
	 *   +----------------------------------------------------------------+
	 * 0 |                     Buffer Address [63:0]                      |
	 *   +----------------------------------------------------------------+
	 * 8 | VLAN tag |  POPTS  | Rsvd | Status | Command | DTYP |  DTALEN  |
	 *   +----------------------------------------------------------------+
	 *   63       48 47     40 39  36 35    32 31     24 23  20 19        0
	 */
296
297
298
	pr_info("Tl[desc]     [address 63:0  ] [SpeCssSCmCsLen] [bi->dma       ] leng  ntw timestamp        bi->skb <-- Legacy format\n");
	pr_info("Tc[desc]     [Ce CoCsIpceCoS] [MssHlRSCm0Plen] [bi->dma       ] leng  ntw timestamp        bi->skb <-- Ext Context format\n");
	pr_info("Td[desc]     [address 63:0  ] [VlaPoRSCm1Dlen] [bi->dma       ] leng  ntw timestamp        bi->skb <-- Ext Data format\n");
299
	for (i = 0; tx_ring->desc && (i < tx_ring->count); i++) {
300
		const char *next_desc;
301
302
303
304
		tx_desc = E1000_TX_DESC(*tx_ring, i);
		buffer_info = &tx_ring->buffer_info[i];
		u0 = (struct my_u0 *)tx_desc;
		if (i == tx_ring->next_to_use && i == tx_ring->next_to_clean)
305
			next_desc = " NTC/U";
306
		else if (i == tx_ring->next_to_use)
307
			next_desc = " NTU";
308
		else if (i == tx_ring->next_to_clean)
309
			next_desc = " NTC";
310
		else
311
312
313
314
315
316
317
318
319
320
321
			next_desc = "";
		pr_info("T%c[0x%03X]    %016llX %016llX %016llX %04X  %3X %016llX %p%s\n",
			(!(le64_to_cpu(u0->b) & (1 << 29)) ? 'l' :
			 ((le64_to_cpu(u0->b) & (1 << 20)) ? 'd' : 'c')),
			i,
			(unsigned long long)le64_to_cpu(u0->a),
			(unsigned long long)le64_to_cpu(u0->b),
			(unsigned long long)buffer_info->dma,
			buffer_info->length, buffer_info->next_to_watch,
			(unsigned long long)buffer_info->time_stamp,
			buffer_info->skb, next_desc);
322

323
		if (netif_msg_pktdata(adapter) && buffer_info->skb)
324
			print_hex_dump(KERN_INFO, "", DUMP_PREFIX_ADDRESS,
325
326
				       16, 1, buffer_info->skb->data,
				       buffer_info->skb->len, true);
327
328
	}

329
	/* Print Rx Ring Summary */
330
rx_ring_summary:
331
	dev_info(&adapter->pdev->dev, "Rx Ring Summary\n");
332
333
334
	pr_info("Queue [NTU] [NTC]\n");
	pr_info(" %5d %5X %5X\n",
		0, rx_ring->next_to_use, rx_ring->next_to_clean);
335

336
	/* Print Rx Ring */
337
	if (!netif_msg_rx_status(adapter))
338
		return;
339

340
	dev_info(&adapter->pdev->dev, "Rx Ring Dump\n");
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
	switch (adapter->rx_ps_pages) {
	case 1:
	case 2:
	case 3:
		/* [Extended] Packet Split Receive Descriptor Format
		 *
		 *    +-----------------------------------------------------+
		 *  0 |                Buffer Address 0 [63:0]              |
		 *    +-----------------------------------------------------+
		 *  8 |                Buffer Address 1 [63:0]              |
		 *    +-----------------------------------------------------+
		 * 16 |                Buffer Address 2 [63:0]              |
		 *    +-----------------------------------------------------+
		 * 24 |                Buffer Address 3 [63:0]              |
		 *    +-----------------------------------------------------+
		 */
357
		pr_info("R  [desc]      [buffer 0 63:0 ] [buffer 1 63:0 ] [buffer 2 63:0 ] [buffer 3 63:0 ] [bi->dma       ] [bi->skb] <-- Ext Pkt Split format\n");
358
359
360
361
362
363
364
365
366
367
368
		/* [Extended] Receive Descriptor (Write-Back) Format
		 *
		 *   63       48 47    32 31     13 12    8 7    4 3        0
		 *   +------------------------------------------------------+
		 * 0 | Packet   | IP     |  Rsvd   | MRQ   | Rsvd | MRQ RSS |
		 *   | Checksum | Ident  |         | Queue |      |  Type   |
		 *   +------------------------------------------------------+
		 * 8 | VLAN Tag | Length | Extended Error | Extended Status |
		 *   +------------------------------------------------------+
		 *   63       48 47    32 31            20 19               0
		 */
369
		pr_info("RWB[desc]      [ck ipid mrqhsh] [vl   l0 ee  es] [ l3  l2  l1 hs] [reserved      ] ---------------- [bi->skb] <-- Ext Rx Write-Back format\n");
370
		for (i = 0; i < rx_ring->count; i++) {
371
			const char *next_desc;
372
373
374
375
			buffer_info = &rx_ring->buffer_info[i];
			rx_desc_ps = E1000_RX_DESC_PS(*rx_ring, i);
			u1 = (struct my_u1 *)rx_desc_ps;
			staterr =
376
			    le32_to_cpu(rx_desc_ps->wb.middle.status_error);
377
378
379
380
381
382
383
384

			if (i == rx_ring->next_to_use)
				next_desc = " NTU";
			else if (i == rx_ring->next_to_clean)
				next_desc = " NTC";
			else
				next_desc = "";

385
386
			if (staterr & E1000_RXD_STAT_DD) {
				/* Descriptor Done */
387
388
389
390
391
392
393
				pr_info("%s[0x%03X]     %016llX %016llX %016llX %016llX ---------------- %p%s\n",
					"RWB", i,
					(unsigned long long)le64_to_cpu(u1->a),
					(unsigned long long)le64_to_cpu(u1->b),
					(unsigned long long)le64_to_cpu(u1->c),
					(unsigned long long)le64_to_cpu(u1->d),
					buffer_info->skb, next_desc);
394
			} else {
395
396
397
398
399
400
401
402
				pr_info("%s[0x%03X]     %016llX %016llX %016llX %016llX %016llX %p%s\n",
					"R  ", i,
					(unsigned long long)le64_to_cpu(u1->a),
					(unsigned long long)le64_to_cpu(u1->b),
					(unsigned long long)le64_to_cpu(u1->c),
					(unsigned long long)le64_to_cpu(u1->d),
					(unsigned long long)buffer_info->dma,
					buffer_info->skb, next_desc);
403
404

				if (netif_msg_pktdata(adapter))
405
406
					e1000e_dump_ps_pages(adapter,
							     buffer_info);
407
408
409
410
411
			}
		}
		break;
	default:
	case 0:
412
		/* Extended Receive Descriptor (Read) Format
413
		 *
414
415
416
417
418
		 *   +-----------------------------------------------------+
		 * 0 |                Buffer Address [63:0]                |
		 *   +-----------------------------------------------------+
		 * 8 |                      Reserved                       |
		 *   +-----------------------------------------------------+
419
		 */
420
		pr_info("R  [desc]      [buf addr 63:0 ] [reserved 63:0 ] [bi->dma       ] [bi->skb] <-- Ext (Read) format\n");
421
422
423
424
425
426
427
428
429
430
431
432
433
		/* Extended Receive Descriptor (Write-Back) Format
		 *
		 *   63       48 47    32 31    24 23            4 3        0
		 *   +------------------------------------------------------+
		 *   |     RSS Hash      |        |               |         |
		 * 0 +-------------------+  Rsvd  |   Reserved    | MRQ RSS |
		 *   | Packet   | IP     |        |               |  Type   |
		 *   | Checksum | Ident  |        |               |         |
		 *   +------------------------------------------------------+
		 * 8 | VLAN Tag | Length | Extended Error | Extended Status |
		 *   +------------------------------------------------------+
		 *   63       48 47    32 31            20 19               0
		 */
434
		pr_info("RWB[desc]      [cs ipid    mrq] [vt   ln xe  xs] [bi->skb] <-- Ext (Write-Back) format\n");
435
436

		for (i = 0; i < rx_ring->count; i++) {
437
438
			const char *next_desc;

439
			buffer_info = &rx_ring->buffer_info[i];
440
441
442
			rx_desc = E1000_RX_DESC_EXT(*rx_ring, i);
			u1 = (struct my_u1 *)rx_desc;
			staterr = le32_to_cpu(rx_desc->wb.upper.status_error);
443
444
445
446
447
448
449
450

			if (i == rx_ring->next_to_use)
				next_desc = " NTU";
			else if (i == rx_ring->next_to_clean)
				next_desc = " NTC";
			else
				next_desc = "";

451
452
			if (staterr & E1000_RXD_STAT_DD) {
				/* Descriptor Done */
453
454
455
456
457
				pr_info("%s[0x%03X]     %016llX %016llX ---------------- %p%s\n",
					"RWB", i,
					(unsigned long long)le64_to_cpu(u1->a),
					(unsigned long long)le64_to_cpu(u1->b),
					buffer_info->skb, next_desc);
458
			} else {
459
460
461
462
463
464
				pr_info("%s[0x%03X]     %016llX %016llX %016llX %p%s\n",
					"R  ", i,
					(unsigned long long)le64_to_cpu(u1->a),
					(unsigned long long)le64_to_cpu(u1->b),
					(unsigned long long)buffer_info->dma,
					buffer_info->skb, next_desc);
465

466
467
				if (netif_msg_pktdata(adapter) &&
				    buffer_info->skb)
468
469
470
					print_hex_dump(KERN_INFO, "",
						       DUMP_PREFIX_ADDRESS, 16,
						       1,
471
						       buffer_info->skb->data,
472
473
474
						       adapter->rx_buffer_len,
						       true);
			}
475
476
477
478
		}
	}
}

479
480
481
482
483
484
485
486
487
488
489
/**
 * e1000_desc_unused - calculate if we have unused descriptors
 **/
static int e1000_desc_unused(struct e1000_ring *ring)
{
	if (ring->next_to_clean > ring->next_to_use)
		return ring->next_to_clean - ring->next_to_use - 1;

	return ring->count + ring->next_to_clean - ring->next_to_use - 1;
}

490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
/**
 * e1000e_systim_to_hwtstamp - convert system time value to hw time stamp
 * @adapter: board private structure
 * @hwtstamps: time stamp structure to update
 * @systim: unsigned 64bit system time value.
 *
 * Convert the system time value stored in the RX/TXSTMP registers into a
 * hwtstamp which can be used by the upper level time stamping functions.
 *
 * The 'systim_lock' spinlock is used to protect the consistency of the
 * system time value. This is needed because reading the 64 bit time
 * value involves reading two 32 bit registers. The first read latches the
 * value.
 **/
static void e1000e_systim_to_hwtstamp(struct e1000_adapter *adapter,
				      struct skb_shared_hwtstamps *hwtstamps,
				      u64 systim)
{
	u64 ns;
	unsigned long flags;

	spin_lock_irqsave(&adapter->systim_lock, flags);
	ns = timecounter_cyc2time(&adapter->tc, systim);
	spin_unlock_irqrestore(&adapter->systim_lock, flags);

	memset(hwtstamps, 0, sizeof(*hwtstamps));
	hwtstamps->hwtstamp = ns_to_ktime(ns);
}

/**
 * e1000e_rx_hwtstamp - utility function which checks for Rx time stamp
 * @adapter: board private structure
 * @status: descriptor extended error and status field
 * @skb: particular skb to include time stamp
 *
 * If the time stamp is valid, convert it into the timecounter ns value
 * and store that result into the shhwtstamps structure which is passed
 * up the network stack.
 **/
static void e1000e_rx_hwtstamp(struct e1000_adapter *adapter, u32 status,
			       struct sk_buff *skb)
{
	struct e1000_hw *hw = &adapter->hw;
	u64 rxstmp;

	if (!(adapter->flags & FLAG_HAS_HW_TIMESTAMP) ||
	    !(status & E1000_RXDEXT_STATERR_TST) ||
	    !(er32(TSYNCRXCTL) & E1000_TSYNCRXCTL_VALID))
		return;

	/* The Rx time stamp registers contain the time stamp.  No other
	 * received packet will be time stamped until the Rx time stamp
	 * registers are read.  Because only one packet can be time stamped
	 * at a time, the register values must belong to this packet and
	 * therefore none of the other additional attributes need to be
	 * compared.
	 */
	rxstmp = (u64)er32(RXSTMPL);
	rxstmp |= (u64)er32(RXSTMPH) << 32;
	e1000e_systim_to_hwtstamp(adapter, skb_hwtstamps(skb), rxstmp);

	adapter->flags2 &= ~FLAG2_CHECK_RX_HWTSTAMP;
}

554
/**
555
 * e1000_receive_skb - helper function to handle Rx indications
556
 * @adapter: board private structure
557
 * @staterr: descriptor extended error and status field as written by hardware
558
559
560
561
 * @vlan: descriptor vlan field as written by hardware (no le/be conversion)
 * @skb: pointer to sk_buff to be indicated to stack
 **/
static void e1000_receive_skb(struct e1000_adapter *adapter,
562
			      struct net_device *netdev, struct sk_buff *skb,
563
			      u32 staterr, __le16 vlan)
564
{
565
	u16 tag = le16_to_cpu(vlan);
566
567
568

	e1000e_rx_hwtstamp(adapter, staterr, skb);

569
570
	skb->protocol = eth_type_trans(skb, netdev);

571
	if (staterr & E1000_RXD_STAT_VP)
572
573
574
		__vlan_hwaccel_put_tag(skb, tag);

	napi_gro_receive(&adapter->napi, skb);
575
576
577
}

/**
578
 * e1000_rx_checksum - Receive Checksum Offload
579
580
581
582
 * @adapter: board private structure
 * @status_err: receive descriptor status and error fields
 * @csum: receive descriptor csum field
 * @sk_buff: socket buffer with received data
583
584
 **/
static void e1000_rx_checksum(struct e1000_adapter *adapter, u32 status_err,
585
			      struct sk_buff *skb)
586
587
588
{
	u16 status = (u16)status_err;
	u8 errors = (u8)(status_err >> 24);
589
590

	skb_checksum_none_assert(skb);
591

592
593
594
595
	/* Rx checksum disabled */
	if (!(adapter->netdev->features & NETIF_F_RXCSUM))
		return;

596
597
598
	/* Ignore Checksum bit is set */
	if (status & E1000_RXD_STAT_IXSM)
		return;
599

600
601
	/* TCP/UDP checksum error bit or IP checksum error bit is set */
	if (errors & (E1000_RXD_ERR_TCPE | E1000_RXD_ERR_IPE)) {
602
603
604
605
606
607
608
609
610
611
		/* let the stack verify checksum errors */
		adapter->hw_csum_err++;
		return;
	}

	/* TCP/UDP Checksum has not been calculated */
	if (!(status & (E1000_RXD_STAT_TCPCS | E1000_RXD_STAT_UDPCS)))
		return;

	/* It must be a TCP or UDP packet with a valid checksum */
612
	skb->ip_summed = CHECKSUM_UNNECESSARY;
613
614
615
	adapter->hw_csum_good++;
}

616
static void e1000e_update_rdt_wa(struct e1000_ring *rx_ring, unsigned int i)
617
{
618
	struct e1000_adapter *adapter = rx_ring->adapter;
619
	struct e1000_hw *hw = &adapter->hw;
620
621
622
	s32 ret_val = __ew32_prepare(hw);

	writel(i, rx_ring->tail);
623

624
	if (unlikely(!ret_val && (i != readl(rx_ring->tail)))) {
625
626
627
628
629
630
631
		u32 rctl = er32(RCTL);
		ew32(RCTL, rctl & ~E1000_RCTL_EN);
		e_err("ME firmware caused invalid RDT - resetting\n");
		schedule_work(&adapter->reset_task);
	}
}

632
static void e1000e_update_tdt_wa(struct e1000_ring *tx_ring, unsigned int i)
633
{
634
	struct e1000_adapter *adapter = tx_ring->adapter;
635
	struct e1000_hw *hw = &adapter->hw;
636
	s32 ret_val = __ew32_prepare(hw);
637

638
639
640
	writel(i, tx_ring->tail);

	if (unlikely(!ret_val && (i != readl(tx_ring->tail)))) {
641
642
643
644
645
646
647
		u32 tctl = er32(TCTL);
		ew32(TCTL, tctl & ~E1000_TCTL_EN);
		e_err("ME firmware caused invalid TDT - resetting\n");
		schedule_work(&adapter->reset_task);
	}
}

648
/**
649
 * e1000_alloc_rx_buffers - Replace used receive buffers
650
 * @rx_ring: Rx descriptor ring
651
 **/
652
static void e1000_alloc_rx_buffers(struct e1000_ring *rx_ring,
653
				   int cleaned_count, gfp_t gfp)
654
{
655
	struct e1000_adapter *adapter = rx_ring->adapter;
656
657
	struct net_device *netdev = adapter->netdev;
	struct pci_dev *pdev = adapter->pdev;
658
	union e1000_rx_desc_extended *rx_desc;
659
660
661
	struct e1000_buffer *buffer_info;
	struct sk_buff *skb;
	unsigned int i;
662
	unsigned int bufsz = adapter->rx_buffer_len;
663
664
665
666
667
668
669
670
671
672
673

	i = rx_ring->next_to_use;
	buffer_info = &rx_ring->buffer_info[i];

	while (cleaned_count--) {
		skb = buffer_info->skb;
		if (skb) {
			skb_trim(skb, 0);
			goto map_skb;
		}

674
		skb = __netdev_alloc_skb_ip_align(netdev, bufsz, gfp);
675
676
677
678
679
680
681
682
		if (!skb) {
			/* Better luck next round */
			adapter->alloc_rx_buff_failed++;
			break;
		}

		buffer_info->skb = skb;
map_skb:
683
		buffer_info->dma = dma_map_single(&pdev->dev, skb->data,
684
						  adapter->rx_buffer_len,
685
686
						  DMA_FROM_DEVICE);
		if (dma_mapping_error(&pdev->dev, buffer_info->dma)) {
687
			dev_err(&pdev->dev, "Rx DMA map failed\n");
688
689
690
691
			adapter->rx_dma_failed++;
			break;
		}

692
693
		rx_desc = E1000_RX_DESC_EXT(*rx_ring, i);
		rx_desc->read.buffer_addr = cpu_to_le64(buffer_info->dma);
694

695
		if (unlikely(!(i & (E1000_RX_BUFFER_WRITE - 1)))) {
696
			/* Force memory writes to complete before letting h/w
697
698
699
700
701
			 * know there are new descriptors to fetch.  (Only
			 * applicable for weak-ordered memory model archs,
			 * such as IA-64).
			 */
			wmb();
702
			if (adapter->flags2 & FLAG2_PCIM2PCI_ARBITER_WA)
703
				e1000e_update_rdt_wa(rx_ring, i);
704
			else
705
				writel(i, rx_ring->tail);
706
		}
707
708
709
710
711
712
		i++;
		if (i == rx_ring->count)
			i = 0;
		buffer_info = &rx_ring->buffer_info[i];
	}

713
	rx_ring->next_to_use = i;
714
715
716
717
}

/**
 * e1000_alloc_rx_buffers_ps - Replace used receive buffers; packet split
718
 * @rx_ring: Rx descriptor ring
719
 **/
720
static void e1000_alloc_rx_buffers_ps(struct e1000_ring *rx_ring,
721
				      int cleaned_count, gfp_t gfp)
722
{
723
	struct e1000_adapter *adapter = rx_ring->adapter;
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
	struct net_device *netdev = adapter->netdev;
	struct pci_dev *pdev = adapter->pdev;
	union e1000_rx_desc_packet_split *rx_desc;
	struct e1000_buffer *buffer_info;
	struct e1000_ps_page *ps_page;
	struct sk_buff *skb;
	unsigned int i, j;

	i = rx_ring->next_to_use;
	buffer_info = &rx_ring->buffer_info[i];

	while (cleaned_count--) {
		rx_desc = E1000_RX_DESC_PS(*rx_ring, i);

		for (j = 0; j < PS_PAGE_BUFFERS; j++) {
739
740
741
			ps_page = &buffer_info->ps_pages[j];
			if (j >= adapter->rx_ps_pages) {
				/* all unused desc entries get hw null ptr */
742
743
				rx_desc->read.buffer_addr[j + 1] =
				    ~cpu_to_le64(0);
744
745
746
				continue;
			}
			if (!ps_page->page) {
747
				ps_page->page = alloc_page(gfp);
748
				if (!ps_page->page) {
749
750
751
					adapter->alloc_rx_buff_failed++;
					goto no_buffers;
				}
752
753
754
755
756
757
				ps_page->dma = dma_map_page(&pdev->dev,
							    ps_page->page,
							    0, PAGE_SIZE,
							    DMA_FROM_DEVICE);
				if (dma_mapping_error(&pdev->dev,
						      ps_page->dma)) {
758
					dev_err(&adapter->pdev->dev,
759
						"Rx DMA page map failed\n");
760
761
					adapter->rx_dma_failed++;
					goto no_buffers;
762
763
				}
			}
764
			/* Refresh the desc even if buffer_addrs
765
766
767
			 * didn't change because each write-back
			 * erases this info.
			 */
768
769
			rx_desc->read.buffer_addr[j + 1] =
			    cpu_to_le64(ps_page->dma);
770
771
		}

772
773
774
		skb = __netdev_alloc_skb_ip_align(netdev,
						  adapter->rx_ps_bsize0,
						  gfp);
775
776
777
778
779
780
781

		if (!skb) {
			adapter->alloc_rx_buff_failed++;
			break;
		}

		buffer_info->skb = skb;
782
		buffer_info->dma = dma_map_single(&pdev->dev, skb->data,
783
						  adapter->rx_ps_bsize0,
784
785
						  DMA_FROM_DEVICE);
		if (dma_mapping_error(&pdev->dev, buffer_info->dma)) {
786
			dev_err(&pdev->dev, "Rx DMA map failed\n");
787
788
789
790
791
792
793
794
795
			adapter->rx_dma_failed++;
			/* cleanup skb */
			dev_kfree_skb_any(skb);
			buffer_info->skb = NULL;
			break;
		}

		rx_desc->read.buffer_addr[0] = cpu_to_le64(buffer_info->dma);

796
		if (unlikely(!(i & (E1000_RX_BUFFER_WRITE - 1)))) {
797
			/* Force memory writes to complete before letting h/w
798
799
800
801
802
			 * know there are new descriptors to fetch.  (Only
			 * applicable for weak-ordered memory model archs,
			 * such as IA-64).
			 */
			wmb();
803
			if (adapter->flags2 & FLAG2_PCIM2PCI_ARBITER_WA)
804
				e1000e_update_rdt_wa(rx_ring, i << 1);
805
			else
806
				writel(i << 1, rx_ring->tail);
807
808
		}

809
810
811
812
813
814
815
		i++;
		if (i == rx_ring->count)
			i = 0;
		buffer_info = &rx_ring->buffer_info[i];
	}

no_buffers:
816
	rx_ring->next_to_use = i;
817
818
}

819
820
/**
 * e1000_alloc_jumbo_rx_buffers - Replace used jumbo receive buffers
821
 * @rx_ring: Rx descriptor ring
822
823
824
 * @cleaned_count: number of buffers to allocate this pass
 **/

825
static void e1000_alloc_jumbo_rx_buffers(struct e1000_ring *rx_ring,
826
					 int cleaned_count, gfp_t gfp)
827
{
828
	struct e1000_adapter *adapter = rx_ring->adapter;
829
830
	struct net_device *netdev = adapter->netdev;
	struct pci_dev *pdev = adapter->pdev;
831
	union e1000_rx_desc_extended *rx_desc;
832
833
834
	struct e1000_buffer *buffer_info;
	struct sk_buff *skb;
	unsigned int i;
835
	unsigned int bufsz = 256 - 16;	/* for skb_reserve */
836
837
838
839
840
841
842
843
844
845
846

	i = rx_ring->next_to_use;
	buffer_info = &rx_ring->buffer_info[i];

	while (cleaned_count--) {
		skb = buffer_info->skb;
		if (skb) {
			skb_trim(skb, 0);
			goto check_page;
		}

847
		skb = __netdev_alloc_skb_ip_align(netdev, bufsz, gfp);
848
849
850
851
852
853
854
855
856
857
		if (unlikely(!skb)) {
			/* Better luck next round */
			adapter->alloc_rx_buff_failed++;
			break;
		}

		buffer_info->skb = skb;
check_page:
		/* allocate a new page if necessary */
		if (!buffer_info->page) {
858
			buffer_info->page = alloc_page(gfp);
859
860
861
862
863
864
865
			if (unlikely(!buffer_info->page)) {
				adapter->alloc_rx_buff_failed++;
				break;
			}
		}

		if (!buffer_info->dma)
866
			buffer_info->dma = dma_map_page(&pdev->dev,
867
868
			                                buffer_info->page, 0,
			                                PAGE_SIZE,
869
							DMA_FROM_DEVICE);
870

871
872
		rx_desc = E1000_RX_DESC_EXT(*rx_ring, i);
		rx_desc->read.buffer_addr = cpu_to_le64(buffer_info->dma);
873
874
875
876
877
878
879
880
881
882
883
884
885
886

		if (unlikely(++i == rx_ring->count))
			i = 0;
		buffer_info = &rx_ring->buffer_info[i];
	}

	if (likely(rx_ring->next_to_use != i)) {
		rx_ring->next_to_use = i;
		if (unlikely(i-- == 0))
			i = (rx_ring->count - 1);

		/* Force memory writes to complete before letting h/w
		 * know there are new descriptors to fetch.  (Only
		 * applicable for weak-ordered memory model archs,
887
888
		 * such as IA-64).
		 */
889
		wmb();
890
		if (adapter->flags2 & FLAG2_PCIM2PCI_ARBITER_WA)
891
			e1000e_update_rdt_wa(rx_ring, i);
892
		else
893
			writel(i, rx_ring->tail);
894
895
896
	}
}

897
898
899
900
901
902
903
static inline void e1000_rx_hash(struct net_device *netdev, __le32 rss,
				 struct sk_buff *skb)
{
	if (netdev->features & NETIF_F_RXHASH)
		skb->rxhash = le32_to_cpu(rss);
}

904
/**
905
906
 * e1000_clean_rx_irq - Send received data up the network stack
 * @rx_ring: Rx descriptor ring
907
908
909
910
 *
 * the return value indicates whether actual cleaning was done, there
 * is no guarantee that everything was cleaned
 **/
911
912
static bool e1000_clean_rx_irq(struct e1000_ring *rx_ring, int *work_done,
			       int work_to_do)
913
{
914
	struct e1000_adapter *adapter = rx_ring->adapter;
915
916
	struct net_device *netdev = adapter->netdev;
	struct pci_dev *pdev = adapter->pdev;
917
	struct e1000_hw *hw = &adapter->hw;
918
	union e1000_rx_desc_extended *rx_desc, *next_rxd;
919
	struct e1000_buffer *buffer_info, *next_buffer;
920
	u32 length, staterr;
921
922
	unsigned int i;
	int cleaned_count = 0;
923
	bool cleaned = false;
924
925
926
	unsigned int total_rx_bytes = 0, total_rx_packets = 0;

	i = rx_ring->next_to_clean;
927
928
	rx_desc = E1000_RX_DESC_EXT(*rx_ring, i);
	staterr = le32_to_cpu(rx_desc->wb.upper.status_error);
929
930
	buffer_info = &rx_ring->buffer_info[i];

931
	while (staterr & E1000_RXD_STAT_DD) {
932
933
934
935
936
		struct sk_buff *skb;

		if (*work_done >= work_to_do)
			break;
		(*work_done)++;
937
		rmb();	/* read descriptor and rx_buffer_info after status DD */
938
939
940
941
942
943
944
945
946

		skb = buffer_info->skb;
		buffer_info->skb = NULL;

		prefetch(skb->data - NET_IP_ALIGN);

		i++;
		if (i == rx_ring->count)
			i = 0;
947
		next_rxd = E1000_RX_DESC_EXT(*rx_ring, i);
948
949
950
951
		prefetch(next_rxd);

		next_buffer = &rx_ring->buffer_info[i];

952
		cleaned = true;
953
		cleaned_count++;
954
		dma_unmap_single(&pdev->dev,
955
956
				 buffer_info->dma,
				 adapter->rx_buffer_len,
957
				 DMA_FROM_DEVICE);
958
959
		buffer_info->dma = 0;

960
		length = le16_to_cpu(rx_desc->wb.upper.length);
961

962
		/* !EOP means multiple descriptors were used to store a single
963
964
965
966
967
		 * packet, if that's the case we need to toss it.  In fact, we
		 * need to toss every packet with the EOP bit clear and the
		 * next frame that _does_ have the EOP bit set, as it is by
		 * definition only a frame fragment
		 */
968
		if (unlikely(!(staterr & E1000_RXD_STAT_EOP)))
969
970
971
			adapter->flags2 |= FLAG2_IS_DISCARDING;

		if (adapter->flags2 & FLAG2_IS_DISCARDING) {
972
			/* All receives must fit into a single buffer */
973
			e_dbg("Receive packet consumed multiple buffers\n");
974
975
			/* recycle */
			buffer_info->skb = skb;
976
			if (staterr & E1000_RXD_STAT_EOP)
977
				adapter->flags2 &= ~FLAG2_IS_DISCARDING;
978
979
980
			goto next_desc;
		}

981
982
		if (unlikely((staterr & E1000_RXDEXT_ERR_FRAME_ERR_MASK) &&
			     !(netdev->features & NETIF_F_RXALL))) {
983
984
985
986
987
			/* recycle */
			buffer_info->skb = skb;
			goto next_desc;
		}

Jeff Kirsher's avatar
Jeff Kirsher committed
988
		/* adjust length to remove Ethernet CRC */
989
990
991
992
993
994
995
996
997
998
		if (!(adapter->flags2 & FLAG2_CRC_STRIPPING)) {
			/* If configured to store CRC, don't subtract FCS,
			 * but keep the FCS bytes out of the total_rx_bytes
			 * counter
			 */
			if (netdev->features & NETIF_F_RXFCS)
				total_rx_bytes -= 4;
			else
				length -= 4;
		}
Jeff Kirsher's avatar
Jeff Kirsher committed
999

1000
		total_rx_bytes += length;