netdev.c 204 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
/* Intel PRO/1000 Linux driver
 * Copyright(c) 1999 - 2014 Intel Corporation.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms and conditions of the GNU General Public License,
 * version 2, as published by the Free Software Foundation.
 *
 * This program is distributed in the hope it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
 * more details.
 *
 * The full GNU General Public License is included in this distribution in
 * the file called "COPYING".
 *
 * Contact Information:
 * Linux NICS <linux.nics@intel.com>
 * e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
 * Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
 */
21

22
23
#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt

24
25
26
27
28
29
30
31
#include <linux/module.h>
#include <linux/types.h>
#include <linux/init.h>
#include <linux/pci.h>
#include <linux/vmalloc.h>
#include <linux/pagemap.h>
#include <linux/delay.h>
#include <linux/netdevice.h>
32
#include <linux/interrupt.h>
33
34
#include <linux/tcp.h>
#include <linux/ipv6.h>
35
#include <linux/slab.h>
36
37
38
39
40
41
#include <net/checksum.h>
#include <net/ip6_checksum.h>
#include <linux/ethtool.h>
#include <linux/if_vlan.h>
#include <linux/cpu.h>
#include <linux/smp.h>
42
#include <linux/pm_qos.h>
43
#include <linux/pm_runtime.h>
Jesse Brandeburg's avatar
Jesse Brandeburg committed
44
#include <linux/aer.h>
45
#include <linux/prefetch.h>
46
47
48

#include "e1000.h"

Bruce Allan's avatar
Bruce Allan committed
49
#define DRV_EXTRAVERSION "-k"
50

51
#define DRV_VERSION "2.3.2" DRV_EXTRAVERSION
52
53
54
char e1000e_driver_name[] = "e1000e";
const char e1000e_driver_version[] = DRV_VERSION;

55
56
57
58
59
#define DEFAULT_MSG_ENABLE (NETIF_MSG_DRV|NETIF_MSG_PROBE|NETIF_MSG_LINK)
static int debug = -1;
module_param(debug, int, 0);
MODULE_PARM_DESC(debug, "Debug level (0=none,...,16=all)");

60
61
62
63
static const struct e1000_info *e1000_info_tbl[] = {
	[board_82571]		= &e1000_82571_info,
	[board_82572]		= &e1000_82572_info,
	[board_82573]		= &e1000_82573_info,
64
	[board_82574]		= &e1000_82574_info,
65
	[board_82583]		= &e1000_82583_info,
66
67
68
	[board_80003es2lan]	= &e1000_es2_info,
	[board_ich8lan]		= &e1000_ich8_info,
	[board_ich9lan]		= &e1000_ich9_info,
69
	[board_ich10lan]	= &e1000_ich10_info,
70
	[board_pchlan]		= &e1000_pch_info,
71
	[board_pch2lan]		= &e1000_pch2_info,
Bruce Allan's avatar
Bruce Allan committed
72
	[board_pch_lpt]		= &e1000_pch_lpt_info,
73
	[board_pch_spt]		= &e1000_pch_spt_info,
74
75
};

76
77
78
79
80
81
82
83
84
85
86
87
88
89
struct e1000_reg_info {
	u32 ofs;
	char *name;
};

static const struct e1000_reg_info e1000_reg_info_tbl[] = {
	/* General Registers */
	{E1000_CTRL, "CTRL"},
	{E1000_STATUS, "STATUS"},
	{E1000_CTRL_EXT, "CTRL_EXT"},

	/* Interrupt Registers */
	{E1000_ICR, "ICR"},

90
	/* Rx Registers */
91
	{E1000_RCTL, "RCTL"},
92
93
94
	{E1000_RDLEN(0), "RDLEN"},
	{E1000_RDH(0), "RDH"},
	{E1000_RDT(0), "RDT"},
95
96
97
	{E1000_RDTR, "RDTR"},
	{E1000_RXDCTL(0), "RXDCTL"},
	{E1000_ERT, "ERT"},
98
99
	{E1000_RDBAL(0), "RDBAL"},
	{E1000_RDBAH(0), "RDBAH"},
100
101
102
103
104
105
	{E1000_RDFH, "RDFH"},
	{E1000_RDFT, "RDFT"},
	{E1000_RDFHS, "RDFHS"},
	{E1000_RDFTS, "RDFTS"},
	{E1000_RDFPC, "RDFPC"},

106
	/* Tx Registers */
107
	{E1000_TCTL, "TCTL"},
108
109
110
111
112
	{E1000_TDBAL(0), "TDBAL"},
	{E1000_TDBAH(0), "TDBAH"},
	{E1000_TDLEN(0), "TDLEN"},
	{E1000_TDH(0), "TDH"},
	{E1000_TDT(0), "TDT"},
113
114
115
116
117
118
119
120
121
122
123
	{E1000_TIDV, "TIDV"},
	{E1000_TXDCTL(0), "TXDCTL"},
	{E1000_TADV, "TADV"},
	{E1000_TARC(0), "TARC"},
	{E1000_TDFH, "TDFH"},
	{E1000_TDFT, "TDFT"},
	{E1000_TDFHS, "TDFHS"},
	{E1000_TDFTS, "TDFTS"},
	{E1000_TDFPC, "TDFPC"},

	/* List Terminator */
124
	{0, NULL}
125
126
};

127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
/**
 * __ew32_prepare - prepare to write to MAC CSR register on certain parts
 * @hw: pointer to the HW structure
 *
 * When updating the MAC CSR registers, the Manageability Engine (ME) could
 * be accessing the registers at the same time.  Normally, this is handled in
 * h/w by an arbiter but on some parts there is a bug that acknowledges Host
 * accesses later than it should which could result in the register to have
 * an incorrect value.  Workaround this by checking the FWSM register which
 * has bit 24 set while ME is accessing MAC CSR registers, wait if it is set
 * and try again a number of times.
 **/
s32 __ew32_prepare(struct e1000_hw *hw)
{
	s32 i = E1000_ICH_FWSM_PCIM2PCI_COUNT;

	while ((er32(FWSM) & E1000_ICH_FWSM_PCIM2PCI) && --i)
		udelay(50);

	return i;
}

void __ew32(struct e1000_hw *hw, unsigned long reg, u32 val)
{
	if (hw->adapter->flags2 & FLAG2_PCIM2PCI_ARBITER_WA)
		__ew32_prepare(hw);

	writel(val, hw->hw_addr + reg);
}

157
/**
158
 * e1000_regdump - register printout routine
159
160
161
 * @hw: pointer to the HW structure
 * @reginfo: pointer to the register info table
 **/
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
static void e1000_regdump(struct e1000_hw *hw, struct e1000_reg_info *reginfo)
{
	int n = 0;
	char rname[16];
	u32 regs[8];

	switch (reginfo->ofs) {
	case E1000_RXDCTL(0):
		for (n = 0; n < 2; n++)
			regs[n] = __er32(hw, E1000_RXDCTL(n));
		break;
	case E1000_TXDCTL(0):
		for (n = 0; n < 2; n++)
			regs[n] = __er32(hw, E1000_TXDCTL(n));
		break;
	case E1000_TARC(0):
		for (n = 0; n < 2; n++)
			regs[n] = __er32(hw, E1000_TARC(n));
		break;
	default:
182
183
		pr_info("%-15s %08x\n",
			reginfo->name, __er32(hw, reginfo->ofs));
184
185
186
187
		return;
	}

	snprintf(rname, 16, "%s%s", reginfo->name, "[0-1]");
188
	pr_info("%-15s %08x %08x\n", rname, regs[0], regs[1]);
189
190
}

191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
static void e1000e_dump_ps_pages(struct e1000_adapter *adapter,
				 struct e1000_buffer *bi)
{
	int i;
	struct e1000_ps_page *ps_page;

	for (i = 0; i < adapter->rx_ps_pages; i++) {
		ps_page = &bi->ps_pages[i];

		if (ps_page->page) {
			pr_info("packet dump for ps_page %d:\n", i);
			print_hex_dump(KERN_INFO, "", DUMP_PREFIX_ADDRESS,
				       16, 1, page_address(ps_page->page),
				       PAGE_SIZE, true);
		}
	}
}

209
/**
210
 * e1000e_dump - Print registers, Tx-ring and Rx-ring
211
212
 * @adapter: board private structure
 **/
213
214
215
216
217
218
219
static void e1000e_dump(struct e1000_adapter *adapter)
{
	struct net_device *netdev = adapter->netdev;
	struct e1000_hw *hw = &adapter->hw;
	struct e1000_reg_info *reginfo;
	struct e1000_ring *tx_ring = adapter->tx_ring;
	struct e1000_tx_desc *tx_desc;
220
	struct my_u0 {
221
222
		__le64 a;
		__le64 b;
223
	} *u0;
224
225
226
	struct e1000_buffer *buffer_info;
	struct e1000_ring *rx_ring = adapter->rx_ring;
	union e1000_rx_desc_packet_split *rx_desc_ps;
227
	union e1000_rx_desc_extended *rx_desc;
228
	struct my_u1 {
229
230
231
232
		__le64 a;
		__le64 b;
		__le64 c;
		__le64 d;
233
	} *u1;
234
235
236
237
238
239
240
241
242
	u32 staterr;
	int i = 0;

	if (!netif_msg_hw(adapter))
		return;

	/* Print netdevice Info */
	if (netdev) {
		dev_info(&adapter->pdev->dev, "Net device Info\n");
243
		pr_info("Device Name     state            trans_start      last_rx\n");
244
245
		pr_info("%-15s %016lX %016lX %016lX\n", netdev->name,
			netdev->state, netdev->trans_start, netdev->last_rx);
246
247
248
249
	}

	/* Print Registers */
	dev_info(&adapter->pdev->dev, "Register Dump\n");
250
	pr_info(" Register Name   Value\n");
251
252
253
254
255
	for (reginfo = (struct e1000_reg_info *)e1000_reg_info_tbl;
	     reginfo->name; reginfo++) {
		e1000_regdump(hw, reginfo);
	}

256
	/* Print Tx Ring Summary */
257
	if (!netdev || !netif_running(netdev))
258
		return;
259

260
	dev_info(&adapter->pdev->dev, "Tx Ring Summary\n");
261
	pr_info("Queue [NTU] [NTC] [bi(ntc)->dma  ] leng ntw timestamp\n");
262
	buffer_info = &tx_ring->buffer_info[tx_ring->next_to_clean];
263
264
265
266
267
268
	pr_info(" %5d %5X %5X %016llX %04X %3X %016llX\n",
		0, tx_ring->next_to_use, tx_ring->next_to_clean,
		(unsigned long long)buffer_info->dma,
		buffer_info->length,
		buffer_info->next_to_watch,
		(unsigned long long)buffer_info->time_stamp);
269

270
	/* Print Tx Ring */
271
272
273
	if (!netif_msg_tx_done(adapter))
		goto rx_ring_summary;

274
	dev_info(&adapter->pdev->dev, "Tx Ring Dump\n");
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302

	/* Transmit Descriptor Formats - DEXT[29] is 0 (Legacy) or 1 (Extended)
	 *
	 * Legacy Transmit Descriptor
	 *   +--------------------------------------------------------------+
	 * 0 |         Buffer Address [63:0] (Reserved on Write Back)       |
	 *   +--------------------------------------------------------------+
	 * 8 | Special  |    CSS     | Status |  CMD    |  CSO   |  Length  |
	 *   +--------------------------------------------------------------+
	 *   63       48 47        36 35    32 31     24 23    16 15        0
	 *
	 * Extended Context Descriptor (DTYP=0x0) for TSO or checksum offload
	 *   63      48 47    40 39       32 31             16 15    8 7      0
	 *   +----------------------------------------------------------------+
	 * 0 |  TUCSE  | TUCS0  |   TUCSS   |     IPCSE       | IPCS0 | IPCSS |
	 *   +----------------------------------------------------------------+
	 * 8 |   MSS   | HDRLEN | RSV | STA | TUCMD | DTYP |      PAYLEN      |
	 *   +----------------------------------------------------------------+
	 *   63      48 47    40 39 36 35 32 31   24 23  20 19                0
	 *
	 * Extended Data Descriptor (DTYP=0x1)
	 *   +----------------------------------------------------------------+
	 * 0 |                     Buffer Address [63:0]                      |
	 *   +----------------------------------------------------------------+
	 * 8 | VLAN tag |  POPTS  | Rsvd | Status | Command | DTYP |  DTALEN  |
	 *   +----------------------------------------------------------------+
	 *   63       48 47     40 39  36 35    32 31     24 23  20 19        0
	 */
303
304
305
	pr_info("Tl[desc]     [address 63:0  ] [SpeCssSCmCsLen] [bi->dma       ] leng  ntw timestamp        bi->skb <-- Legacy format\n");
	pr_info("Tc[desc]     [Ce CoCsIpceCoS] [MssHlRSCm0Plen] [bi->dma       ] leng  ntw timestamp        bi->skb <-- Ext Context format\n");
	pr_info("Td[desc]     [address 63:0  ] [VlaPoRSCm1Dlen] [bi->dma       ] leng  ntw timestamp        bi->skb <-- Ext Data format\n");
306
	for (i = 0; tx_ring->desc && (i < tx_ring->count); i++) {
307
		const char *next_desc;
308
309
310
311
		tx_desc = E1000_TX_DESC(*tx_ring, i);
		buffer_info = &tx_ring->buffer_info[i];
		u0 = (struct my_u0 *)tx_desc;
		if (i == tx_ring->next_to_use && i == tx_ring->next_to_clean)
312
			next_desc = " NTC/U";
313
		else if (i == tx_ring->next_to_use)
314
			next_desc = " NTU";
315
		else if (i == tx_ring->next_to_clean)
316
			next_desc = " NTC";
317
		else
318
319
320
321
322
323
324
325
326
327
328
			next_desc = "";
		pr_info("T%c[0x%03X]    %016llX %016llX %016llX %04X  %3X %016llX %p%s\n",
			(!(le64_to_cpu(u0->b) & (1 << 29)) ? 'l' :
			 ((le64_to_cpu(u0->b) & (1 << 20)) ? 'd' : 'c')),
			i,
			(unsigned long long)le64_to_cpu(u0->a),
			(unsigned long long)le64_to_cpu(u0->b),
			(unsigned long long)buffer_info->dma,
			buffer_info->length, buffer_info->next_to_watch,
			(unsigned long long)buffer_info->time_stamp,
			buffer_info->skb, next_desc);
329

330
		if (netif_msg_pktdata(adapter) && buffer_info->skb)
331
			print_hex_dump(KERN_INFO, "", DUMP_PREFIX_ADDRESS,
332
333
				       16, 1, buffer_info->skb->data,
				       buffer_info->skb->len, true);
334
335
	}

336
	/* Print Rx Ring Summary */
337
rx_ring_summary:
338
	dev_info(&adapter->pdev->dev, "Rx Ring Summary\n");
339
340
341
	pr_info("Queue [NTU] [NTC]\n");
	pr_info(" %5d %5X %5X\n",
		0, rx_ring->next_to_use, rx_ring->next_to_clean);
342

343
	/* Print Rx Ring */
344
	if (!netif_msg_rx_status(adapter))
345
		return;
346

347
	dev_info(&adapter->pdev->dev, "Rx Ring Dump\n");
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
	switch (adapter->rx_ps_pages) {
	case 1:
	case 2:
	case 3:
		/* [Extended] Packet Split Receive Descriptor Format
		 *
		 *    +-----------------------------------------------------+
		 *  0 |                Buffer Address 0 [63:0]              |
		 *    +-----------------------------------------------------+
		 *  8 |                Buffer Address 1 [63:0]              |
		 *    +-----------------------------------------------------+
		 * 16 |                Buffer Address 2 [63:0]              |
		 *    +-----------------------------------------------------+
		 * 24 |                Buffer Address 3 [63:0]              |
		 *    +-----------------------------------------------------+
		 */
364
		pr_info("R  [desc]      [buffer 0 63:0 ] [buffer 1 63:0 ] [buffer 2 63:0 ] [buffer 3 63:0 ] [bi->dma       ] [bi->skb] <-- Ext Pkt Split format\n");
365
366
367
368
369
370
371
372
373
374
375
		/* [Extended] Receive Descriptor (Write-Back) Format
		 *
		 *   63       48 47    32 31     13 12    8 7    4 3        0
		 *   +------------------------------------------------------+
		 * 0 | Packet   | IP     |  Rsvd   | MRQ   | Rsvd | MRQ RSS |
		 *   | Checksum | Ident  |         | Queue |      |  Type   |
		 *   +------------------------------------------------------+
		 * 8 | VLAN Tag | Length | Extended Error | Extended Status |
		 *   +------------------------------------------------------+
		 *   63       48 47    32 31            20 19               0
		 */
376
		pr_info("RWB[desc]      [ck ipid mrqhsh] [vl   l0 ee  es] [ l3  l2  l1 hs] [reserved      ] ---------------- [bi->skb] <-- Ext Rx Write-Back format\n");
377
		for (i = 0; i < rx_ring->count; i++) {
378
			const char *next_desc;
379
380
381
382
			buffer_info = &rx_ring->buffer_info[i];
			rx_desc_ps = E1000_RX_DESC_PS(*rx_ring, i);
			u1 = (struct my_u1 *)rx_desc_ps;
			staterr =
383
			    le32_to_cpu(rx_desc_ps->wb.middle.status_error);
384
385
386
387
388
389
390
391

			if (i == rx_ring->next_to_use)
				next_desc = " NTU";
			else if (i == rx_ring->next_to_clean)
				next_desc = " NTC";
			else
				next_desc = "";

392
393
			if (staterr & E1000_RXD_STAT_DD) {
				/* Descriptor Done */
394
395
396
397
398
399
400
				pr_info("%s[0x%03X]     %016llX %016llX %016llX %016llX ---------------- %p%s\n",
					"RWB", i,
					(unsigned long long)le64_to_cpu(u1->a),
					(unsigned long long)le64_to_cpu(u1->b),
					(unsigned long long)le64_to_cpu(u1->c),
					(unsigned long long)le64_to_cpu(u1->d),
					buffer_info->skb, next_desc);
401
			} else {
402
403
404
405
406
407
408
409
				pr_info("%s[0x%03X]     %016llX %016llX %016llX %016llX %016llX %p%s\n",
					"R  ", i,
					(unsigned long long)le64_to_cpu(u1->a),
					(unsigned long long)le64_to_cpu(u1->b),
					(unsigned long long)le64_to_cpu(u1->c),
					(unsigned long long)le64_to_cpu(u1->d),
					(unsigned long long)buffer_info->dma,
					buffer_info->skb, next_desc);
410
411

				if (netif_msg_pktdata(adapter))
412
413
					e1000e_dump_ps_pages(adapter,
							     buffer_info);
414
415
416
417
418
			}
		}
		break;
	default:
	case 0:
419
		/* Extended Receive Descriptor (Read) Format
420
		 *
421
422
423
424
425
		 *   +-----------------------------------------------------+
		 * 0 |                Buffer Address [63:0]                |
		 *   +-----------------------------------------------------+
		 * 8 |                      Reserved                       |
		 *   +-----------------------------------------------------+
426
		 */
427
		pr_info("R  [desc]      [buf addr 63:0 ] [reserved 63:0 ] [bi->dma       ] [bi->skb] <-- Ext (Read) format\n");
428
429
430
431
432
433
434
435
436
437
438
439
440
		/* Extended Receive Descriptor (Write-Back) Format
		 *
		 *   63       48 47    32 31    24 23            4 3        0
		 *   +------------------------------------------------------+
		 *   |     RSS Hash      |        |               |         |
		 * 0 +-------------------+  Rsvd  |   Reserved    | MRQ RSS |
		 *   | Packet   | IP     |        |               |  Type   |
		 *   | Checksum | Ident  |        |               |         |
		 *   +------------------------------------------------------+
		 * 8 | VLAN Tag | Length | Extended Error | Extended Status |
		 *   +------------------------------------------------------+
		 *   63       48 47    32 31            20 19               0
		 */
441
		pr_info("RWB[desc]      [cs ipid    mrq] [vt   ln xe  xs] [bi->skb] <-- Ext (Write-Back) format\n");
442
443

		for (i = 0; i < rx_ring->count; i++) {
444
445
			const char *next_desc;

446
			buffer_info = &rx_ring->buffer_info[i];
447
448
449
			rx_desc = E1000_RX_DESC_EXT(*rx_ring, i);
			u1 = (struct my_u1 *)rx_desc;
			staterr = le32_to_cpu(rx_desc->wb.upper.status_error);
450
451
452
453
454
455
456
457

			if (i == rx_ring->next_to_use)
				next_desc = " NTU";
			else if (i == rx_ring->next_to_clean)
				next_desc = " NTC";
			else
				next_desc = "";

458
459
			if (staterr & E1000_RXD_STAT_DD) {
				/* Descriptor Done */
460
461
462
463
464
				pr_info("%s[0x%03X]     %016llX %016llX ---------------- %p%s\n",
					"RWB", i,
					(unsigned long long)le64_to_cpu(u1->a),
					(unsigned long long)le64_to_cpu(u1->b),
					buffer_info->skb, next_desc);
465
			} else {
466
467
468
469
470
471
				pr_info("%s[0x%03X]     %016llX %016llX %016llX %p%s\n",
					"R  ", i,
					(unsigned long long)le64_to_cpu(u1->a),
					(unsigned long long)le64_to_cpu(u1->b),
					(unsigned long long)buffer_info->dma,
					buffer_info->skb, next_desc);
472

473
474
				if (netif_msg_pktdata(adapter) &&
				    buffer_info->skb)
475
476
477
					print_hex_dump(KERN_INFO, "",
						       DUMP_PREFIX_ADDRESS, 16,
						       1,
478
						       buffer_info->skb->data,
479
480
481
						       adapter->rx_buffer_len,
						       true);
			}
482
483
484
485
		}
	}
}

486
487
488
489
490
491
492
493
494
495
496
/**
 * e1000_desc_unused - calculate if we have unused descriptors
 **/
static int e1000_desc_unused(struct e1000_ring *ring)
{
	if (ring->next_to_clean > ring->next_to_use)
		return ring->next_to_clean - ring->next_to_use - 1;

	return ring->count + ring->next_to_clean - ring->next_to_use - 1;
}

497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
/**
 * e1000e_systim_to_hwtstamp - convert system time value to hw time stamp
 * @adapter: board private structure
 * @hwtstamps: time stamp structure to update
 * @systim: unsigned 64bit system time value.
 *
 * Convert the system time value stored in the RX/TXSTMP registers into a
 * hwtstamp which can be used by the upper level time stamping functions.
 *
 * The 'systim_lock' spinlock is used to protect the consistency of the
 * system time value. This is needed because reading the 64 bit time
 * value involves reading two 32 bit registers. The first read latches the
 * value.
 **/
static void e1000e_systim_to_hwtstamp(struct e1000_adapter *adapter,
				      struct skb_shared_hwtstamps *hwtstamps,
				      u64 systim)
{
	u64 ns;
	unsigned long flags;

	spin_lock_irqsave(&adapter->systim_lock, flags);
	ns = timecounter_cyc2time(&adapter->tc, systim);
	spin_unlock_irqrestore(&adapter->systim_lock, flags);

	memset(hwtstamps, 0, sizeof(*hwtstamps));
	hwtstamps->hwtstamp = ns_to_ktime(ns);
}

/**
 * e1000e_rx_hwtstamp - utility function which checks for Rx time stamp
 * @adapter: board private structure
 * @status: descriptor extended error and status field
 * @skb: particular skb to include time stamp
 *
 * If the time stamp is valid, convert it into the timecounter ns value
 * and store that result into the shhwtstamps structure which is passed
 * up the network stack.
 **/
static void e1000e_rx_hwtstamp(struct e1000_adapter *adapter, u32 status,
			       struct sk_buff *skb)
{
	struct e1000_hw *hw = &adapter->hw;
	u64 rxstmp;

	if (!(adapter->flags & FLAG_HAS_HW_TIMESTAMP) ||
	    !(status & E1000_RXDEXT_STATERR_TST) ||
	    !(er32(TSYNCRXCTL) & E1000_TSYNCRXCTL_VALID))
		return;

	/* The Rx time stamp registers contain the time stamp.  No other
	 * received packet will be time stamped until the Rx time stamp
	 * registers are read.  Because only one packet can be time stamped
	 * at a time, the register values must belong to this packet and
	 * therefore none of the other additional attributes need to be
	 * compared.
	 */
	rxstmp = (u64)er32(RXSTMPL);
	rxstmp |= (u64)er32(RXSTMPH) << 32;
	e1000e_systim_to_hwtstamp(adapter, skb_hwtstamps(skb), rxstmp);

	adapter->flags2 &= ~FLAG2_CHECK_RX_HWTSTAMP;
}

561
/**
562
 * e1000_receive_skb - helper function to handle Rx indications
563
 * @adapter: board private structure
564
 * @staterr: descriptor extended error and status field as written by hardware
565
566
567
568
 * @vlan: descriptor vlan field as written by hardware (no le/be conversion)
 * @skb: pointer to sk_buff to be indicated to stack
 **/
static void e1000_receive_skb(struct e1000_adapter *adapter,
569
			      struct net_device *netdev, struct sk_buff *skb,
570
			      u32 staterr, __le16 vlan)
571
{
572
	u16 tag = le16_to_cpu(vlan);
573
574
575

	e1000e_rx_hwtstamp(adapter, staterr, skb);

576
577
	skb->protocol = eth_type_trans(skb, netdev);

578
	if (staterr & E1000_RXD_STAT_VP)
579
		__vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), tag);
580
581

	napi_gro_receive(&adapter->napi, skb);
582
583
584
}

/**
585
 * e1000_rx_checksum - Receive Checksum Offload
586
587
588
589
 * @adapter: board private structure
 * @status_err: receive descriptor status and error fields
 * @csum: receive descriptor csum field
 * @sk_buff: socket buffer with received data
590
591
 **/
static void e1000_rx_checksum(struct e1000_adapter *adapter, u32 status_err,
592
			      struct sk_buff *skb)
593
594
595
{
	u16 status = (u16)status_err;
	u8 errors = (u8)(status_err >> 24);
596
597

	skb_checksum_none_assert(skb);
598

599
600
601
602
	/* Rx checksum disabled */
	if (!(adapter->netdev->features & NETIF_F_RXCSUM))
		return;

603
604
605
	/* Ignore Checksum bit is set */
	if (status & E1000_RXD_STAT_IXSM)
		return;
606

607
608
	/* TCP/UDP checksum error bit or IP checksum error bit is set */
	if (errors & (E1000_RXD_ERR_TCPE | E1000_RXD_ERR_IPE)) {
609
610
611
612
613
614
615
616
617
618
		/* let the stack verify checksum errors */
		adapter->hw_csum_err++;
		return;
	}

	/* TCP/UDP Checksum has not been calculated */
	if (!(status & (E1000_RXD_STAT_TCPCS | E1000_RXD_STAT_UDPCS)))
		return;

	/* It must be a TCP or UDP packet with a valid checksum */
619
	skb->ip_summed = CHECKSUM_UNNECESSARY;
620
621
622
	adapter->hw_csum_good++;
}

623
static void e1000e_update_rdt_wa(struct e1000_ring *rx_ring, unsigned int i)
624
{
625
	struct e1000_adapter *adapter = rx_ring->adapter;
626
	struct e1000_hw *hw = &adapter->hw;
627
628
629
	s32 ret_val = __ew32_prepare(hw);

	writel(i, rx_ring->tail);
630

631
	if (unlikely(!ret_val && (i != readl(rx_ring->tail)))) {
632
		u32 rctl = er32(RCTL);
633

634
635
636
637
638
639
		ew32(RCTL, rctl & ~E1000_RCTL_EN);
		e_err("ME firmware caused invalid RDT - resetting\n");
		schedule_work(&adapter->reset_task);
	}
}

640
static void e1000e_update_tdt_wa(struct e1000_ring *tx_ring, unsigned int i)
641
{
642
	struct e1000_adapter *adapter = tx_ring->adapter;
643
	struct e1000_hw *hw = &adapter->hw;
644
	s32 ret_val = __ew32_prepare(hw);
645

646
647
648
	writel(i, tx_ring->tail);

	if (unlikely(!ret_val && (i != readl(tx_ring->tail)))) {
649
		u32 tctl = er32(TCTL);
650

651
652
653
654
655
656
		ew32(TCTL, tctl & ~E1000_TCTL_EN);
		e_err("ME firmware caused invalid TDT - resetting\n");
		schedule_work(&adapter->reset_task);
	}
}

657
/**
658
 * e1000_alloc_rx_buffers - Replace used receive buffers
659
 * @rx_ring: Rx descriptor ring
660
 **/
661
static void e1000_alloc_rx_buffers(struct e1000_ring *rx_ring,
662
				   int cleaned_count, gfp_t gfp)
663
{
664
	struct e1000_adapter *adapter = rx_ring->adapter;
665
666
	struct net_device *netdev = adapter->netdev;
	struct pci_dev *pdev = adapter->pdev;
667
	union e1000_rx_desc_extended *rx_desc;
668
669
670
	struct e1000_buffer *buffer_info;
	struct sk_buff *skb;
	unsigned int i;
671
	unsigned int bufsz = adapter->rx_buffer_len;
672
673
674
675
676
677
678
679
680
681
682

	i = rx_ring->next_to_use;
	buffer_info = &rx_ring->buffer_info[i];

	while (cleaned_count--) {
		skb = buffer_info->skb;
		if (skb) {
			skb_trim(skb, 0);
			goto map_skb;
		}

683
		skb = __netdev_alloc_skb_ip_align(netdev, bufsz, gfp);
684
685
686
687
688
689
690
691
		if (!skb) {
			/* Better luck next round */
			adapter->alloc_rx_buff_failed++;
			break;
		}

		buffer_info->skb = skb;
map_skb:
692
		buffer_info->dma = dma_map_single(&pdev->dev, skb->data,
693
						  adapter->rx_buffer_len,
694
695
						  DMA_FROM_DEVICE);
		if (dma_mapping_error(&pdev->dev, buffer_info->dma)) {
696
			dev_err(&pdev->dev, "Rx DMA map failed\n");
697
698
699
700
			adapter->rx_dma_failed++;
			break;
		}

701
702
		rx_desc = E1000_RX_DESC_EXT(*rx_ring, i);
		rx_desc->read.buffer_addr = cpu_to_le64(buffer_info->dma);
703

704
		if (unlikely(!(i & (E1000_RX_BUFFER_WRITE - 1)))) {
705
			/* Force memory writes to complete before letting h/w
706
707
708
709
710
			 * know there are new descriptors to fetch.  (Only
			 * applicable for weak-ordered memory model archs,
			 * such as IA-64).
			 */
			wmb();
711
			if (adapter->flags2 & FLAG2_PCIM2PCI_ARBITER_WA)
712
				e1000e_update_rdt_wa(rx_ring, i);
713
			else
714
				writel(i, rx_ring->tail);
715
		}
716
717
718
719
720
721
		i++;
		if (i == rx_ring->count)
			i = 0;
		buffer_info = &rx_ring->buffer_info[i];
	}

722
	rx_ring->next_to_use = i;
723
724
725
726
}

/**
 * e1000_alloc_rx_buffers_ps - Replace used receive buffers; packet split
727
 * @rx_ring: Rx descriptor ring
728
 **/
729
static void e1000_alloc_rx_buffers_ps(struct e1000_ring *rx_ring,
730
				      int cleaned_count, gfp_t gfp)
731
{
732
	struct e1000_adapter *adapter = rx_ring->adapter;
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
	struct net_device *netdev = adapter->netdev;
	struct pci_dev *pdev = adapter->pdev;
	union e1000_rx_desc_packet_split *rx_desc;
	struct e1000_buffer *buffer_info;
	struct e1000_ps_page *ps_page;
	struct sk_buff *skb;
	unsigned int i, j;

	i = rx_ring->next_to_use;
	buffer_info = &rx_ring->buffer_info[i];

	while (cleaned_count--) {
		rx_desc = E1000_RX_DESC_PS(*rx_ring, i);

		for (j = 0; j < PS_PAGE_BUFFERS; j++) {
748
749
750
			ps_page = &buffer_info->ps_pages[j];
			if (j >= adapter->rx_ps_pages) {
				/* all unused desc entries get hw null ptr */
751
752
				rx_desc->read.buffer_addr[j + 1] =
				    ~cpu_to_le64(0);
753
754
755
				continue;
			}
			if (!ps_page->page) {
756
				ps_page->page = alloc_page(gfp);
757
				if (!ps_page->page) {
758
759
760
					adapter->alloc_rx_buff_failed++;
					goto no_buffers;
				}
761
762
763
764
765
766
				ps_page->dma = dma_map_page(&pdev->dev,
							    ps_page->page,
							    0, PAGE_SIZE,
							    DMA_FROM_DEVICE);
				if (dma_mapping_error(&pdev->dev,
						      ps_page->dma)) {
767
					dev_err(&adapter->pdev->dev,
768
						"Rx DMA page map failed\n");
769
770
					adapter->rx_dma_failed++;
					goto no_buffers;
771
772
				}
			}
773
			/* Refresh the desc even if buffer_addrs
774
775
776
			 * didn't change because each write-back
			 * erases this info.
			 */
777
778
			rx_desc->read.buffer_addr[j + 1] =
			    cpu_to_le64(ps_page->dma);
779
780
		}

781
		skb = __netdev_alloc_skb_ip_align(netdev, adapter->rx_ps_bsize0,
782
						  gfp);
783
784
785
786
787
788
789

		if (!skb) {
			adapter->alloc_rx_buff_failed++;
			break;
		}

		buffer_info->skb = skb;
790
		buffer_info->dma = dma_map_single(&pdev->dev, skb->data,
791
						  adapter->rx_ps_bsize0,
792
793
						  DMA_FROM_DEVICE);
		if (dma_mapping_error(&pdev->dev, buffer_info->dma)) {
794
			dev_err(&pdev->dev, "Rx DMA map failed\n");
795
796
797
798
799
800
801
802
803
			adapter->rx_dma_failed++;
			/* cleanup skb */
			dev_kfree_skb_any(skb);
			buffer_info->skb = NULL;
			break;
		}

		rx_desc->read.buffer_addr[0] = cpu_to_le64(buffer_info->dma);

804
		if (unlikely(!(i & (E1000_RX_BUFFER_WRITE - 1)))) {
805
			/* Force memory writes to complete before letting h/w
806
807
808
809
810
			 * know there are new descriptors to fetch.  (Only
			 * applicable for weak-ordered memory model archs,
			 * such as IA-64).
			 */
			wmb();
811
			if (adapter->flags2 & FLAG2_PCIM2PCI_ARBITER_WA)
812
				e1000e_update_rdt_wa(rx_ring, i << 1);
813
			else
814
				writel(i << 1, rx_ring->tail);
815
816
		}

817
818
819
820
821
822
823
		i++;
		if (i == rx_ring->count)
			i = 0;
		buffer_info = &rx_ring->buffer_info[i];
	}

no_buffers:
824
	rx_ring->next_to_use = i;
825
826
}

827
828
/**
 * e1000_alloc_jumbo_rx_buffers - Replace used jumbo receive buffers
829
 * @rx_ring: Rx descriptor ring
830
831
832
 * @cleaned_count: number of buffers to allocate this pass
 **/

833
static void e1000_alloc_jumbo_rx_buffers(struct e1000_ring *rx_ring,
834
					 int cleaned_count, gfp_t gfp)
835
{
836
	struct e1000_adapter *adapter = rx_ring->adapter;
837
838
	struct net_device *netdev = adapter->netdev;
	struct pci_dev *pdev = adapter->pdev;
839
	union e1000_rx_desc_extended *rx_desc;
840
841
842
	struct e1000_buffer *buffer_info;
	struct sk_buff *skb;
	unsigned int i;
843
	unsigned int bufsz = 256 - 16;	/* for skb_reserve */
844
845
846
847
848
849
850
851
852
853
854

	i = rx_ring->next_to_use;
	buffer_info = &rx_ring->buffer_info[i];

	while (cleaned_count--) {
		skb = buffer_info->skb;
		if (skb) {
			skb_trim(skb, 0);
			goto check_page;
		}

855
		skb = __netdev_alloc_skb_ip_align(netdev, bufsz, gfp);
856
857
858
859
860
861
862
863
864
865
		if (unlikely(!skb)) {
			/* Better luck next round */
			adapter->alloc_rx_buff_failed++;
			break;
		}

		buffer_info->skb = skb;
check_page:
		/* allocate a new page if necessary */
		if (!buffer_info->page) {
866
			buffer_info->page = alloc_page(gfp);
867
868
869
870
871
872
			if (unlikely(!buffer_info->page)) {
				adapter->alloc_rx_buff_failed++;
				break;
			}
		}

873
		if (!buffer_info->dma) {
874
			buffer_info->dma = dma_map_page(&pdev->dev,
875
876
							buffer_info->page, 0,
							PAGE_SIZE,
877
							DMA_FROM_DEVICE);
878
879
880
881
882
			if (dma_mapping_error(&pdev->dev, buffer_info->dma)) {
				adapter->alloc_rx_buff_failed++;
				break;
			}
		}
883

884
885
		rx_desc = E1000_RX_DESC_EXT(*rx_ring, i);
		rx_desc->read.buffer_addr = cpu_to_le64(buffer_info->dma);
886
887
888
889
890
891
892
893
894
895
896
897
898
899

		if (unlikely(++i == rx_ring->count))
			i = 0;
		buffer_info = &rx_ring->buffer_info[i];
	}

	if (likely(rx_ring->next_to_use != i)) {
		rx_ring->next_to_use = i;
		if (unlikely(i-- == 0))
			i = (rx_ring->count - 1);

		/* Force memory writes to complete before letting h/w
		 * know there are new descriptors to fetch.  (Only
		 * applicable for weak-ordered memory model archs,
900
901
		 * such as IA-64).
		 */
902
		wmb();
903
		if (adapter->flags2 & FLAG2_PCIM2PCI_ARBITER_WA)
904
			e1000e_update_rdt_wa(rx_ring, i);
905
		else
906
			writel(i, rx_ring->tail);
907
908
909
	}
}

910
911
912
913
static inline void e1000_rx_hash(struct net_device *netdev, __le32 rss,
				 struct sk_buff *skb)
{
	if (netdev->features & NETIF_F_RXHASH)
Tom Herbert's avatar
Tom Herbert committed
914
		skb_set_hash(skb, le32_to_cpu(rss), PKT_HASH_TYPE_L3);
915
916
}

917
/**
918
919
 * e1000_clean_rx_irq - Send received data up the network stack
 * @rx_ring: Rx descriptor ring
920
921
922
923
 *
 * the return value indicates whether actual cleaning was done, there
 * is no guarantee that everything was cleaned
 **/
924
925
static bool e1000_clean_rx_irq(struct e1000_ring *rx_ring, int *work_done,
			       int work_to_do)
926
{
927
	struct e1000_adapter *adapter = rx_ring->adapter;
928
929
	struct net_device *netdev = adapter->netdev;
	struct pci_dev *pdev = adapter->pdev;
930
	struct e1000_hw *hw = &adapter->hw;
931
	union e1000_rx_desc_extended *rx_desc, *next_rxd;
932
	struct e1000_buffer *buffer_info, *next_buffer;
933
	u32 length, staterr;
934
935
	unsigned int i;
	int cleaned_count = 0;
936
	bool cleaned = false;
937
938
939
	unsigned int total_rx_bytes = 0, total_rx_packets = 0;

	i = rx_ring->next_to_clean;
940
941
	rx_desc = E1000_RX_DESC_EXT(*rx_ring, i);
	staterr = le32_to_cpu(rx_desc->wb.upper.status_error);
942
943
	buffer_info = &rx_ring->buffer_info[i];

944
	while (staterr & E1000_RXD_STAT_DD) {
945
946
947
948
949
		struct sk_buff *skb;

		if (*work_done >= work_to_do)
			break;
		(*work_done)++;
950
		rmb();	/* read descriptor and rx_buffer_info after status DD */
951
952
953
954
955
956
957
958
959

		skb = buffer_info->skb;
		buffer_info->skb = NULL;

		prefetch(skb->data - NET_IP_ALIGN);

		i++;
		if (i == rx_ring->count)
			i = 0;
960
		next_rxd = E1000_RX_DESC_EXT(*rx_ring, i);
961
962
963
964
		prefetch(next_rxd);

		next_buffer = &rx_ring->buffer_info[i];

965
		cleaned = true;
966
		cleaned_count++;
967
968
		dma_unmap_single(&pdev->dev, buffer_info->dma,
				 adapter->rx_buffer_len, DMA_FROM_DEVICE);
969
970
		buffer_info->dma = 0;

971
		length = le16_to_cpu(rx_desc->wb.upper.length);
972

973
		/* !EOP means multiple descriptors were used to store a single
974
975
976
977
978
		 * packet, if that's the case we need to toss it.  In fact, we
		 * need to toss every packet with the EOP bit clear and the
		 * next frame that _does_ have the EOP bit set, as it is by
		 * definition only a frame fragment
		 */
979
		if (unlikely(!(staterr & E1000_RXD_STAT_EOP)))
980
981
982
			adapter->flags2 |= FLAG2_IS_DISCARDING;

		if (adapter->flags2 & FLAG2_IS_DISCARDING) {
983
			/* All receives must fit into a single buffer */
984
			e_dbg("Receive packet consumed multiple buffers\n");
985
986
			/* recycle */
			buffer_info->skb = skb;
987
			if (staterr & E1000_RXD_STAT_EOP)
988
				adapter->flags2 &= ~FLAG2_IS_DISCARDING;
989
990
991
			goto next_desc;
		}

992
993
		if (unlikely((staterr & E1000_RXDEXT_ERR_FRAME_ERR_MASK) &&
			     !(netdev->features & NETIF_F_RXALL))) {
994
995
996
997
998
			/* recycle */
			buffer_info->skb = skb;
			goto next_desc;
		}

Jeff Kirsher's avatar
Jeff Kirsher committed
999
		/* adjust length to remove Ethernet CRC */
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
		if (!(adapter->flags2 & FLAG2_CRC_STRIPPING)) {
			/* If configured to store CRC, don't subtract FCS,
			 * but keep the FCS bytes out of the total_rx_bytes
			 * counter
			 */
			if (netdev->features & NETIF_F_RXFCS)
				total_rx_bytes -= 4;
			else
				length -= 4;
		}
Jeff Kirsher's avatar
Jeff Kirsher committed
1010

1011
1012
1013
		total_rx_bytes += length;
		total_rx_packets++;

1014
		/* code added for copybreak, this should improve
1015
		 * performance for small packets with large amounts
1016
1017
		 * of reassembly being done in the stack
		 */
1018
1019
		if (length < copybreak) {
			struct sk_buff *new_skb =
1020
				napi_alloc_skb(&adapter->napi, length);
1021
			if (new_skb) {
1022
1023
1024
1025
1026
1027
				skb_copy_to_linear_data_offset(new_skb,
							       -NET_IP_ALIGN,
							       (skb->data -
								NET_IP_ALIGN),
							       (length +
								NET_IP_ALIGN));
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
				/* save the skb in buffer_info as good */
				buffer_info->skb = skb;
				skb = new_skb;
			}
			/* else just continue with the old one */
		}
		/* end copybreak code */
		skb_put(skb, length);

		/* Receive Checksum Offload */
1038
		e1000_rx_checksum(adapter, staterr, skb);
1039

1040
1041
		e1000_rx_hash(netdev, rx_desc->wb.lower.hi_dword.rss, skb);

1042
1043
		e1000_receive_skb(adapter, netdev, skb, staterr,
				  rx_desc->wb.upper.vlan);
1044
1045

next_desc:
1046
		rx_desc->wb.upper.status_error &= cpu_to_le32(~0xFF);
1047
1048
1049

		/* return some buffers to hardware, one at a time is too slow */
		if (cleaned_count >= E1000_RX_BUFFER_WRITE) {
1050
			adapter->alloc_rx_buf(rx_ring, cleaned_count,
1051
					      GFP_ATOMIC);
1052
1053
1054
1055
1056
1057
			cleaned_count = 0;
		}

		/* use prefetched values */
		rx_desc = next_rxd;
		buffer_info = next_buffer;
1058
1059

		staterr = le32_to_cpu(rx_desc->wb.upper.status_error);
1060
1061
1062
1063
1064
	}
	rx_ring->next_to_clean = i;

	cleaned_count = e1000_desc_unused(rx_ring);
	if (cleaned_count)
1065
		adapter->alloc_rx_buf(rx_ring, cleaned_count, GFP_ATOMIC);
1066
1067

	adapter->total_rx_bytes += total_rx_bytes;
1068
	adapter->total_rx_packets += total_rx_packets;
1069
1070
1071
	return cleaned;
}

1072
1073
static void e1000_put_txbuf(struct e1000_ring *tx_ring,
			    struct e1000_buffer *buffer_info)
1074
{
1075
1076
	struct e1000_adapter *adapter = tx_ring->adapter;

1077
1078
	if (buffer_info->dma) {
		if (buffer_info->mapped_as_page)
1079
1080
			dma_unmap_page(&adapter->pdev->dev, buffer_info->dma,
				       buffer_info->length, DMA_TO_DEVICE);
1081
		else
1082
1083
			dma_unmap_single(&adapter->pdev->dev, buffer_info->dma,
					 buffer_info->length, DMA_TO_DEVICE);
1084
1085
		buffer_info->dma = 0;
	}
1086
1087
1088
1089
	if (buffer_info->skb) {
		dev_kfree_skb_any(buffer_info->skb);
		buffer_info->skb = NULL;
	}
1090
	buffer_info->time_stamp = 0;
1091
1092
}

1093
static void e1000_print_hw_hang(struct work_struct *work)
1094
{
1095
	struct e1000_adapter *adapter = container_of(work,
1096
1097
						     struct e1000_adapter,
						     print_hang_task);
1098
	struct net_device *netdev = adapter->netdev;
1099
1100
1101
1102
	struct e1000_ring *tx_ring = adapter->tx_ring;
	unsigned int i = tx_ring->next_to_clean;
	unsigned int eop = tx_ring->buffer_info[i].next_to_watch;
	struct e1000_tx_desc *eop_desc = E1000_TX_DESC(*tx_ring, eop);
1103
1104
1105
1106
	struct e1000_hw *hw = &adapter->hw;
	u16 phy_status, phy_1000t_status, phy_ext_status;
	u16 pci_status;

1107
1108
1109
	if (test_bit(__E1000_DOWN, &adapter->state))
		return;

1110
	if (!adapter->tx_hang_recheck && (adapter->flags2 & FLAG2_DMA_BURST)) {
1111
		/* May be block on write-back, flush and detect again
1112
1113
1114
1115
1116
		 * flush pending descriptor writebacks to memory
		 */
		ew32(TIDV, adapter->tx_int_delay | E1000_TIDV_FPD);
		/* execute the writes immediately */
		e1e_flush();
1117
		/* Due to rare timing issues, write to TIDV again to ensure
1118
1119
1120
1121
1122
		 * the write is successful
		 */
		ew32(TIDV, adapter->tx_int_delay | E1000_TIDV_FPD);
		/* execute the writes immediately */
		e1e_flush();
1123
1124
1125
1126
		adapter->tx_hang_recheck = true;
		return;
	}
	adapter->tx_hang_recheck = false;
David Ertman's avatar
David Ertman committed
1127
1128
1129
1130
1131
1132
1133

	if (er32(TDH(0)) == er32(TDT(0))) {
		e_dbg("false hang detected, ignoring\n");
		return;
	}

	/* Real hang detected */
1134
1135
	netif_stop_queue(netdev);