falcon.c 92.2 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
/****************************************************************************
 * Driver for Solarflare Solarstorm network controllers and boards
 * Copyright 2005-2006 Fen Systems Ltd.
 * Copyright 2006-2008 Solarflare Communications Inc.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License version 2 as published
 * by the Free Software Foundation, incorporated herein by reference.
 */

#include <linux/bitops.h>
#include <linux/delay.h>
#include <linux/pci.h>
#include <linux/module.h>
#include <linux/seq_file.h>
16
#include <linux/i2c.h>
17
#include <linux/mii.h>
18
19
20
21
22
23
#include "net_driver.h"
#include "bitfield.h"
#include "efx.h"
#include "mac.h"
#include "spi.h"
#include "falcon.h"
24
#include "regs.h"
25
#include "io.h"
26
27
28
29
#include "mdio_10g.h"
#include "phy.h"
#include "workarounds.h"

Ben Hutchings's avatar
Ben Hutchings committed
30
/* Hardware control for SFC4000 (aka Falcon). */
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46

/**************************************************************************
 *
 * Configurable values
 *
 **************************************************************************
 */

/* This is set to 16 for a good reason.  In summary, if larger than
 * 16, the descriptor cache holds more than a default socket
 * buffer's worth of packets (for UDP we can only have at most one
 * socket buffer's worth outstanding).  This combined with the fact
 * that we only get 1 TX event per descriptor cache means the NIC
 * goes idle.
 */
#define TX_DC_ENTRIES 16
47
#define TX_DC_ENTRIES_ORDER 1
48
49
50
#define TX_DC_BASE 0x130000

#define RX_DC_ENTRIES 64
51
#define RX_DC_ENTRIES_ORDER 3
52
53
#define RX_DC_BASE 0x100000

54
55
56
57
58
59
60
61
62
63
64
65
66
67
static const unsigned int
/* "Large" EEPROM device: Atmel AT25640 or similar
 * 8 KB, 16-bit address, 32 B write block */
large_eeprom_type = ((13 << SPI_DEV_TYPE_SIZE_LBN)
		     | (2 << SPI_DEV_TYPE_ADDR_LEN_LBN)
		     | (5 << SPI_DEV_TYPE_BLOCK_SIZE_LBN)),
/* Default flash device: Atmel AT25F1024
 * 128 KB, 24-bit address, 32 KB erase block, 256 B write block */
default_flash_type = ((17 << SPI_DEV_TYPE_SIZE_LBN)
		      | (3 << SPI_DEV_TYPE_ADDR_LEN_LBN)
		      | (0x52 << SPI_DEV_TYPE_ERASE_CMD_LBN)
		      | (15 << SPI_DEV_TYPE_ERASE_SIZE_LBN)
		      | (8 << SPI_DEV_TYPE_BLOCK_SIZE_LBN));

68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
/* RX FIFO XOFF watermark
 *
 * When the amount of the RX FIFO increases used increases past this
 * watermark send XOFF. Only used if RX flow control is enabled (ethtool -A)
 * This also has an effect on RX/TX arbitration
 */
static int rx_xoff_thresh_bytes = -1;
module_param(rx_xoff_thresh_bytes, int, 0644);
MODULE_PARM_DESC(rx_xoff_thresh_bytes, "RX fifo XOFF threshold");

/* RX FIFO XON watermark
 *
 * When the amount of the RX FIFO used decreases below this
 * watermark send XON. Only used if TX flow control is enabled (ethtool -A)
 * This also has an effect on RX/TX arbitration
 */
static int rx_xon_thresh_bytes = -1;
module_param(rx_xon_thresh_bytes, int, 0644);
MODULE_PARM_DESC(rx_xon_thresh_bytes, "RX fifo XON threshold");

88
89
90
91
92
93
/* If FALCON_MAX_INT_ERRORS internal errors occur within
 * FALCON_INT_ERROR_EXPIRE seconds, we consider the NIC broken and
 * disable it.
 */
#define FALCON_INT_ERROR_EXPIRE 3600
#define FALCON_MAX_INT_ERRORS 5
94

95
96
97
98
/* We poll for events every FLUSH_INTERVAL ms, and check FLUSH_POLL_COUNT times
 */
#define FALCON_FLUSH_INTERVAL 10
#define FALCON_FLUSH_POLL_COUNT 100
99
100
101
102
103
104
105
106
107
108
109

/**************************************************************************
 *
 * Falcon constants
 *
 **************************************************************************
 */

/* Size and alignment of special buffers (4KB) */
#define FALCON_BUF_SIZE 4096

110
111
112
/* Depth of RX flush request fifo */
#define FALCON_RX_FLUSH_COUNT 4

113
#define FALCON_IS_DUAL_FUNC(efx)		\
114
	(falcon_rev(efx) < FALCON_REV_B0)
115
116
117
118
119
120
121

/**************************************************************************
 *
 * Falcon hardware access
 *
 **************************************************************************/

122
123
124
125
126
127
128
static inline void falcon_write_buf_tbl(struct efx_nic *efx, efx_qword_t *value,
					unsigned int index)
{
	efx_sram_writeq(efx, efx->membase + efx->type->buf_tbl_base,
			value, index);
}

129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
/* Read the current event from the event queue */
static inline efx_qword_t *falcon_event(struct efx_channel *channel,
					unsigned int index)
{
	return (((efx_qword_t *) (channel->eventq.addr)) + index);
}

/* See if an event is present
 *
 * We check both the high and low dword of the event for all ones.  We
 * wrote all ones when we cleared the event, and no valid event can
 * have all ones in either its high or low dwords.  This approach is
 * robust against reordering.
 *
 * Note that using a single 64-bit comparison is incorrect; even
 * though the CPU read will be atomic, the DMA write may not be.
 */
static inline int falcon_event_present(efx_qword_t *event)
{
	return (!(EFX_DWORD_IS_ALL_ONES(event->dword[0]) |
		  EFX_DWORD_IS_ALL_ONES(event->dword[1])));
}

/**************************************************************************
 *
 * I2C bus - this is a bit-bashing interface using GPIO pins
 * Note that it uses the output enables to tristate the outputs
 * SDA is the data pin and SCL is the clock
 *
 **************************************************************************
 */
160
static void falcon_setsda(void *data, int state)
161
{
162
	struct efx_nic *efx = (struct efx_nic *)data;
163
164
	efx_oword_t reg;

165
	efx_reado(efx, &reg, FR_AB_GPIO_CTL);
166
	EFX_SET_OWORD_FIELD(reg, FRF_AB_GPIO3_OEN, !state);
167
	efx_writeo(efx, &reg, FR_AB_GPIO_CTL);
168
169
}

170
static void falcon_setscl(void *data, int state)
171
{
172
	struct efx_nic *efx = (struct efx_nic *)data;
173
174
	efx_oword_t reg;

175
	efx_reado(efx, &reg, FR_AB_GPIO_CTL);
176
	EFX_SET_OWORD_FIELD(reg, FRF_AB_GPIO0_OEN, !state);
177
	efx_writeo(efx, &reg, FR_AB_GPIO_CTL);
178
179
180
181
182
183
184
}

static int falcon_getsda(void *data)
{
	struct efx_nic *efx = (struct efx_nic *)data;
	efx_oword_t reg;

185
	efx_reado(efx, &reg, FR_AB_GPIO_CTL);
186
	return EFX_OWORD_FIELD(reg, FRF_AB_GPIO3_IN);
187
188
}

189
static int falcon_getscl(void *data)
190
{
191
	struct efx_nic *efx = (struct efx_nic *)data;
192
193
	efx_oword_t reg;

194
	efx_reado(efx, &reg, FR_AB_GPIO_CTL);
195
	return EFX_OWORD_FIELD(reg, FRF_AB_GPIO0_IN);
196
197
}

198
199
200
static struct i2c_algo_bit_data falcon_i2c_bit_operations = {
	.setsda		= falcon_setsda,
	.setscl		= falcon_setscl,
201
202
	.getsda		= falcon_getsda,
	.getscl		= falcon_getscl,
203
	.udelay		= 5,
204
205
	/* Wait up to 50 ms for slave to let us pull SCL high */
	.timeout	= DIV_ROUND_UP(HZ, 20),
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
};

/**************************************************************************
 *
 * Falcon special buffer handling
 * Special buffers are used for event queues and the TX and RX
 * descriptor rings.
 *
 *************************************************************************/

/*
 * Initialise a Falcon special buffer
 *
 * This will define a buffer (previously allocated via
 * falcon_alloc_special_buffer()) in Falcon's buffer table, allowing
 * it to be used for event queues, descriptor rings etc.
 */
223
static void
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
falcon_init_special_buffer(struct efx_nic *efx,
			   struct efx_special_buffer *buffer)
{
	efx_qword_t buf_desc;
	int index;
	dma_addr_t dma_addr;
	int i;

	EFX_BUG_ON_PARANOID(!buffer->addr);

	/* Write buffer descriptors to NIC */
	for (i = 0; i < buffer->entries; i++) {
		index = buffer->index + i;
		dma_addr = buffer->dma_addr + (i * 4096);
		EFX_LOG(efx, "mapping special buffer %d at %llx\n",
			index, (unsigned long long)dma_addr);
240
241
242
243
		EFX_POPULATE_QWORD_3(buf_desc,
				     FRF_AZ_BUF_ADR_REGION, 0,
				     FRF_AZ_BUF_ADR_FBUF, dma_addr >> 12,
				     FRF_AZ_BUF_OWNER_ID_FBUF, 0);
244
		falcon_write_buf_tbl(efx, &buf_desc, index);
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
	}
}

/* Unmaps a buffer from Falcon and clears the buffer table entries */
static void
falcon_fini_special_buffer(struct efx_nic *efx,
			   struct efx_special_buffer *buffer)
{
	efx_oword_t buf_tbl_upd;
	unsigned int start = buffer->index;
	unsigned int end = (buffer->index + buffer->entries - 1);

	if (!buffer->entries)
		return;

	EFX_LOG(efx, "unmapping special buffers %d-%d\n",
		buffer->index, buffer->index + buffer->entries - 1);

	EFX_POPULATE_OWORD_4(buf_tbl_upd,
264
265
266
267
			     FRF_AZ_BUF_UPD_CMD, 0,
			     FRF_AZ_BUF_CLR_CMD, 1,
			     FRF_AZ_BUF_CLR_END_ID, end,
			     FRF_AZ_BUF_CLR_START_ID, start);
268
	efx_writeo(efx, &buf_tbl_upd, FR_AZ_BUF_TBL_UPD);
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
}

/*
 * Allocate a new Falcon special buffer
 *
 * This allocates memory for a new buffer, clears it and allocates a
 * new buffer ID range.  It does not write into Falcon's buffer table.
 *
 * This call will allocate 4KB buffers, since Falcon can't use 8KB
 * buffers for event queues and descriptor rings.
 */
static int falcon_alloc_special_buffer(struct efx_nic *efx,
				       struct efx_special_buffer *buffer,
				       unsigned int len)
{
	len = ALIGN(len, FALCON_BUF_SIZE);

	buffer->addr = pci_alloc_consistent(efx->pci_dev, len,
					    &buffer->dma_addr);
	if (!buffer->addr)
		return -ENOMEM;
	buffer->len = len;
	buffer->entries = len / FALCON_BUF_SIZE;
	BUG_ON(buffer->dma_addr & (FALCON_BUF_SIZE - 1));

	/* All zeros is a potentially valid event so memset to 0xff */
	memset(buffer->addr, 0xff, len);

	/* Select new buffer ID */
298
299
	buffer->index = efx->next_buffer_table;
	efx->next_buffer_table += buffer->entries;
300
301

	EFX_LOG(efx, "allocating special buffers %d-%d at %llx+%x "
302
		"(virt %p phys %llx)\n", buffer->index,
303
		buffer->index + buffer->entries - 1,
304
305
		(u64)buffer->dma_addr, len,
		buffer->addr, (u64)virt_to_phys(buffer->addr));
306
307
308
309
310
311
312
313
314
315
316

	return 0;
}

static void falcon_free_special_buffer(struct efx_nic *efx,
				       struct efx_special_buffer *buffer)
{
	if (!buffer->addr)
		return;

	EFX_LOG(efx, "deallocating special buffers %d-%d at %llx+%x "
317
		"(virt %p phys %llx)\n", buffer->index,
318
		buffer->index + buffer->entries - 1,
319
320
		(u64)buffer->dma_addr, buffer->len,
		buffer->addr, (u64)virt_to_phys(buffer->addr));
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376

	pci_free_consistent(efx->pci_dev, buffer->len, buffer->addr,
			    buffer->dma_addr);
	buffer->addr = NULL;
	buffer->entries = 0;
}

/**************************************************************************
 *
 * Falcon generic buffer handling
 * These buffers are used for interrupt status and MAC stats
 *
 **************************************************************************/

static int falcon_alloc_buffer(struct efx_nic *efx,
			       struct efx_buffer *buffer, unsigned int len)
{
	buffer->addr = pci_alloc_consistent(efx->pci_dev, len,
					    &buffer->dma_addr);
	if (!buffer->addr)
		return -ENOMEM;
	buffer->len = len;
	memset(buffer->addr, 0, len);
	return 0;
}

static void falcon_free_buffer(struct efx_nic *efx, struct efx_buffer *buffer)
{
	if (buffer->addr) {
		pci_free_consistent(efx->pci_dev, buffer->len,
				    buffer->addr, buffer->dma_addr);
		buffer->addr = NULL;
	}
}

/**************************************************************************
 *
 * Falcon TX path
 *
 **************************************************************************/

/* Returns a pointer to the specified transmit descriptor in the TX
 * descriptor queue belonging to the specified channel.
 */
static inline efx_qword_t *falcon_tx_desc(struct efx_tx_queue *tx_queue,
					       unsigned int index)
{
	return (((efx_qword_t *) (tx_queue->txd.addr)) + index);
}

/* This writes to the TX_DESC_WPTR; write pointer for TX descriptor ring */
static inline void falcon_notify_tx_desc(struct efx_tx_queue *tx_queue)
{
	unsigned write_ptr;
	efx_dword_t reg;

377
	write_ptr = tx_queue->write_count & EFX_TXQ_MASK;
378
	EFX_POPULATE_DWORD_1(reg, FRF_AZ_TX_DESC_WPTR_DWORD, write_ptr);
379
380
	efx_writed_page(tx_queue->efx, &reg,
			FR_AZ_TX_DESC_UPD_DWORD_P0, tx_queue->queue);
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
}


/* For each entry inserted into the software descriptor ring, create a
 * descriptor in the hardware TX descriptor ring (in host memory), and
 * write a doorbell.
 */
void falcon_push_buffers(struct efx_tx_queue *tx_queue)
{

	struct efx_tx_buffer *buffer;
	efx_qword_t *txd;
	unsigned write_ptr;

	BUG_ON(tx_queue->write_count == tx_queue->insert_count);

	do {
398
		write_ptr = tx_queue->write_count & EFX_TXQ_MASK;
399
400
401
402
403
		buffer = &tx_queue->buffer[write_ptr];
		txd = falcon_tx_desc(tx_queue, write_ptr);
		++tx_queue->write_count;

		/* Create TX descriptor ring entry */
404
405
406
407
408
		EFX_POPULATE_QWORD_4(*txd,
				     FSF_AZ_TX_KER_CONT, buffer->continuation,
				     FSF_AZ_TX_KER_BYTE_COUNT, buffer->len,
				     FSF_AZ_TX_KER_BUF_REGION, 0,
				     FSF_AZ_TX_KER_BUF_ADDR, buffer->dma_addr);
409
410
411
412
413
414
415
416
417
418
	} while (tx_queue->write_count != tx_queue->insert_count);

	wmb(); /* Ensure descriptors are written before they are fetched */
	falcon_notify_tx_desc(tx_queue);
}

/* Allocate hardware resources for a TX queue */
int falcon_probe_tx(struct efx_tx_queue *tx_queue)
{
	struct efx_nic *efx = tx_queue->efx;
419
420
	BUILD_BUG_ON(EFX_TXQ_SIZE < 512 || EFX_TXQ_SIZE > 4096 ||
		     EFX_TXQ_SIZE & EFX_TXQ_MASK);
421
	return falcon_alloc_special_buffer(efx, &tx_queue->txd,
422
					   EFX_TXQ_SIZE * sizeof(efx_qword_t));
423
424
}

425
void falcon_init_tx(struct efx_tx_queue *tx_queue)
426
427
428
429
{
	efx_oword_t tx_desc_ptr;
	struct efx_nic *efx = tx_queue->efx;

430
	tx_queue->flushed = FLUSH_NONE;
431

432
	/* Pin TX descriptor ring */
433
	falcon_init_special_buffer(efx, &tx_queue->txd);
434
435
436

	/* Push TX descriptor ring to card */
	EFX_POPULATE_OWORD_10(tx_desc_ptr,
437
438
439
440
441
442
443
444
			      FRF_AZ_TX_DESCQ_EN, 1,
			      FRF_AZ_TX_ISCSI_DDIG_EN, 0,
			      FRF_AZ_TX_ISCSI_HDIG_EN, 0,
			      FRF_AZ_TX_DESCQ_BUF_BASE_ID, tx_queue->txd.index,
			      FRF_AZ_TX_DESCQ_EVQ_ID,
			      tx_queue->channel->channel,
			      FRF_AZ_TX_DESCQ_OWNER_ID, 0,
			      FRF_AZ_TX_DESCQ_LABEL, tx_queue->queue,
445
446
			      FRF_AZ_TX_DESCQ_SIZE,
			      __ffs(tx_queue->txd.entries),
447
448
			      FRF_AZ_TX_DESCQ_TYPE, 0,
			      FRF_BZ_TX_NON_IP_DROP_DIS, 1);
449

450
	if (falcon_rev(efx) >= FALCON_REV_B0) {
451
		int csum = tx_queue->queue == EFX_TX_QUEUE_OFFLOAD_CSUM;
452
453
454
		EFX_SET_OWORD_FIELD(tx_desc_ptr, FRF_BZ_TX_IP_CHKSM_DIS, !csum);
		EFX_SET_OWORD_FIELD(tx_desc_ptr, FRF_BZ_TX_TCP_CHKSM_DIS,
				    !csum);
455
456
	}

457
458
	efx_writeo_table(efx, &tx_desc_ptr, efx->type->txd_ptr_tbl_base,
			 tx_queue->queue);
459

460
	if (falcon_rev(efx) < FALCON_REV_B0) {
461
462
		efx_oword_t reg;

463
464
		/* Only 128 bits in this register */
		BUILD_BUG_ON(EFX_TX_QUEUE_COUNT >= 128);
465

466
		efx_reado(efx, &reg, FR_AA_TX_CHKSM_CFG);
467
		if (tx_queue->queue == EFX_TX_QUEUE_OFFLOAD_CSUM)
468
469
470
			clear_bit_le(tx_queue->queue, (void *)&reg);
		else
			set_bit_le(tx_queue->queue, (void *)&reg);
471
		efx_writeo(efx, &reg, FR_AA_TX_CHKSM_CFG);
472
473
474
	}
}

475
static void falcon_flush_tx_queue(struct efx_tx_queue *tx_queue)
476
477
478
479
{
	struct efx_nic *efx = tx_queue->efx;
	efx_oword_t tx_flush_descq;

480
481
	tx_queue->flushed = FLUSH_PENDING;

482
483
	/* Post a flush command */
	EFX_POPULATE_OWORD_2(tx_flush_descq,
484
485
			     FRF_AZ_TX_FLUSH_DESCQ_CMD, 1,
			     FRF_AZ_TX_FLUSH_DESCQ, tx_queue->queue);
486
	efx_writeo(efx, &tx_flush_descq, FR_AZ_TX_FLUSH_DESCQ);
487
488
489
490
491
492
493
}

void falcon_fini_tx(struct efx_tx_queue *tx_queue)
{
	struct efx_nic *efx = tx_queue->efx;
	efx_oword_t tx_desc_ptr;

494
	/* The queue should have been flushed */
495
	WARN_ON(tx_queue->flushed != FLUSH_DONE);
496
497
498

	/* Remove TX descriptor ring from card */
	EFX_ZERO_OWORD(tx_desc_ptr);
499
500
	efx_writeo_table(efx, &tx_desc_ptr, efx->type->txd_ptr_tbl_base,
			 tx_queue->queue);
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534

	/* Unpin TX descriptor ring */
	falcon_fini_special_buffer(efx, &tx_queue->txd);
}

/* Free buffers backing TX queue */
void falcon_remove_tx(struct efx_tx_queue *tx_queue)
{
	falcon_free_special_buffer(tx_queue->efx, &tx_queue->txd);
}

/**************************************************************************
 *
 * Falcon RX path
 *
 **************************************************************************/

/* Returns a pointer to the specified descriptor in the RX descriptor queue */
static inline efx_qword_t *falcon_rx_desc(struct efx_rx_queue *rx_queue,
					       unsigned int index)
{
	return (((efx_qword_t *) (rx_queue->rxd.addr)) + index);
}

/* This creates an entry in the RX descriptor queue */
static inline void falcon_build_rx_desc(struct efx_rx_queue *rx_queue,
					unsigned index)
{
	struct efx_rx_buffer *rx_buf;
	efx_qword_t *rxd;

	rxd = falcon_rx_desc(rx_queue, index);
	rx_buf = efx_rx_buffer(rx_queue, index);
	EFX_POPULATE_QWORD_3(*rxd,
535
			     FSF_AZ_RX_KER_BUF_SIZE,
536
537
			     rx_buf->len -
			     rx_queue->efx->type->rx_buffer_padding,
538
539
			     FSF_AZ_RX_KER_BUF_REGION, 0,
			     FSF_AZ_RX_KER_BUF_ADDR, rx_buf->dma_addr);
540
541
542
543
544
545
546
547
548
549
550
551
552
}

/* This writes to the RX_DESC_WPTR register for the specified receive
 * descriptor ring.
 */
void falcon_notify_rx_desc(struct efx_rx_queue *rx_queue)
{
	efx_dword_t reg;
	unsigned write_ptr;

	while (rx_queue->notified_count != rx_queue->added_count) {
		falcon_build_rx_desc(rx_queue,
				     rx_queue->notified_count &
553
				     EFX_RXQ_MASK);
554
555
556
557
		++rx_queue->notified_count;
	}

	wmb();
558
	write_ptr = rx_queue->added_count & EFX_RXQ_MASK;
559
	EFX_POPULATE_DWORD_1(reg, FRF_AZ_RX_DESC_WPTR_DWORD, write_ptr);
560
561
	efx_writed_page(rx_queue->efx, &reg,
			FR_AZ_RX_DESC_UPD_DWORD_P0, rx_queue->queue);
562
563
564
565
566
}

int falcon_probe_rx(struct efx_rx_queue *rx_queue)
{
	struct efx_nic *efx = rx_queue->efx;
567
568
	BUILD_BUG_ON(EFX_RXQ_SIZE < 512 || EFX_RXQ_SIZE > 4096 ||
		     EFX_RXQ_SIZE & EFX_RXQ_MASK);
569
	return falcon_alloc_special_buffer(efx, &rx_queue->rxd,
570
					   EFX_RXQ_SIZE * sizeof(efx_qword_t));
571
572
}

573
void falcon_init_rx(struct efx_rx_queue *rx_queue)
574
575
576
{
	efx_oword_t rx_desc_ptr;
	struct efx_nic *efx = rx_queue->efx;
577
578
	bool is_b0 = falcon_rev(efx) >= FALCON_REV_B0;
	bool iscsi_digest_en = is_b0;
579
580
581
582
583

	EFX_LOG(efx, "RX queue %d ring in special buffers %d-%d\n",
		rx_queue->queue, rx_queue->rxd.index,
		rx_queue->rxd.index + rx_queue->rxd.entries - 1);

584
	rx_queue->flushed = FLUSH_NONE;
585

586
	/* Pin RX descriptor ring */
587
	falcon_init_special_buffer(efx, &rx_queue->rxd);
588
589
590

	/* Push RX descriptor ring to card */
	EFX_POPULATE_OWORD_10(rx_desc_ptr,
591
592
593
594
595
596
597
			      FRF_AZ_RX_ISCSI_DDIG_EN, iscsi_digest_en,
			      FRF_AZ_RX_ISCSI_HDIG_EN, iscsi_digest_en,
			      FRF_AZ_RX_DESCQ_BUF_BASE_ID, rx_queue->rxd.index,
			      FRF_AZ_RX_DESCQ_EVQ_ID,
			      rx_queue->channel->channel,
			      FRF_AZ_RX_DESCQ_OWNER_ID, 0,
			      FRF_AZ_RX_DESCQ_LABEL, rx_queue->queue,
598
599
			      FRF_AZ_RX_DESCQ_SIZE,
			      __ffs(rx_queue->rxd.entries),
600
			      FRF_AZ_RX_DESCQ_TYPE, 0 /* kernel queue */ ,
601
			      /* For >=B0 this is scatter so disable */
602
603
			      FRF_AZ_RX_DESCQ_JUMBO, !is_b0,
			      FRF_AZ_RX_DESCQ_EN, 1);
604
605
	efx_writeo_table(efx, &rx_desc_ptr, efx->type->rxd_ptr_tbl_base,
			 rx_queue->queue);
606
607
}

608
static void falcon_flush_rx_queue(struct efx_rx_queue *rx_queue)
609
610
611
612
{
	struct efx_nic *efx = rx_queue->efx;
	efx_oword_t rx_flush_descq;

613
614
	rx_queue->flushed = FLUSH_PENDING;

615
616
	/* Post a flush command */
	EFX_POPULATE_OWORD_2(rx_flush_descq,
617
618
			     FRF_AZ_RX_FLUSH_DESCQ_CMD, 1,
			     FRF_AZ_RX_FLUSH_DESCQ, rx_queue->queue);
619
	efx_writeo(efx, &rx_flush_descq, FR_AZ_RX_FLUSH_DESCQ);
620
621
622
623
624
625
626
}

void falcon_fini_rx(struct efx_rx_queue *rx_queue)
{
	efx_oword_t rx_desc_ptr;
	struct efx_nic *efx = rx_queue->efx;

627
	/* The queue should already have been flushed */
628
	WARN_ON(rx_queue->flushed != FLUSH_DONE);
629
630
631

	/* Remove RX descriptor ring from card */
	EFX_ZERO_OWORD(rx_desc_ptr);
632
633
	efx_writeo_table(efx, &rx_desc_ptr, efx->type->rxd_ptr_tbl_base,
			 rx_queue->queue);
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665

	/* Unpin RX descriptor ring */
	falcon_fini_special_buffer(efx, &rx_queue->rxd);
}

/* Free buffers backing RX queue */
void falcon_remove_rx(struct efx_rx_queue *rx_queue)
{
	falcon_free_special_buffer(rx_queue->efx, &rx_queue->rxd);
}

/**************************************************************************
 *
 * Falcon event queue processing
 * Event queues are processed by per-channel tasklets.
 *
 **************************************************************************/

/* Update a channel's event queue's read pointer (RPTR) register
 *
 * This writes the EVQ_RPTR_REG register for the specified channel's
 * event queue.
 *
 * Note that EVQ_RPTR_REG contains the index of the "last read" event,
 * whereas channel->eventq_read_ptr contains the index of the "next to
 * read" event.
 */
void falcon_eventq_read_ack(struct efx_channel *channel)
{
	efx_dword_t reg;
	struct efx_nic *efx = channel->efx;

666
	EFX_POPULATE_DWORD_1(reg, FRF_AZ_EVQ_RPTR, channel->eventq_read_ptr);
667
	efx_writed_table(efx, &reg, efx->type->evq_rptr_tbl_base,
668
			    channel->channel);
669
670
671
672
673
674
675
}

/* Use HW to insert a SW defined event */
void falcon_generate_event(struct efx_channel *channel, efx_qword_t *event)
{
	efx_oword_t drv_ev_reg;

676
677
678
679
680
681
682
	BUILD_BUG_ON(FRF_AZ_DRV_EV_DATA_LBN != 0 ||
		     FRF_AZ_DRV_EV_DATA_WIDTH != 64);
	drv_ev_reg.u32[0] = event->u32[0];
	drv_ev_reg.u32[1] = event->u32[1];
	drv_ev_reg.u32[2] = 0;
	drv_ev_reg.u32[3] = 0;
	EFX_SET_OWORD_FIELD(drv_ev_reg, FRF_AZ_DRV_EV_QID, channel->channel);
683
	efx_writeo(channel->efx, &drv_ev_reg, FR_AZ_DRV_EV);
684
685
686
687
688
689
690
}

/* Handle a transmit completion event
 *
 * Falcon batches TX completion events; the message we receive is of
 * the form "complete all TX events up to this index".
 */
691
692
static void falcon_handle_tx_event(struct efx_channel *channel,
				   efx_qword_t *event)
693
694
695
696
697
698
{
	unsigned int tx_ev_desc_ptr;
	unsigned int tx_ev_q_label;
	struct efx_tx_queue *tx_queue;
	struct efx_nic *efx = channel->efx;

699
	if (likely(EFX_QWORD_FIELD(*event, FSF_AZ_TX_EV_COMP))) {
700
		/* Transmit completion */
701
702
		tx_ev_desc_ptr = EFX_QWORD_FIELD(*event, FSF_AZ_TX_EV_DESC_PTR);
		tx_ev_q_label = EFX_QWORD_FIELD(*event, FSF_AZ_TX_EV_Q_LABEL);
703
		tx_queue = &efx->tx_queue[tx_ev_q_label];
704
705
		channel->irq_mod_score +=
			(tx_ev_desc_ptr - tx_queue->read_count) &
706
			EFX_TXQ_MASK;
707
		efx_xmit_done(tx_queue, tx_ev_desc_ptr);
708
	} else if (EFX_QWORD_FIELD(*event, FSF_AZ_TX_EV_WQ_FF_FULL)) {
709
		/* Rewrite the FIFO write pointer */
710
		tx_ev_q_label = EFX_QWORD_FIELD(*event, FSF_AZ_TX_EV_Q_LABEL);
711
712
		tx_queue = &efx->tx_queue[tx_ev_q_label];

713
		if (efx_dev_registered(efx))
714
715
			netif_tx_lock(efx->net_dev);
		falcon_notify_tx_desc(tx_queue);
716
		if (efx_dev_registered(efx))
717
			netif_tx_unlock(efx->net_dev);
718
	} else if (EFX_QWORD_FIELD(*event, FSF_AZ_TX_EV_PKT_ERR) &&
719
720
721
722
723
724
725
726
727
728
729
730
		   EFX_WORKAROUND_10727(efx)) {
		efx_schedule_reset(efx, RESET_TYPE_TX_DESC_FETCH);
	} else {
		EFX_ERR(efx, "channel %d unexpected TX event "
			EFX_QWORD_FMT"\n", channel->channel,
			EFX_QWORD_VAL(*event));
	}
}

/* Detect errors included in the rx_evt_pkt_ok bit. */
static void falcon_handle_rx_not_ok(struct efx_rx_queue *rx_queue,
				    const efx_qword_t *event,
731
732
				    bool *rx_ev_pkt_ok,
				    bool *discard)
733
734
{
	struct efx_nic *efx = rx_queue->efx;
735
736
737
738
739
740
	bool rx_ev_buf_owner_id_err, rx_ev_ip_hdr_chksum_err;
	bool rx_ev_tcp_udp_chksum_err, rx_ev_eth_crc_err;
	bool rx_ev_frm_trunc, rx_ev_drib_nib, rx_ev_tobe_disc;
	bool rx_ev_other_err, rx_ev_pause_frm;
	bool rx_ev_ip_frag_err, rx_ev_hdr_type, rx_ev_mcast_pkt;
	unsigned rx_ev_pkt_type;
741

742
743
744
745
	rx_ev_hdr_type = EFX_QWORD_FIELD(*event, FSF_AZ_RX_EV_HDR_TYPE);
	rx_ev_mcast_pkt = EFX_QWORD_FIELD(*event, FSF_AZ_RX_EV_MCAST_PKT);
	rx_ev_tobe_disc = EFX_QWORD_FIELD(*event, FSF_AZ_RX_EV_TOBE_DISC);
	rx_ev_pkt_type = EFX_QWORD_FIELD(*event, FSF_AZ_RX_EV_PKT_TYPE);
746
	rx_ev_buf_owner_id_err = EFX_QWORD_FIELD(*event,
747
748
						 FSF_AZ_RX_EV_BUF_OWNER_ID_ERR);
	rx_ev_ip_frag_err = EFX_QWORD_FIELD(*event, FSF_AZ_RX_EV_IP_FRAG_ERR);
749
	rx_ev_ip_hdr_chksum_err = EFX_QWORD_FIELD(*event,
750
						  FSF_AZ_RX_EV_IP_HDR_CHKSUM_ERR);
751
	rx_ev_tcp_udp_chksum_err = EFX_QWORD_FIELD(*event,
752
753
754
						   FSF_AZ_RX_EV_TCP_UDP_CHKSUM_ERR);
	rx_ev_eth_crc_err = EFX_QWORD_FIELD(*event, FSF_AZ_RX_EV_ETH_CRC_ERR);
	rx_ev_frm_trunc = EFX_QWORD_FIELD(*event, FSF_AZ_RX_EV_FRM_TRUNC);
755
	rx_ev_drib_nib = ((falcon_rev(efx) >= FALCON_REV_B0) ?
756
757
			  0 : EFX_QWORD_FIELD(*event, FSF_AA_RX_EV_DRIB_NIB));
	rx_ev_pause_frm = EFX_QWORD_FIELD(*event, FSF_AZ_RX_EV_PAUSE_FRM_ERR);
758
759
760
761
762
763

	/* Every error apart from tobe_disc and pause_frm */
	rx_ev_other_err = (rx_ev_drib_nib | rx_ev_tcp_udp_chksum_err |
			   rx_ev_buf_owner_id_err | rx_ev_eth_crc_err |
			   rx_ev_frm_trunc | rx_ev_ip_hdr_chksum_err);

764
765
	/* Count errors that are not in MAC stats.  Ignore expected
	 * checksum errors during self-test. */
766
767
768
769
	if (rx_ev_frm_trunc)
		++rx_queue->channel->n_rx_frm_trunc;
	else if (rx_ev_tobe_disc)
		++rx_queue->channel->n_rx_tobe_disc;
770
771
772
773
774
775
	else if (!efx->loopback_selftest) {
		if (rx_ev_ip_hdr_chksum_err)
			++rx_queue->channel->n_rx_ip_hdr_chksum_err;
		else if (rx_ev_tcp_udp_chksum_err)
			++rx_queue->channel->n_rx_tcp_udp_chksum_err;
	}
776
777
778
779
780
781
782
783
784
785
786
787
788
789
	if (rx_ev_ip_frag_err)
		++rx_queue->channel->n_rx_ip_frag_err;

	/* The frame must be discarded if any of these are true. */
	*discard = (rx_ev_eth_crc_err | rx_ev_frm_trunc | rx_ev_drib_nib |
		    rx_ev_tobe_disc | rx_ev_pause_frm);

	/* TOBE_DISC is expected on unicast mismatches; don't print out an
	 * error message.  FRM_TRUNC indicates RXDP dropped the packet due
	 * to a FIFO overflow.
	 */
#ifdef EFX_ENABLE_DEBUG
	if (rx_ev_other_err) {
		EFX_INFO_RL(efx, " RX queue %d unexpected RX event "
790
			    EFX_QWORD_FMT "%s%s%s%s%s%s%s%s\n",
791
792
793
794
795
796
797
798
799
800
			    rx_queue->queue, EFX_QWORD_VAL(*event),
			    rx_ev_buf_owner_id_err ? " [OWNER_ID_ERR]" : "",
			    rx_ev_ip_hdr_chksum_err ?
			    " [IP_HDR_CHKSUM_ERR]" : "",
			    rx_ev_tcp_udp_chksum_err ?
			    " [TCP_UDP_CHKSUM_ERR]" : "",
			    rx_ev_eth_crc_err ? " [ETH_CRC_ERR]" : "",
			    rx_ev_frm_trunc ? " [FRM_TRUNC]" : "",
			    rx_ev_drib_nib ? " [DRIB_NIB]" : "",
			    rx_ev_tobe_disc ? " [TOBE_DISC]" : "",
801
			    rx_ev_pause_frm ? " [PAUSE]" : "");
802
803
804
805
806
807
808
809
810
811
812
	}
#endif
}

/* Handle receive events that are not in-order. */
static void falcon_handle_rx_bad_index(struct efx_rx_queue *rx_queue,
				       unsigned index)
{
	struct efx_nic *efx = rx_queue->efx;
	unsigned expected, dropped;

813
814
	expected = rx_queue->removed_count & EFX_RXQ_MASK;
	dropped = (index - expected) & EFX_RXQ_MASK;
815
816
817
818
819
820
821
822
823
824
825
826
827
828
	EFX_INFO(efx, "dropped %d events (index=%d expected=%d)\n",
		dropped, index, expected);

	efx_schedule_reset(efx, EFX_WORKAROUND_5676(efx) ?
			   RESET_TYPE_RX_RECOVERY : RESET_TYPE_DISABLE);
}

/* Handle a packet received event
 *
 * Falcon silicon gives a "discard" flag if it's a unicast packet with the
 * wrong destination address
 * Also "is multicast" and "matches multicast filter" flags can be used to
 * discard non-matching multicast packets.
 */
829
830
static void falcon_handle_rx_event(struct efx_channel *channel,
				   const efx_qword_t *event)
831
{
832
	unsigned int rx_ev_desc_ptr, rx_ev_byte_cnt;
833
	unsigned int rx_ev_hdr_type, rx_ev_mcast_pkt;
834
	unsigned expected_ptr;
835
	bool rx_ev_pkt_ok, discard = false, checksummed;
836
837
838
839
	struct efx_rx_queue *rx_queue;
	struct efx_nic *efx = channel->efx;

	/* Basic packet information */
840
841
842
843
844
845
846
	rx_ev_byte_cnt = EFX_QWORD_FIELD(*event, FSF_AZ_RX_EV_BYTE_CNT);
	rx_ev_pkt_ok = EFX_QWORD_FIELD(*event, FSF_AZ_RX_EV_PKT_OK);
	rx_ev_hdr_type = EFX_QWORD_FIELD(*event, FSF_AZ_RX_EV_HDR_TYPE);
	WARN_ON(EFX_QWORD_FIELD(*event, FSF_AZ_RX_EV_JUMBO_CONT));
	WARN_ON(EFX_QWORD_FIELD(*event, FSF_AZ_RX_EV_SOP) != 1);
	WARN_ON(EFX_QWORD_FIELD(*event, FSF_AZ_RX_EV_Q_LABEL) !=
		channel->channel);
847

848
	rx_queue = &efx->rx_queue[channel->channel];
849

850
	rx_ev_desc_ptr = EFX_QWORD_FIELD(*event, FSF_AZ_RX_EV_DESC_PTR);
851
	expected_ptr = rx_queue->removed_count & EFX_RXQ_MASK;
852
	if (unlikely(rx_ev_desc_ptr != expected_ptr))
853
854
855
856
857
858
		falcon_handle_rx_bad_index(rx_queue, rx_ev_desc_ptr);

	if (likely(rx_ev_pkt_ok)) {
		/* If packet is marked as OK and packet type is TCP/IPv4 or
		 * UDP/IPv4, then we can rely on the hardware checksum.
		 */
859
		checksummed =
860
861
862
			efx->rx_checksum_enabled &&
			(rx_ev_hdr_type == FSE_AB_RX_EV_HDR_TYPE_IPV4_TCP ||
			 rx_ev_hdr_type == FSE_AB_RX_EV_HDR_TYPE_IPV4_UDP);
863
864
	} else {
		falcon_handle_rx_not_ok(rx_queue, event, &rx_ev_pkt_ok,
865
					&discard);
866
		checksummed = false;
867
868
869
	}

	/* Detect multicast packets that didn't match the filter */
870
	rx_ev_mcast_pkt = EFX_QWORD_FIELD(*event, FSF_AZ_RX_EV_MCAST_PKT);
871
872
	if (rx_ev_mcast_pkt) {
		unsigned int rx_ev_mcast_hash_match =
873
			EFX_QWORD_FIELD(*event, FSF_AZ_RX_EV_MCAST_HASH_MATCH);
874
875

		if (unlikely(!rx_ev_mcast_hash_match))
876
			discard = true;
877
878
	}

879
880
	channel->irq_mod_score += 2;

881
882
883
884
885
886
887
888
889
890
	/* Handle received packet */
	efx_rx_packet(rx_queue, rx_ev_desc_ptr, rx_ev_byte_cnt,
		      checksummed, discard);
}

/* Global events are basically PHY events */
static void falcon_handle_global_event(struct efx_channel *channel,
				       efx_qword_t *event)
{
	struct efx_nic *efx = channel->efx;
891
	bool handled = false;
892

893
894
895
	if (EFX_QWORD_FIELD(*event, FSF_AB_GLB_EV_G_PHY0_INTR) ||
	    EFX_QWORD_FIELD(*event, FSF_AB_GLB_EV_XG_PHY0_INTR) ||
	    EFX_QWORD_FIELD(*event, FSF_AB_GLB_EV_XFP_PHY0_INTR)) {
Steve Hodgson's avatar
Steve Hodgson committed
896
		/* Ignored */
897
898
		handled = true;
	}
899

900
	if ((falcon_rev(efx) >= FALCON_REV_B0) &&
901
	    EFX_QWORD_FIELD(*event, FSF_BB_GLB_EV_XG_MGT_INTR)) {
902
		efx->xmac_poll_required = true;
903
		handled = true;
904
905
	}

906
	if (falcon_rev(efx) <= FALCON_REV_A1 ?
907
908
	    EFX_QWORD_FIELD(*event, FSF_AA_GLB_EV_RX_RECOVERY) :
	    EFX_QWORD_FIELD(*event, FSF_BB_GLB_EV_RX_RECOVERY)) {
909
910
911
912
913
914
		EFX_ERR(efx, "channel %d seen global RX_RESET "
			"event. Resetting.\n", channel->channel);

		atomic_inc(&efx->rx_reset);
		efx_schedule_reset(efx, EFX_WORKAROUND_6555(efx) ?
				   RESET_TYPE_RX_RECOVERY : RESET_TYPE_DISABLE);
915
		handled = true;
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
	}

	if (!handled)
		EFX_ERR(efx, "channel %d unknown global event "
			EFX_QWORD_FMT "\n", channel->channel,
			EFX_QWORD_VAL(*event));
}

static void falcon_handle_driver_event(struct efx_channel *channel,
				       efx_qword_t *event)
{
	struct efx_nic *efx = channel->efx;
	unsigned int ev_sub_code;
	unsigned int ev_sub_data;

931
932
	ev_sub_code = EFX_QWORD_FIELD(*event, FSF_AZ_DRIVER_EV_SUBCODE);
	ev_sub_data = EFX_QWORD_FIELD(*event, FSF_AZ_DRIVER_EV_SUBDATA);
933
934

	switch (ev_sub_code) {
935
	case FSE_AZ_TX_DESCQ_FLS_DONE_EV:
936
937
938
		EFX_TRACE(efx, "channel %d TXQ %d flushed\n",
			  channel->channel, ev_sub_data);
		break;
939
	case FSE_AZ_RX_DESCQ_FLS_DONE_EV:
940
941
942
		EFX_TRACE(efx, "channel %d RXQ %d flushed\n",
			  channel->channel, ev_sub_data);
		break;
943
	case FSE_AZ_EVQ_INIT_DONE_EV:
944
945
946
		EFX_LOG(efx, "channel %d EVQ %d initialised\n",
			channel->channel, ev_sub_data);
		break;
947
	case FSE_AZ_SRM_UPD_DONE_EV:
948
949
950
		EFX_TRACE(efx, "channel %d SRAM update done\n",
			  channel->channel);
		break;
951
	case FSE_AZ_WAKE_UP_EV:
952
953
954
		EFX_TRACE(efx, "channel %d RXQ %d wakeup event\n",
			  channel->channel, ev_sub_data);
		break;
955
	case FSE_AZ_TIMER_EV:
956
957
958
		EFX_TRACE(efx, "channel %d RX queue %d timer expired\n",
			  channel->channel, ev_sub_data);
		break;
959
	case FSE_AA_RX_RECOVER_EV:
960
961
		EFX_ERR(efx, "channel %d seen DRIVER RX_RESET event. "
			"Resetting.\n", channel->channel);
962
		atomic_inc(&efx->rx_reset);
963
964
965
966
967
		efx_schedule_reset(efx,
				   EFX_WORKAROUND_6555(efx) ?
				   RESET_TYPE_RX_RECOVERY :
				   RESET_TYPE_DISABLE);
		break;
968
	case FSE_BZ_RX_DSC_ERROR_EV:
969
970
971
972
		EFX_ERR(efx, "RX DMA Q %d reports descriptor fetch error."
			" RX Q %d is disabled.\n", ev_sub_data, ev_sub_data);
		efx_schedule_reset(efx, RESET_TYPE_RX_DESC_FETCH);
		break;
973
	case FSE_BZ_TX_DSC_ERROR_EV:
974
975
976
977
978
979
980
981
982
983
984
985
		EFX_ERR(efx, "TX DMA Q %d reports descriptor fetch error."
			" TX Q %d is disabled.\n", ev_sub_data, ev_sub_data);
		efx_schedule_reset(efx, RESET_TYPE_TX_DESC_FETCH);
		break;
	default:
		EFX_TRACE(efx, "channel %d unknown driver event code %d "
			  "data %04x\n", channel->channel, ev_sub_code,
			  ev_sub_data);
		break;
	}
}

986
int falcon_process_eventq(struct efx_channel *channel, int rx_quota)
987
988
989
990
{
	unsigned int read_ptr;
	efx_qword_t event, *p_event;
	int ev_code;
991
	int rx_packets = 0;
992
993
994
995
996
997
998
999
1000

	read_ptr = channel->eventq_read_ptr;

	do {
		p_event = falcon_event(channel, read_ptr);
		event = *p_event;

		if (!falcon_event_present(&event))
			/* End of events */