ahci.c 49.9 KB
Newer Older
Linus Torvalds's avatar
Linus Torvalds committed
1
2
3
/*
 *  ahci.c - AHCI SATA support
 *
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
 *  Maintained by:  Jeff Garzik <jgarzik@pobox.com>
 *    		    Please ALWAYS copy linux-ide@vger.kernel.org
 *		    on emails.
 *
 *  Copyright 2004-2005 Red Hat, Inc.
 *
 *
 *  This program is free software; you can redistribute it and/or modify
 *  it under the terms of the GNU General Public License as published by
 *  the Free Software Foundation; either version 2, or (at your option)
 *  any later version.
 *
 *  This program is distributed in the hope that it will be useful,
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *  GNU General Public License for more details.
 *
 *  You should have received a copy of the GNU General Public License
 *  along with this program; see the file COPYING.  If not, write to
 *  the Free Software Foundation, 675 Mass Ave, Cambridge, MA 02139, USA.
 *
 *
 * libata documentation is available via 'make {ps|pdf}docs',
 * as Documentation/DocBook/libata.*
 *
 * AHCI hardware documentation:
Linus Torvalds's avatar
Linus Torvalds committed
30
 * http://www.intel.com/technology/serialata/pdf/rev1_0.pdf
31
 * http://www.intel.com/technology/serialata/pdf/rev1_1.pdf
Linus Torvalds's avatar
Linus Torvalds committed
32
33
34
35
36
37
38
39
40
41
 *
 */

#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/pci.h>
#include <linux/init.h>
#include <linux/blkdev.h>
#include <linux/delay.h>
#include <linux/interrupt.h>
42
#include <linux/dma-mapping.h>
43
#include <linux/device.h>
Linus Torvalds's avatar
Linus Torvalds committed
44
#include <scsi/scsi_host.h>
45
#include <scsi/scsi_cmnd.h>
Linus Torvalds's avatar
Linus Torvalds committed
46
47
48
#include <linux/libata.h>

#define DRV_NAME	"ahci"
49
#define DRV_VERSION	"2.3"
Linus Torvalds's avatar
Linus Torvalds committed
50
51
52
53


enum {
	AHCI_PCI_BAR		= 5,
54
	AHCI_MAX_PORTS		= 32,
Linus Torvalds's avatar
Linus Torvalds committed
55
56
	AHCI_MAX_SG		= 168, /* hardware max is 64K */
	AHCI_DMA_BOUNDARY	= 0xffffffff,
57
	AHCI_USE_CLUSTERING	= 1,
58
	AHCI_MAX_CMDS		= 32,
59
	AHCI_CMD_SZ		= 32,
60
	AHCI_CMD_SLOT_SZ	= AHCI_MAX_CMDS * AHCI_CMD_SZ,
Linus Torvalds's avatar
Linus Torvalds committed
61
	AHCI_RX_FIS_SZ		= 256,
62
	AHCI_CMD_TBL_CDB	= 0x40,
63
64
65
66
	AHCI_CMD_TBL_HDR_SZ	= 0x80,
	AHCI_CMD_TBL_SZ		= AHCI_CMD_TBL_HDR_SZ + (AHCI_MAX_SG * 16),
	AHCI_CMD_TBL_AR_SZ	= AHCI_CMD_TBL_SZ * AHCI_MAX_CMDS,
	AHCI_PORT_PRIV_DMA_SZ	= AHCI_CMD_SLOT_SZ + AHCI_CMD_TBL_AR_SZ +
Linus Torvalds's avatar
Linus Torvalds committed
67
68
69
70
				  AHCI_RX_FIS_SZ,
	AHCI_IRQ_ON_SG		= (1 << 31),
	AHCI_CMD_ATAPI		= (1 << 5),
	AHCI_CMD_WRITE		= (1 << 6),
71
	AHCI_CMD_PREFETCH	= (1 << 7),
72
73
	AHCI_CMD_RESET		= (1 << 8),
	AHCI_CMD_CLR_BUSY	= (1 << 10),
Linus Torvalds's avatar
Linus Torvalds committed
74
75

	RX_FIS_D2H_REG		= 0x40,	/* offset of D2H Register FIS data */
76
	RX_FIS_SDB		= 0x58, /* offset of SDB FIS data */
Tejun Heo's avatar
Tejun Heo committed
77
	RX_FIS_UNK		= 0x60, /* offset of Unknown FIS data */
Linus Torvalds's avatar
Linus Torvalds committed
78
79

	board_ahci		= 0,
80
81
82
	board_ahci_pi		= 1,
	board_ahci_vt8251	= 2,
	board_ahci_ign_iferr	= 3,
83
	board_ahci_sb600	= 4,
84
	board_ahci_mv		= 5,
Linus Torvalds's avatar
Linus Torvalds committed
85
86
87
88
89
90
91
92
93
94
95
96
97
98

	/* global controller registers */
	HOST_CAP		= 0x00, /* host capabilities */
	HOST_CTL		= 0x04, /* global host control */
	HOST_IRQ_STAT		= 0x08, /* interrupt status */
	HOST_PORTS_IMPL		= 0x0c, /* bitmap of implemented ports */
	HOST_VERSION		= 0x10, /* AHCI spec. version compliancy */

	/* HOST_CTL bits */
	HOST_RESET		= (1 << 0),  /* reset controller; self-clear */
	HOST_IRQ_EN		= (1 << 1),  /* global IRQ enable */
	HOST_AHCI_EN		= (1 << 31), /* AHCI enabled */

	/* HOST_CAP bits */
99
	HOST_CAP_SSC		= (1 << 14), /* Slumber capable */
100
	HOST_CAP_CLO		= (1 << 24), /* Command List Override support */
101
	HOST_CAP_SSS		= (1 << 27), /* Staggered Spin-up */
102
	HOST_CAP_SNTF		= (1 << 29), /* SNotification register */
103
	HOST_CAP_NCQ		= (1 << 30), /* Native Command Queueing */
104
	HOST_CAP_64		= (1 << 31), /* PCI DAC (64-bit DMA) support */
Linus Torvalds's avatar
Linus Torvalds committed
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120

	/* registers for each SATA port */
	PORT_LST_ADDR		= 0x00, /* command list DMA addr */
	PORT_LST_ADDR_HI	= 0x04, /* command list DMA addr hi */
	PORT_FIS_ADDR		= 0x08, /* FIS rx buf addr */
	PORT_FIS_ADDR_HI	= 0x0c, /* FIS rx buf addr hi */
	PORT_IRQ_STAT		= 0x10, /* interrupt status */
	PORT_IRQ_MASK		= 0x14, /* interrupt enable/disable mask */
	PORT_CMD		= 0x18, /* port command */
	PORT_TFDATA		= 0x20,	/* taskfile data */
	PORT_SIG		= 0x24,	/* device TF signature */
	PORT_CMD_ISSUE		= 0x38, /* command issue */
	PORT_SCR_STAT		= 0x28, /* SATA phy register: SStatus */
	PORT_SCR_CTL		= 0x2c, /* SATA phy register: SControl */
	PORT_SCR_ERR		= 0x30, /* SATA phy register: SError */
	PORT_SCR_ACT		= 0x34, /* SATA phy register: SActive */
121
	PORT_SCR_NTF		= 0x3c, /* SATA phy register: SNotification */
Linus Torvalds's avatar
Linus Torvalds committed
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142

	/* PORT_IRQ_{STAT,MASK} bits */
	PORT_IRQ_COLD_PRES	= (1 << 31), /* cold presence detect */
	PORT_IRQ_TF_ERR		= (1 << 30), /* task file error */
	PORT_IRQ_HBUS_ERR	= (1 << 29), /* host bus fatal error */
	PORT_IRQ_HBUS_DATA_ERR	= (1 << 28), /* host bus data error */
	PORT_IRQ_IF_ERR		= (1 << 27), /* interface fatal error */
	PORT_IRQ_IF_NONFATAL	= (1 << 26), /* interface non-fatal error */
	PORT_IRQ_OVERFLOW	= (1 << 24), /* xfer exhausted available S/G */
	PORT_IRQ_BAD_PMP	= (1 << 23), /* incorrect port multiplier */

	PORT_IRQ_PHYRDY		= (1 << 22), /* PhyRdy changed */
	PORT_IRQ_DEV_ILCK	= (1 << 7), /* device interlock */
	PORT_IRQ_CONNECT	= (1 << 6), /* port connect change status */
	PORT_IRQ_SG_DONE	= (1 << 5), /* descriptor processed */
	PORT_IRQ_UNK_FIS	= (1 << 4), /* unknown FIS rx'd */
	PORT_IRQ_SDB_FIS	= (1 << 3), /* Set Device Bits FIS rx'd */
	PORT_IRQ_DMAS_FIS	= (1 << 2), /* DMA Setup FIS rx'd */
	PORT_IRQ_PIOS_FIS	= (1 << 1), /* PIO Setup FIS rx'd */
	PORT_IRQ_D2H_REG_FIS	= (1 << 0), /* D2H Register FIS rx'd */

Tejun Heo's avatar
Tejun Heo committed
143
144
145
	PORT_IRQ_FREEZE		= PORT_IRQ_HBUS_ERR |
				  PORT_IRQ_IF_ERR |
				  PORT_IRQ_CONNECT |
146
				  PORT_IRQ_PHYRDY |
Tejun Heo's avatar
Tejun Heo committed
147
148
149
150
151
152
153
				  PORT_IRQ_UNK_FIS,
	PORT_IRQ_ERROR		= PORT_IRQ_FREEZE |
				  PORT_IRQ_TF_ERR |
				  PORT_IRQ_HBUS_DATA_ERR,
	DEF_PORT_IRQ		= PORT_IRQ_ERROR | PORT_IRQ_SG_DONE |
				  PORT_IRQ_SDB_FIS | PORT_IRQ_DMAS_FIS |
				  PORT_IRQ_PIOS_FIS | PORT_IRQ_D2H_REG_FIS,
Linus Torvalds's avatar
Linus Torvalds committed
154
155

	/* PORT_CMD bits */
156
	PORT_CMD_ATAPI		= (1 << 24), /* Device is ATAPI */
Linus Torvalds's avatar
Linus Torvalds committed
157
158
159
	PORT_CMD_LIST_ON	= (1 << 15), /* cmd list DMA engine running */
	PORT_CMD_FIS_ON		= (1 << 14), /* FIS DMA engine running */
	PORT_CMD_FIS_RX		= (1 << 4), /* Enable FIS receive DMA engine */
160
	PORT_CMD_CLO		= (1 << 3), /* Command list override */
Linus Torvalds's avatar
Linus Torvalds committed
161
162
163
164
	PORT_CMD_POWER_ON	= (1 << 2), /* Power up device */
	PORT_CMD_SPIN_UP	= (1 << 1), /* Spin up device */
	PORT_CMD_START		= (1 << 0), /* Enable port DMA engine */

165
	PORT_CMD_ICC_MASK	= (0xf << 28), /* i/f ICC state mask */
Linus Torvalds's avatar
Linus Torvalds committed
166
167
168
	PORT_CMD_ICC_ACTIVE	= (0x1 << 28), /* Put i/f in active state */
	PORT_CMD_ICC_PARTIAL	= (0x2 << 28), /* Put i/f in partial state */
	PORT_CMD_ICC_SLUMBER	= (0x6 << 28), /* Put i/f in slumber state */
169

170
	/* ap->flags bits */
171
172
	AHCI_FLAG_NO_NCQ		= (1 << 24),
	AHCI_FLAG_IGN_IRQ_IF_ERR	= (1 << 25), /* ignore IRQ_IF_ERR */
173
	AHCI_FLAG_HONOR_PI		= (1 << 26), /* honor PORTS_IMPL */
174
	AHCI_FLAG_IGN_SERR_INTERNAL	= (1 << 27), /* ignore SERR_INTERNAL */
Tejun Heo's avatar
Tejun Heo committed
175
	AHCI_FLAG_32BIT_ONLY		= (1 << 28), /* force 32bit */
176
177
	AHCI_FLAG_MV_PATA		= (1 << 29), /* PATA port */
	AHCI_FLAG_NO_MSI		= (1 << 30), /* no PCI MSI */
178
179
180

	AHCI_FLAG_COMMON		= ATA_FLAG_SATA | ATA_FLAG_NO_LEGACY |
					  ATA_FLAG_MMIO | ATA_FLAG_PIO_DMA |
181
182
					  ATA_FLAG_SKIP_D2H_BSY |
					  ATA_FLAG_ACPI_SATA,
Linus Torvalds's avatar
Linus Torvalds committed
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
};

struct ahci_cmd_hdr {
	u32			opts;
	u32			status;
	u32			tbl_addr;
	u32			tbl_addr_hi;
	u32			reserved[4];
};

struct ahci_sg {
	u32			addr;
	u32			addr_hi;
	u32			reserved;
	u32			flags_size;
};

struct ahci_host_priv {
201
202
203
204
	u32			cap;		/* cap to use */
	u32			port_map;	/* port map to use */
	u32			saved_cap;	/* saved initial cap */
	u32			saved_port_map;	/* saved initial port_map */
Linus Torvalds's avatar
Linus Torvalds committed
205
206
207
208
209
210
211
212
213
};

struct ahci_port_priv {
	struct ahci_cmd_hdr	*cmd_slot;
	dma_addr_t		cmd_slot_dma;
	void			*cmd_tbl;
	dma_addr_t		cmd_tbl_dma;
	void			*rx_fis;
	dma_addr_t		rx_fis_dma;
214
215
216
	/* for NCQ spurious interrupt analysis */
	unsigned int		ncq_saw_d2h:1;
	unsigned int		ncq_saw_dmas:1;
217
	unsigned int		ncq_saw_sdb:1;
Linus Torvalds's avatar
Linus Torvalds committed
218
219
};

220
221
static int ahci_scr_read(struct ata_port *ap, unsigned int sc_reg, u32 *val);
static int ahci_scr_write(struct ata_port *ap, unsigned int sc_reg, u32 val);
Linus Torvalds's avatar
Linus Torvalds committed
222
static int ahci_init_one (struct pci_dev *pdev, const struct pci_device_id *ent);
223
static unsigned int ahci_qc_issue(struct ata_queued_cmd *qc);
Linus Torvalds's avatar
Linus Torvalds committed
224
225
226
227
228
229
static void ahci_irq_clear(struct ata_port *ap);
static int ahci_port_start(struct ata_port *ap);
static void ahci_port_stop(struct ata_port *ap);
static void ahci_tf_read(struct ata_port *ap, struct ata_taskfile *tf);
static void ahci_qc_prep(struct ata_queued_cmd *qc);
static u8 ahci_check_status(struct ata_port *ap);
Tejun Heo's avatar
Tejun Heo committed
230
231
232
static void ahci_freeze(struct ata_port *ap);
static void ahci_thaw(struct ata_port *ap);
static void ahci_error_handler(struct ata_port *ap);
233
static void ahci_vt8251_error_handler(struct ata_port *ap);
Tejun Heo's avatar
Tejun Heo committed
234
static void ahci_post_internal_cmd(struct ata_queued_cmd *qc);
235
static int ahci_port_resume(struct ata_port *ap);
236
237
238
static unsigned int ahci_fill_sg(struct ata_queued_cmd *qc, void *cmd_tbl);
static void ahci_fill_cmd_slot(struct ahci_port_priv *pp, unsigned int tag,
			       u32 opts);
239
#ifdef CONFIG_PM
240
241
242
static int ahci_port_suspend(struct ata_port *ap, pm_message_t mesg);
static int ahci_pci_device_suspend(struct pci_dev *pdev, pm_message_t mesg);
static int ahci_pci_device_resume(struct pci_dev *pdev);
243
#endif
Linus Torvalds's avatar
Linus Torvalds committed
244

245
static struct scsi_host_template ahci_sht = {
Linus Torvalds's avatar
Linus Torvalds committed
246
247
248
249
	.module			= THIS_MODULE,
	.name			= DRV_NAME,
	.ioctl			= ata_scsi_ioctl,
	.queuecommand		= ata_scsi_queuecmd,
250
251
	.change_queue_depth	= ata_scsi_change_queue_depth,
	.can_queue		= AHCI_MAX_CMDS - 1,
Linus Torvalds's avatar
Linus Torvalds committed
252
253
254
255
256
257
258
259
	.this_id		= ATA_SHT_THIS_ID,
	.sg_tablesize		= AHCI_MAX_SG,
	.cmd_per_lun		= ATA_SHT_CMD_PER_LUN,
	.emulated		= ATA_SHT_EMULATED,
	.use_clustering		= AHCI_USE_CLUSTERING,
	.proc_name		= DRV_NAME,
	.dma_boundary		= AHCI_DMA_BOUNDARY,
	.slave_configure	= ata_scsi_slave_config,
Tejun Heo's avatar
Tejun Heo committed
260
	.slave_destroy		= ata_scsi_slave_destroy,
Linus Torvalds's avatar
Linus Torvalds committed
261
262
263
	.bios_param		= ata_std_bios_param,
};

264
static const struct ata_port_operations ahci_ops = {
Linus Torvalds's avatar
Linus Torvalds committed
265
266
267
268
269
270
271
272
273
274
275
276
	.port_disable		= ata_port_disable,

	.check_status		= ahci_check_status,
	.check_altstatus	= ahci_check_status,
	.dev_select		= ata_noop_dev_select,

	.tf_read		= ahci_tf_read,

	.qc_prep		= ahci_qc_prep,
	.qc_issue		= ahci_qc_issue,

	.irq_clear		= ahci_irq_clear,
277
278
	.irq_on			= ata_dummy_irq_on,
	.irq_ack		= ata_dummy_irq_ack,
Linus Torvalds's avatar
Linus Torvalds committed
279
280
281
282

	.scr_read		= ahci_scr_read,
	.scr_write		= ahci_scr_write,

Tejun Heo's avatar
Tejun Heo committed
283
284
285
286
287
288
	.freeze			= ahci_freeze,
	.thaw			= ahci_thaw,

	.error_handler		= ahci_error_handler,
	.post_internal_cmd	= ahci_post_internal_cmd,

289
#ifdef CONFIG_PM
290
291
	.port_suspend		= ahci_port_suspend,
	.port_resume		= ahci_port_resume,
292
#endif
293

Linus Torvalds's avatar
Linus Torvalds committed
294
295
296
297
	.port_start		= ahci_port_start,
	.port_stop		= ahci_port_stop,
};

298
299
300
301
302
303
304
305
306
307
308
309
310
static const struct ata_port_operations ahci_vt8251_ops = {
	.port_disable		= ata_port_disable,

	.check_status		= ahci_check_status,
	.check_altstatus	= ahci_check_status,
	.dev_select		= ata_noop_dev_select,

	.tf_read		= ahci_tf_read,

	.qc_prep		= ahci_qc_prep,
	.qc_issue		= ahci_qc_issue,

	.irq_clear		= ahci_irq_clear,
311
312
	.irq_on			= ata_dummy_irq_on,
	.irq_ack		= ata_dummy_irq_ack,
313
314
315
316
317
318
319
320
321
322

	.scr_read		= ahci_scr_read,
	.scr_write		= ahci_scr_write,

	.freeze			= ahci_freeze,
	.thaw			= ahci_thaw,

	.error_handler		= ahci_vt8251_error_handler,
	.post_internal_cmd	= ahci_post_internal_cmd,

323
#ifdef CONFIG_PM
324
325
	.port_suspend		= ahci_port_suspend,
	.port_resume		= ahci_port_resume,
326
#endif
327
328
329
330
331

	.port_start		= ahci_port_start,
	.port_stop		= ahci_port_stop,
};

332
static const struct ata_port_info ahci_port_info[] = {
Linus Torvalds's avatar
Linus Torvalds committed
333
334
	/* board_ahci */
	{
335
		.flags		= AHCI_FLAG_COMMON,
336
		.pio_mask	= 0x1f, /* pio0-4 */
337
		.udma_mask	= ATA_UDMA6,
Linus Torvalds's avatar
Linus Torvalds committed
338
339
		.port_ops	= &ahci_ops,
	},
340
341
	/* board_ahci_pi */
	{
342
		.flags		= AHCI_FLAG_COMMON | AHCI_FLAG_HONOR_PI,
343
		.pio_mask	= 0x1f, /* pio0-4 */
344
		.udma_mask	= ATA_UDMA6,
345
346
		.port_ops	= &ahci_ops,
	},
347
348
	/* board_ahci_vt8251 */
	{
349
350
		.flags		= AHCI_FLAG_COMMON | ATA_FLAG_HRST_TO_RESUME |
				  AHCI_FLAG_NO_NCQ,
351
		.pio_mask	= 0x1f, /* pio0-4 */
352
		.udma_mask	= ATA_UDMA6,
353
		.port_ops	= &ahci_vt8251_ops,
354
	},
355
356
	/* board_ahci_ign_iferr */
	{
357
		.flags		= AHCI_FLAG_COMMON | AHCI_FLAG_IGN_IRQ_IF_ERR,
358
		.pio_mask	= 0x1f, /* pio0-4 */
359
		.udma_mask	= ATA_UDMA6,
360
361
		.port_ops	= &ahci_ops,
	},
362
363
	/* board_ahci_sb600 */
	{
364
		.flags		= AHCI_FLAG_COMMON |
Tejun Heo's avatar
Tejun Heo committed
365
366
				  AHCI_FLAG_IGN_SERR_INTERNAL |
				  AHCI_FLAG_32BIT_ONLY,
367
		.pio_mask	= 0x1f, /* pio0-4 */
368
		.udma_mask	= ATA_UDMA6,
369
370
		.port_ops	= &ahci_ops,
	},
371
372
373
374
375
376
377
378
379
380
381
382
	/* board_ahci_mv */
	{
		.sht		= &ahci_sht,
		.flags		= ATA_FLAG_SATA | ATA_FLAG_NO_LEGACY |
				  ATA_FLAG_MMIO | ATA_FLAG_PIO_DMA |
				  ATA_FLAG_SKIP_D2H_BSY | AHCI_FLAG_HONOR_PI |
				  AHCI_FLAG_NO_NCQ | AHCI_FLAG_NO_MSI |
				  AHCI_FLAG_MV_PATA,
		.pio_mask	= 0x1f, /* pio0-4 */
		.udma_mask	= ATA_UDMA6,
		.port_ops	= &ahci_ops,
	},
Linus Torvalds's avatar
Linus Torvalds committed
383
384
};

385
static const struct pci_device_id ahci_pci_tbl[] = {
386
	/* Intel */
387
388
389
390
391
	{ PCI_VDEVICE(INTEL, 0x2652), board_ahci }, /* ICH6 */
	{ PCI_VDEVICE(INTEL, 0x2653), board_ahci }, /* ICH6M */
	{ PCI_VDEVICE(INTEL, 0x27c1), board_ahci }, /* ICH7 */
	{ PCI_VDEVICE(INTEL, 0x27c5), board_ahci }, /* ICH7M */
	{ PCI_VDEVICE(INTEL, 0x27c3), board_ahci }, /* ICH7R */
392
	{ PCI_VDEVICE(AL, 0x5288), board_ahci_ign_iferr }, /* ULi M5288 */
393
394
395
396
	{ PCI_VDEVICE(INTEL, 0x2681), board_ahci }, /* ESB2 */
	{ PCI_VDEVICE(INTEL, 0x2682), board_ahci }, /* ESB2 */
	{ PCI_VDEVICE(INTEL, 0x2683), board_ahci }, /* ESB2 */
	{ PCI_VDEVICE(INTEL, 0x27c6), board_ahci }, /* ICH7-M DH */
397
398
399
400
401
402
403
404
405
406
407
408
409
	{ PCI_VDEVICE(INTEL, 0x2821), board_ahci_pi }, /* ICH8 */
	{ PCI_VDEVICE(INTEL, 0x2822), board_ahci_pi }, /* ICH8 */
	{ PCI_VDEVICE(INTEL, 0x2824), board_ahci_pi }, /* ICH8 */
	{ PCI_VDEVICE(INTEL, 0x2829), board_ahci_pi }, /* ICH8M */
	{ PCI_VDEVICE(INTEL, 0x282a), board_ahci_pi }, /* ICH8M */
	{ PCI_VDEVICE(INTEL, 0x2922), board_ahci_pi }, /* ICH9 */
	{ PCI_VDEVICE(INTEL, 0x2923), board_ahci_pi }, /* ICH9 */
	{ PCI_VDEVICE(INTEL, 0x2924), board_ahci_pi }, /* ICH9 */
	{ PCI_VDEVICE(INTEL, 0x2925), board_ahci_pi }, /* ICH9 */
	{ PCI_VDEVICE(INTEL, 0x2927), board_ahci_pi }, /* ICH9 */
	{ PCI_VDEVICE(INTEL, 0x2929), board_ahci_pi }, /* ICH9M */
	{ PCI_VDEVICE(INTEL, 0x292a), board_ahci_pi }, /* ICH9M */
	{ PCI_VDEVICE(INTEL, 0x292b), board_ahci_pi }, /* ICH9M */
410
	{ PCI_VDEVICE(INTEL, 0x292c), board_ahci_pi }, /* ICH9M */
411
412
413
	{ PCI_VDEVICE(INTEL, 0x292f), board_ahci_pi }, /* ICH9M */
	{ PCI_VDEVICE(INTEL, 0x294d), board_ahci_pi }, /* ICH9 */
	{ PCI_VDEVICE(INTEL, 0x294e), board_ahci_pi }, /* ICH9M */
414

415
416
417
	/* JMicron 360/1/3/5/6, match class to avoid IDE function */
	{ PCI_VENDOR_ID_JMICRON, PCI_ANY_ID, PCI_ANY_ID, PCI_ANY_ID,
	  PCI_CLASS_STORAGE_SATA_AHCI, 0xffffff, board_ahci_ign_iferr },
418
419

	/* ATI */
420
	{ PCI_VDEVICE(ATI, 0x4380), board_ahci_sb600 }, /* ATI SB600 */
421
422
423
424
425
426
	{ PCI_VDEVICE(ATI, 0x4390), board_ahci_sb600 }, /* ATI SB700/800 */
	{ PCI_VDEVICE(ATI, 0x4391), board_ahci_sb600 }, /* ATI SB700/800 */
	{ PCI_VDEVICE(ATI, 0x4392), board_ahci_sb600 }, /* ATI SB700/800 */
	{ PCI_VDEVICE(ATI, 0x4393), board_ahci_sb600 }, /* ATI SB700/800 */
	{ PCI_VDEVICE(ATI, 0x4394), board_ahci_sb600 }, /* ATI SB700/800 */
	{ PCI_VDEVICE(ATI, 0x4395), board_ahci_sb600 }, /* ATI SB700/800 */
427
428

	/* VIA */
429
	{ PCI_VDEVICE(VIA, 0x3349), board_ahci_vt8251 }, /* VIA VT8251 */
Tejun Heo's avatar
Tejun Heo committed
430
	{ PCI_VDEVICE(VIA, 0x6287), board_ahci_vt8251 }, /* VIA VT8251 */
431
432

	/* NVIDIA */
433
434
435
436
	{ PCI_VDEVICE(NVIDIA, 0x044c), board_ahci },		/* MCP65 */
	{ PCI_VDEVICE(NVIDIA, 0x044d), board_ahci },		/* MCP65 */
	{ PCI_VDEVICE(NVIDIA, 0x044e), board_ahci },		/* MCP65 */
	{ PCI_VDEVICE(NVIDIA, 0x044f), board_ahci },		/* MCP65 */
437
438
439
440
441
442
443
444
	{ PCI_VDEVICE(NVIDIA, 0x045c), board_ahci },		/* MCP65 */
	{ PCI_VDEVICE(NVIDIA, 0x045d), board_ahci },		/* MCP65 */
	{ PCI_VDEVICE(NVIDIA, 0x045e), board_ahci },		/* MCP65 */
	{ PCI_VDEVICE(NVIDIA, 0x045f), board_ahci },		/* MCP65 */
	{ PCI_VDEVICE(NVIDIA, 0x0550), board_ahci },		/* MCP67 */
	{ PCI_VDEVICE(NVIDIA, 0x0551), board_ahci },		/* MCP67 */
	{ PCI_VDEVICE(NVIDIA, 0x0552), board_ahci },		/* MCP67 */
	{ PCI_VDEVICE(NVIDIA, 0x0553), board_ahci },		/* MCP67 */
445
446
447
448
449
450
451
452
	{ PCI_VDEVICE(NVIDIA, 0x0554), board_ahci },		/* MCP67 */
	{ PCI_VDEVICE(NVIDIA, 0x0555), board_ahci },		/* MCP67 */
	{ PCI_VDEVICE(NVIDIA, 0x0556), board_ahci },		/* MCP67 */
	{ PCI_VDEVICE(NVIDIA, 0x0557), board_ahci },		/* MCP67 */
	{ PCI_VDEVICE(NVIDIA, 0x0558), board_ahci },		/* MCP67 */
	{ PCI_VDEVICE(NVIDIA, 0x0559), board_ahci },		/* MCP67 */
	{ PCI_VDEVICE(NVIDIA, 0x055a), board_ahci },		/* MCP67 */
	{ PCI_VDEVICE(NVIDIA, 0x055b), board_ahci },		/* MCP67 */
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
	{ PCI_VDEVICE(NVIDIA, 0x07f0), board_ahci },		/* MCP73 */
	{ PCI_VDEVICE(NVIDIA, 0x07f1), board_ahci },		/* MCP73 */
	{ PCI_VDEVICE(NVIDIA, 0x07f2), board_ahci },		/* MCP73 */
	{ PCI_VDEVICE(NVIDIA, 0x07f3), board_ahci },		/* MCP73 */
	{ PCI_VDEVICE(NVIDIA, 0x07f4), board_ahci },		/* MCP73 */
	{ PCI_VDEVICE(NVIDIA, 0x07f5), board_ahci },		/* MCP73 */
	{ PCI_VDEVICE(NVIDIA, 0x07f6), board_ahci },		/* MCP73 */
	{ PCI_VDEVICE(NVIDIA, 0x07f7), board_ahci },		/* MCP73 */
	{ PCI_VDEVICE(NVIDIA, 0x07f8), board_ahci },		/* MCP73 */
	{ PCI_VDEVICE(NVIDIA, 0x07f9), board_ahci },		/* MCP73 */
	{ PCI_VDEVICE(NVIDIA, 0x07fa), board_ahci },		/* MCP73 */
	{ PCI_VDEVICE(NVIDIA, 0x07fb), board_ahci },		/* MCP73 */
	{ PCI_VDEVICE(NVIDIA, 0x0ad0), board_ahci },		/* MCP77 */
	{ PCI_VDEVICE(NVIDIA, 0x0ad1), board_ahci },		/* MCP77 */
	{ PCI_VDEVICE(NVIDIA, 0x0ad2), board_ahci },		/* MCP77 */
	{ PCI_VDEVICE(NVIDIA, 0x0ad3), board_ahci },		/* MCP77 */
	{ PCI_VDEVICE(NVIDIA, 0x0ad4), board_ahci },		/* MCP77 */
	{ PCI_VDEVICE(NVIDIA, 0x0ad5), board_ahci },		/* MCP77 */
	{ PCI_VDEVICE(NVIDIA, 0x0ad6), board_ahci },		/* MCP77 */
	{ PCI_VDEVICE(NVIDIA, 0x0ad7), board_ahci },		/* MCP77 */
	{ PCI_VDEVICE(NVIDIA, 0x0ad8), board_ahci },		/* MCP77 */
	{ PCI_VDEVICE(NVIDIA, 0x0ad9), board_ahci },		/* MCP77 */
	{ PCI_VDEVICE(NVIDIA, 0x0ada), board_ahci },		/* MCP77 */
	{ PCI_VDEVICE(NVIDIA, 0x0adb), board_ahci },		/* MCP77 */
477

Jeff Garzik's avatar
Jeff Garzik committed
478
	/* SiS */
479
480
481
	{ PCI_VDEVICE(SI, 0x1184), board_ahci }, /* SiS 966 */
	{ PCI_VDEVICE(SI, 0x1185), board_ahci }, /* SiS 966 */
	{ PCI_VDEVICE(SI, 0x0186), board_ahci }, /* SiS 968 */
Jeff Garzik's avatar
Jeff Garzik committed
482

483
484
485
	/* Marvell */
	{ PCI_VDEVICE(MARVELL, 0x6145), board_ahci_mv },	/* 6145 */

486
487
	/* Generic, PCI class code for AHCI */
	{ PCI_ANY_ID, PCI_ANY_ID, PCI_ANY_ID, PCI_ANY_ID,
488
	  PCI_CLASS_STORAGE_SATA_AHCI, 0xffffff, board_ahci },
489

Linus Torvalds's avatar
Linus Torvalds committed
490
491
492
493
494
495
496
497
	{ }	/* terminate list */
};


static struct pci_driver ahci_pci_driver = {
	.name			= DRV_NAME,
	.id_table		= ahci_pci_tbl,
	.probe			= ahci_init_one,
498
	.remove			= ata_pci_remove_one,
499
#ifdef CONFIG_PM
500
501
	.suspend		= ahci_pci_device_suspend,
	.resume			= ahci_pci_device_resume,
502
#endif
Linus Torvalds's avatar
Linus Torvalds committed
503
504
505
};


506
507
508
509
510
static inline int ahci_nr_ports(u32 cap)
{
	return (cap & 0x1f) + 1;
}

511
512
static inline void __iomem *__ahci_port_base(struct ata_host *host,
					     unsigned int port_no)
Linus Torvalds's avatar
Linus Torvalds committed
513
{
514
	void __iomem *mmio = host->iomap[AHCI_PCI_BAR];
515

516
517
518
519
520
521
	return mmio + 0x100 + (port_no * 0x80);
}

static inline void __iomem *ahci_port_base(struct ata_port *ap)
{
	return __ahci_port_base(ap->host, ap->port_no);
Linus Torvalds's avatar
Linus Torvalds committed
522
523
}

524
525
/**
 *	ahci_save_initial_config - Save and fixup initial config values
526
527
528
 *	@pdev: target PCI device
 *	@pi: associated ATA port info
 *	@hpriv: host private area to store config values
529
530
531
532
533
534
535
536
537
538
539
 *
 *	Some registers containing configuration info might be setup by
 *	BIOS and might be cleared on reset.  This function saves the
 *	initial values of those registers into @hpriv such that they
 *	can be restored after controller reset.
 *
 *	If inconsistent, config values are fixed up by this function.
 *
 *	LOCKING:
 *	None.
 */
540
541
542
static void ahci_save_initial_config(struct pci_dev *pdev,
				     const struct ata_port_info *pi,
				     struct ahci_host_priv *hpriv)
543
{
544
	void __iomem *mmio = pcim_iomap_table(pdev)[AHCI_PCI_BAR];
545
	u32 cap, port_map;
546
	int i;
547
548
549
550
551
552
553

	/* Values prefixed with saved_ are written back to host after
	 * reset.  Values without are used for driver operation.
	 */
	hpriv->saved_cap = cap = readl(mmio + HOST_CAP);
	hpriv->saved_port_map = port_map = readl(mmio + HOST_PORTS_IMPL);

554
	/* some chips have errata preventing 64bit use */
Tejun Heo's avatar
Tejun Heo committed
555
556
557
558
559
560
	if ((cap & HOST_CAP_64) && (pi->flags & AHCI_FLAG_32BIT_ONLY)) {
		dev_printk(KERN_INFO, &pdev->dev,
			   "controller can't do 64bit DMA, forcing 32bit\n");
		cap &= ~HOST_CAP_64;
	}

561
562
563
564
565
566
	if ((cap & HOST_CAP_NCQ) && (pi->flags & AHCI_FLAG_NO_NCQ)) {
		dev_printk(KERN_INFO, &pdev->dev,
			   "controller can't do NCQ, turning off CAP_NCQ\n");
		cap &= ~HOST_CAP_NCQ;
	}

567
568
	/* fixup zero port_map */
	if (!port_map) {
Tejun Heo's avatar
Tejun Heo committed
569
		port_map = (1 << ahci_nr_ports(cap)) - 1;
570
		dev_printk(KERN_WARNING, &pdev->dev,
571
572
573
574
575
576
			   "PORTS_IMPL is zero, forcing 0x%x\n", port_map);

		/* write the fixed up value to the PI register */
		hpriv->saved_port_map = port_map;
	}

577
578
579
580
581
582
583
584
585
586
587
588
589
590
	/*
	 * Temporary Marvell 6145 hack: PATA port presence
	 * is asserted through the standard AHCI port
	 * presence register, as bit 4 (counting from 0)
	 */
	if (pi->flags & AHCI_FLAG_MV_PATA) {
		dev_printk(KERN_ERR, &pdev->dev,
			   "MV_AHCI HACK: port_map %x -> %x\n",
			   hpriv->port_map,
			   hpriv->port_map & 0xf);

		port_map &= 0xf;
	}

591
	/* cross check port_map and cap.n_ports */
592
	if (pi->flags & AHCI_FLAG_HONOR_PI) {
593
594
595
596
597
598
599
600
601
602
603
604
605
606
		u32 tmp_port_map = port_map;
		int n_ports = ahci_nr_ports(cap);

		for (i = 0; i < AHCI_MAX_PORTS && n_ports; i++) {
			if (tmp_port_map & (1 << i)) {
				n_ports--;
				tmp_port_map &= ~(1 << i);
			}
		}

		/* Whine if inconsistent.  No need to update cap.
		 * port_map is used to determine number of ports.
		 */
		if (n_ports || tmp_port_map)
607
			dev_printk(KERN_WARNING, &pdev->dev,
608
609
610
611
612
613
614
615
				   "nr_ports (%u) and implemented port map "
				   "(0x%x) don't match\n",
				   ahci_nr_ports(cap), port_map);
	} else {
		/* fabricate port_map from cap.nr_ports */
		port_map = (1 << ahci_nr_ports(cap)) - 1;
	}

616
617
618
619
620
621
622
	/* record values to use during operation */
	hpriv->cap = cap;
	hpriv->port_map = port_map;
}

/**
 *	ahci_restore_initial_config - Restore initial config
623
 *	@host: target ATA host
624
625
626
627
628
629
 *
 *	Restore initial config stored by ahci_save_initial_config().
 *
 *	LOCKING:
 *	None.
 */
630
static void ahci_restore_initial_config(struct ata_host *host)
631
{
632
633
634
	struct ahci_host_priv *hpriv = host->private_data;
	void __iomem *mmio = host->iomap[AHCI_PCI_BAR];

635
636
637
638
639
	writel(hpriv->saved_cap, mmio + HOST_CAP);
	writel(hpriv->saved_port_map, mmio + HOST_PORTS_IMPL);
	(void) readl(mmio + HOST_PORTS_IMPL);	/* flush */
}

640
static unsigned ahci_scr_offset(struct ata_port *ap, unsigned int sc_reg)
Linus Torvalds's avatar
Linus Torvalds committed
641
{
642
643
644
645
646
647
648
649
	static const int offset[] = {
		[SCR_STATUS]		= PORT_SCR_STAT,
		[SCR_CONTROL]		= PORT_SCR_CTL,
		[SCR_ERROR]		= PORT_SCR_ERR,
		[SCR_ACTIVE]		= PORT_SCR_ACT,
		[SCR_NOTIFICATION]	= PORT_SCR_NTF,
	};
	struct ahci_host_priv *hpriv = ap->host->private_data;
Linus Torvalds's avatar
Linus Torvalds committed
650

651
652
653
	if (sc_reg < ARRAY_SIZE(offset) &&
	    (sc_reg != SCR_NOTIFICATION || (hpriv->cap & HOST_CAP_SNTF)))
		return offset[sc_reg];
654
	return 0;
Linus Torvalds's avatar
Linus Torvalds committed
655
656
}

657
static int ahci_scr_read(struct ata_port *ap, unsigned int sc_reg, u32 *val)
Linus Torvalds's avatar
Linus Torvalds committed
658
{
659
660
661
662
663
664
	void __iomem *port_mmio = ahci_port_base(ap);
	int offset = ahci_scr_offset(ap, sc_reg);

	if (offset) {
		*val = readl(port_mmio + offset);
		return 0;
Linus Torvalds's avatar
Linus Torvalds committed
665
	}
666
667
	return -EINVAL;
}
Linus Torvalds's avatar
Linus Torvalds committed
668

669
670
671
672
673
674
675
676
677
678
static int ahci_scr_write(struct ata_port *ap, unsigned int sc_reg, u32 val)
{
	void __iomem *port_mmio = ahci_port_base(ap);
	int offset = ahci_scr_offset(ap, sc_reg);

	if (offset) {
		writel(val, port_mmio + offset);
		return 0;
	}
	return -EINVAL;
Linus Torvalds's avatar
Linus Torvalds committed
679
680
}

681
static void ahci_start_engine(struct ata_port *ap)
682
{
683
	void __iomem *port_mmio = ahci_port_base(ap);
684
685
	u32 tmp;

686
	/* start DMA */
687
	tmp = readl(port_mmio + PORT_CMD);
688
689
690
691
692
	tmp |= PORT_CMD_START;
	writel(tmp, port_mmio + PORT_CMD);
	readl(port_mmio + PORT_CMD); /* flush */
}

693
static int ahci_stop_engine(struct ata_port *ap)
694
{
695
	void __iomem *port_mmio = ahci_port_base(ap);
696
697
698
699
	u32 tmp;

	tmp = readl(port_mmio + PORT_CMD);

700
	/* check if the HBA is idle */
701
702
703
	if ((tmp & (PORT_CMD_START | PORT_CMD_LIST_ON)) == 0)
		return 0;

704
	/* setting HBA to idle */
705
706
707
	tmp &= ~PORT_CMD_START;
	writel(tmp, port_mmio + PORT_CMD);

708
	/* wait for engine to stop. This could be as long as 500 msec */
709
710
	tmp = ata_wait_register(port_mmio + PORT_CMD,
			        PORT_CMD_LIST_ON, PORT_CMD_LIST_ON, 1, 500);
711
	if (tmp & PORT_CMD_LIST_ON)
712
713
714
715
716
		return -EIO;

	return 0;
}

717
static void ahci_start_fis_rx(struct ata_port *ap)
718
{
719
720
721
	void __iomem *port_mmio = ahci_port_base(ap);
	struct ahci_host_priv *hpriv = ap->host->private_data;
	struct ahci_port_priv *pp = ap->private_data;
722
723
724
	u32 tmp;

	/* set FIS registers */
725
726
727
728
	if (hpriv->cap & HOST_CAP_64)
		writel((pp->cmd_slot_dma >> 16) >> 16,
		       port_mmio + PORT_LST_ADDR_HI);
	writel(pp->cmd_slot_dma & 0xffffffff, port_mmio + PORT_LST_ADDR);
729

730
731
732
733
	if (hpriv->cap & HOST_CAP_64)
		writel((pp->rx_fis_dma >> 16) >> 16,
		       port_mmio + PORT_FIS_ADDR_HI);
	writel(pp->rx_fis_dma & 0xffffffff, port_mmio + PORT_FIS_ADDR);
734
735
736
737
738
739
740
741
742
743

	/* enable FIS reception */
	tmp = readl(port_mmio + PORT_CMD);
	tmp |= PORT_CMD_FIS_RX;
	writel(tmp, port_mmio + PORT_CMD);

	/* flush */
	readl(port_mmio + PORT_CMD);
}

744
static int ahci_stop_fis_rx(struct ata_port *ap)
745
{
746
	void __iomem *port_mmio = ahci_port_base(ap);
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
	u32 tmp;

	/* disable FIS reception */
	tmp = readl(port_mmio + PORT_CMD);
	tmp &= ~PORT_CMD_FIS_RX;
	writel(tmp, port_mmio + PORT_CMD);

	/* wait for completion, spec says 500ms, give it 1000 */
	tmp = ata_wait_register(port_mmio + PORT_CMD, PORT_CMD_FIS_ON,
				PORT_CMD_FIS_ON, 10, 1000);
	if (tmp & PORT_CMD_FIS_ON)
		return -EBUSY;

	return 0;
}

763
static void ahci_power_up(struct ata_port *ap)
764
{
765
766
	struct ahci_host_priv *hpriv = ap->host->private_data;
	void __iomem *port_mmio = ahci_port_base(ap);
767
768
769
770
771
	u32 cmd;

	cmd = readl(port_mmio + PORT_CMD) & ~PORT_CMD_ICC_MASK;

	/* spin up device */
772
	if (hpriv->cap & HOST_CAP_SSS) {
773
774
775
776
777
778
779
780
		cmd |= PORT_CMD_SPIN_UP;
		writel(cmd, port_mmio + PORT_CMD);
	}

	/* wake up link */
	writel(cmd | PORT_CMD_ICC_ACTIVE, port_mmio + PORT_CMD);
}

781
#ifdef CONFIG_PM
782
static void ahci_power_down(struct ata_port *ap)
783
{
784
785
	struct ahci_host_priv *hpriv = ap->host->private_data;
	void __iomem *port_mmio = ahci_port_base(ap);
786
787
	u32 cmd, scontrol;

788
	if (!(hpriv->cap & HOST_CAP_SSS))
789
		return;
790

791
792
793
794
	/* put device into listen mode, first set PxSCTL.DET to 0 */
	scontrol = readl(port_mmio + PORT_SCR_CTL);
	scontrol &= ~0xf;
	writel(scontrol, port_mmio + PORT_SCR_CTL);
795

796
797
798
799
	/* then set PxCMD.SUD to 0 */
	cmd = readl(port_mmio + PORT_CMD) & ~PORT_CMD_ICC_MASK;
	cmd &= ~PORT_CMD_SPIN_UP;
	writel(cmd, port_mmio + PORT_CMD);
800
}
801
#endif
802

803
static void ahci_start_port(struct ata_port *ap)
804
805
{
	/* enable FIS reception */
806
	ahci_start_fis_rx(ap);
807
808

	/* enable DMA */
809
	ahci_start_engine(ap);
810
811
}

812
static int ahci_deinit_port(struct ata_port *ap, const char **emsg)
813
814
815
816
{
	int rc;

	/* disable DMA */
817
	rc = ahci_stop_engine(ap);
818
819
820
821
822
823
	if (rc) {
		*emsg = "failed to stop engine";
		return rc;
	}

	/* disable FIS reception */
824
	rc = ahci_stop_fis_rx(ap);
825
826
827
828
829
830
831
832
	if (rc) {
		*emsg = "failed stop FIS RX";
		return rc;
	}

	return 0;
}

833
static int ahci_reset_controller(struct ata_host *host)
834
{
835
836
	struct pci_dev *pdev = to_pci_dev(host->dev);
	void __iomem *mmio = host->iomap[AHCI_PCI_BAR];
837
	u32 tmp;
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852

	/* global controller reset */
	tmp = readl(mmio + HOST_CTL);
	if ((tmp & HOST_RESET) == 0) {
		writel(tmp | HOST_RESET, mmio + HOST_CTL);
		readl(mmio + HOST_CTL); /* flush */
	}

	/* reset must complete within 1 second, or
	 * the hardware should be considered fried.
	 */
	ssleep(1);

	tmp = readl(mmio + HOST_CTL);
	if (tmp & HOST_RESET) {
853
		dev_printk(KERN_ERR, host->dev,
854
855
856
857
			   "controller reset failed (0x%x)\n", tmp);
		return -EIO;
	}

858
	/* turn on AHCI mode */
859
860
	writel(HOST_AHCI_EN, mmio + HOST_CTL);
	(void) readl(mmio + HOST_CTL);	/* flush */
861

862
	/* some registers might be cleared on reset.  restore initial values */
863
	ahci_restore_initial_config(host);
864
865
866
867
868
869
870
871
872
873
874
875
876

	if (pdev->vendor == PCI_VENDOR_ID_INTEL) {
		u16 tmp16;

		/* configure PCS */
		pci_read_config_word(pdev, 0x92, &tmp16);
		tmp16 |= 0xf;
		pci_write_config_word(pdev, 0x92, tmp16);
	}

	return 0;
}

877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
static void ahci_port_init(struct pci_dev *pdev, struct ata_port *ap,
			   int port_no, void __iomem *mmio,
			   void __iomem *port_mmio)
{
	const char *emsg = NULL;
	int rc;
	u32 tmp;

	/* make sure port is not active */
	rc = ahci_deinit_port(ap, &emsg);
	if (rc)
		dev_printk(KERN_WARNING, &pdev->dev,
			   "%s (%d)\n", emsg, rc);

	/* clear SError */
	tmp = readl(port_mmio + PORT_SCR_ERR);
	VPRINTK("PORT_SCR_ERR 0x%x\n", tmp);
	writel(tmp, port_mmio + PORT_SCR_ERR);

	/* clear port IRQ */
	tmp = readl(port_mmio + PORT_IRQ_STAT);
	VPRINTK("PORT_IRQ_STAT 0x%x\n", tmp);
	if (tmp)
		writel(tmp, port_mmio + PORT_IRQ_STAT);

	writel(1 << port_no, mmio + HOST_IRQ_STAT);
}

905
static void ahci_init_controller(struct ata_host *host)
906
{
907
908
	struct pci_dev *pdev = to_pci_dev(host->dev);
	void __iomem *mmio = host->iomap[AHCI_PCI_BAR];
909
	int i;
910
	void __iomem *port_mmio;
911
912
	u32 tmp;

913
914
915
916
917
918
919
920
921
922
923
924
	if (host->ports[0]->flags & AHCI_FLAG_MV_PATA) {
		port_mmio = __ahci_port_base(host, 4);

		writel(0, port_mmio + PORT_IRQ_MASK);

		/* clear port IRQ */
		tmp = readl(port_mmio + PORT_IRQ_STAT);
		VPRINTK("PORT_IRQ_STAT 0x%x\n", tmp);
		if (tmp)
			writel(tmp, port_mmio + PORT_IRQ_STAT);
	}

925
926
	for (i = 0; i < host->n_ports; i++) {
		struct ata_port *ap = host->ports[i];
927

928
		port_mmio = ahci_port_base(ap);
929
		if (ata_port_is_dummy(ap))
930
931
			continue;

932
		ahci_port_init(pdev, ap, i, mmio, port_mmio);
933
934
935
936
937
938
939
940
941
	}

	tmp = readl(mmio + HOST_CTL);
	VPRINTK("HOST_CTL 0x%x\n", tmp);
	writel(tmp | HOST_IRQ_EN, mmio + HOST_CTL);
	tmp = readl(mmio + HOST_CTL);
	VPRINTK("HOST_CTL 0x%x\n", tmp);
}

942
static unsigned int ahci_dev_classify(struct ata_port *ap)
Linus Torvalds's avatar
Linus Torvalds committed
943
{
944
	void __iomem *port_mmio = ahci_port_base(ap);
Linus Torvalds's avatar
Linus Torvalds committed
945
	struct ata_taskfile tf;
946
947
948
949
950
951
952
953
954
955
956
	u32 tmp;

	tmp = readl(port_mmio + PORT_SIG);
	tf.lbah		= (tmp >> 24)	& 0xff;
	tf.lbam		= (tmp >> 16)	& 0xff;
	tf.lbal		= (tmp >> 8)	& 0xff;
	tf.nsect	= (tmp)		& 0xff;

	return ata_dev_classify(&tf);
}

957
958
static void ahci_fill_cmd_slot(struct ahci_port_priv *pp, unsigned int tag,
			       u32 opts)
959
{
960
961
962
963
964
965
966
967
	dma_addr_t cmd_tbl_dma;

	cmd_tbl_dma = pp->cmd_tbl_dma + tag * AHCI_CMD_TBL_SZ;

	pp->cmd_slot[tag].opts = cpu_to_le32(opts);
	pp->cmd_slot[tag].status = 0;
	pp->cmd_slot[tag].tbl_addr = cpu_to_le32(cmd_tbl_dma & 0xffffffff);
	pp->cmd_slot[tag].tbl_addr_hi = cpu_to_le32((cmd_tbl_dma >> 16) >> 16);
968
969
}

970
static int ahci_kick_engine(struct ata_port *ap, int force_restart)
Tejun Heo's avatar
Tejun Heo committed
971
{
Tejun Heo's avatar
Tejun Heo committed
972
	void __iomem *port_mmio = ap->ioaddr.cmd_addr;
Jeff Garzik's avatar
Jeff Garzik committed
973
	struct ahci_host_priv *hpriv = ap->host->private_data;
974
	u32 tmp;
975
	int busy, rc;
976

977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
	/* do we need to kick the port? */
	busy = ahci_check_status(ap) & (ATA_BUSY | ATA_DRQ);
	if (!busy && !force_restart)
		return 0;

	/* stop engine */
	rc = ahci_stop_engine(ap);
	if (rc)
		goto out_restart;

	/* need to do CLO? */
	if (!busy) {
		rc = 0;
		goto out_restart;
	}

	if (!(hpriv->cap & HOST_CAP_CLO)) {
		rc = -EOPNOTSUPP;
		goto out_restart;
	}
997

998
	/* perform CLO */
999
1000
1001
1002
	tmp = readl(port_mmio + PORT_CMD);
	tmp |= PORT_CMD_CLO;
	writel(tmp, port_mmio + PORT_CMD);

1003
	rc = 0;
1004
1005
1006
	tmp = ata_wait_register(port_mmio + PORT_CMD,
				PORT_CMD_CLO, PORT_CMD_CLO, 1, 500);
	if (tmp & PORT_CMD_CLO)
1007
		rc = -EIO;
1008

1009
1010
1011
1012
	/* restart engine */
 out_restart:
	ahci_start_engine(ap);
	return rc;
1013
1014
}

1015
1016
1017
static int ahci_exec_polled_cmd(struct ata_port *ap, int pmp,
				struct ata_taskfile *tf, int is_cmd, u16 flags,
				unsigned long timeout_msec)
1018
{
1019
	const u32 cmd_fis_len = 5; /* five dwords */
Tejun Heo's avatar
Tejun Heo committed
1020
	struct ahci_port_priv *pp = ap->private_data;
1021
	void __iomem *port_mmio = ahci_port_base(ap);
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
	u8 *fis = pp->cmd_tbl;
	u32 tmp;

	/* prep the command */
	ata_tf_to_fis(tf, pmp, is_cmd, fis);
	ahci_fill_cmd_slot(pp, 0, cmd_fis_len | flags | (pmp << 12));

	/* issue & wait */
	writel(1, port_mmio + PORT_CMD_ISSUE);

	if (timeout_msec) {
		tmp = ata_wait_register(port_mmio + PORT_CMD_ISSUE, 0x1, 0x1,
					1, timeout_msec);
		if (tmp & 0x1) {
			ahci_kick_engine(ap, 1);
			return -EBUSY;
		}
	} else
		readl(port_mmio + PORT_CMD_ISSUE);	/* flush */

	return 0;
}

1045
1046
static int ahci_do_softreset(struct ata_port *ap, unsigned int *class,
			     int pmp, unsigned long deadline)
1047
{
Tejun Heo's avatar
Tejun Heo committed
1048
	const char *reason = NULL;
1049
	unsigned long now, msecs;
Tejun Heo's avatar
Tejun Heo committed
1050
1051
1052
1053
1054
	struct ata_taskfile tf;
	int rc;

	DPRINTK("ENTER\n");

1055
	if (ata_link_offline(&ap->link)) {
1056
1057
1058
1059
1060
		DPRINTK("PHY reports no device\n");
		*class = ATA_DEV_NONE;
		return 0;
	}

Tejun Heo's avatar
Tejun Heo committed
1061
	/* prepare for SRST (AHCI-1.1 10.4.1) */
1062
1063
1064
1065
	rc = ahci_kick_engine(ap, 1);
	if (rc)
		ata_port_printk(ap, KERN_WARNING,
				"failed to reset engine (errno=%d)", rc);
Tejun Heo's avatar
Tejun Heo committed
1066

Tejun Heo's avatar
Tejun Heo committed
1067
	ata_tf_init(ap->link.device, &tf);
Tejun Heo's avatar
Tejun Heo committed
1068
1069

	/* issue the first D2H Register FIS */
1070
1071
1072
1073
1074
	msecs = 0;
	now = jiffies;
	if (time_after(now, deadline))
		msecs = jiffies_to_msecs(deadline - now);

Tejun Heo's avatar
Tejun Heo committed
1075
	tf.ctl |= ATA_SRST;
1076
	if (ahci_exec_polled_cmd(ap, pmp, &tf, 0,
1077
				 AHCI_CMD_RESET | AHCI_CMD_CLR_BUSY, msecs)) {
Tejun Heo's avatar
Tejun Heo committed
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
		rc = -EIO;
		reason = "1st FIS failed";
		goto fail;
	}

	/* spec says at least 5us, but be generous and sleep for 1ms */
	msleep(1);

	/* issue the second D2H Register FIS */
	tf.ctl &= ~ATA_SRST;
1088
	ahci_exec_polled_cmd(ap, pmp, &tf, 0, 0, 0);
Tejun Heo's avatar
Tejun Heo committed
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099

	/* spec mandates ">= 2ms" before checking status.
	 * We wait 150ms, because that was the magic delay used for
	 * ATAPI devices in Hale Landis's ATADRVR, for the period of time
	 * between when the ATA command register is written, and then
	 * status is checked.  Because waiting for "a while" before
	 * checking status is fine, post SRST, we perform this magic
	 * delay here as well.
	 */
	msleep(150);

1100
1101
1102
1103
1104
	rc = ata_wait_ready(ap, deadline);
	/* link occupied, -ENODEV too is an error */
	if (rc) {
		reason = "device not ready";
		goto fail;
Tejun Heo's avatar
Tejun Heo committed
1105
	}
1106
	*class = ahci_dev_classify(ap);
Tejun Heo's avatar
Tejun Heo committed
1107
1108
1109
1110
1111

	DPRINTK("EXIT, class=%u\n", *class);
	return 0;

 fail:
1112
	ata_port_printk(ap, KERN_ERR, "softreset failed (%s)\n", reason);
Tejun Heo's avatar
Tejun Heo committed
1113
1114
1115
	return rc;
}

1116
1117
1118
1119
1120
1121
static int ahci_softreset(struct ata_port *ap, unsigned int *class,
			  unsigned long deadline)
{
	return ahci_do_softreset(ap, class, 0, deadline);
}

1122
1123
static int ahci_hardreset(struct ata_port *ap, unsigned int *class,
			  unsigned long deadline)
1124
{
1125
1126
1127
	struct ahci_port_priv *pp = ap->private_data;
	u8 *d2h_fis = pp->rx_fis + RX_FIS_D2H_REG;
	struct ata_taskfile tf;
1128
1129
1130
	int rc;

	DPRINTK("ENTER\n");
Linus Torvalds's avatar
Linus Torvalds committed
1131

1132
	ahci_stop_engine(ap);
1133
1134

	/* clear D2H reception area to properly wait for D2H FIS */
Tejun Heo's avatar
Tejun Heo committed
1135
	ata_tf_init(ap->link.device, &tf);
1136
	tf.command = 0x80;
1137
	ata_tf_to_fis(&tf, 0, 0, d2h_fis);
1138

1139
	rc = sata_std_hardreset(ap, class, deadline);
1140

1141
	ahci_start_engine(ap);
Linus Torvalds's avatar
Linus Torvalds committed
1142

1143
	if (rc == 0 && ata_link_online(&ap->link))
1144
1145
1146
		*class = ahci_dev_classify(ap);
	if (*class == ATA_DEV_UNKNOWN)
		*class = ATA_DEV_NONE;
Linus Torvalds's avatar
Linus Torvalds committed
1147

1148
1149
1150
1151
	DPRINTK("EXIT, rc=%d, class=%u\n", rc, *class);
	return rc;
}

1152
1153
static int ahci_vt8251_hardreset(struct ata_port *ap, unsigned int *class,
				 unsigned long deadline)
1154
{
1155
	u32 serror;
1156
1157
1158
1159
	int rc;

	DPRINTK("ENTER\n");

1160
	ahci_stop_engine(ap);
1161

Tejun Heo's avatar
Tejun Heo committed
1162
	rc = sata_port_hardreset(ap, sata_ehc_deb_timing(&ap->link.eh_context),
1163
				 deadline);
1164
1165

	/* vt8251 needs SError cleared for the port to operate */
1166
1167
	ahci_scr_read(ap, SCR_ERROR, &serror);
	ahci_scr_write(ap, SCR_ERROR, serror);
1168

1169
	ahci_start_engine(ap);
1170
1171
1172
1173
1174
1175
1176
1177
1178

	DPRINTK("EXIT, rc=%d, class=%u\n", rc, *class);

	/* vt8251 doesn't clear BSY on signature FIS reception,
	 * request follow-up softreset.
	 */
	return rc ?: -EAGAIN;
}

1179
1180
static void ahci_postreset(struct ata_port *ap, unsigned int *class)
{
1181
	void __iomem *port_mmio = ahci_port_base(ap);
1182
1183
1184
	u32 new_tmp, tmp;

	ata_std_postreset(ap, class);
1185
1186
1187

	/* Make sure port's ATAPI bit is set appropriately */
	new_tmp = tmp = readl(port_mmio + PORT_CMD);
1188
	if (*class == ATA_DEV_ATAPI)
1189
1190
1191
1192
1193
1194
1195
		new_tmp |= PORT_CMD_ATAPI;
	else
		new_tmp &= ~PORT_CMD_ATAPI;
	if (new_tmp != tmp) {
		writel(new_tmp, port_mmio + PORT_CMD);
		readl(port_mmio + PORT_CMD); /* flush */
	}
Linus Torvalds's avatar
Linus Torvalds committed
1196
1197
1198
1199
}

static u8 ahci_check_status(struct ata_port *ap)
{
Tejun Heo's avatar
Tejun Heo committed
1200
	void __iomem *mmio = ap->ioaddr.cmd_addr;