ahci.c 62.3 KB
Newer Older
Linus Torvalds's avatar
Linus Torvalds committed
1
2
3
/*
 *  ahci.c - AHCI SATA support
 *
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
 *  Maintained by:  Jeff Garzik <jgarzik@pobox.com>
 *    		    Please ALWAYS copy linux-ide@vger.kernel.org
 *		    on emails.
 *
 *  Copyright 2004-2005 Red Hat, Inc.
 *
 *
 *  This program is free software; you can redistribute it and/or modify
 *  it under the terms of the GNU General Public License as published by
 *  the Free Software Foundation; either version 2, or (at your option)
 *  any later version.
 *
 *  This program is distributed in the hope that it will be useful,
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *  GNU General Public License for more details.
 *
 *  You should have received a copy of the GNU General Public License
 *  along with this program; see the file COPYING.  If not, write to
 *  the Free Software Foundation, 675 Mass Ave, Cambridge, MA 02139, USA.
 *
 *
 * libata documentation is available via 'make {ps|pdf}docs',
 * as Documentation/DocBook/libata.*
 *
 * AHCI hardware documentation:
Linus Torvalds's avatar
Linus Torvalds committed
30
 * http://www.intel.com/technology/serialata/pdf/rev1_0.pdf
31
 * http://www.intel.com/technology/serialata/pdf/rev1_1.pdf
Linus Torvalds's avatar
Linus Torvalds committed
32
33
34
35
36
37
38
39
40
41
 *
 */

#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/pci.h>
#include <linux/init.h>
#include <linux/blkdev.h>
#include <linux/delay.h>
#include <linux/interrupt.h>
42
#include <linux/dma-mapping.h>
43
#include <linux/device.h>
44
#include <linux/dmi.h>
Linus Torvalds's avatar
Linus Torvalds committed
45
#include <scsi/scsi_host.h>
46
#include <scsi/scsi_cmnd.h>
Linus Torvalds's avatar
Linus Torvalds committed
47
48
49
#include <linux/libata.h>

#define DRV_NAME	"ahci"
Tejun Heo's avatar
Tejun Heo committed
50
#define DRV_VERSION	"3.0"
Linus Torvalds's avatar
Linus Torvalds committed
51

52
53
54
static int ahci_enable_alpm(struct ata_port *ap,
		enum link_pm policy);
static void ahci_disable_alpm(struct ata_port *ap);
Linus Torvalds's avatar
Linus Torvalds committed
55
56
57

enum {
	AHCI_PCI_BAR		= 5,
58
	AHCI_MAX_PORTS		= 32,
Linus Torvalds's avatar
Linus Torvalds committed
59
60
	AHCI_MAX_SG		= 168, /* hardware max is 64K */
	AHCI_DMA_BOUNDARY	= 0xffffffff,
61
	AHCI_USE_CLUSTERING	= 1,
62
	AHCI_MAX_CMDS		= 32,
63
	AHCI_CMD_SZ		= 32,
64
	AHCI_CMD_SLOT_SZ	= AHCI_MAX_CMDS * AHCI_CMD_SZ,
Linus Torvalds's avatar
Linus Torvalds committed
65
	AHCI_RX_FIS_SZ		= 256,
66
	AHCI_CMD_TBL_CDB	= 0x40,
67
68
69
70
	AHCI_CMD_TBL_HDR_SZ	= 0x80,
	AHCI_CMD_TBL_SZ		= AHCI_CMD_TBL_HDR_SZ + (AHCI_MAX_SG * 16),
	AHCI_CMD_TBL_AR_SZ	= AHCI_CMD_TBL_SZ * AHCI_MAX_CMDS,
	AHCI_PORT_PRIV_DMA_SZ	= AHCI_CMD_SLOT_SZ + AHCI_CMD_TBL_AR_SZ +
Linus Torvalds's avatar
Linus Torvalds committed
71
72
73
74
				  AHCI_RX_FIS_SZ,
	AHCI_IRQ_ON_SG		= (1 << 31),
	AHCI_CMD_ATAPI		= (1 << 5),
	AHCI_CMD_WRITE		= (1 << 6),
75
	AHCI_CMD_PREFETCH	= (1 << 7),
76
77
	AHCI_CMD_RESET		= (1 << 8),
	AHCI_CMD_CLR_BUSY	= (1 << 10),
Linus Torvalds's avatar
Linus Torvalds committed
78
79

	RX_FIS_D2H_REG		= 0x40,	/* offset of D2H Register FIS data */
80
	RX_FIS_SDB		= 0x58, /* offset of SDB FIS data */
Tejun Heo's avatar
Tejun Heo committed
81
	RX_FIS_UNK		= 0x60, /* offset of Unknown FIS data */
Linus Torvalds's avatar
Linus Torvalds committed
82
83

	board_ahci		= 0,
84
85
86
87
	board_ahci_vt8251	= 1,
	board_ahci_ign_iferr	= 2,
	board_ahci_sb600	= 3,
	board_ahci_mv		= 4,
Linus Torvalds's avatar
Linus Torvalds committed
88
89
90
91
92
93
94
95
96
97
98
99
100
101

	/* global controller registers */
	HOST_CAP		= 0x00, /* host capabilities */
	HOST_CTL		= 0x04, /* global host control */
	HOST_IRQ_STAT		= 0x08, /* interrupt status */
	HOST_PORTS_IMPL		= 0x0c, /* bitmap of implemented ports */
	HOST_VERSION		= 0x10, /* AHCI spec. version compliancy */

	/* HOST_CTL bits */
	HOST_RESET		= (1 << 0),  /* reset controller; self-clear */
	HOST_IRQ_EN		= (1 << 1),  /* global IRQ enable */
	HOST_AHCI_EN		= (1 << 31), /* AHCI enabled */

	/* HOST_CAP bits */
102
	HOST_CAP_SSC		= (1 << 14), /* Slumber capable */
Tejun Heo's avatar
Tejun Heo committed
103
	HOST_CAP_PMP		= (1 << 17), /* Port Multiplier support */
104
	HOST_CAP_CLO		= (1 << 24), /* Command List Override support */
105
	HOST_CAP_ALPM		= (1 << 26), /* Aggressive Link PM support */
106
	HOST_CAP_SSS		= (1 << 27), /* Staggered Spin-up */
107
	HOST_CAP_SNTF		= (1 << 29), /* SNotification register */
108
	HOST_CAP_NCQ		= (1 << 30), /* Native Command Queueing */
109
	HOST_CAP_64		= (1 << 31), /* PCI DAC (64-bit DMA) support */
Linus Torvalds's avatar
Linus Torvalds committed
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125

	/* registers for each SATA port */
	PORT_LST_ADDR		= 0x00, /* command list DMA addr */
	PORT_LST_ADDR_HI	= 0x04, /* command list DMA addr hi */
	PORT_FIS_ADDR		= 0x08, /* FIS rx buf addr */
	PORT_FIS_ADDR_HI	= 0x0c, /* FIS rx buf addr hi */
	PORT_IRQ_STAT		= 0x10, /* interrupt status */
	PORT_IRQ_MASK		= 0x14, /* interrupt enable/disable mask */
	PORT_CMD		= 0x18, /* port command */
	PORT_TFDATA		= 0x20,	/* taskfile data */
	PORT_SIG		= 0x24,	/* device TF signature */
	PORT_CMD_ISSUE		= 0x38, /* command issue */
	PORT_SCR_STAT		= 0x28, /* SATA phy register: SStatus */
	PORT_SCR_CTL		= 0x2c, /* SATA phy register: SControl */
	PORT_SCR_ERR		= 0x30, /* SATA phy register: SError */
	PORT_SCR_ACT		= 0x34, /* SATA phy register: SActive */
126
	PORT_SCR_NTF		= 0x3c, /* SATA phy register: SNotification */
Linus Torvalds's avatar
Linus Torvalds committed
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147

	/* PORT_IRQ_{STAT,MASK} bits */
	PORT_IRQ_COLD_PRES	= (1 << 31), /* cold presence detect */
	PORT_IRQ_TF_ERR		= (1 << 30), /* task file error */
	PORT_IRQ_HBUS_ERR	= (1 << 29), /* host bus fatal error */
	PORT_IRQ_HBUS_DATA_ERR	= (1 << 28), /* host bus data error */
	PORT_IRQ_IF_ERR		= (1 << 27), /* interface fatal error */
	PORT_IRQ_IF_NONFATAL	= (1 << 26), /* interface non-fatal error */
	PORT_IRQ_OVERFLOW	= (1 << 24), /* xfer exhausted available S/G */
	PORT_IRQ_BAD_PMP	= (1 << 23), /* incorrect port multiplier */

	PORT_IRQ_PHYRDY		= (1 << 22), /* PhyRdy changed */
	PORT_IRQ_DEV_ILCK	= (1 << 7), /* device interlock */
	PORT_IRQ_CONNECT	= (1 << 6), /* port connect change status */
	PORT_IRQ_SG_DONE	= (1 << 5), /* descriptor processed */
	PORT_IRQ_UNK_FIS	= (1 << 4), /* unknown FIS rx'd */
	PORT_IRQ_SDB_FIS	= (1 << 3), /* Set Device Bits FIS rx'd */
	PORT_IRQ_DMAS_FIS	= (1 << 2), /* DMA Setup FIS rx'd */
	PORT_IRQ_PIOS_FIS	= (1 << 1), /* PIO Setup FIS rx'd */
	PORT_IRQ_D2H_REG_FIS	= (1 << 0), /* D2H Register FIS rx'd */

Tejun Heo's avatar
Tejun Heo committed
148
149
150
	PORT_IRQ_FREEZE		= PORT_IRQ_HBUS_ERR |
				  PORT_IRQ_IF_ERR |
				  PORT_IRQ_CONNECT |
151
				  PORT_IRQ_PHYRDY |
Tejun Heo's avatar
Tejun Heo committed
152
153
				  PORT_IRQ_UNK_FIS |
				  PORT_IRQ_BAD_PMP,
Tejun Heo's avatar
Tejun Heo committed
154
155
156
157
158
159
	PORT_IRQ_ERROR		= PORT_IRQ_FREEZE |
				  PORT_IRQ_TF_ERR |
				  PORT_IRQ_HBUS_DATA_ERR,
	DEF_PORT_IRQ		= PORT_IRQ_ERROR | PORT_IRQ_SG_DONE |
				  PORT_IRQ_SDB_FIS | PORT_IRQ_DMAS_FIS |
				  PORT_IRQ_PIOS_FIS | PORT_IRQ_D2H_REG_FIS,
Linus Torvalds's avatar
Linus Torvalds committed
160
161

	/* PORT_CMD bits */
162
163
	PORT_CMD_ASP		= (1 << 27), /* Aggressive Slumber/Partial */
	PORT_CMD_ALPE		= (1 << 26), /* Aggressive Link PM enable */
164
	PORT_CMD_ATAPI		= (1 << 24), /* Device is ATAPI */
Tejun Heo's avatar
Tejun Heo committed
165
	PORT_CMD_PMP		= (1 << 17), /* PMP attached */
Linus Torvalds's avatar
Linus Torvalds committed
166
167
168
	PORT_CMD_LIST_ON	= (1 << 15), /* cmd list DMA engine running */
	PORT_CMD_FIS_ON		= (1 << 14), /* FIS DMA engine running */
	PORT_CMD_FIS_RX		= (1 << 4), /* Enable FIS receive DMA engine */
169
	PORT_CMD_CLO		= (1 << 3), /* Command list override */
Linus Torvalds's avatar
Linus Torvalds committed
170
171
172
173
	PORT_CMD_POWER_ON	= (1 << 2), /* Power up device */
	PORT_CMD_SPIN_UP	= (1 << 1), /* Spin up device */
	PORT_CMD_START		= (1 << 0), /* Enable port DMA engine */

174
	PORT_CMD_ICC_MASK	= (0xf << 28), /* i/f ICC state mask */
Linus Torvalds's avatar
Linus Torvalds committed
175
176
177
	PORT_CMD_ICC_ACTIVE	= (0x1 << 28), /* Put i/f in active state */
	PORT_CMD_ICC_PARTIAL	= (0x2 << 28), /* Put i/f in partial state */
	PORT_CMD_ICC_SLUMBER	= (0x6 << 28), /* Put i/f in slumber state */
178

179
180
181
182
183
184
185
	/* hpriv->flags bits */
	AHCI_HFLAG_NO_NCQ		= (1 << 0),
	AHCI_HFLAG_IGN_IRQ_IF_ERR	= (1 << 1), /* ignore IRQ_IF_ERR */
	AHCI_HFLAG_IGN_SERR_INTERNAL	= (1 << 2), /* ignore SERR_INTERNAL */
	AHCI_HFLAG_32BIT_ONLY		= (1 << 3), /* force 32bit */
	AHCI_HFLAG_MV_PATA		= (1 << 4), /* PATA port */
	AHCI_HFLAG_NO_MSI		= (1 << 5), /* no PCI MSI */
Tejun Heo's avatar
Tejun Heo committed
186
	AHCI_HFLAG_NO_PMP		= (1 << 6), /* no PMP */
187
	AHCI_HFLAG_NO_HOTPLUG		= (1 << 7), /* ignore PxSERR.DIAG.N */
188

189
	/* ap->flags bits */
190
191
192

	AHCI_FLAG_COMMON		= ATA_FLAG_SATA | ATA_FLAG_NO_LEGACY |
					  ATA_FLAG_MMIO | ATA_FLAG_PIO_DMA |
193
194
					  ATA_FLAG_ACPI_SATA | ATA_FLAG_AN |
					  ATA_FLAG_IPM,
195
	AHCI_LFLAG_COMMON		= ATA_LFLAG_SKIP_D2H_BSY,
196
197

	ICH_MAP				= 0x90, /* ICH MAP register */
Linus Torvalds's avatar
Linus Torvalds committed
198
199
200
};

struct ahci_cmd_hdr {
Al Viro's avatar
Al Viro committed
201
202
203
204
205
	__le32			opts;
	__le32			status;
	__le32			tbl_addr;
	__le32			tbl_addr_hi;
	__le32			reserved[4];
Linus Torvalds's avatar
Linus Torvalds committed
206
207
208
};

struct ahci_sg {
Al Viro's avatar
Al Viro committed
209
210
211
212
	__le32			addr;
	__le32			addr_hi;
	__le32			reserved;
	__le32			flags_size;
Linus Torvalds's avatar
Linus Torvalds committed
213
214
215
};

struct ahci_host_priv {
216
	unsigned int		flags;		/* AHCI_HFLAG_* */
217
218
219
220
	u32			cap;		/* cap to use */
	u32			port_map;	/* port map to use */
	u32			saved_cap;	/* saved initial cap */
	u32			saved_port_map;	/* saved initial port_map */
Linus Torvalds's avatar
Linus Torvalds committed
221
222
223
};

struct ahci_port_priv {
Tejun Heo's avatar
Tejun Heo committed
224
	struct ata_link		*active_link;
Linus Torvalds's avatar
Linus Torvalds committed
225
226
227
228
229
230
	struct ahci_cmd_hdr	*cmd_slot;
	dma_addr_t		cmd_slot_dma;
	void			*cmd_tbl;
	dma_addr_t		cmd_tbl_dma;
	void			*rx_fis;
	dma_addr_t		rx_fis_dma;
231
232
233
	/* for NCQ spurious interrupt analysis */
	unsigned int		ncq_saw_d2h:1;
	unsigned int		ncq_saw_dmas:1;
234
	unsigned int		ncq_saw_sdb:1;
235
	u32 			intr_mask;	/* interrupts to enable */
Linus Torvalds's avatar
Linus Torvalds committed
236
237
};

238
239
static int ahci_scr_read(struct ata_port *ap, unsigned int sc_reg, u32 *val);
static int ahci_scr_write(struct ata_port *ap, unsigned int sc_reg, u32 val);
240
static int ahci_init_one(struct pci_dev *pdev, const struct pci_device_id *ent);
241
static unsigned int ahci_qc_issue(struct ata_queued_cmd *qc);
Linus Torvalds's avatar
Linus Torvalds committed
242
243
244
245
246
247
static void ahci_irq_clear(struct ata_port *ap);
static int ahci_port_start(struct ata_port *ap);
static void ahci_port_stop(struct ata_port *ap);
static void ahci_tf_read(struct ata_port *ap, struct ata_taskfile *tf);
static void ahci_qc_prep(struct ata_queued_cmd *qc);
static u8 ahci_check_status(struct ata_port *ap);
Tejun Heo's avatar
Tejun Heo committed
248
249
static void ahci_freeze(struct ata_port *ap);
static void ahci_thaw(struct ata_port *ap);
Tejun Heo's avatar
Tejun Heo committed
250
251
static void ahci_pmp_attach(struct ata_port *ap);
static void ahci_pmp_detach(struct ata_port *ap);
Tejun Heo's avatar
Tejun Heo committed
252
static void ahci_error_handler(struct ata_port *ap);
253
static void ahci_vt8251_error_handler(struct ata_port *ap);
254
static void ahci_p5wdh_error_handler(struct ata_port *ap);
Tejun Heo's avatar
Tejun Heo committed
255
static void ahci_post_internal_cmd(struct ata_queued_cmd *qc);
256
static int ahci_port_resume(struct ata_port *ap);
257
258
259
static unsigned int ahci_fill_sg(struct ata_queued_cmd *qc, void *cmd_tbl);
static void ahci_fill_cmd_slot(struct ahci_port_priv *pp, unsigned int tag,
			       u32 opts);
260
#ifdef CONFIG_PM
261
262
263
static int ahci_port_suspend(struct ata_port *ap, pm_message_t mesg);
static int ahci_pci_device_suspend(struct pci_dev *pdev, pm_message_t mesg);
static int ahci_pci_device_resume(struct pci_dev *pdev);
264
#endif
Linus Torvalds's avatar
Linus Torvalds committed
265

266
267
268
269
270
static struct class_device_attribute *ahci_shost_attrs[] = {
	&class_device_attr_link_power_management_policy,
	NULL
};

271
static struct scsi_host_template ahci_sht = {
Linus Torvalds's avatar
Linus Torvalds committed
272
273
274
275
	.module			= THIS_MODULE,
	.name			= DRV_NAME,
	.ioctl			= ata_scsi_ioctl,
	.queuecommand		= ata_scsi_queuecmd,
276
277
	.change_queue_depth	= ata_scsi_change_queue_depth,
	.can_queue		= AHCI_MAX_CMDS - 1,
Linus Torvalds's avatar
Linus Torvalds committed
278
279
280
281
282
283
284
285
	.this_id		= ATA_SHT_THIS_ID,
	.sg_tablesize		= AHCI_MAX_SG,
	.cmd_per_lun		= ATA_SHT_CMD_PER_LUN,
	.emulated		= ATA_SHT_EMULATED,
	.use_clustering		= AHCI_USE_CLUSTERING,
	.proc_name		= DRV_NAME,
	.dma_boundary		= AHCI_DMA_BOUNDARY,
	.slave_configure	= ata_scsi_slave_config,
Tejun Heo's avatar
Tejun Heo committed
286
	.slave_destroy		= ata_scsi_slave_destroy,
Linus Torvalds's avatar
Linus Torvalds committed
287
	.bios_param		= ata_std_bios_param,
288
	.shost_attrs		= ahci_shost_attrs,
Linus Torvalds's avatar
Linus Torvalds committed
289
290
};

291
static const struct ata_port_operations ahci_ops = {
Linus Torvalds's avatar
Linus Torvalds committed
292
293
294
295
296
297
	.check_status		= ahci_check_status,
	.check_altstatus	= ahci_check_status,
	.dev_select		= ata_noop_dev_select,

	.tf_read		= ahci_tf_read,

Tejun Heo's avatar
Tejun Heo committed
298
	.qc_defer		= sata_pmp_qc_defer_cmd_switch,
Linus Torvalds's avatar
Linus Torvalds committed
299
300
301
302
303
304
305
306
	.qc_prep		= ahci_qc_prep,
	.qc_issue		= ahci_qc_issue,

	.irq_clear		= ahci_irq_clear,

	.scr_read		= ahci_scr_read,
	.scr_write		= ahci_scr_write,

Tejun Heo's avatar
Tejun Heo committed
307
308
309
310
311
312
	.freeze			= ahci_freeze,
	.thaw			= ahci_thaw,

	.error_handler		= ahci_error_handler,
	.post_internal_cmd	= ahci_post_internal_cmd,

Tejun Heo's avatar
Tejun Heo committed
313
314
315
	.pmp_attach		= ahci_pmp_attach,
	.pmp_detach		= ahci_pmp_detach,

316
#ifdef CONFIG_PM
317
318
	.port_suspend		= ahci_port_suspend,
	.port_resume		= ahci_port_resume,
319
#endif
320
321
	.enable_pm		= ahci_enable_alpm,
	.disable_pm		= ahci_disable_alpm,
322

Linus Torvalds's avatar
Linus Torvalds committed
323
324
325
326
	.port_start		= ahci_port_start,
	.port_stop		= ahci_port_stop,
};

327
328
329
330
331
332
333
static const struct ata_port_operations ahci_vt8251_ops = {
	.check_status		= ahci_check_status,
	.check_altstatus	= ahci_check_status,
	.dev_select		= ata_noop_dev_select,

	.tf_read		= ahci_tf_read,

Tejun Heo's avatar
Tejun Heo committed
334
	.qc_defer		= sata_pmp_qc_defer_cmd_switch,
335
336
337
338
339
340
341
342
343
344
345
346
347
348
	.qc_prep		= ahci_qc_prep,
	.qc_issue		= ahci_qc_issue,

	.irq_clear		= ahci_irq_clear,

	.scr_read		= ahci_scr_read,
	.scr_write		= ahci_scr_write,

	.freeze			= ahci_freeze,
	.thaw			= ahci_thaw,

	.error_handler		= ahci_vt8251_error_handler,
	.post_internal_cmd	= ahci_post_internal_cmd,

Tejun Heo's avatar
Tejun Heo committed
349
350
351
	.pmp_attach		= ahci_pmp_attach,
	.pmp_detach		= ahci_pmp_detach,

352
#ifdef CONFIG_PM
353
354
	.port_suspend		= ahci_port_suspend,
	.port_resume		= ahci_port_resume,
355
#endif
356
357
358
359
360

	.port_start		= ahci_port_start,
	.port_stop		= ahci_port_stop,
};

361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
static const struct ata_port_operations ahci_p5wdh_ops = {
	.check_status		= ahci_check_status,
	.check_altstatus	= ahci_check_status,
	.dev_select		= ata_noop_dev_select,

	.tf_read		= ahci_tf_read,

	.qc_defer		= sata_pmp_qc_defer_cmd_switch,
	.qc_prep		= ahci_qc_prep,
	.qc_issue		= ahci_qc_issue,

	.irq_clear		= ahci_irq_clear,

	.scr_read		= ahci_scr_read,
	.scr_write		= ahci_scr_write,

	.freeze			= ahci_freeze,
	.thaw			= ahci_thaw,

	.error_handler		= ahci_p5wdh_error_handler,
	.post_internal_cmd	= ahci_post_internal_cmd,

	.pmp_attach		= ahci_pmp_attach,
	.pmp_detach		= ahci_pmp_detach,

#ifdef CONFIG_PM
	.port_suspend		= ahci_port_suspend,
	.port_resume		= ahci_port_resume,
#endif

	.port_start		= ahci_port_start,
	.port_stop		= ahci_port_stop,
};

395
396
#define AHCI_HFLAGS(flags)	.private_data	= (void *)(flags)

397
static const struct ata_port_info ahci_port_info[] = {
Linus Torvalds's avatar
Linus Torvalds committed
398
399
	/* board_ahci */
	{
400
		.flags		= AHCI_FLAG_COMMON,
401
		.link_flags	= AHCI_LFLAG_COMMON,
402
		.pio_mask	= 0x1f, /* pio0-4 */
403
		.udma_mask	= ATA_UDMA6,
Linus Torvalds's avatar
Linus Torvalds committed
404
405
		.port_ops	= &ahci_ops,
	},
406
407
	/* board_ahci_vt8251 */
	{
Tejun Heo's avatar
Tejun Heo committed
408
		AHCI_HFLAGS	(AHCI_HFLAG_NO_NCQ | AHCI_HFLAG_NO_PMP),
409
		.flags		= AHCI_FLAG_COMMON,
410
		.link_flags	= AHCI_LFLAG_COMMON | ATA_LFLAG_HRST_TO_RESUME,
411
		.pio_mask	= 0x1f, /* pio0-4 */
412
		.udma_mask	= ATA_UDMA6,
413
		.port_ops	= &ahci_vt8251_ops,
414
	},
415
416
	/* board_ahci_ign_iferr */
	{
417
418
		AHCI_HFLAGS	(AHCI_HFLAG_IGN_IRQ_IF_ERR),
		.flags		= AHCI_FLAG_COMMON,
419
		.link_flags	= AHCI_LFLAG_COMMON,
420
		.pio_mask	= 0x1f, /* pio0-4 */
421
		.udma_mask	= ATA_UDMA6,
422
423
		.port_ops	= &ahci_ops,
	},
424
425
	/* board_ahci_sb600 */
	{
426
		AHCI_HFLAGS	(AHCI_HFLAG_IGN_SERR_INTERNAL |
Tejun Heo's avatar
Tejun Heo committed
427
				 AHCI_HFLAG_32BIT_ONLY | AHCI_HFLAG_NO_PMP),
428
		.flags		= AHCI_FLAG_COMMON,
429
		.link_flags	= AHCI_LFLAG_COMMON,
430
		.pio_mask	= 0x1f, /* pio0-4 */
431
		.udma_mask	= ATA_UDMA6,
432
433
		.port_ops	= &ahci_ops,
	},
434
435
	/* board_ahci_mv */
	{
436
437
		AHCI_HFLAGS	(AHCI_HFLAG_NO_NCQ | AHCI_HFLAG_NO_MSI |
				 AHCI_HFLAG_MV_PATA),
438
		.flags		= ATA_FLAG_SATA | ATA_FLAG_NO_LEGACY |
439
				  ATA_FLAG_MMIO | ATA_FLAG_PIO_DMA,
440
		.link_flags	= AHCI_LFLAG_COMMON,
441
442
443
444
		.pio_mask	= 0x1f, /* pio0-4 */
		.udma_mask	= ATA_UDMA6,
		.port_ops	= &ahci_ops,
	},
Linus Torvalds's avatar
Linus Torvalds committed
445
446
};

447
static const struct pci_device_id ahci_pci_tbl[] = {
448
	/* Intel */
449
450
451
452
453
	{ PCI_VDEVICE(INTEL, 0x2652), board_ahci }, /* ICH6 */
	{ PCI_VDEVICE(INTEL, 0x2653), board_ahci }, /* ICH6M */
	{ PCI_VDEVICE(INTEL, 0x27c1), board_ahci }, /* ICH7 */
	{ PCI_VDEVICE(INTEL, 0x27c5), board_ahci }, /* ICH7M */
	{ PCI_VDEVICE(INTEL, 0x27c3), board_ahci }, /* ICH7R */
454
	{ PCI_VDEVICE(AL, 0x5288), board_ahci_ign_iferr }, /* ULi M5288 */
455
456
457
458
	{ PCI_VDEVICE(INTEL, 0x2681), board_ahci }, /* ESB2 */
	{ PCI_VDEVICE(INTEL, 0x2682), board_ahci }, /* ESB2 */
	{ PCI_VDEVICE(INTEL, 0x2683), board_ahci }, /* ESB2 */
	{ PCI_VDEVICE(INTEL, 0x27c6), board_ahci }, /* ICH7-M DH */
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
	{ PCI_VDEVICE(INTEL, 0x2821), board_ahci }, /* ICH8 */
	{ PCI_VDEVICE(INTEL, 0x2822), board_ahci }, /* ICH8 */
	{ PCI_VDEVICE(INTEL, 0x2824), board_ahci }, /* ICH8 */
	{ PCI_VDEVICE(INTEL, 0x2829), board_ahci }, /* ICH8M */
	{ PCI_VDEVICE(INTEL, 0x282a), board_ahci }, /* ICH8M */
	{ PCI_VDEVICE(INTEL, 0x2922), board_ahci }, /* ICH9 */
	{ PCI_VDEVICE(INTEL, 0x2923), board_ahci }, /* ICH9 */
	{ PCI_VDEVICE(INTEL, 0x2924), board_ahci }, /* ICH9 */
	{ PCI_VDEVICE(INTEL, 0x2925), board_ahci }, /* ICH9 */
	{ PCI_VDEVICE(INTEL, 0x2927), board_ahci }, /* ICH9 */
	{ PCI_VDEVICE(INTEL, 0x2929), board_ahci }, /* ICH9M */
	{ PCI_VDEVICE(INTEL, 0x292a), board_ahci }, /* ICH9M */
	{ PCI_VDEVICE(INTEL, 0x292b), board_ahci }, /* ICH9M */
	{ PCI_VDEVICE(INTEL, 0x292c), board_ahci }, /* ICH9M */
	{ PCI_VDEVICE(INTEL, 0x292f), board_ahci }, /* ICH9M */
	{ PCI_VDEVICE(INTEL, 0x294d), board_ahci }, /* ICH9 */
	{ PCI_VDEVICE(INTEL, 0x294e), board_ahci }, /* ICH9M */
476
477
	{ PCI_VDEVICE(INTEL, 0x502a), board_ahci }, /* Tolapai */
	{ PCI_VDEVICE(INTEL, 0x502b), board_ahci }, /* Tolapai */
478

479
480
481
	/* JMicron 360/1/3/5/6, match class to avoid IDE function */
	{ PCI_VENDOR_ID_JMICRON, PCI_ANY_ID, PCI_ANY_ID, PCI_ANY_ID,
	  PCI_CLASS_STORAGE_SATA_AHCI, 0xffffff, board_ahci_ign_iferr },
482
483

	/* ATI */
484
	{ PCI_VDEVICE(ATI, 0x4380), board_ahci_sb600 }, /* ATI SB600 */
485
486
487
488
489
490
	{ PCI_VDEVICE(ATI, 0x4390), board_ahci_sb600 }, /* ATI SB700/800 */
	{ PCI_VDEVICE(ATI, 0x4391), board_ahci_sb600 }, /* ATI SB700/800 */
	{ PCI_VDEVICE(ATI, 0x4392), board_ahci_sb600 }, /* ATI SB700/800 */
	{ PCI_VDEVICE(ATI, 0x4393), board_ahci_sb600 }, /* ATI SB700/800 */
	{ PCI_VDEVICE(ATI, 0x4394), board_ahci_sb600 }, /* ATI SB700/800 */
	{ PCI_VDEVICE(ATI, 0x4395), board_ahci_sb600 }, /* ATI SB700/800 */
491
492

	/* VIA */
493
	{ PCI_VDEVICE(VIA, 0x3349), board_ahci_vt8251 }, /* VIA VT8251 */
Tejun Heo's avatar
Tejun Heo committed
494
	{ PCI_VDEVICE(VIA, 0x6287), board_ahci_vt8251 }, /* VIA VT8251 */
495
496

	/* NVIDIA */
497
498
499
500
	{ PCI_VDEVICE(NVIDIA, 0x044c), board_ahci },		/* MCP65 */
	{ PCI_VDEVICE(NVIDIA, 0x044d), board_ahci },		/* MCP65 */
	{ PCI_VDEVICE(NVIDIA, 0x044e), board_ahci },		/* MCP65 */
	{ PCI_VDEVICE(NVIDIA, 0x044f), board_ahci },		/* MCP65 */
501
502
503
504
505
506
507
508
	{ PCI_VDEVICE(NVIDIA, 0x045c), board_ahci },		/* MCP65 */
	{ PCI_VDEVICE(NVIDIA, 0x045d), board_ahci },		/* MCP65 */
	{ PCI_VDEVICE(NVIDIA, 0x045e), board_ahci },		/* MCP65 */
	{ PCI_VDEVICE(NVIDIA, 0x045f), board_ahci },		/* MCP65 */
	{ PCI_VDEVICE(NVIDIA, 0x0550), board_ahci },		/* MCP67 */
	{ PCI_VDEVICE(NVIDIA, 0x0551), board_ahci },		/* MCP67 */
	{ PCI_VDEVICE(NVIDIA, 0x0552), board_ahci },		/* MCP67 */
	{ PCI_VDEVICE(NVIDIA, 0x0553), board_ahci },		/* MCP67 */
509
510
511
512
513
514
515
516
	{ PCI_VDEVICE(NVIDIA, 0x0554), board_ahci },		/* MCP67 */
	{ PCI_VDEVICE(NVIDIA, 0x0555), board_ahci },		/* MCP67 */
	{ PCI_VDEVICE(NVIDIA, 0x0556), board_ahci },		/* MCP67 */
	{ PCI_VDEVICE(NVIDIA, 0x0557), board_ahci },		/* MCP67 */
	{ PCI_VDEVICE(NVIDIA, 0x0558), board_ahci },		/* MCP67 */
	{ PCI_VDEVICE(NVIDIA, 0x0559), board_ahci },		/* MCP67 */
	{ PCI_VDEVICE(NVIDIA, 0x055a), board_ahci },		/* MCP67 */
	{ PCI_VDEVICE(NVIDIA, 0x055b), board_ahci },		/* MCP67 */
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
	{ PCI_VDEVICE(NVIDIA, 0x07f0), board_ahci },		/* MCP73 */
	{ PCI_VDEVICE(NVIDIA, 0x07f1), board_ahci },		/* MCP73 */
	{ PCI_VDEVICE(NVIDIA, 0x07f2), board_ahci },		/* MCP73 */
	{ PCI_VDEVICE(NVIDIA, 0x07f3), board_ahci },		/* MCP73 */
	{ PCI_VDEVICE(NVIDIA, 0x07f4), board_ahci },		/* MCP73 */
	{ PCI_VDEVICE(NVIDIA, 0x07f5), board_ahci },		/* MCP73 */
	{ PCI_VDEVICE(NVIDIA, 0x07f6), board_ahci },		/* MCP73 */
	{ PCI_VDEVICE(NVIDIA, 0x07f7), board_ahci },		/* MCP73 */
	{ PCI_VDEVICE(NVIDIA, 0x07f8), board_ahci },		/* MCP73 */
	{ PCI_VDEVICE(NVIDIA, 0x07f9), board_ahci },		/* MCP73 */
	{ PCI_VDEVICE(NVIDIA, 0x07fa), board_ahci },		/* MCP73 */
	{ PCI_VDEVICE(NVIDIA, 0x07fb), board_ahci },		/* MCP73 */
	{ PCI_VDEVICE(NVIDIA, 0x0ad0), board_ahci },		/* MCP77 */
	{ PCI_VDEVICE(NVIDIA, 0x0ad1), board_ahci },		/* MCP77 */
	{ PCI_VDEVICE(NVIDIA, 0x0ad2), board_ahci },		/* MCP77 */
	{ PCI_VDEVICE(NVIDIA, 0x0ad3), board_ahci },		/* MCP77 */
	{ PCI_VDEVICE(NVIDIA, 0x0ad4), board_ahci },		/* MCP77 */
	{ PCI_VDEVICE(NVIDIA, 0x0ad5), board_ahci },		/* MCP77 */
	{ PCI_VDEVICE(NVIDIA, 0x0ad6), board_ahci },		/* MCP77 */
	{ PCI_VDEVICE(NVIDIA, 0x0ad7), board_ahci },		/* MCP77 */
	{ PCI_VDEVICE(NVIDIA, 0x0ad8), board_ahci },		/* MCP77 */
	{ PCI_VDEVICE(NVIDIA, 0x0ad9), board_ahci },		/* MCP77 */
	{ PCI_VDEVICE(NVIDIA, 0x0ada), board_ahci },		/* MCP77 */
	{ PCI_VDEVICE(NVIDIA, 0x0adb), board_ahci },		/* MCP77 */
541
542
543
544
	{ PCI_VDEVICE(NVIDIA, 0x0ab4), board_ahci },		/* MCP79 */
	{ PCI_VDEVICE(NVIDIA, 0x0ab5), board_ahci },		/* MCP79 */
	{ PCI_VDEVICE(NVIDIA, 0x0ab6), board_ahci },		/* MCP79 */
	{ PCI_VDEVICE(NVIDIA, 0x0ab7), board_ahci },		/* MCP79 */
545
546
547
548
549
550
551
552
	{ PCI_VDEVICE(NVIDIA, 0x0ab8), board_ahci },		/* MCP79 */
	{ PCI_VDEVICE(NVIDIA, 0x0ab9), board_ahci },		/* MCP79 */
	{ PCI_VDEVICE(NVIDIA, 0x0aba), board_ahci },		/* MCP79 */
	{ PCI_VDEVICE(NVIDIA, 0x0abb), board_ahci },		/* MCP79 */
	{ PCI_VDEVICE(NVIDIA, 0x0abc), board_ahci },		/* MCP79 */
	{ PCI_VDEVICE(NVIDIA, 0x0abd), board_ahci },		/* MCP79 */
	{ PCI_VDEVICE(NVIDIA, 0x0abe), board_ahci },		/* MCP79 */
	{ PCI_VDEVICE(NVIDIA, 0x0abf), board_ahci },		/* MCP79 */
553

Jeff Garzik's avatar
Jeff Garzik committed
554
	/* SiS */
555
556
557
	{ PCI_VDEVICE(SI, 0x1184), board_ahci }, /* SiS 966 */
	{ PCI_VDEVICE(SI, 0x1185), board_ahci }, /* SiS 966 */
	{ PCI_VDEVICE(SI, 0x0186), board_ahci }, /* SiS 968 */
Jeff Garzik's avatar
Jeff Garzik committed
558

559
560
561
	/* Marvell */
	{ PCI_VDEVICE(MARVELL, 0x6145), board_ahci_mv },	/* 6145 */

562
563
	/* Generic, PCI class code for AHCI */
	{ PCI_ANY_ID, PCI_ANY_ID, PCI_ANY_ID, PCI_ANY_ID,
564
	  PCI_CLASS_STORAGE_SATA_AHCI, 0xffffff, board_ahci },
565

Linus Torvalds's avatar
Linus Torvalds committed
566
567
568
569
570
571
572
573
	{ }	/* terminate list */
};


static struct pci_driver ahci_pci_driver = {
	.name			= DRV_NAME,
	.id_table		= ahci_pci_tbl,
	.probe			= ahci_init_one,
574
	.remove			= ata_pci_remove_one,
575
#ifdef CONFIG_PM
576
577
	.suspend		= ahci_pci_device_suspend,
	.resume			= ahci_pci_device_resume,
578
#endif
Linus Torvalds's avatar
Linus Torvalds committed
579
580
581
};


582
583
584
585
586
static inline int ahci_nr_ports(u32 cap)
{
	return (cap & 0x1f) + 1;
}

587
588
static inline void __iomem *__ahci_port_base(struct ata_host *host,
					     unsigned int port_no)
Linus Torvalds's avatar
Linus Torvalds committed
589
{
590
	void __iomem *mmio = host->iomap[AHCI_PCI_BAR];
591

592
593
594
595
596
597
	return mmio + 0x100 + (port_no * 0x80);
}

static inline void __iomem *ahci_port_base(struct ata_port *ap)
{
	return __ahci_port_base(ap->host, ap->port_no);
Linus Torvalds's avatar
Linus Torvalds committed
598
599
}

600
601
602
603
604
605
606
607
608
609
610
611
612
613
static void ahci_enable_ahci(void __iomem *mmio)
{
	u32 tmp;

	/* turn on AHCI_EN */
	tmp = readl(mmio + HOST_CTL);
	if (!(tmp & HOST_AHCI_EN)) {
		tmp |= HOST_AHCI_EN;
		writel(tmp, mmio + HOST_CTL);
		tmp = readl(mmio + HOST_CTL);	/* flush && sanity check */
		WARN_ON(!(tmp & HOST_AHCI_EN));
	}
}

614
615
/**
 *	ahci_save_initial_config - Save and fixup initial config values
616
617
 *	@pdev: target PCI device
 *	@hpriv: host private area to store config values
618
619
620
621
622
623
624
625
626
627
628
 *
 *	Some registers containing configuration info might be setup by
 *	BIOS and might be cleared on reset.  This function saves the
 *	initial values of those registers into @hpriv such that they
 *	can be restored after controller reset.
 *
 *	If inconsistent, config values are fixed up by this function.
 *
 *	LOCKING:
 *	None.
 */
629
630
static void ahci_save_initial_config(struct pci_dev *pdev,
				     struct ahci_host_priv *hpriv)
631
{
632
	void __iomem *mmio = pcim_iomap_table(pdev)[AHCI_PCI_BAR];
633
	u32 cap, port_map;
634
	int i;
635

636
637
638
	/* make sure AHCI mode is enabled before accessing CAP */
	ahci_enable_ahci(mmio);

639
640
641
642
643
644
	/* Values prefixed with saved_ are written back to host after
	 * reset.  Values without are used for driver operation.
	 */
	hpriv->saved_cap = cap = readl(mmio + HOST_CAP);
	hpriv->saved_port_map = port_map = readl(mmio + HOST_PORTS_IMPL);

645
	/* some chips have errata preventing 64bit use */
646
	if ((cap & HOST_CAP_64) && (hpriv->flags & AHCI_HFLAG_32BIT_ONLY)) {
Tejun Heo's avatar
Tejun Heo committed
647
648
649
650
651
		dev_printk(KERN_INFO, &pdev->dev,
			   "controller can't do 64bit DMA, forcing 32bit\n");
		cap &= ~HOST_CAP_64;
	}

652
	if ((cap & HOST_CAP_NCQ) && (hpriv->flags & AHCI_HFLAG_NO_NCQ)) {
653
654
655
656
657
		dev_printk(KERN_INFO, &pdev->dev,
			   "controller can't do NCQ, turning off CAP_NCQ\n");
		cap &= ~HOST_CAP_NCQ;
	}

Tejun Heo's avatar
Tejun Heo committed
658
659
660
661
662
663
	if ((cap && HOST_CAP_PMP) && (hpriv->flags & AHCI_HFLAG_NO_PMP)) {
		dev_printk(KERN_INFO, &pdev->dev,
			   "controller can't do PMP, turning off CAP_PMP\n");
		cap &= ~HOST_CAP_PMP;
	}

664
665
666
667
668
	/*
	 * Temporary Marvell 6145 hack: PATA port presence
	 * is asserted through the standard AHCI port
	 * presence register, as bit 4 (counting from 0)
	 */
669
	if (hpriv->flags & AHCI_HFLAG_MV_PATA) {
670
671
672
673
674
675
676
677
		dev_printk(KERN_ERR, &pdev->dev,
			   "MV_AHCI HACK: port_map %x -> %x\n",
			   hpriv->port_map,
			   hpriv->port_map & 0xf);

		port_map &= 0xf;
	}

678
	/* cross check port_map and cap.n_ports */
679
	if (port_map) {
680
681
682
683
684
685
686
687
688
689
		u32 tmp_port_map = port_map;
		int n_ports = ahci_nr_ports(cap);

		for (i = 0; i < AHCI_MAX_PORTS && n_ports; i++) {
			if (tmp_port_map & (1 << i)) {
				n_ports--;
				tmp_port_map &= ~(1 << i);
			}
		}

690
691
		/* If n_ports and port_map are inconsistent, whine and
		 * clear port_map and let it be generated from n_ports.
692
		 */
693
		if (n_ports || tmp_port_map) {
694
			dev_printk(KERN_WARNING, &pdev->dev,
695
				   "nr_ports (%u) and implemented port map "
696
				   "(0x%x) don't match, using nr_ports\n",
697
				   ahci_nr_ports(cap), port_map);
698
699
700
701
702
703
			port_map = 0;
		}
	}

	/* fabricate port_map from cap.nr_ports */
	if (!port_map) {
704
		port_map = (1 << ahci_nr_ports(cap)) - 1;
705
706
707
708
709
		dev_printk(KERN_WARNING, &pdev->dev,
			   "forcing PORTS_IMPL to 0x%x\n", port_map);

		/* write the fixed up value to the PI register */
		hpriv->saved_port_map = port_map;
710
711
	}

712
713
714
715
716
717
718
	/* record values to use during operation */
	hpriv->cap = cap;
	hpriv->port_map = port_map;
}

/**
 *	ahci_restore_initial_config - Restore initial config
719
 *	@host: target ATA host
720
721
722
723
724
725
 *
 *	Restore initial config stored by ahci_save_initial_config().
 *
 *	LOCKING:
 *	None.
 */
726
static void ahci_restore_initial_config(struct ata_host *host)
727
{
728
729
730
	struct ahci_host_priv *hpriv = host->private_data;
	void __iomem *mmio = host->iomap[AHCI_PCI_BAR];

731
732
733
734
735
	writel(hpriv->saved_cap, mmio + HOST_CAP);
	writel(hpriv->saved_port_map, mmio + HOST_PORTS_IMPL);
	(void) readl(mmio + HOST_PORTS_IMPL);	/* flush */
}

736
static unsigned ahci_scr_offset(struct ata_port *ap, unsigned int sc_reg)
Linus Torvalds's avatar
Linus Torvalds committed
737
{
738
739
740
741
742
743
744
745
	static const int offset[] = {
		[SCR_STATUS]		= PORT_SCR_STAT,
		[SCR_CONTROL]		= PORT_SCR_CTL,
		[SCR_ERROR]		= PORT_SCR_ERR,
		[SCR_ACTIVE]		= PORT_SCR_ACT,
		[SCR_NOTIFICATION]	= PORT_SCR_NTF,
	};
	struct ahci_host_priv *hpriv = ap->host->private_data;
Linus Torvalds's avatar
Linus Torvalds committed
746

747
748
749
	if (sc_reg < ARRAY_SIZE(offset) &&
	    (sc_reg != SCR_NOTIFICATION || (hpriv->cap & HOST_CAP_SNTF)))
		return offset[sc_reg];
750
	return 0;
Linus Torvalds's avatar
Linus Torvalds committed
751
752
}

753
static int ahci_scr_read(struct ata_port *ap, unsigned int sc_reg, u32 *val)
Linus Torvalds's avatar
Linus Torvalds committed
754
{
755
756
757
758
759
760
	void __iomem *port_mmio = ahci_port_base(ap);
	int offset = ahci_scr_offset(ap, sc_reg);

	if (offset) {
		*val = readl(port_mmio + offset);
		return 0;
Linus Torvalds's avatar
Linus Torvalds committed
761
	}
762
763
	return -EINVAL;
}
Linus Torvalds's avatar
Linus Torvalds committed
764

765
766
767
768
769
770
771
772
773
774
static int ahci_scr_write(struct ata_port *ap, unsigned int sc_reg, u32 val)
{
	void __iomem *port_mmio = ahci_port_base(ap);
	int offset = ahci_scr_offset(ap, sc_reg);

	if (offset) {
		writel(val, port_mmio + offset);
		return 0;
	}
	return -EINVAL;
Linus Torvalds's avatar
Linus Torvalds committed
775
776
}

777
static void ahci_start_engine(struct ata_port *ap)
778
{
779
	void __iomem *port_mmio = ahci_port_base(ap);
780
781
	u32 tmp;

782
	/* start DMA */
783
	tmp = readl(port_mmio + PORT_CMD);
784
785
786
787
788
	tmp |= PORT_CMD_START;
	writel(tmp, port_mmio + PORT_CMD);
	readl(port_mmio + PORT_CMD); /* flush */
}

789
static int ahci_stop_engine(struct ata_port *ap)
790
{
791
	void __iomem *port_mmio = ahci_port_base(ap);
792
793
794
795
	u32 tmp;

	tmp = readl(port_mmio + PORT_CMD);

796
	/* check if the HBA is idle */
797
798
799
	if ((tmp & (PORT_CMD_START | PORT_CMD_LIST_ON)) == 0)
		return 0;

800
	/* setting HBA to idle */
801
802
803
	tmp &= ~PORT_CMD_START;
	writel(tmp, port_mmio + PORT_CMD);

804
	/* wait for engine to stop. This could be as long as 500 msec */
805
	tmp = ata_wait_register(port_mmio + PORT_CMD,
806
				PORT_CMD_LIST_ON, PORT_CMD_LIST_ON, 1, 500);
807
	if (tmp & PORT_CMD_LIST_ON)
808
809
810
811
812
		return -EIO;

	return 0;
}

813
static void ahci_start_fis_rx(struct ata_port *ap)
814
{
815
816
817
	void __iomem *port_mmio = ahci_port_base(ap);
	struct ahci_host_priv *hpriv = ap->host->private_data;
	struct ahci_port_priv *pp = ap->private_data;
818
819
820
	u32 tmp;

	/* set FIS registers */
821
822
823
824
	if (hpriv->cap & HOST_CAP_64)
		writel((pp->cmd_slot_dma >> 16) >> 16,
		       port_mmio + PORT_LST_ADDR_HI);
	writel(pp->cmd_slot_dma & 0xffffffff, port_mmio + PORT_LST_ADDR);
825

826
827
828
829
	if (hpriv->cap & HOST_CAP_64)
		writel((pp->rx_fis_dma >> 16) >> 16,
		       port_mmio + PORT_FIS_ADDR_HI);
	writel(pp->rx_fis_dma & 0xffffffff, port_mmio + PORT_FIS_ADDR);
830
831
832
833
834
835
836
837
838
839

	/* enable FIS reception */
	tmp = readl(port_mmio + PORT_CMD);
	tmp |= PORT_CMD_FIS_RX;
	writel(tmp, port_mmio + PORT_CMD);

	/* flush */
	readl(port_mmio + PORT_CMD);
}

840
static int ahci_stop_fis_rx(struct ata_port *ap)
841
{
842
	void __iomem *port_mmio = ahci_port_base(ap);
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
	u32 tmp;

	/* disable FIS reception */
	tmp = readl(port_mmio + PORT_CMD);
	tmp &= ~PORT_CMD_FIS_RX;
	writel(tmp, port_mmio + PORT_CMD);

	/* wait for completion, spec says 500ms, give it 1000 */
	tmp = ata_wait_register(port_mmio + PORT_CMD, PORT_CMD_FIS_ON,
				PORT_CMD_FIS_ON, 10, 1000);
	if (tmp & PORT_CMD_FIS_ON)
		return -EBUSY;

	return 0;
}

859
static void ahci_power_up(struct ata_port *ap)
860
{
861
862
	struct ahci_host_priv *hpriv = ap->host->private_data;
	void __iomem *port_mmio = ahci_port_base(ap);
863
864
865
866
867
	u32 cmd;

	cmd = readl(port_mmio + PORT_CMD) & ~PORT_CMD_ICC_MASK;

	/* spin up device */
868
	if (hpriv->cap & HOST_CAP_SSS) {
869
870
871
872
873
874
875
876
		cmd |= PORT_CMD_SPIN_UP;
		writel(cmd, port_mmio + PORT_CMD);
	}

	/* wake up link */
	writel(cmd | PORT_CMD_ICC_ACTIVE, port_mmio + PORT_CMD);
}

877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
static void ahci_disable_alpm(struct ata_port *ap)
{
	struct ahci_host_priv *hpriv = ap->host->private_data;
	void __iomem *port_mmio = ahci_port_base(ap);
	u32 cmd;
	struct ahci_port_priv *pp = ap->private_data;

	/* IPM bits should be disabled by libata-core */
	/* get the existing command bits */
	cmd = readl(port_mmio + PORT_CMD);

	/* disable ALPM and ASP */
	cmd &= ~PORT_CMD_ASP;
	cmd &= ~PORT_CMD_ALPE;

	/* force the interface back to active */
	cmd |= PORT_CMD_ICC_ACTIVE;

	/* write out new cmd value */
	writel(cmd, port_mmio + PORT_CMD);
	cmd = readl(port_mmio + PORT_CMD);

	/* wait 10ms to be sure we've come out of any low power state */
	msleep(10);

	/* clear out any PhyRdy stuff from interrupt status */
	writel(PORT_IRQ_PHYRDY, port_mmio + PORT_IRQ_STAT);

	/* go ahead and clean out PhyRdy Change from Serror too */
	ahci_scr_write(ap, SCR_ERROR, ((1 << 16) | (1 << 18)));

	/*
 	 * Clear flag to indicate that we should ignore all PhyRdy
 	 * state changes
 	 */
	hpriv->flags &= ~AHCI_HFLAG_NO_HOTPLUG;

	/*
 	 * Enable interrupts on Phy Ready.
 	 */
	pp->intr_mask |= PORT_IRQ_PHYRDY;
	writel(pp->intr_mask, port_mmio + PORT_IRQ_MASK);

	/*
 	 * don't change the link pm policy - we can be called
 	 * just to turn of link pm temporarily
 	 */
}

static int ahci_enable_alpm(struct ata_port *ap,
	enum link_pm policy)
{
	struct ahci_host_priv *hpriv = ap->host->private_data;
	void __iomem *port_mmio = ahci_port_base(ap);
	u32 cmd;
	struct ahci_port_priv *pp = ap->private_data;
	u32 asp;

	/* Make sure the host is capable of link power management */
	if (!(hpriv->cap & HOST_CAP_ALPM))
		return -EINVAL;

	switch (policy) {
	case MAX_PERFORMANCE:
	case NOT_AVAILABLE:
		/*
 		 * if we came here with NOT_AVAILABLE,
 		 * it just means this is the first time we
 		 * have tried to enable - default to max performance,
 		 * and let the user go to lower power modes on request.
 		 */
		ahci_disable_alpm(ap);
		return 0;
	case MIN_POWER:
		/* configure HBA to enter SLUMBER */
		asp = PORT_CMD_ASP;
		break;
	case MEDIUM_POWER:
		/* configure HBA to enter PARTIAL */
		asp = 0;
		break;
	default:
		return -EINVAL;
	}

	/*
 	 * Disable interrupts on Phy Ready. This keeps us from
 	 * getting woken up due to spurious phy ready interrupts
	 * TBD - Hot plug should be done via polling now, is
	 * that even supported?
 	 */
	pp->intr_mask &= ~PORT_IRQ_PHYRDY;
	writel(pp->intr_mask, port_mmio + PORT_IRQ_MASK);

	/*
 	 * Set a flag to indicate that we should ignore all PhyRdy
 	 * state changes since these can happen now whenever we
 	 * change link state
 	 */
	hpriv->flags |= AHCI_HFLAG_NO_HOTPLUG;

	/* get the existing command bits */
	cmd = readl(port_mmio + PORT_CMD);

	/*
 	 * Set ASP based on Policy
 	 */
	cmd |= asp;

	/*
 	 * Setting this bit will instruct the HBA to aggressively
 	 * enter a lower power link state when it's appropriate and
 	 * based on the value set above for ASP
 	 */
	cmd |= PORT_CMD_ALPE;

	/* write out new cmd value */
	writel(cmd, port_mmio + PORT_CMD);
	cmd = readl(port_mmio + PORT_CMD);

	/* IPM bits should be set by libata-core */
	return 0;
}

1001
#ifdef CONFIG_PM
1002
static void ahci_power_down(struct ata_port *ap)
1003
{
1004
1005
	struct ahci_host_priv *hpriv = ap->host->private_data;
	void __iomem *port_mmio = ahci_port_base(ap);
1006
1007
	u32 cmd, scontrol;

1008
	if (!(hpriv->cap & HOST_CAP_SSS))
1009
		return;
1010

1011
1012
1013
1014
	/* put device into listen mode, first set PxSCTL.DET to 0 */
	scontrol = readl(port_mmio + PORT_SCR_CTL);
	scontrol &= ~0xf;
	writel(scontrol, port_mmio + PORT_SCR_CTL);
1015

1016
1017
1018
1019
	/* then set PxCMD.SUD to 0 */
	cmd = readl(port_mmio + PORT_CMD) & ~PORT_CMD_ICC_MASK;
	cmd &= ~PORT_CMD_SPIN_UP;
	writel(cmd, port_mmio + PORT_CMD);
1020
}
1021
#endif
1022

1023
static void ahci_start_port(struct ata_port *ap)
1024
1025
{
	/* enable FIS reception */
1026
	ahci_start_fis_rx(ap);
1027
1028

	/* enable DMA */
1029
	ahci_start_engine(ap);
1030
1031
}

1032
static int ahci_deinit_port(struct ata_port *ap, const char **emsg)
1033
1034
1035
1036
{
	int rc;

	/* disable DMA */
1037
	rc = ahci_stop_engine(ap);
1038
1039
1040
1041
1042
1043
	if (rc) {
		*emsg = "failed to stop engine";
		return rc;
	}

	/* disable FIS reception */
1044
	rc = ahci_stop_fis_rx(ap);
1045
1046
1047
1048
1049
1050
1051
1052
	if (rc) {
		*emsg = "failed stop FIS RX";
		return rc;
	}

	return 0;
}

1053
static int ahci_reset_controller(struct ata_host *host)
1054
{
1055
	struct pci_dev *pdev = to_pci_dev(host->dev);
Tejun Heo's avatar
Tejun Heo committed
1056
	struct ahci_host_priv *hpriv = host->private_data;
1057
	void __iomem *mmio = host->iomap[AHCI_PCI_BAR];
1058
	u32 tmp;
1059

1060
1061
1062
	/* we must be in AHCI mode, before using anything
	 * AHCI-specific, such as HOST_RESET.
	 */
1063
	ahci_enable_ahci(mmio);
1064
1065

	/* global controller reset */
1066
	tmp = readl(mmio + HOST_CTL);
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
	if ((tmp & HOST_RESET) == 0) {
		writel(tmp | HOST_RESET, mmio + HOST_CTL);
		readl(mmio + HOST_CTL); /* flush */
	}

	/* reset must complete within 1 second, or
	 * the hardware should be considered fried.
	 */
	ssleep(1);

	tmp = readl(mmio + HOST_CTL);
	if (tmp & HOST_RESET) {
1079
		dev_printk(KERN_ERR, host->dev,
1080
1081
1082
1083
			   "controller reset failed (0x%x)\n", tmp);
		return -EIO;
	}

1084
	/* turn on AHCI mode */
1085
	ahci_enable_ahci(mmio);
1086

1087
	/* some registers might be cleared on reset.  restore initial values */
1088
	ahci_restore_initial_config(host);
1089
1090
1091
1092
1093
1094

	if (pdev->vendor == PCI_VENDOR_ID_INTEL) {
		u16 tmp16;

		/* configure PCS */
		pci_read_config_word(pdev, 0x92, &tmp16);
Tejun Heo's avatar
Tejun Heo committed
1095
1096
1097
1098
		if ((tmp16 & hpriv->port_map) != hpriv->port_map) {
			tmp16 |= hpriv->port_map;
			pci_write_config_word(pdev, 0x92, tmp16);
		}
1099
1100
1101
1102
1103
	}

	return 0;
}

1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
static void ahci_port_init(struct pci_dev *pdev, struct ata_port *ap,
			   int port_no, void __iomem *mmio,
			   void __iomem *port_mmio)
{
	const char *emsg = NULL;
	int rc;
	u32 tmp;

	/* make sure port is not active */
	rc = ahci_deinit_port(ap, &emsg);
	if (rc)
		dev_printk(KERN_WARNING, &pdev->dev,
			   "%s (%d)\n", emsg, rc);

	/* clear SError */
	tmp = readl(port_mmio + PORT_SCR_ERR);
	VPRINTK("PORT_SCR_ERR 0x%x\n", tmp);
	writel(tmp, port_mmio + PORT_SCR_ERR);

	/* clear port IRQ */
	tmp = readl(port_mmio + PORT_IRQ_STAT);
	VPRINTK("PORT_IRQ_STAT 0x%x\n", tmp);
	if (tmp)
		writel(tmp, port_mmio + PORT_IRQ_STAT);

	writel(1 << port_no, mmio + HOST_IRQ_STAT);
}

1132
static void ahci_init_controller(struct ata_host *host)
1133
{
1134
	struct ahci_host_priv *hpriv = host->private_data;
1135
1136
	struct pci_dev *pdev = to_pci_dev(host->dev);
	void __iomem *mmio = host->iomap[AHCI_PCI_BAR];
1137
	int i;
1138
	void __iomem *port_mmio;
1139
1140
	u32 tmp;

1141
	if (hpriv->flags & AHCI_HFLAG_MV_PATA) {
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
		port_mmio = __ahci_port_base(host, 4);

		writel(0, port_mmio + PORT_IRQ_MASK);

		/* clear port IRQ */
		tmp = readl(port_mmio + PORT_IRQ_STAT);
		VPRINTK("PORT_IRQ_STAT 0x%x\n", tmp);
		if (tmp)
			writel(tmp, port_mmio + PORT_IRQ_STAT);
	}

1153
1154
	for (i = 0; i < host->n_ports; i++) {
		struct ata_port *ap = host->ports[i];
1155

1156
		port_mmio = ahci_port_base(ap);
1157
		if (ata_port_is_dummy(ap))
1158
1159
			continue;

1160
		ahci_port_init(pdev, ap, i, mmio, port_mmio);
1161
1162
1163
1164
1165
1166
1167
1168
1169
	}

	tmp = readl(mmio + HOST_CTL);
	VPRINTK("HOST_CTL 0x%x\n", tmp);
	writel(tmp | HOST_IRQ_EN, mmio + HOST_CTL);
	tmp = readl(mmio + HOST_CTL);
	VPRINTK("HOST_CTL 0x%x\n", tmp);
}

1170
static unsigned int ahci_dev_classify(struct ata_port *ap)
Linus Torvalds's avatar
Linus Torvalds committed
1171
{
1172
	void __iomem *port_mmio = ahci_port_base(ap);
Linus Torvalds's avatar
Linus Torvalds committed
1173
	struct ata_taskfile tf;
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
	u32 tmp;

	tmp = readl(port_mmio + PORT_SIG);
	tf.lbah		= (tmp >> 24)	& 0xff;
	tf.lbam		= (tmp >> 16)	& 0xff;
	tf.lbal		= (tmp >> 8)	& 0xff;
	tf.nsect	= (tmp)		& 0xff;

	return ata_dev_classify(&tf);
}

1185
1186
static void ahci_fill_cmd_slot(struct ahci_port_priv *pp, unsigned int tag,
			       u32 opts)
1187
{
1188
1189
1190
1191
1192
1193
1194
1195
	dma_addr_t cmd_tbl_dma;

	cmd_tbl_dma = pp->cmd_tbl_dma + tag * AHCI_CMD_TBL_SZ;

	pp->cmd_slot[tag].opts = cpu_to_le32(opts);
	pp->cmd_slot[tag].status = 0;
	pp->cmd_slot[tag].tbl_addr = cpu_to_le32(cmd_tbl_dma & 0xffffffff);
	pp->cmd_slot[tag].tbl_addr_hi = cpu_to_le32((cmd_tbl_dma >> 16) >> 16);
1196
1197
}

1198
static int ahci_kick_engine(struct ata_port *ap, int force_restart)
Tejun Heo's avatar
Tejun Heo committed
1199
{
Tejun Heo's avatar
Tejun Heo committed
1200
	void __iomem *port_mmio = ap->ioaddr.cmd_addr;
Jeff Garzik's avatar
Jeff Garzik committed
1201
	struct ahci_host_priv *hpriv = ap->host->private_data;
1202
	u32 tmp;
1203
	int busy, rc;
1204

1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
	/* do we need to kick the port? */
	busy = ahci_check_status(ap) & (ATA_BUSY | ATA_DRQ);
	if (!busy && !force_restart)
		return 0;

	/* stop engine */
	rc = ahci_stop_engine(ap);
	if (rc)
		goto out_restart;

	/* need to do CLO? */
	if (!busy) {
		rc = 0;
		goto out_restart;
	}

	if (!(hpriv->cap & HOST_CAP_CLO)) {
		rc = -EOPNOTSUPP;
		goto out_restart;
	}
1225

1226
	/* perform CLO */
1227
1228
1229
1230
	tmp = readl(port_mmio + PORT_CMD);
	tmp |= PORT_CMD_CLO;
	writel(tmp, port_mmio + PORT_CMD);

1231
	rc = 0;
1232
1233
1234
	tmp = ata_wait_register(port_mmio + PORT_CMD,
				PORT_CMD_CLO, PORT_CMD_CLO, 1, 500);
	if (tmp & PORT_CMD_CLO)
1235
		rc = -EIO;
1236

1237
1238
1239
1240
	/* restart engine */
 out_restart:
	ahci_start_engine(ap);
	return rc;
1241
1242
}

1243
1244
1245
static int ahci_exec_polled_cmd(struct ata_port *ap, int pmp,
				struct ata_taskfile *tf, int is_cmd, u16 flags,
				unsigned long timeout_msec)
1246
{