ahci.c 62.6 KB
Newer Older
Linus Torvalds's avatar
Linus Torvalds committed
1
2
3
/*
 *  ahci.c - AHCI SATA support
 *
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
 *  Maintained by:  Jeff Garzik <jgarzik@pobox.com>
 *    		    Please ALWAYS copy linux-ide@vger.kernel.org
 *		    on emails.
 *
 *  Copyright 2004-2005 Red Hat, Inc.
 *
 *
 *  This program is free software; you can redistribute it and/or modify
 *  it under the terms of the GNU General Public License as published by
 *  the Free Software Foundation; either version 2, or (at your option)
 *  any later version.
 *
 *  This program is distributed in the hope that it will be useful,
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *  GNU General Public License for more details.
 *
 *  You should have received a copy of the GNU General Public License
 *  along with this program; see the file COPYING.  If not, write to
 *  the Free Software Foundation, 675 Mass Ave, Cambridge, MA 02139, USA.
 *
 *
 * libata documentation is available via 'make {ps|pdf}docs',
 * as Documentation/DocBook/libata.*
 *
 * AHCI hardware documentation:
Linus Torvalds's avatar
Linus Torvalds committed
30
 * http://www.intel.com/technology/serialata/pdf/rev1_0.pdf
31
 * http://www.intel.com/technology/serialata/pdf/rev1_1.pdf
Linus Torvalds's avatar
Linus Torvalds committed
32
33
34
35
36
37
38
39
40
41
 *
 */

#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/pci.h>
#include <linux/init.h>
#include <linux/blkdev.h>
#include <linux/delay.h>
#include <linux/interrupt.h>
42
#include <linux/dma-mapping.h>
43
#include <linux/device.h>
44
#include <linux/dmi.h>
Linus Torvalds's avatar
Linus Torvalds committed
45
#include <scsi/scsi_host.h>
46
#include <scsi/scsi_cmnd.h>
Linus Torvalds's avatar
Linus Torvalds committed
47
48
49
#include <linux/libata.h>

#define DRV_NAME	"ahci"
Tejun Heo's avatar
Tejun Heo committed
50
#define DRV_VERSION	"3.0"
Linus Torvalds's avatar
Linus Torvalds committed
51

52
53
54
static int ahci_enable_alpm(struct ata_port *ap,
		enum link_pm policy);
static void ahci_disable_alpm(struct ata_port *ap);
Linus Torvalds's avatar
Linus Torvalds committed
55
56
57

enum {
	AHCI_PCI_BAR		= 5,
58
	AHCI_MAX_PORTS		= 32,
Linus Torvalds's avatar
Linus Torvalds committed
59
60
	AHCI_MAX_SG		= 168, /* hardware max is 64K */
	AHCI_DMA_BOUNDARY	= 0xffffffff,
61
	AHCI_USE_CLUSTERING	= 1,
62
	AHCI_MAX_CMDS		= 32,
63
	AHCI_CMD_SZ		= 32,
64
	AHCI_CMD_SLOT_SZ	= AHCI_MAX_CMDS * AHCI_CMD_SZ,
Linus Torvalds's avatar
Linus Torvalds committed
65
	AHCI_RX_FIS_SZ		= 256,
66
	AHCI_CMD_TBL_CDB	= 0x40,
67
68
69
70
	AHCI_CMD_TBL_HDR_SZ	= 0x80,
	AHCI_CMD_TBL_SZ		= AHCI_CMD_TBL_HDR_SZ + (AHCI_MAX_SG * 16),
	AHCI_CMD_TBL_AR_SZ	= AHCI_CMD_TBL_SZ * AHCI_MAX_CMDS,
	AHCI_PORT_PRIV_DMA_SZ	= AHCI_CMD_SLOT_SZ + AHCI_CMD_TBL_AR_SZ +
Linus Torvalds's avatar
Linus Torvalds committed
71
72
73
74
				  AHCI_RX_FIS_SZ,
	AHCI_IRQ_ON_SG		= (1 << 31),
	AHCI_CMD_ATAPI		= (1 << 5),
	AHCI_CMD_WRITE		= (1 << 6),
75
	AHCI_CMD_PREFETCH	= (1 << 7),
76
77
	AHCI_CMD_RESET		= (1 << 8),
	AHCI_CMD_CLR_BUSY	= (1 << 10),
Linus Torvalds's avatar
Linus Torvalds committed
78
79

	RX_FIS_D2H_REG		= 0x40,	/* offset of D2H Register FIS data */
80
	RX_FIS_SDB		= 0x58, /* offset of SDB FIS data */
Tejun Heo's avatar
Tejun Heo committed
81
	RX_FIS_UNK		= 0x60, /* offset of Unknown FIS data */
Linus Torvalds's avatar
Linus Torvalds committed
82
83

	board_ahci		= 0,
84
85
86
87
	board_ahci_vt8251	= 1,
	board_ahci_ign_iferr	= 2,
	board_ahci_sb600	= 3,
	board_ahci_mv		= 4,
Linus Torvalds's avatar
Linus Torvalds committed
88
89
90
91
92
93
94
95
96
97
98
99
100
101

	/* global controller registers */
	HOST_CAP		= 0x00, /* host capabilities */
	HOST_CTL		= 0x04, /* global host control */
	HOST_IRQ_STAT		= 0x08, /* interrupt status */
	HOST_PORTS_IMPL		= 0x0c, /* bitmap of implemented ports */
	HOST_VERSION		= 0x10, /* AHCI spec. version compliancy */

	/* HOST_CTL bits */
	HOST_RESET		= (1 << 0),  /* reset controller; self-clear */
	HOST_IRQ_EN		= (1 << 1),  /* global IRQ enable */
	HOST_AHCI_EN		= (1 << 31), /* AHCI enabled */

	/* HOST_CAP bits */
102
	HOST_CAP_SSC		= (1 << 14), /* Slumber capable */
Tejun Heo's avatar
Tejun Heo committed
103
	HOST_CAP_PMP		= (1 << 17), /* Port Multiplier support */
104
	HOST_CAP_CLO		= (1 << 24), /* Command List Override support */
105
	HOST_CAP_ALPM		= (1 << 26), /* Aggressive Link PM support */
106
	HOST_CAP_SSS		= (1 << 27), /* Staggered Spin-up */
107
	HOST_CAP_SNTF		= (1 << 29), /* SNotification register */
108
	HOST_CAP_NCQ		= (1 << 30), /* Native Command Queueing */
109
	HOST_CAP_64		= (1 << 31), /* PCI DAC (64-bit DMA) support */
Linus Torvalds's avatar
Linus Torvalds committed
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125

	/* registers for each SATA port */
	PORT_LST_ADDR		= 0x00, /* command list DMA addr */
	PORT_LST_ADDR_HI	= 0x04, /* command list DMA addr hi */
	PORT_FIS_ADDR		= 0x08, /* FIS rx buf addr */
	PORT_FIS_ADDR_HI	= 0x0c, /* FIS rx buf addr hi */
	PORT_IRQ_STAT		= 0x10, /* interrupt status */
	PORT_IRQ_MASK		= 0x14, /* interrupt enable/disable mask */
	PORT_CMD		= 0x18, /* port command */
	PORT_TFDATA		= 0x20,	/* taskfile data */
	PORT_SIG		= 0x24,	/* device TF signature */
	PORT_CMD_ISSUE		= 0x38, /* command issue */
	PORT_SCR_STAT		= 0x28, /* SATA phy register: SStatus */
	PORT_SCR_CTL		= 0x2c, /* SATA phy register: SControl */
	PORT_SCR_ERR		= 0x30, /* SATA phy register: SError */
	PORT_SCR_ACT		= 0x34, /* SATA phy register: SActive */
126
	PORT_SCR_NTF		= 0x3c, /* SATA phy register: SNotification */
Linus Torvalds's avatar
Linus Torvalds committed
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147

	/* PORT_IRQ_{STAT,MASK} bits */
	PORT_IRQ_COLD_PRES	= (1 << 31), /* cold presence detect */
	PORT_IRQ_TF_ERR		= (1 << 30), /* task file error */
	PORT_IRQ_HBUS_ERR	= (1 << 29), /* host bus fatal error */
	PORT_IRQ_HBUS_DATA_ERR	= (1 << 28), /* host bus data error */
	PORT_IRQ_IF_ERR		= (1 << 27), /* interface fatal error */
	PORT_IRQ_IF_NONFATAL	= (1 << 26), /* interface non-fatal error */
	PORT_IRQ_OVERFLOW	= (1 << 24), /* xfer exhausted available S/G */
	PORT_IRQ_BAD_PMP	= (1 << 23), /* incorrect port multiplier */

	PORT_IRQ_PHYRDY		= (1 << 22), /* PhyRdy changed */
	PORT_IRQ_DEV_ILCK	= (1 << 7), /* device interlock */
	PORT_IRQ_CONNECT	= (1 << 6), /* port connect change status */
	PORT_IRQ_SG_DONE	= (1 << 5), /* descriptor processed */
	PORT_IRQ_UNK_FIS	= (1 << 4), /* unknown FIS rx'd */
	PORT_IRQ_SDB_FIS	= (1 << 3), /* Set Device Bits FIS rx'd */
	PORT_IRQ_DMAS_FIS	= (1 << 2), /* DMA Setup FIS rx'd */
	PORT_IRQ_PIOS_FIS	= (1 << 1), /* PIO Setup FIS rx'd */
	PORT_IRQ_D2H_REG_FIS	= (1 << 0), /* D2H Register FIS rx'd */

Tejun Heo's avatar
Tejun Heo committed
148
149
150
	PORT_IRQ_FREEZE		= PORT_IRQ_HBUS_ERR |
				  PORT_IRQ_IF_ERR |
				  PORT_IRQ_CONNECT |
151
				  PORT_IRQ_PHYRDY |
Tejun Heo's avatar
Tejun Heo committed
152
153
				  PORT_IRQ_UNK_FIS |
				  PORT_IRQ_BAD_PMP,
Tejun Heo's avatar
Tejun Heo committed
154
155
156
157
158
159
	PORT_IRQ_ERROR		= PORT_IRQ_FREEZE |
				  PORT_IRQ_TF_ERR |
				  PORT_IRQ_HBUS_DATA_ERR,
	DEF_PORT_IRQ		= PORT_IRQ_ERROR | PORT_IRQ_SG_DONE |
				  PORT_IRQ_SDB_FIS | PORT_IRQ_DMAS_FIS |
				  PORT_IRQ_PIOS_FIS | PORT_IRQ_D2H_REG_FIS,
Linus Torvalds's avatar
Linus Torvalds committed
160
161

	/* PORT_CMD bits */
162
163
	PORT_CMD_ASP		= (1 << 27), /* Aggressive Slumber/Partial */
	PORT_CMD_ALPE		= (1 << 26), /* Aggressive Link PM enable */
164
	PORT_CMD_ATAPI		= (1 << 24), /* Device is ATAPI */
Tejun Heo's avatar
Tejun Heo committed
165
	PORT_CMD_PMP		= (1 << 17), /* PMP attached */
Linus Torvalds's avatar
Linus Torvalds committed
166
167
168
	PORT_CMD_LIST_ON	= (1 << 15), /* cmd list DMA engine running */
	PORT_CMD_FIS_ON		= (1 << 14), /* FIS DMA engine running */
	PORT_CMD_FIS_RX		= (1 << 4), /* Enable FIS receive DMA engine */
169
	PORT_CMD_CLO		= (1 << 3), /* Command list override */
Linus Torvalds's avatar
Linus Torvalds committed
170
171
172
173
	PORT_CMD_POWER_ON	= (1 << 2), /* Power up device */
	PORT_CMD_SPIN_UP	= (1 << 1), /* Spin up device */
	PORT_CMD_START		= (1 << 0), /* Enable port DMA engine */

174
	PORT_CMD_ICC_MASK	= (0xf << 28), /* i/f ICC state mask */
Linus Torvalds's avatar
Linus Torvalds committed
175
176
177
	PORT_CMD_ICC_ACTIVE	= (0x1 << 28), /* Put i/f in active state */
	PORT_CMD_ICC_PARTIAL	= (0x2 << 28), /* Put i/f in partial state */
	PORT_CMD_ICC_SLUMBER	= (0x6 << 28), /* Put i/f in slumber state */
178

179
180
181
182
183
184
185
	/* hpriv->flags bits */
	AHCI_HFLAG_NO_NCQ		= (1 << 0),
	AHCI_HFLAG_IGN_IRQ_IF_ERR	= (1 << 1), /* ignore IRQ_IF_ERR */
	AHCI_HFLAG_IGN_SERR_INTERNAL	= (1 << 2), /* ignore SERR_INTERNAL */
	AHCI_HFLAG_32BIT_ONLY		= (1 << 3), /* force 32bit */
	AHCI_HFLAG_MV_PATA		= (1 << 4), /* PATA port */
	AHCI_HFLAG_NO_MSI		= (1 << 5), /* no PCI MSI */
Tejun Heo's avatar
Tejun Heo committed
186
	AHCI_HFLAG_NO_PMP		= (1 << 6), /* no PMP */
187
	AHCI_HFLAG_NO_HOTPLUG		= (1 << 7), /* ignore PxSERR.DIAG.N */
188

189
	/* ap->flags bits */
190
191
192

	AHCI_FLAG_COMMON		= ATA_FLAG_SATA | ATA_FLAG_NO_LEGACY |
					  ATA_FLAG_MMIO | ATA_FLAG_PIO_DMA |
193
194
					  ATA_FLAG_ACPI_SATA | ATA_FLAG_AN |
					  ATA_FLAG_IPM,
195
	AHCI_LFLAG_COMMON		= ATA_LFLAG_SKIP_D2H_BSY,
196
197

	ICH_MAP				= 0x90, /* ICH MAP register */
Linus Torvalds's avatar
Linus Torvalds committed
198
199
200
};

struct ahci_cmd_hdr {
Al Viro's avatar
Al Viro committed
201
202
203
204
205
	__le32			opts;
	__le32			status;
	__le32			tbl_addr;
	__le32			tbl_addr_hi;
	__le32			reserved[4];
Linus Torvalds's avatar
Linus Torvalds committed
206
207
208
};

struct ahci_sg {
Al Viro's avatar
Al Viro committed
209
210
211
212
	__le32			addr;
	__le32			addr_hi;
	__le32			reserved;
	__le32			flags_size;
Linus Torvalds's avatar
Linus Torvalds committed
213
214
215
};

struct ahci_host_priv {
216
	unsigned int		flags;		/* AHCI_HFLAG_* */
217
218
219
220
	u32			cap;		/* cap to use */
	u32			port_map;	/* port map to use */
	u32			saved_cap;	/* saved initial cap */
	u32			saved_port_map;	/* saved initial port_map */
Linus Torvalds's avatar
Linus Torvalds committed
221
222
223
};

struct ahci_port_priv {
Tejun Heo's avatar
Tejun Heo committed
224
	struct ata_link		*active_link;
Linus Torvalds's avatar
Linus Torvalds committed
225
226
227
228
229
230
	struct ahci_cmd_hdr	*cmd_slot;
	dma_addr_t		cmd_slot_dma;
	void			*cmd_tbl;
	dma_addr_t		cmd_tbl_dma;
	void			*rx_fis;
	dma_addr_t		rx_fis_dma;
231
232
233
	/* for NCQ spurious interrupt analysis */
	unsigned int		ncq_saw_d2h:1;
	unsigned int		ncq_saw_dmas:1;
234
	unsigned int		ncq_saw_sdb:1;
235
	u32 			intr_mask;	/* interrupts to enable */
Linus Torvalds's avatar
Linus Torvalds committed
236
237
};

238
239
static int ahci_scr_read(struct ata_port *ap, unsigned int sc_reg, u32 *val);
static int ahci_scr_write(struct ata_port *ap, unsigned int sc_reg, u32 val);
240
static int ahci_init_one(struct pci_dev *pdev, const struct pci_device_id *ent);
241
static unsigned int ahci_qc_issue(struct ata_queued_cmd *qc);
Linus Torvalds's avatar
Linus Torvalds committed
242
243
244
245
246
247
static void ahci_irq_clear(struct ata_port *ap);
static int ahci_port_start(struct ata_port *ap);
static void ahci_port_stop(struct ata_port *ap);
static void ahci_tf_read(struct ata_port *ap, struct ata_taskfile *tf);
static void ahci_qc_prep(struct ata_queued_cmd *qc);
static u8 ahci_check_status(struct ata_port *ap);
Tejun Heo's avatar
Tejun Heo committed
248
249
static void ahci_freeze(struct ata_port *ap);
static void ahci_thaw(struct ata_port *ap);
Tejun Heo's avatar
Tejun Heo committed
250
251
static void ahci_pmp_attach(struct ata_port *ap);
static void ahci_pmp_detach(struct ata_port *ap);
Tejun Heo's avatar
Tejun Heo committed
252
static void ahci_error_handler(struct ata_port *ap);
253
static void ahci_vt8251_error_handler(struct ata_port *ap);
254
static void ahci_p5wdh_error_handler(struct ata_port *ap);
Tejun Heo's avatar
Tejun Heo committed
255
static void ahci_post_internal_cmd(struct ata_queued_cmd *qc);
256
static int ahci_port_resume(struct ata_port *ap);
257
258
259
static unsigned int ahci_fill_sg(struct ata_queued_cmd *qc, void *cmd_tbl);
static void ahci_fill_cmd_slot(struct ahci_port_priv *pp, unsigned int tag,
			       u32 opts);
260
#ifdef CONFIG_PM
261
262
263
static int ahci_port_suspend(struct ata_port *ap, pm_message_t mesg);
static int ahci_pci_device_suspend(struct pci_dev *pdev, pm_message_t mesg);
static int ahci_pci_device_resume(struct pci_dev *pdev);
264
#endif
Linus Torvalds's avatar
Linus Torvalds committed
265

266
267
268
269
270
static struct class_device_attribute *ahci_shost_attrs[] = {
	&class_device_attr_link_power_management_policy,
	NULL
};

271
static struct scsi_host_template ahci_sht = {
Linus Torvalds's avatar
Linus Torvalds committed
272
273
274
275
	.module			= THIS_MODULE,
	.name			= DRV_NAME,
	.ioctl			= ata_scsi_ioctl,
	.queuecommand		= ata_scsi_queuecmd,
276
277
	.change_queue_depth	= ata_scsi_change_queue_depth,
	.can_queue		= AHCI_MAX_CMDS - 1,
Linus Torvalds's avatar
Linus Torvalds committed
278
279
280
281
282
283
284
285
	.this_id		= ATA_SHT_THIS_ID,
	.sg_tablesize		= AHCI_MAX_SG,
	.cmd_per_lun		= ATA_SHT_CMD_PER_LUN,
	.emulated		= ATA_SHT_EMULATED,
	.use_clustering		= AHCI_USE_CLUSTERING,
	.proc_name		= DRV_NAME,
	.dma_boundary		= AHCI_DMA_BOUNDARY,
	.slave_configure	= ata_scsi_slave_config,
Tejun Heo's avatar
Tejun Heo committed
286
	.slave_destroy		= ata_scsi_slave_destroy,
Linus Torvalds's avatar
Linus Torvalds committed
287
	.bios_param		= ata_std_bios_param,
288
	.shost_attrs		= ahci_shost_attrs,
Linus Torvalds's avatar
Linus Torvalds committed
289
290
};

291
static const struct ata_port_operations ahci_ops = {
Linus Torvalds's avatar
Linus Torvalds committed
292
293
294
295
296
297
	.check_status		= ahci_check_status,
	.check_altstatus	= ahci_check_status,
	.dev_select		= ata_noop_dev_select,

	.tf_read		= ahci_tf_read,

Tejun Heo's avatar
Tejun Heo committed
298
	.qc_defer		= sata_pmp_qc_defer_cmd_switch,
Linus Torvalds's avatar
Linus Torvalds committed
299
300
301
302
303
304
305
306
	.qc_prep		= ahci_qc_prep,
	.qc_issue		= ahci_qc_issue,

	.irq_clear		= ahci_irq_clear,

	.scr_read		= ahci_scr_read,
	.scr_write		= ahci_scr_write,

Tejun Heo's avatar
Tejun Heo committed
307
308
309
310
311
312
	.freeze			= ahci_freeze,
	.thaw			= ahci_thaw,

	.error_handler		= ahci_error_handler,
	.post_internal_cmd	= ahci_post_internal_cmd,

Tejun Heo's avatar
Tejun Heo committed
313
314
315
	.pmp_attach		= ahci_pmp_attach,
	.pmp_detach		= ahci_pmp_detach,

316
#ifdef CONFIG_PM
317
318
	.port_suspend		= ahci_port_suspend,
	.port_resume		= ahci_port_resume,
319
#endif
320
321
	.enable_pm		= ahci_enable_alpm,
	.disable_pm		= ahci_disable_alpm,
322

Linus Torvalds's avatar
Linus Torvalds committed
323
324
325
326
	.port_start		= ahci_port_start,
	.port_stop		= ahci_port_stop,
};

327
328
329
330
331
332
333
static const struct ata_port_operations ahci_vt8251_ops = {
	.check_status		= ahci_check_status,
	.check_altstatus	= ahci_check_status,
	.dev_select		= ata_noop_dev_select,

	.tf_read		= ahci_tf_read,

Tejun Heo's avatar
Tejun Heo committed
334
	.qc_defer		= sata_pmp_qc_defer_cmd_switch,
335
336
337
338
339
340
341
342
343
344
345
346
347
348
	.qc_prep		= ahci_qc_prep,
	.qc_issue		= ahci_qc_issue,

	.irq_clear		= ahci_irq_clear,

	.scr_read		= ahci_scr_read,
	.scr_write		= ahci_scr_write,

	.freeze			= ahci_freeze,
	.thaw			= ahci_thaw,

	.error_handler		= ahci_vt8251_error_handler,
	.post_internal_cmd	= ahci_post_internal_cmd,

Tejun Heo's avatar
Tejun Heo committed
349
350
351
	.pmp_attach		= ahci_pmp_attach,
	.pmp_detach		= ahci_pmp_detach,

352
#ifdef CONFIG_PM
353
354
	.port_suspend		= ahci_port_suspend,
	.port_resume		= ahci_port_resume,
355
#endif
356
357
358
359
360

	.port_start		= ahci_port_start,
	.port_stop		= ahci_port_stop,
};

361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
static const struct ata_port_operations ahci_p5wdh_ops = {
	.check_status		= ahci_check_status,
	.check_altstatus	= ahci_check_status,
	.dev_select		= ata_noop_dev_select,

	.tf_read		= ahci_tf_read,

	.qc_defer		= sata_pmp_qc_defer_cmd_switch,
	.qc_prep		= ahci_qc_prep,
	.qc_issue		= ahci_qc_issue,

	.irq_clear		= ahci_irq_clear,

	.scr_read		= ahci_scr_read,
	.scr_write		= ahci_scr_write,

	.freeze			= ahci_freeze,
	.thaw			= ahci_thaw,

	.error_handler		= ahci_p5wdh_error_handler,
	.post_internal_cmd	= ahci_post_internal_cmd,

	.pmp_attach		= ahci_pmp_attach,
	.pmp_detach		= ahci_pmp_detach,

#ifdef CONFIG_PM
	.port_suspend		= ahci_port_suspend,
	.port_resume		= ahci_port_resume,
#endif

	.port_start		= ahci_port_start,
	.port_stop		= ahci_port_stop,
};

395
396
#define AHCI_HFLAGS(flags)	.private_data	= (void *)(flags)

397
static const struct ata_port_info ahci_port_info[] = {
Linus Torvalds's avatar
Linus Torvalds committed
398
399
	/* board_ahci */
	{
400
		.flags		= AHCI_FLAG_COMMON,
401
		.link_flags	= AHCI_LFLAG_COMMON,
402
		.pio_mask	= 0x1f, /* pio0-4 */
403
		.udma_mask	= ATA_UDMA6,
Linus Torvalds's avatar
Linus Torvalds committed
404
405
		.port_ops	= &ahci_ops,
	},
406
407
	/* board_ahci_vt8251 */
	{
Tejun Heo's avatar
Tejun Heo committed
408
		AHCI_HFLAGS	(AHCI_HFLAG_NO_NCQ | AHCI_HFLAG_NO_PMP),
409
		.flags		= AHCI_FLAG_COMMON,
410
		.link_flags	= AHCI_LFLAG_COMMON | ATA_LFLAG_HRST_TO_RESUME,
411
		.pio_mask	= 0x1f, /* pio0-4 */
412
		.udma_mask	= ATA_UDMA6,
413
		.port_ops	= &ahci_vt8251_ops,
414
	},
415
416
	/* board_ahci_ign_iferr */
	{
417
418
		AHCI_HFLAGS	(AHCI_HFLAG_IGN_IRQ_IF_ERR),
		.flags		= AHCI_FLAG_COMMON,
419
		.link_flags	= AHCI_LFLAG_COMMON,
420
		.pio_mask	= 0x1f, /* pio0-4 */
421
		.udma_mask	= ATA_UDMA6,
422
423
		.port_ops	= &ahci_ops,
	},
424
425
	/* board_ahci_sb600 */
	{
426
		AHCI_HFLAGS	(AHCI_HFLAG_IGN_SERR_INTERNAL |
Tejun Heo's avatar
Tejun Heo committed
427
				 AHCI_HFLAG_32BIT_ONLY | AHCI_HFLAG_NO_PMP),
428
		.flags		= AHCI_FLAG_COMMON,
429
		.link_flags	= AHCI_LFLAG_COMMON,
430
		.pio_mask	= 0x1f, /* pio0-4 */
431
		.udma_mask	= ATA_UDMA6,
432
433
		.port_ops	= &ahci_ops,
	},
434
435
	/* board_ahci_mv */
	{
436
437
		AHCI_HFLAGS	(AHCI_HFLAG_NO_NCQ | AHCI_HFLAG_NO_MSI |
				 AHCI_HFLAG_MV_PATA),
438
		.flags		= ATA_FLAG_SATA | ATA_FLAG_NO_LEGACY |
439
				  ATA_FLAG_MMIO | ATA_FLAG_PIO_DMA,
440
		.link_flags	= AHCI_LFLAG_COMMON,
441
442
443
444
		.pio_mask	= 0x1f, /* pio0-4 */
		.udma_mask	= ATA_UDMA6,
		.port_ops	= &ahci_ops,
	},
Linus Torvalds's avatar
Linus Torvalds committed
445
446
};

447
static const struct pci_device_id ahci_pci_tbl[] = {
448
	/* Intel */
449
450
451
452
453
	{ PCI_VDEVICE(INTEL, 0x2652), board_ahci }, /* ICH6 */
	{ PCI_VDEVICE(INTEL, 0x2653), board_ahci }, /* ICH6M */
	{ PCI_VDEVICE(INTEL, 0x27c1), board_ahci }, /* ICH7 */
	{ PCI_VDEVICE(INTEL, 0x27c5), board_ahci }, /* ICH7M */
	{ PCI_VDEVICE(INTEL, 0x27c3), board_ahci }, /* ICH7R */
454
	{ PCI_VDEVICE(AL, 0x5288), board_ahci_ign_iferr }, /* ULi M5288 */
455
456
457
458
	{ PCI_VDEVICE(INTEL, 0x2681), board_ahci }, /* ESB2 */
	{ PCI_VDEVICE(INTEL, 0x2682), board_ahci }, /* ESB2 */
	{ PCI_VDEVICE(INTEL, 0x2683), board_ahci }, /* ESB2 */
	{ PCI_VDEVICE(INTEL, 0x27c6), board_ahci }, /* ICH7-M DH */
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
	{ PCI_VDEVICE(INTEL, 0x2821), board_ahci }, /* ICH8 */
	{ PCI_VDEVICE(INTEL, 0x2822), board_ahci }, /* ICH8 */
	{ PCI_VDEVICE(INTEL, 0x2824), board_ahci }, /* ICH8 */
	{ PCI_VDEVICE(INTEL, 0x2829), board_ahci }, /* ICH8M */
	{ PCI_VDEVICE(INTEL, 0x282a), board_ahci }, /* ICH8M */
	{ PCI_VDEVICE(INTEL, 0x2922), board_ahci }, /* ICH9 */
	{ PCI_VDEVICE(INTEL, 0x2923), board_ahci }, /* ICH9 */
	{ PCI_VDEVICE(INTEL, 0x2924), board_ahci }, /* ICH9 */
	{ PCI_VDEVICE(INTEL, 0x2925), board_ahci }, /* ICH9 */
	{ PCI_VDEVICE(INTEL, 0x2927), board_ahci }, /* ICH9 */
	{ PCI_VDEVICE(INTEL, 0x2929), board_ahci }, /* ICH9M */
	{ PCI_VDEVICE(INTEL, 0x292a), board_ahci }, /* ICH9M */
	{ PCI_VDEVICE(INTEL, 0x292b), board_ahci }, /* ICH9M */
	{ PCI_VDEVICE(INTEL, 0x292c), board_ahci }, /* ICH9M */
	{ PCI_VDEVICE(INTEL, 0x292f), board_ahci }, /* ICH9M */
	{ PCI_VDEVICE(INTEL, 0x294d), board_ahci }, /* ICH9 */
	{ PCI_VDEVICE(INTEL, 0x294e), board_ahci }, /* ICH9M */
476
477
	{ PCI_VDEVICE(INTEL, 0x502a), board_ahci }, /* Tolapai */
	{ PCI_VDEVICE(INTEL, 0x502b), board_ahci }, /* Tolapai */
478
479
	{ PCI_VDEVICE(INTEL, 0x3a05), board_ahci }, /* ICH10 */
	{ PCI_VDEVICE(INTEL, 0x3a25), board_ahci }, /* ICH10 */
480

481
482
483
	/* JMicron 360/1/3/5/6, match class to avoid IDE function */
	{ PCI_VENDOR_ID_JMICRON, PCI_ANY_ID, PCI_ANY_ID, PCI_ANY_ID,
	  PCI_CLASS_STORAGE_SATA_AHCI, 0xffffff, board_ahci_ign_iferr },
484
485

	/* ATI */
486
	{ PCI_VDEVICE(ATI, 0x4380), board_ahci_sb600 }, /* ATI SB600 */
487
488
489
490
491
492
	{ PCI_VDEVICE(ATI, 0x4390), board_ahci_sb600 }, /* ATI SB700/800 */
	{ PCI_VDEVICE(ATI, 0x4391), board_ahci_sb600 }, /* ATI SB700/800 */
	{ PCI_VDEVICE(ATI, 0x4392), board_ahci_sb600 }, /* ATI SB700/800 */
	{ PCI_VDEVICE(ATI, 0x4393), board_ahci_sb600 }, /* ATI SB700/800 */
	{ PCI_VDEVICE(ATI, 0x4394), board_ahci_sb600 }, /* ATI SB700/800 */
	{ PCI_VDEVICE(ATI, 0x4395), board_ahci_sb600 }, /* ATI SB700/800 */
493
494

	/* VIA */
495
	{ PCI_VDEVICE(VIA, 0x3349), board_ahci_vt8251 }, /* VIA VT8251 */
Tejun Heo's avatar
Tejun Heo committed
496
	{ PCI_VDEVICE(VIA, 0x6287), board_ahci_vt8251 }, /* VIA VT8251 */
497
498

	/* NVIDIA */
499
500
501
502
	{ PCI_VDEVICE(NVIDIA, 0x044c), board_ahci },		/* MCP65 */
	{ PCI_VDEVICE(NVIDIA, 0x044d), board_ahci },		/* MCP65 */
	{ PCI_VDEVICE(NVIDIA, 0x044e), board_ahci },		/* MCP65 */
	{ PCI_VDEVICE(NVIDIA, 0x044f), board_ahci },		/* MCP65 */
503
504
505
506
507
508
509
510
	{ PCI_VDEVICE(NVIDIA, 0x045c), board_ahci },		/* MCP65 */
	{ PCI_VDEVICE(NVIDIA, 0x045d), board_ahci },		/* MCP65 */
	{ PCI_VDEVICE(NVIDIA, 0x045e), board_ahci },		/* MCP65 */
	{ PCI_VDEVICE(NVIDIA, 0x045f), board_ahci },		/* MCP65 */
	{ PCI_VDEVICE(NVIDIA, 0x0550), board_ahci },		/* MCP67 */
	{ PCI_VDEVICE(NVIDIA, 0x0551), board_ahci },		/* MCP67 */
	{ PCI_VDEVICE(NVIDIA, 0x0552), board_ahci },		/* MCP67 */
	{ PCI_VDEVICE(NVIDIA, 0x0553), board_ahci },		/* MCP67 */
511
512
513
514
515
516
517
518
	{ PCI_VDEVICE(NVIDIA, 0x0554), board_ahci },		/* MCP67 */
	{ PCI_VDEVICE(NVIDIA, 0x0555), board_ahci },		/* MCP67 */
	{ PCI_VDEVICE(NVIDIA, 0x0556), board_ahci },		/* MCP67 */
	{ PCI_VDEVICE(NVIDIA, 0x0557), board_ahci },		/* MCP67 */
	{ PCI_VDEVICE(NVIDIA, 0x0558), board_ahci },		/* MCP67 */
	{ PCI_VDEVICE(NVIDIA, 0x0559), board_ahci },		/* MCP67 */
	{ PCI_VDEVICE(NVIDIA, 0x055a), board_ahci },		/* MCP67 */
	{ PCI_VDEVICE(NVIDIA, 0x055b), board_ahci },		/* MCP67 */
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
	{ PCI_VDEVICE(NVIDIA, 0x07f0), board_ahci },		/* MCP73 */
	{ PCI_VDEVICE(NVIDIA, 0x07f1), board_ahci },		/* MCP73 */
	{ PCI_VDEVICE(NVIDIA, 0x07f2), board_ahci },		/* MCP73 */
	{ PCI_VDEVICE(NVIDIA, 0x07f3), board_ahci },		/* MCP73 */
	{ PCI_VDEVICE(NVIDIA, 0x07f4), board_ahci },		/* MCP73 */
	{ PCI_VDEVICE(NVIDIA, 0x07f5), board_ahci },		/* MCP73 */
	{ PCI_VDEVICE(NVIDIA, 0x07f6), board_ahci },		/* MCP73 */
	{ PCI_VDEVICE(NVIDIA, 0x07f7), board_ahci },		/* MCP73 */
	{ PCI_VDEVICE(NVIDIA, 0x07f8), board_ahci },		/* MCP73 */
	{ PCI_VDEVICE(NVIDIA, 0x07f9), board_ahci },		/* MCP73 */
	{ PCI_VDEVICE(NVIDIA, 0x07fa), board_ahci },		/* MCP73 */
	{ PCI_VDEVICE(NVIDIA, 0x07fb), board_ahci },		/* MCP73 */
	{ PCI_VDEVICE(NVIDIA, 0x0ad0), board_ahci },		/* MCP77 */
	{ PCI_VDEVICE(NVIDIA, 0x0ad1), board_ahci },		/* MCP77 */
	{ PCI_VDEVICE(NVIDIA, 0x0ad2), board_ahci },		/* MCP77 */
	{ PCI_VDEVICE(NVIDIA, 0x0ad3), board_ahci },		/* MCP77 */
	{ PCI_VDEVICE(NVIDIA, 0x0ad4), board_ahci },		/* MCP77 */
	{ PCI_VDEVICE(NVIDIA, 0x0ad5), board_ahci },		/* MCP77 */
	{ PCI_VDEVICE(NVIDIA, 0x0ad6), board_ahci },		/* MCP77 */
	{ PCI_VDEVICE(NVIDIA, 0x0ad7), board_ahci },		/* MCP77 */
	{ PCI_VDEVICE(NVIDIA, 0x0ad8), board_ahci },		/* MCP77 */
	{ PCI_VDEVICE(NVIDIA, 0x0ad9), board_ahci },		/* MCP77 */
	{ PCI_VDEVICE(NVIDIA, 0x0ada), board_ahci },		/* MCP77 */
	{ PCI_VDEVICE(NVIDIA, 0x0adb), board_ahci },		/* MCP77 */
543
544
545
546
	{ PCI_VDEVICE(NVIDIA, 0x0ab4), board_ahci },		/* MCP79 */
	{ PCI_VDEVICE(NVIDIA, 0x0ab5), board_ahci },		/* MCP79 */
	{ PCI_VDEVICE(NVIDIA, 0x0ab6), board_ahci },		/* MCP79 */
	{ PCI_VDEVICE(NVIDIA, 0x0ab7), board_ahci },		/* MCP79 */
547
548
549
550
551
552
553
554
	{ PCI_VDEVICE(NVIDIA, 0x0ab8), board_ahci },		/* MCP79 */
	{ PCI_VDEVICE(NVIDIA, 0x0ab9), board_ahci },		/* MCP79 */
	{ PCI_VDEVICE(NVIDIA, 0x0aba), board_ahci },		/* MCP79 */
	{ PCI_VDEVICE(NVIDIA, 0x0abb), board_ahci },		/* MCP79 */
	{ PCI_VDEVICE(NVIDIA, 0x0abc), board_ahci },		/* MCP79 */
	{ PCI_VDEVICE(NVIDIA, 0x0abd), board_ahci },		/* MCP79 */
	{ PCI_VDEVICE(NVIDIA, 0x0abe), board_ahci },		/* MCP79 */
	{ PCI_VDEVICE(NVIDIA, 0x0abf), board_ahci },		/* MCP79 */
555

Jeff Garzik's avatar
Jeff Garzik committed
556
	/* SiS */
557
558
559
	{ PCI_VDEVICE(SI, 0x1184), board_ahci }, /* SiS 966 */
	{ PCI_VDEVICE(SI, 0x1185), board_ahci }, /* SiS 966 */
	{ PCI_VDEVICE(SI, 0x0186), board_ahci }, /* SiS 968 */
Jeff Garzik's avatar
Jeff Garzik committed
560

561
562
563
	/* Marvell */
	{ PCI_VDEVICE(MARVELL, 0x6145), board_ahci_mv },	/* 6145 */

564
565
	/* Generic, PCI class code for AHCI */
	{ PCI_ANY_ID, PCI_ANY_ID, PCI_ANY_ID, PCI_ANY_ID,
566
	  PCI_CLASS_STORAGE_SATA_AHCI, 0xffffff, board_ahci },
567

Linus Torvalds's avatar
Linus Torvalds committed
568
569
570
571
572
573
574
575
	{ }	/* terminate list */
};


static struct pci_driver ahci_pci_driver = {
	.name			= DRV_NAME,
	.id_table		= ahci_pci_tbl,
	.probe			= ahci_init_one,
576
	.remove			= ata_pci_remove_one,
577
#ifdef CONFIG_PM
578
579
	.suspend		= ahci_pci_device_suspend,
	.resume			= ahci_pci_device_resume,
580
#endif
Linus Torvalds's avatar
Linus Torvalds committed
581
582
583
};


584
585
586
587
588
static inline int ahci_nr_ports(u32 cap)
{
	return (cap & 0x1f) + 1;
}

589
590
static inline void __iomem *__ahci_port_base(struct ata_host *host,
					     unsigned int port_no)
Linus Torvalds's avatar
Linus Torvalds committed
591
{
592
	void __iomem *mmio = host->iomap[AHCI_PCI_BAR];
593

594
595
596
597
598
599
	return mmio + 0x100 + (port_no * 0x80);
}

static inline void __iomem *ahci_port_base(struct ata_port *ap)
{
	return __ahci_port_base(ap->host, ap->port_no);
Linus Torvalds's avatar
Linus Torvalds committed
600
601
}

602
603
604
605
606
607
608
609
610
611
612
613
614
615
static void ahci_enable_ahci(void __iomem *mmio)
{
	u32 tmp;

	/* turn on AHCI_EN */
	tmp = readl(mmio + HOST_CTL);
	if (!(tmp & HOST_AHCI_EN)) {
		tmp |= HOST_AHCI_EN;
		writel(tmp, mmio + HOST_CTL);
		tmp = readl(mmio + HOST_CTL);	/* flush && sanity check */
		WARN_ON(!(tmp & HOST_AHCI_EN));
	}
}

616
617
/**
 *	ahci_save_initial_config - Save and fixup initial config values
618
619
 *	@pdev: target PCI device
 *	@hpriv: host private area to store config values
620
621
622
623
624
625
626
627
628
629
630
 *
 *	Some registers containing configuration info might be setup by
 *	BIOS and might be cleared on reset.  This function saves the
 *	initial values of those registers into @hpriv such that they
 *	can be restored after controller reset.
 *
 *	If inconsistent, config values are fixed up by this function.
 *
 *	LOCKING:
 *	None.
 */
631
632
static void ahci_save_initial_config(struct pci_dev *pdev,
				     struct ahci_host_priv *hpriv)
633
{
634
	void __iomem *mmio = pcim_iomap_table(pdev)[AHCI_PCI_BAR];
635
	u32 cap, port_map;
636
	int i;
637

638
639
640
	/* make sure AHCI mode is enabled before accessing CAP */
	ahci_enable_ahci(mmio);

641
642
643
644
645
646
	/* Values prefixed with saved_ are written back to host after
	 * reset.  Values without are used for driver operation.
	 */
	hpriv->saved_cap = cap = readl(mmio + HOST_CAP);
	hpriv->saved_port_map = port_map = readl(mmio + HOST_PORTS_IMPL);

647
	/* some chips have errata preventing 64bit use */
648
	if ((cap & HOST_CAP_64) && (hpriv->flags & AHCI_HFLAG_32BIT_ONLY)) {
Tejun Heo's avatar
Tejun Heo committed
649
650
651
652
653
		dev_printk(KERN_INFO, &pdev->dev,
			   "controller can't do 64bit DMA, forcing 32bit\n");
		cap &= ~HOST_CAP_64;
	}

654
	if ((cap & HOST_CAP_NCQ) && (hpriv->flags & AHCI_HFLAG_NO_NCQ)) {
655
656
657
658
659
		dev_printk(KERN_INFO, &pdev->dev,
			   "controller can't do NCQ, turning off CAP_NCQ\n");
		cap &= ~HOST_CAP_NCQ;
	}

Tejun Heo's avatar
Tejun Heo committed
660
661
662
663
664
665
	if ((cap && HOST_CAP_PMP) && (hpriv->flags & AHCI_HFLAG_NO_PMP)) {
		dev_printk(KERN_INFO, &pdev->dev,
			   "controller can't do PMP, turning off CAP_PMP\n");
		cap &= ~HOST_CAP_PMP;
	}

666
667
668
669
670
	/*
	 * Temporary Marvell 6145 hack: PATA port presence
	 * is asserted through the standard AHCI port
	 * presence register, as bit 4 (counting from 0)
	 */
671
	if (hpriv->flags & AHCI_HFLAG_MV_PATA) {
672
673
674
675
676
677
678
679
		dev_printk(KERN_ERR, &pdev->dev,
			   "MV_AHCI HACK: port_map %x -> %x\n",
			   hpriv->port_map,
			   hpriv->port_map & 0xf);

		port_map &= 0xf;
	}

680
	/* cross check port_map and cap.n_ports */
681
	if (port_map) {
Tejun Heo's avatar
Tejun Heo committed
682
		int map_ports = 0;
683

Tejun Heo's avatar
Tejun Heo committed
684
685
686
		for (i = 0; i < AHCI_MAX_PORTS; i++)
			if (port_map & (1 << i))
				map_ports++;
687

Tejun Heo's avatar
Tejun Heo committed
688
689
		/* If PI has more ports than n_ports, whine, clear
		 * port_map and let it be generated from n_ports.
690
		 */
Tejun Heo's avatar
Tejun Heo committed
691
		if (map_ports > ahci_nr_ports(cap)) {
692
			dev_printk(KERN_WARNING, &pdev->dev,
Tejun Heo's avatar
Tejun Heo committed
693
694
695
				   "implemented port map (0x%x) contains more "
				   "ports than nr_ports (%u), using nr_ports\n",
				   port_map, ahci_nr_ports(cap));
696
697
698
699
700
701
			port_map = 0;
		}
	}

	/* fabricate port_map from cap.nr_ports */
	if (!port_map) {
702
		port_map = (1 << ahci_nr_ports(cap)) - 1;
703
704
705
706
707
		dev_printk(KERN_WARNING, &pdev->dev,
			   "forcing PORTS_IMPL to 0x%x\n", port_map);

		/* write the fixed up value to the PI register */
		hpriv->saved_port_map = port_map;
708
709
	}

710
711
712
713
714
715
716
	/* record values to use during operation */
	hpriv->cap = cap;
	hpriv->port_map = port_map;
}

/**
 *	ahci_restore_initial_config - Restore initial config
717
 *	@host: target ATA host
718
719
720
721
722
723
 *
 *	Restore initial config stored by ahci_save_initial_config().
 *
 *	LOCKING:
 *	None.
 */
724
static void ahci_restore_initial_config(struct ata_host *host)
725
{
726
727
728
	struct ahci_host_priv *hpriv = host->private_data;
	void __iomem *mmio = host->iomap[AHCI_PCI_BAR];

729
730
731
732
733
	writel(hpriv->saved_cap, mmio + HOST_CAP);
	writel(hpriv->saved_port_map, mmio + HOST_PORTS_IMPL);
	(void) readl(mmio + HOST_PORTS_IMPL);	/* flush */
}

734
static unsigned ahci_scr_offset(struct ata_port *ap, unsigned int sc_reg)
Linus Torvalds's avatar
Linus Torvalds committed
735
{
736
737
738
739
740
741
742
743
	static const int offset[] = {
		[SCR_STATUS]		= PORT_SCR_STAT,
		[SCR_CONTROL]		= PORT_SCR_CTL,
		[SCR_ERROR]		= PORT_SCR_ERR,
		[SCR_ACTIVE]		= PORT_SCR_ACT,
		[SCR_NOTIFICATION]	= PORT_SCR_NTF,
	};
	struct ahci_host_priv *hpriv = ap->host->private_data;
Linus Torvalds's avatar
Linus Torvalds committed
744

745
746
747
	if (sc_reg < ARRAY_SIZE(offset) &&
	    (sc_reg != SCR_NOTIFICATION || (hpriv->cap & HOST_CAP_SNTF)))
		return offset[sc_reg];
748
	return 0;
Linus Torvalds's avatar
Linus Torvalds committed
749
750
}

751
static int ahci_scr_read(struct ata_port *ap, unsigned int sc_reg, u32 *val)
Linus Torvalds's avatar
Linus Torvalds committed
752
{
753
754
755
756
757
758
	void __iomem *port_mmio = ahci_port_base(ap);
	int offset = ahci_scr_offset(ap, sc_reg);

	if (offset) {
		*val = readl(port_mmio + offset);
		return 0;
Linus Torvalds's avatar
Linus Torvalds committed
759
	}
760
761
	return -EINVAL;
}
Linus Torvalds's avatar
Linus Torvalds committed
762

763
764
765
766
767
768
769
770
771
772
static int ahci_scr_write(struct ata_port *ap, unsigned int sc_reg, u32 val)
{
	void __iomem *port_mmio = ahci_port_base(ap);
	int offset = ahci_scr_offset(ap, sc_reg);

	if (offset) {
		writel(val, port_mmio + offset);
		return 0;
	}
	return -EINVAL;
Linus Torvalds's avatar
Linus Torvalds committed
773
774
}

775
static void ahci_start_engine(struct ata_port *ap)
776
{
777
	void __iomem *port_mmio = ahci_port_base(ap);
778
779
	u32 tmp;

780
	/* start DMA */
781
	tmp = readl(port_mmio + PORT_CMD);
782
783
784
785
786
	tmp |= PORT_CMD_START;
	writel(tmp, port_mmio + PORT_CMD);
	readl(port_mmio + PORT_CMD); /* flush */
}

787
static int ahci_stop_engine(struct ata_port *ap)
788
{
789
	void __iomem *port_mmio = ahci_port_base(ap);
790
791
792
793
	u32 tmp;

	tmp = readl(port_mmio + PORT_CMD);

794
	/* check if the HBA is idle */
795
796
797
	if ((tmp & (PORT_CMD_START | PORT_CMD_LIST_ON)) == 0)
		return 0;

798
	/* setting HBA to idle */
799
800
801
	tmp &= ~PORT_CMD_START;
	writel(tmp, port_mmio + PORT_CMD);

802
	/* wait for engine to stop. This could be as long as 500 msec */
803
	tmp = ata_wait_register(port_mmio + PORT_CMD,
804
				PORT_CMD_LIST_ON, PORT_CMD_LIST_ON, 1, 500);
805
	if (tmp & PORT_CMD_LIST_ON)
806
807
808
809
810
		return -EIO;

	return 0;
}

811
static void ahci_start_fis_rx(struct ata_port *ap)
812
{
813
814
815
	void __iomem *port_mmio = ahci_port_base(ap);
	struct ahci_host_priv *hpriv = ap->host->private_data;
	struct ahci_port_priv *pp = ap->private_data;
816
817
818
	u32 tmp;

	/* set FIS registers */
819
820
821
822
	if (hpriv->cap & HOST_CAP_64)
		writel((pp->cmd_slot_dma >> 16) >> 16,
		       port_mmio + PORT_LST_ADDR_HI);
	writel(pp->cmd_slot_dma & 0xffffffff, port_mmio + PORT_LST_ADDR);
823

824
825
826
827
	if (hpriv->cap & HOST_CAP_64)
		writel((pp->rx_fis_dma >> 16) >> 16,
		       port_mmio + PORT_FIS_ADDR_HI);
	writel(pp->rx_fis_dma & 0xffffffff, port_mmio + PORT_FIS_ADDR);
828
829
830
831
832
833
834
835
836
837

	/* enable FIS reception */
	tmp = readl(port_mmio + PORT_CMD);
	tmp |= PORT_CMD_FIS_RX;
	writel(tmp, port_mmio + PORT_CMD);

	/* flush */
	readl(port_mmio + PORT_CMD);
}

838
static int ahci_stop_fis_rx(struct ata_port *ap)
839
{
840
	void __iomem *port_mmio = ahci_port_base(ap);
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
	u32 tmp;

	/* disable FIS reception */
	tmp = readl(port_mmio + PORT_CMD);
	tmp &= ~PORT_CMD_FIS_RX;
	writel(tmp, port_mmio + PORT_CMD);

	/* wait for completion, spec says 500ms, give it 1000 */
	tmp = ata_wait_register(port_mmio + PORT_CMD, PORT_CMD_FIS_ON,
				PORT_CMD_FIS_ON, 10, 1000);
	if (tmp & PORT_CMD_FIS_ON)
		return -EBUSY;

	return 0;
}

857
static void ahci_power_up(struct ata_port *ap)
858
{
859
860
	struct ahci_host_priv *hpriv = ap->host->private_data;
	void __iomem *port_mmio = ahci_port_base(ap);
861
862
863
864
865
	u32 cmd;

	cmd = readl(port_mmio + PORT_CMD) & ~PORT_CMD_ICC_MASK;

	/* spin up device */
866
	if (hpriv->cap & HOST_CAP_SSS) {
867
868
869
870
871
872
873
874
		cmd |= PORT_CMD_SPIN_UP;
		writel(cmd, port_mmio + PORT_CMD);
	}

	/* wake up link */
	writel(cmd | PORT_CMD_ICC_ACTIVE, port_mmio + PORT_CMD);
}

875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
static void ahci_disable_alpm(struct ata_port *ap)
{
	struct ahci_host_priv *hpriv = ap->host->private_data;
	void __iomem *port_mmio = ahci_port_base(ap);
	u32 cmd;
	struct ahci_port_priv *pp = ap->private_data;

	/* IPM bits should be disabled by libata-core */
	/* get the existing command bits */
	cmd = readl(port_mmio + PORT_CMD);

	/* disable ALPM and ASP */
	cmd &= ~PORT_CMD_ASP;
	cmd &= ~PORT_CMD_ALPE;

	/* force the interface back to active */
	cmd |= PORT_CMD_ICC_ACTIVE;

	/* write out new cmd value */
	writel(cmd, port_mmio + PORT_CMD);
	cmd = readl(port_mmio + PORT_CMD);

	/* wait 10ms to be sure we've come out of any low power state */
	msleep(10);

	/* clear out any PhyRdy stuff from interrupt status */
	writel(PORT_IRQ_PHYRDY, port_mmio + PORT_IRQ_STAT);

	/* go ahead and clean out PhyRdy Change from Serror too */
	ahci_scr_write(ap, SCR_ERROR, ((1 << 16) | (1 << 18)));

	/*
 	 * Clear flag to indicate that we should ignore all PhyRdy
 	 * state changes
 	 */
	hpriv->flags &= ~AHCI_HFLAG_NO_HOTPLUG;

	/*
 	 * Enable interrupts on Phy Ready.
 	 */
	pp->intr_mask |= PORT_IRQ_PHYRDY;
	writel(pp->intr_mask, port_mmio + PORT_IRQ_MASK);

	/*
 	 * don't change the link pm policy - we can be called
 	 * just to turn of link pm temporarily
 	 */
}

static int ahci_enable_alpm(struct ata_port *ap,
	enum link_pm policy)
{
	struct ahci_host_priv *hpriv = ap->host->private_data;
	void __iomem *port_mmio = ahci_port_base(ap);
	u32 cmd;
	struct ahci_port_priv *pp = ap->private_data;
	u32 asp;

	/* Make sure the host is capable of link power management */
	if (!(hpriv->cap & HOST_CAP_ALPM))
		return -EINVAL;

	switch (policy) {
	case MAX_PERFORMANCE:
	case NOT_AVAILABLE:
		/*
 		 * if we came here with NOT_AVAILABLE,
 		 * it just means this is the first time we
 		 * have tried to enable - default to max performance,
 		 * and let the user go to lower power modes on request.
 		 */
		ahci_disable_alpm(ap);
		return 0;
	case MIN_POWER:
		/* configure HBA to enter SLUMBER */
		asp = PORT_CMD_ASP;
		break;
	case MEDIUM_POWER:
		/* configure HBA to enter PARTIAL */
		asp = 0;
		break;
	default:
		return -EINVAL;
	}

	/*
 	 * Disable interrupts on Phy Ready. This keeps us from
 	 * getting woken up due to spurious phy ready interrupts
	 * TBD - Hot plug should be done via polling now, is
	 * that even supported?
 	 */
	pp->intr_mask &= ~PORT_IRQ_PHYRDY;
	writel(pp->intr_mask, port_mmio + PORT_IRQ_MASK);

	/*
 	 * Set a flag to indicate that we should ignore all PhyRdy
 	 * state changes since these can happen now whenever we
 	 * change link state
 	 */
	hpriv->flags |= AHCI_HFLAG_NO_HOTPLUG;

	/* get the existing command bits */
	cmd = readl(port_mmio + PORT_CMD);

	/*
 	 * Set ASP based on Policy
 	 */
	cmd |= asp;

	/*
 	 * Setting this bit will instruct the HBA to aggressively
 	 * enter a lower power link state when it's appropriate and
 	 * based on the value set above for ASP
 	 */
	cmd |= PORT_CMD_ALPE;

	/* write out new cmd value */
	writel(cmd, port_mmio + PORT_CMD);
	cmd = readl(port_mmio + PORT_CMD);

	/* IPM bits should be set by libata-core */
	return 0;
}

999
#ifdef CONFIG_PM
1000
static void ahci_power_down(struct ata_port *ap)
1001
{
1002
1003
	struct ahci_host_priv *hpriv = ap->host->private_data;
	void __iomem *port_mmio = ahci_port_base(ap);
1004
1005
	u32 cmd, scontrol;

1006
	if (!(hpriv->cap & HOST_CAP_SSS))
1007
		return;
1008

1009
1010
1011
1012
	/* put device into listen mode, first set PxSCTL.DET to 0 */
	scontrol = readl(port_mmio + PORT_SCR_CTL);
	scontrol &= ~0xf;
	writel(scontrol, port_mmio + PORT_SCR_CTL);
1013

1014
1015
1016
1017
	/* then set PxCMD.SUD to 0 */
	cmd = readl(port_mmio + PORT_CMD) & ~PORT_CMD_ICC_MASK;
	cmd &= ~PORT_CMD_SPIN_UP;
	writel(cmd, port_mmio + PORT_CMD);
1018
}
1019
#endif
1020

1021
static void ahci_start_port(struct ata_port *ap)
1022
1023
{
	/* enable FIS reception */
1024
	ahci_start_fis_rx(ap);
1025
1026

	/* enable DMA */
1027
	ahci_start_engine(ap);
1028
1029
}

1030
static int ahci_deinit_port(struct ata_port *ap, const char **emsg)
1031
1032
1033
1034
{
	int rc;

	/* disable DMA */
1035
	rc = ahci_stop_engine(ap);
1036
1037
1038
1039
1040
1041
	if (rc) {
		*emsg = "failed to stop engine";
		return rc;
	}

	/* disable FIS reception */
1042
	rc = ahci_stop_fis_rx(ap);
1043
1044
1045
1046
1047
1048
1049
1050
	if (rc) {
		*emsg = "failed stop FIS RX";
		return rc;
	}

	return 0;
}

1051
static int ahci_reset_controller(struct ata_host *host)
1052
{
1053
	struct pci_dev *pdev = to_pci_dev(host->dev);
Tejun Heo's avatar
Tejun Heo committed
1054
	struct ahci_host_priv *hpriv = host->private_data;
1055
	void __iomem *mmio = host->iomap[AHCI_PCI_BAR];
1056
	u32 tmp;
1057

1058
1059
1060
	/* we must be in AHCI mode, before using anything
	 * AHCI-specific, such as HOST_RESET.
	 */
1061
	ahci_enable_ahci(mmio);
1062
1063

	/* global controller reset */
1064
	tmp = readl(mmio + HOST_CTL);
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
	if ((tmp & HOST_RESET) == 0) {
		writel(tmp | HOST_RESET, mmio + HOST_CTL);
		readl(mmio + HOST_CTL); /* flush */
	}

	/* reset must complete within 1 second, or
	 * the hardware should be considered fried.
	 */
	ssleep(1);

	tmp = readl(mmio + HOST_CTL);
	if (tmp & HOST_RESET) {
1077
		dev_printk(KERN_ERR, host->dev,
1078
1079
1080
1081
			   "controller reset failed (0x%x)\n", tmp);
		return -EIO;
	}

1082
	/* turn on AHCI mode */
1083
	ahci_enable_ahci(mmio);
1084

1085
	/* some registers might be cleared on reset.  restore initial values */
1086
	ahci_restore_initial_config(host);
1087
1088
1089
1090
1091
1092

	if (pdev->vendor == PCI_VENDOR_ID_INTEL) {
		u16 tmp16;

		/* configure PCS */
		pci_read_config_word(pdev, 0x92, &tmp16);
Tejun Heo's avatar
Tejun Heo committed
1093
1094
1095
1096
		if ((tmp16 & hpriv->port_map) != hpriv->port_map) {
			tmp16 |= hpriv->port_map;
			pci_write_config_word(pdev, 0x92, tmp16);
		}
1097
1098
1099
1100
1101
	}

	return 0;
}

1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
static void ahci_port_init(struct pci_dev *pdev, struct ata_port *ap,
			   int port_no, void __iomem *mmio,
			   void __iomem *port_mmio)
{
	const char *emsg = NULL;
	int rc;
	u32 tmp;

	/* make sure port is not active */
	rc = ahci_deinit_port(ap, &emsg);
	if (rc)
		dev_printk(KERN_WARNING, &pdev->dev,
			   "%s (%d)\n", emsg, rc);

	/* clear SError */
	tmp = readl(port_mmio + PORT_SCR_ERR);
	VPRINTK("PORT_SCR_ERR 0x%x\n", tmp);
	writel(tmp, port_mmio + PORT_SCR_ERR);

	/* clear port IRQ */
	tmp = readl(port_mmio + PORT_IRQ_STAT);
	VPRINTK("PORT_IRQ_STAT 0x%x\n", tmp);
	if (tmp)
		writel(tmp, port_mmio + PORT_IRQ_STAT);

	writel(1 << port_no, mmio + HOST_IRQ_STAT);
}

1130
static void ahci_init_controller(struct ata_host *host)
1131
{
1132
	struct ahci_host_priv *hpriv = host->private_data;
1133
1134
	struct pci_dev *pdev = to_pci_dev(host->dev);
	void __iomem *mmio = host->iomap[AHCI_PCI_BAR];
1135
	int i;
1136
	void __iomem *port_mmio;
1137
1138
	u32 tmp;

1139
	if (hpriv->flags & AHCI_HFLAG_MV_PATA) {
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
		port_mmio = __ahci_port_base(host, 4);

		writel(0, port_mmio + PORT_IRQ_MASK);

		/* clear port IRQ */
		tmp = readl(port_mmio + PORT_IRQ_STAT);
		VPRINTK("PORT_IRQ_STAT 0x%x\n", tmp);
		if (tmp)
			writel(tmp, port_mmio + PORT_IRQ_STAT);
	}

1151
1152
	for (i = 0; i < host->n_ports; i++) {
		struct ata_port *ap = host->ports[i];
1153

1154
		port_mmio = ahci_port_base(ap);
1155
		if (ata_port_is_dummy(ap))
1156
1157
			continue;

1158
		ahci_port_init(pdev, ap, i, mmio, port_mmio);
1159
1160
1161
1162
1163
1164
1165
1166
1167
	}

	tmp = readl(mmio + HOST_CTL);
	VPRINTK("HOST_CTL 0x%x\n", tmp);
	writel(tmp | HOST_IRQ_EN, mmio + HOST_CTL);
	tmp = readl(mmio + HOST_CTL);
	VPRINTK("HOST_CTL 0x%x\n", tmp);
}

1168
static unsigned int ahci_dev_classify(struct ata_port *ap)
Linus Torvalds's avatar
Linus Torvalds committed
1169
{
1170
	void __iomem *port_mmio = ahci_port_base(ap);
Linus Torvalds's avatar
Linus Torvalds committed
1171
	struct ata_taskfile tf;
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
	u32 tmp;

	tmp =