projects / deliverable / linux.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
libata: implement and use ops inheritance
[deliverable/linux.git] / drivers / ata / sata_nv.c
CommitLineData
1da177e4
LT		 1/*
2 * sata_nv.c - NVIDIA nForce SATA
3 *
4 * Copyright 2004 NVIDIA Corp. All rights reserved.
5 * Copyright 2004 Andrew Chew
6 *
aa7e16d6
JG		 7 *
8 * This program is free software; you can redistribute it and/or modify
9 * it under the terms of the GNU General Public License as published by
10 * the Free Software Foundation; either version 2, or (at your option)
11 * any later version.
12 *
13 * This program is distributed in the hope that it will be useful,
14 * but WITHOUT ANY WARRANTY; without even the implied warranty of
15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 * GNU General Public License for more details.
17 *
18 * You should have received a copy of the GNU General Public License
19 * along with this program; see the file COPYING. If not, write to
20 * the Free Software Foundation, 675 Mass Ave, Cambridge, MA 02139, USA.
1da177e4 21 *
af36d7f0
JG		 22 *
23 * libata documentation is available via 'make {ps|pdf}docs',
24 * as Documentation/DocBook/libata.*
25 *
26 * No hardware documentation available outside of NVIDIA.
27 * This driver programs the NVIDIA SATA controller in a similar
28 * fashion as with other PCI IDE BMDMA controllers, with a few
29 * NV-specific details such as register offsets, SATA phy location,
30 * hotplug info, etc.
31 *
fbbb262d
RH		 32 * CK804/MCP04 controllers support an alternate programming interface
33 * similar to the ADMA specification (with some modifications).
34 * This allows the use of NCQ. Non-DMA-mapped ATA commands are still
35 * sent through the legacy interface.
36 *
1da177e4
LT		 37 */
38
1da177e4
LT		 39#include <linux/kernel.h>
40#include <linux/module.h>
41#include <linux/pci.h>
42#include <linux/init.h>
43#include <linux/blkdev.h>
44#include <linux/delay.h>
45#include <linux/interrupt.h>
a9524a76 46#include <linux/device.h>
1da177e4 47#include <scsi/scsi_host.h>
fbbb262d 48#include <scsi/scsi_device.h>
1da177e4
LT		 49#include <linux/libata.h>
50
51#define DRV_NAME "sata_nv"
2a3103ce 52#define DRV_VERSION "3.5"
fbbb262d
RH		 53
54#define NV_ADMA_DMA_BOUNDARY 0xffffffffUL
1da177e4 55
10ad05df 56enum {
0d5ff566
TH		 57 NV_MMIO_BAR = 5,
58
10ad05df
JG		 59 NV_PORTS = 2,
60 NV_PIO_MASK = 0x1f,
61 NV_MWDMA_MASK = 0x07,
62 NV_UDMA_MASK = 0x7f,
63 NV_PORT0_SCR_REG_OFFSET = 0x00,
64 NV_PORT1_SCR_REG_OFFSET = 0x40,
1da177e4 65
27e4b274 66 /* INT_STATUS/ENABLE */
10ad05df 67 NV_INT_STATUS = 0x10,
10ad05df 68 NV_INT_ENABLE = 0x11,
27e4b274 69 NV_INT_STATUS_CK804 = 0x440,
10ad05df 70 NV_INT_ENABLE_CK804 = 0x441,
1da177e4 71
27e4b274
TH		 72 /* INT_STATUS/ENABLE bits */
73 NV_INT_DEV = 0x01,
74 NV_INT_PM = 0x02,
75 NV_INT_ADDED = 0x04,
76 NV_INT_REMOVED = 0x08,
77
78 NV_INT_PORT_SHIFT = 4, /* each port occupies 4 bits */
79
39f87582 80 NV_INT_ALL = 0x0f,
5a44efff
TH		 81 NV_INT_MASK = NV_INT_DEV |
82 NV_INT_ADDED | NV_INT_REMOVED,
39f87582 83
27e4b274 84 /* INT_CONFIG */
10ad05df
JG		 85 NV_INT_CONFIG = 0x12,
86 NV_INT_CONFIG_METHD = 0x01, // 0 = INT, 1 = SMI
1da177e4 87
10ad05df
JG		 88 // For PCI config register 20
89 NV_MCP_SATA_CFG_20 = 0x50,
90 NV_MCP_SATA_CFG_20_SATA_SPACE_EN = 0x04,
fbbb262d
RH		 91 NV_MCP_SATA_CFG_20_PORT0_EN = (1 << 17),
92 NV_MCP_SATA_CFG_20_PORT1_EN = (1 << 16),
93 NV_MCP_SATA_CFG_20_PORT0_PWB_EN = (1 << 14),
94 NV_MCP_SATA_CFG_20_PORT1_PWB_EN = (1 << 12),
95
96 NV_ADMA_MAX_CPBS = 32,
97 NV_ADMA_CPB_SZ = 128,
98 NV_ADMA_APRD_SZ = 16,
99 NV_ADMA_SGTBL_LEN = (1024 - NV_ADMA_CPB_SZ) /
100 NV_ADMA_APRD_SZ,
101 NV_ADMA_SGTBL_TOTAL_LEN = NV_ADMA_SGTBL_LEN + 5,
102 NV_ADMA_SGTBL_SZ = NV_ADMA_SGTBL_LEN * NV_ADMA_APRD_SZ,
103 NV_ADMA_PORT_PRIV_DMA_SZ = NV_ADMA_MAX_CPBS *
104 (NV_ADMA_CPB_SZ + NV_ADMA_SGTBL_SZ),
105
106 /* BAR5 offset to ADMA general registers */
107 NV_ADMA_GEN = 0x400,
108 NV_ADMA_GEN_CTL = 0x00,
109 NV_ADMA_NOTIFIER_CLEAR = 0x30,
110
111 /* BAR5 offset to ADMA ports */
112 NV_ADMA_PORT = 0x480,
113
114 /* size of ADMA port register space */
115 NV_ADMA_PORT_SIZE = 0x100,
116
117 /* ADMA port registers */
118 NV_ADMA_CTL = 0x40,
119 NV_ADMA_CPB_COUNT = 0x42,
120 NV_ADMA_NEXT_CPB_IDX = 0x43,
121 NV_ADMA_STAT = 0x44,
122 NV_ADMA_CPB_BASE_LOW = 0x48,
123 NV_ADMA_CPB_BASE_HIGH = 0x4C,
124 NV_ADMA_APPEND = 0x50,
125 NV_ADMA_NOTIFIER = 0x68,
126 NV_ADMA_NOTIFIER_ERROR = 0x6C,
127
128 /* NV_ADMA_CTL register bits */
129 NV_ADMA_CTL_HOTPLUG_IEN = (1 << 0),
130 NV_ADMA_CTL_CHANNEL_RESET = (1 << 5),
131 NV_ADMA_CTL_GO = (1 << 7),
132 NV_ADMA_CTL_AIEN = (1 << 8),
133 NV_ADMA_CTL_READ_NON_COHERENT = (1 << 11),
134 NV_ADMA_CTL_WRITE_NON_COHERENT = (1 << 12),
135
136 /* CPB response flag bits */
137 NV_CPB_RESP_DONE = (1 << 0),
138 NV_CPB_RESP_ATA_ERR = (1 << 3),
139 NV_CPB_RESP_CMD_ERR = (1 << 4),
140 NV_CPB_RESP_CPB_ERR = (1 << 7),
141
142 /* CPB control flag bits */
143 NV_CPB_CTL_CPB_VALID = (1 << 0),
144 NV_CPB_CTL_QUEUE = (1 << 1),
145 NV_CPB_CTL_APRD_VALID = (1 << 2),
146 NV_CPB_CTL_IEN = (1 << 3),
147 NV_CPB_CTL_FPDMA = (1 << 4),
148
149 /* APRD flags */
150 NV_APRD_WRITE = (1 << 1),
151 NV_APRD_END = (1 << 2),
152 NV_APRD_CONT = (1 << 3),
153
154 /* NV_ADMA_STAT flags */
155 NV_ADMA_STAT_TIMEOUT = (1 << 0),
156 NV_ADMA_STAT_HOTUNPLUG = (1 << 1),
157 NV_ADMA_STAT_HOTPLUG = (1 << 2),
158 NV_ADMA_STAT_CPBERR = (1 << 4),
159 NV_ADMA_STAT_SERROR = (1 << 5),
160 NV_ADMA_STAT_CMD_COMPLETE = (1 << 6),
161 NV_ADMA_STAT_IDLE = (1 << 8),
162 NV_ADMA_STAT_LEGACY = (1 << 9),
163 NV_ADMA_STAT_STOPPED = (1 << 10),
164 NV_ADMA_STAT_DONE = (1 << 12),
165 NV_ADMA_STAT_ERR = NV_ADMA_STAT_CPBERR |
2dcb407e 166 NV_ADMA_STAT_TIMEOUT,
fbbb262d
RH		 167
168 /* port flags */
169 NV_ADMA_PORT_REGISTER_MODE = (1 << 0),
2dec7555 170 NV_ADMA_ATAPI_SETUP_COMPLETE = (1 << 1),
fbbb262d 171
f140f0f1
KL		 172 /* MCP55 reg offset */
173 NV_CTL_MCP55 = 0x400,
174 NV_INT_STATUS_MCP55 = 0x440,
175 NV_INT_ENABLE_MCP55 = 0x444,
176 NV_NCQ_REG_MCP55 = 0x448,
177
178 /* MCP55 */
179 NV_INT_ALL_MCP55 = 0xffff,
180 NV_INT_PORT_SHIFT_MCP55 = 16, /* each port occupies 16 bits */
181 NV_INT_MASK_MCP55 = NV_INT_ALL_MCP55 & 0xfffd,
182
183 /* SWNCQ ENABLE BITS*/
184 NV_CTL_PRI_SWNCQ = 0x02,
185 NV_CTL_SEC_SWNCQ = 0x04,
186
187 /* SW NCQ status bits*/
188 NV_SWNCQ_IRQ_DEV = (1 << 0),
189 NV_SWNCQ_IRQ_PM = (1 << 1),
190 NV_SWNCQ_IRQ_ADDED = (1 << 2),
191 NV_SWNCQ_IRQ_REMOVED = (1 << 3),
192
193 NV_SWNCQ_IRQ_BACKOUT = (1 << 4),
194 NV_SWNCQ_IRQ_SDBFIS = (1 << 5),
195 NV_SWNCQ_IRQ_DHREGFIS = (1 << 6),
196 NV_SWNCQ_IRQ_DMASETUP = (1 << 7),
197
198 NV_SWNCQ_IRQ_HOTPLUG = NV_SWNCQ_IRQ_ADDED |
199 NV_SWNCQ_IRQ_REMOVED,
200
fbbb262d
RH		 201};
202
203/* ADMA Physical Region Descriptor - one SG segment */
204struct nv_adma_prd {
205 __le64 addr;
206 __le32 len;
207 u8 flags;
208 u8 packet_len;
209 __le16 reserved;
210};
211
212enum nv_adma_regbits {
213 CMDEND = (1 << 15), /* end of command list */
214 WNB = (1 << 14), /* wait-not-BSY */
215 IGN = (1 << 13), /* ignore this entry */
216 CS1n = (1 << (4 + 8)), /* std. PATA signals follow... */
217 DA2 = (1 << (2 + 8)),
218 DA1 = (1 << (1 + 8)),
219 DA0 = (1 << (0 + 8)),
220};
221
222/* ADMA Command Parameter Block
223 The first 5 SG segments are stored inside the Command Parameter Block itself.
224 If there are more than 5 segments the remainder are stored in a separate
225 memory area indicated by next_aprd. */
226struct nv_adma_cpb {
227 u8 resp_flags; /* 0 */
228 u8 reserved1; /* 1 */
229 u8 ctl_flags; /* 2 */
230 /* len is length of taskfile in 64 bit words */
2dcb407e 231 u8 len; /* 3 */
fbbb262d
RH		 232 u8 tag; /* 4 */
233 u8 next_cpb_idx; /* 5 */
234 __le16 reserved2; /* 6-7 */
235 __le16 tf[12]; /* 8-31 */
236 struct nv_adma_prd aprd[5]; /* 32-111 */
237 __le64 next_aprd; /* 112-119 */
238 __le64 reserved3; /* 120-127 */
10ad05df 239};
1da177e4 240
fbbb262d
RH		 241
242struct nv_adma_port_priv {
243 struct nv_adma_cpb *cpb;
244 dma_addr_t cpb_dma;
245 struct nv_adma_prd *aprd;
246 dma_addr_t aprd_dma;
2dcb407e
JG		 247 void __iomem *ctl_block;
248 void __iomem *gen_block;
249 void __iomem *notifier_clear_block;
8959d300 250 u64 adma_dma_mask;
fbbb262d 251 u8 flags;
5e5c74a5 252 int last_issue_ncq;
fbbb262d
RH		 253};
254
cdf56bcf
RH		 255struct nv_host_priv {
256 unsigned long type;
257};
258
f140f0f1
KL		 259struct defer_queue {
260 u32 defer_bits;
261 unsigned int head;
262 unsigned int tail;
263 unsigned int tag[ATA_MAX_QUEUE];
264};
265
266enum ncq_saw_flag_list {
267 ncq_saw_d2h = (1U << 0),
268 ncq_saw_dmas = (1U << 1),
269 ncq_saw_sdb = (1U << 2),
270 ncq_saw_backout = (1U << 3),
271};
272
273struct nv_swncq_port_priv {
274 struct ata_prd *prd; /* our SG list */
275 dma_addr_t prd_dma; /* and its DMA mapping */
276 void __iomem *sactive_block;
277 void __iomem *irq_block;
278 void __iomem *tag_block;
279 u32 qc_active;
280
281 unsigned int last_issue_tag;
282
283 /* fifo circular queue to store deferral command */
284 struct defer_queue defer_queue;
285
286 /* for NCQ interrupt analysis */
287 u32 dhfis_bits;
288 u32 dmafis_bits;
289 u32 sdbfis_bits;
290
291 unsigned int ncq_flags;
292};
293
294
5796d1c4 295#define NV_ADMA_CHECK_INTR(GCTL, PORT) ((GCTL) & (1 << (19 + (12 * (PORT)))))
fbbb262d 296
2dcb407e 297static int nv_init_one(struct pci_dev *pdev, const struct pci_device_id *ent);
438ac6d5 298#ifdef CONFIG_PM
cdf56bcf 299static int nv_pci_device_resume(struct pci_dev *pdev);
438ac6d5 300#endif
cca3974e 301static void nv_ck804_host_stop(struct ata_host *host);
7d12e780
DH		 302static irqreturn_t nv_generic_interrupt(int irq, void *dev_instance);
303static irqreturn_t nv_nf2_interrupt(int irq, void *dev_instance);
304static irqreturn_t nv_ck804_interrupt(int irq, void *dev_instance);
2dcb407e
JG		 305static int nv_scr_read(struct ata_port *ap, unsigned int sc_reg, u32 *val);
306static int nv_scr_write(struct ata_port *ap, unsigned int sc_reg, u32 val);
1da177e4 307
39f87582
TH		 308static void nv_nf2_freeze(struct ata_port *ap);
309static void nv_nf2_thaw(struct ata_port *ap);
310static void nv_ck804_freeze(struct ata_port *ap);
311static void nv_ck804_thaw(struct ata_port *ap);
312static void nv_error_handler(struct ata_port *ap);
fbbb262d 313static int nv_adma_slave_config(struct scsi_device *sdev);
2dec7555 314static int nv_adma_check_atapi_dma(struct ata_queued_cmd *qc);
fbbb262d
RH		 315static void nv_adma_qc_prep(struct ata_queued_cmd *qc);
316static unsigned int nv_adma_qc_issue(struct ata_queued_cmd *qc);
317static irqreturn_t nv_adma_interrupt(int irq, void *dev_instance);
318static void nv_adma_irq_clear(struct ata_port *ap);
319static int nv_adma_port_start(struct ata_port *ap);
320static void nv_adma_port_stop(struct ata_port *ap);
438ac6d5 321#ifdef CONFIG_PM
cdf56bcf
RH		 322static int nv_adma_port_suspend(struct ata_port *ap, pm_message_t mesg);
323static int nv_adma_port_resume(struct ata_port *ap);
438ac6d5 324#endif
53014e25
RH		 325static void nv_adma_freeze(struct ata_port *ap);
326static void nv_adma_thaw(struct ata_port *ap);
fbbb262d
RH		 327static void nv_adma_error_handler(struct ata_port *ap);
328static void nv_adma_host_stop(struct ata_host *host);
f5ecac2d 329static void nv_adma_post_internal_cmd(struct ata_queued_cmd *qc);
f2fb344b 330static void nv_adma_tf_read(struct ata_port *ap, struct ata_taskfile *tf);
39f87582 331
f140f0f1
KL		 332static void nv_mcp55_thaw(struct ata_port *ap);
333static void nv_mcp55_freeze(struct ata_port *ap);
334static void nv_swncq_error_handler(struct ata_port *ap);
335static int nv_swncq_slave_config(struct scsi_device *sdev);
336static int nv_swncq_port_start(struct ata_port *ap);
337static void nv_swncq_qc_prep(struct ata_queued_cmd *qc);
338static void nv_swncq_fill_sg(struct ata_queued_cmd *qc);
339static unsigned int nv_swncq_qc_issue(struct ata_queued_cmd *qc);
340static void nv_swncq_irq_clear(struct ata_port *ap, u16 fis);
341static irqreturn_t nv_swncq_interrupt(int irq, void *dev_instance);
342#ifdef CONFIG_PM
343static int nv_swncq_port_suspend(struct ata_port *ap, pm_message_t mesg);
344static int nv_swncq_port_resume(struct ata_port *ap);
345#endif
346
1da177e4
LT		 347enum nv_host_type
348{
349 GENERIC,
350 NFORCE2,
27e4b274 351 NFORCE3 = NFORCE2, /* NF2 == NF3 as far as sata_nv is concerned */
fbbb262d 352 CK804,
f140f0f1
KL		 353 ADMA,
354 SWNCQ,
1da177e4
LT		 355};
356
3b7d697d 357static const struct pci_device_id nv_pci_tbl[] = {
54bb3a94
JG		 358 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE2S_SATA), NFORCE2 },
359 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE3S_SATA), NFORCE3 },
360 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE3S_SATA2), NFORCE3 },
361 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_CK804_SATA), CK804 },
362 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_CK804_SATA2), CK804 },
363 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_MCP04_SATA), CK804 },
364 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_MCP04_SATA2), CK804 },
f140f0f1
KL		 365 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_MCP51_SATA), SWNCQ },
366 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_MCP51_SATA2), SWNCQ },
367 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_MCP55_SATA), SWNCQ },
368 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_MCP55_SATA2), SWNCQ },
e2e031eb
KL		 369 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_MCP61_SATA), GENERIC },
370 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_MCP61_SATA2), GENERIC },
371 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_MCP61_SATA3), GENERIC },
2d2744fc
JG		 372
373 { } /* terminate list */
1da177e4
LT		 374};
375
1da177e4
LT		 376static struct pci_driver nv_pci_driver = {
377 .name = DRV_NAME,
378 .id_table = nv_pci_tbl,
379 .probe = nv_init_one,
438ac6d5 380#ifdef CONFIG_PM
cdf56bcf
RH		 381 .suspend = ata_pci_device_suspend,
382 .resume = nv_pci_device_resume,
438ac6d5 383#endif
1daf9ce7 384 .remove = ata_pci_remove_one,
1da177e4
LT		 385};
386
193515d5 387static struct scsi_host_template nv_sht = {
68d1d07b 388 ATA_BMDMA_SHT(DRV_NAME),
1da177e4
LT		 389};
390
fbbb262d 391static struct scsi_host_template nv_adma_sht = {
68d1d07b 392 ATA_NCQ_SHT(DRV_NAME),
fbbb262d 393 .can_queue = NV_ADMA_MAX_CPBS,
fbbb262d 394 .sg_tablesize = NV_ADMA_SGTBL_TOTAL_LEN,
fbbb262d
RH		 395 .dma_boundary = NV_ADMA_DMA_BOUNDARY,
396 .slave_configure = nv_adma_slave_config,
fbbb262d
RH		 397};
398
f140f0f1 399static struct scsi_host_template nv_swncq_sht = {
68d1d07b 400 ATA_NCQ_SHT(DRV_NAME),
f140f0f1 401 .can_queue = ATA_MAX_QUEUE,
f140f0f1 402 .sg_tablesize = LIBATA_MAX_PRD,
f140f0f1
KL		 403 .dma_boundary = ATA_DMA_BOUNDARY,
404 .slave_configure = nv_swncq_slave_config,
f140f0f1
KL		 405};
406
029cfd6b
TH		 407static struct ata_port_operations nv_generic_ops = {
408 .inherits = &ata_bmdma_port_ops,
39f87582 409 .error_handler = nv_error_handler,
1da177e4
LT		 410 .scr_read = nv_scr_read,
411 .scr_write = nv_scr_write,
1da177e4
LT		 412};
413
029cfd6b
TH		 414static struct ata_port_operations nv_nf2_ops = {
415 .inherits = &nv_generic_ops,
39f87582
TH		 416 .freeze = nv_nf2_freeze,
417 .thaw = nv_nf2_thaw,
ada364e8
TH		 418};
419
029cfd6b
TH		 420static struct ata_port_operations nv_ck804_ops = {
421 .inherits = &nv_generic_ops,
39f87582
TH		 422 .freeze = nv_ck804_freeze,
423 .thaw = nv_ck804_thaw,
ada364e8
TH		 424 .host_stop = nv_ck804_host_stop,
425};
426
029cfd6b
TH		 427static struct ata_port_operations nv_adma_ops = {
428 .inherits = &nv_generic_ops,
429
2dec7555 430 .check_atapi_dma = nv_adma_check_atapi_dma,
029cfd6b 431 .tf_read = nv_adma_tf_read,
31cc23b3 432 .qc_defer = ata_std_qc_defer,
fbbb262d
RH		 433 .qc_prep = nv_adma_qc_prep,
434 .qc_issue = nv_adma_qc_issue,
029cfd6b
TH		 435 .irq_clear = nv_adma_irq_clear,
436
53014e25
RH		 437 .freeze = nv_adma_freeze,
438 .thaw = nv_adma_thaw,
fbbb262d 439 .error_handler = nv_adma_error_handler,
f5ecac2d 440 .post_internal_cmd = nv_adma_post_internal_cmd,
029cfd6b 441
fbbb262d
RH		 442 .port_start = nv_adma_port_start,
443 .port_stop = nv_adma_port_stop,
438ac6d5 444#ifdef CONFIG_PM
cdf56bcf
RH		 445 .port_suspend = nv_adma_port_suspend,
446 .port_resume = nv_adma_port_resume,
438ac6d5 447#endif
fbbb262d
RH		 448 .host_stop = nv_adma_host_stop,
449};
450
029cfd6b
TH		 451static struct ata_port_operations nv_swncq_ops = {
452 .inherits = &nv_generic_ops,
453
f140f0f1
KL		 454 .qc_defer = ata_std_qc_defer,
455 .qc_prep = nv_swncq_qc_prep,
456 .qc_issue = nv_swncq_qc_issue,
029cfd6b 457
f140f0f1
KL		 458 .freeze = nv_mcp55_freeze,
459 .thaw = nv_mcp55_thaw,
460 .error_handler = nv_swncq_error_handler,
029cfd6b 461
f140f0f1
KL		 462#ifdef CONFIG_PM
463 .port_suspend = nv_swncq_port_suspend,
464 .port_resume = nv_swncq_port_resume,
465#endif
466 .port_start = nv_swncq_port_start,
467};
468
1626aeb8 469static const struct ata_port_info nv_port_info[] = {
ada364e8
TH		 470 /* generic */
471 {
472 .sht = &nv_sht,
0c88758b 473 .flags = ATA_FLAG_SATA | ATA_FLAG_NO_LEGACY,
ada364e8
TH		 474 .pio_mask = NV_PIO_MASK,
475 .mwdma_mask = NV_MWDMA_MASK,
476 .udma_mask = NV_UDMA_MASK,
477 .port_ops = &nv_generic_ops,
9a829ccf 478 .irq_handler = nv_generic_interrupt,
ada364e8
TH		 479 },
480 /* nforce2/3 */
481 {
482 .sht = &nv_sht,
0c88758b 483 .flags = ATA_FLAG_SATA | ATA_FLAG_NO_LEGACY,
ada364e8
TH		 484 .pio_mask = NV_PIO_MASK,
485 .mwdma_mask = NV_MWDMA_MASK,
486 .udma_mask = NV_UDMA_MASK,
487 .port_ops = &nv_nf2_ops,
9a829ccf 488 .irq_handler = nv_nf2_interrupt,
ada364e8
TH		 489 },
490 /* ck804 */
491 {
492 .sht = &nv_sht,
0c88758b 493 .flags = ATA_FLAG_SATA | ATA_FLAG_NO_LEGACY,
ada364e8
TH		 494 .pio_mask = NV_PIO_MASK,
495 .mwdma_mask = NV_MWDMA_MASK,
496 .udma_mask = NV_UDMA_MASK,
497 .port_ops = &nv_ck804_ops,
9a829ccf 498 .irq_handler = nv_ck804_interrupt,
ada364e8 499 },
fbbb262d
RH		 500 /* ADMA */
501 {
502 .sht = &nv_adma_sht,
503 .flags = ATA_FLAG_SATA | ATA_FLAG_NO_LEGACY |
504 ATA_FLAG_MMIO | ATA_FLAG_NCQ,
505 .pio_mask = NV_PIO_MASK,
506 .mwdma_mask = NV_MWDMA_MASK,
507 .udma_mask = NV_UDMA_MASK,
508 .port_ops = &nv_adma_ops,
9a829ccf 509 .irq_handler = nv_adma_interrupt,
fbbb262d 510 },
f140f0f1
KL		 511 /* SWNCQ */
512 {
513 .sht = &nv_swncq_sht,
514 .flags = ATA_FLAG_SATA | ATA_FLAG_NO_LEGACY |
515 ATA_FLAG_NCQ,
f140f0f1
KL		 516 .pio_mask = NV_PIO_MASK,
517 .mwdma_mask = NV_MWDMA_MASK,
518 .udma_mask = NV_UDMA_MASK,
519 .port_ops = &nv_swncq_ops,
520 .irq_handler = nv_swncq_interrupt,
521 },
1da177e4
LT		 522};
523
524MODULE_AUTHOR("NVIDIA");
525MODULE_DESCRIPTION("low-level driver for NVIDIA nForce SATA controller");
526MODULE_LICENSE("GPL");
527MODULE_DEVICE_TABLE(pci, nv_pci_tbl);
528MODULE_VERSION(DRV_VERSION);
529
fbbb262d 530static int adma_enabled = 1;
f140f0f1 531static int swncq_enabled;
fbbb262d 532
2dec7555
RH		 533static void nv_adma_register_mode(struct ata_port *ap)
534{
2dec7555 535 struct nv_adma_port_priv *pp = ap->private_data;
cdf56bcf 536 void __iomem *mmio = pp->ctl_block;
a2cfe81a
RH		 537 u16 tmp, status;
538 int count = 0;
2dec7555
RH		 539
540 if (pp->flags & NV_ADMA_PORT_REGISTER_MODE)
541 return;
542
a2cfe81a 543 status = readw(mmio + NV_ADMA_STAT);
2dcb407e 544 while (!(status & NV_ADMA_STAT_IDLE) && count < 20) {
a2cfe81a
RH		 545 ndelay(50);
546 status = readw(mmio + NV_ADMA_STAT);
547 count++;
548 }
2dcb407e 549 if (count == 20)
a2cfe81a
RH		 550 ata_port_printk(ap, KERN_WARNING,
551 "timeout waiting for ADMA IDLE, stat=0x%hx\n",
552 status);
553
2dec7555
RH		 554 tmp = readw(mmio + NV_ADMA_CTL);
555 writew(tmp & ~NV_ADMA_CTL_GO, mmio + NV_ADMA_CTL);
556
a2cfe81a
RH		 557 count = 0;
558 status = readw(mmio + NV_ADMA_STAT);
2dcb407e 559 while (!(status & NV_ADMA_STAT_LEGACY) && count < 20) {
a2cfe81a
RH		 560 ndelay(50);
561 status = readw(mmio + NV_ADMA_STAT);
562 count++;
563 }
2dcb407e 564 if (count == 20)
a2cfe81a
RH		 565 ata_port_printk(ap, KERN_WARNING,
566 "timeout waiting for ADMA LEGACY, stat=0x%hx\n",
567 status);
568
2dec7555
RH		 569 pp->flags |= NV_ADMA_PORT_REGISTER_MODE;
570}
571
572static void nv_adma_mode(struct ata_port *ap)
573{
2dec7555 574 struct nv_adma_port_priv *pp = ap->private_data;
cdf56bcf 575 void __iomem *mmio = pp->ctl_block;
a2cfe81a
RH		 576 u16 tmp, status;
577 int count = 0;
2dec7555
RH		 578
579 if (!(pp->flags & NV_ADMA_PORT_REGISTER_MODE))
580 return;
f20b16ff 581
2dec7555
RH		 582 WARN_ON(pp->flags & NV_ADMA_ATAPI_SETUP_COMPLETE);
583
584 tmp = readw(mmio + NV_ADMA_CTL);
585 writew(tmp | NV_ADMA_CTL_GO, mmio + NV_ADMA_CTL);
586
a2cfe81a 587 status = readw(mmio + NV_ADMA_STAT);
2dcb407e 588 while (((status & NV_ADMA_STAT_LEGACY) ||
a2cfe81a
RH		 589 !(status & NV_ADMA_STAT_IDLE)) && count < 20) {
590 ndelay(50);
591 status = readw(mmio + NV_ADMA_STAT);
592 count++;
593 }
2dcb407e 594 if (count == 20)
a2cfe81a
RH		 595 ata_port_printk(ap, KERN_WARNING,
596 "timeout waiting for ADMA LEGACY clear and IDLE, stat=0x%hx\n",
597 status);
598
2dec7555
RH		 599 pp->flags &= ~NV_ADMA_PORT_REGISTER_MODE;
600}
601
fbbb262d
RH		 602static int nv_adma_slave_config(struct scsi_device *sdev)
603{
604 struct ata_port *ap = ata_shost_to_port(sdev->host);
2dec7555 605 struct nv_adma_port_priv *pp = ap->private_data;
8959d300
RH		 606 struct nv_adma_port_priv *port0, *port1;
607 struct scsi_device *sdev0, *sdev1;
2dec7555 608 struct pci_dev *pdev = to_pci_dev(ap->host->dev);
8959d300 609 unsigned long segment_boundary, flags;
fbbb262d
RH		 610 unsigned short sg_tablesize;
611 int rc;
2dec7555
RH		 612 int adma_enable;
613 u32 current_reg, new_reg, config_mask;
fbbb262d
RH		 614
615 rc = ata_scsi_slave_config(sdev);
616
617 if (sdev->id >= ATA_MAX_DEVICES || sdev->channel || sdev->lun)
618 /* Not a proper libata device, ignore */
619 return rc;
620
8959d300
RH		 621 spin_lock_irqsave(ap->lock, flags);
622
9af5c9c9 623 if (ap->link.device[sdev->id].class == ATA_DEV_ATAPI) {
fbbb262d
RH		 624 /*
625 * NVIDIA reports that ADMA mode does not support ATAPI commands.
626 * Therefore ATAPI commands are sent through the legacy interface.
627 * However, the legacy interface only supports 32-bit DMA.
628 * Restrict DMA parameters as required by the legacy interface
629 * when an ATAPI device is connected.
630 */
fbbb262d
RH		 631 segment_boundary = ATA_DMA_BOUNDARY;
632 /* Subtract 1 since an extra entry may be needed for padding, see
633 libata-scsi.c */
634 sg_tablesize = LIBATA_MAX_PRD - 1;
f20b16ff 635
2dec7555
RH		 636 /* Since the legacy DMA engine is in use, we need to disable ADMA
637 on the port. */
638 adma_enable = 0;
639 nv_adma_register_mode(ap);
2dcb407e 640 } else {
fbbb262d
RH		 641 segment_boundary = NV_ADMA_DMA_BOUNDARY;
642 sg_tablesize = NV_ADMA_SGTBL_TOTAL_LEN;
2dec7555 643 adma_enable = 1;
fbbb262d 644 }
f20b16ff 645
2dec7555
RH		 646 pci_read_config_dword(pdev, NV_MCP_SATA_CFG_20, &current_reg);
647
2dcb407e 648 if (ap->port_no == 1)
2dec7555
RH		 649 config_mask = NV_MCP_SATA_CFG_20_PORT1_EN |
650 NV_MCP_SATA_CFG_20_PORT1_PWB_EN;
651 else
652 config_mask = NV_MCP_SATA_CFG_20_PORT0_EN |
653 NV_MCP_SATA_CFG_20_PORT0_PWB_EN;
f20b16ff 654
2dcb407e 655 if (adma_enable) {
2dec7555
RH		 656 new_reg = current_reg | config_mask;
657 pp->flags &= ~NV_ADMA_ATAPI_SETUP_COMPLETE;
2dcb407e 658 } else {
2dec7555
RH		 659 new_reg = current_reg & ~config_mask;
660 pp->flags |= NV_ADMA_ATAPI_SETUP_COMPLETE;
661 }
f20b16ff 662
2dcb407e 663 if (current_reg != new_reg)
2dec7555 664 pci_write_config_dword(pdev, NV_MCP_SATA_CFG_20, new_reg);
f20b16ff 665
8959d300
RH		 666 port0 = ap->host->ports[0]->private_data;
667 port1 = ap->host->ports[1]->private_data;
668 sdev0 = ap->host->ports[0]->link.device[0].sdev;
669 sdev1 = ap->host->ports[1]->link.device[0].sdev;
670 if ((port0->flags & NV_ADMA_ATAPI_SETUP_COMPLETE) ||
671 (port1->flags & NV_ADMA_ATAPI_SETUP_COMPLETE)) {
672 /** We have to set the DMA mask to 32-bit if either port is in
673 ATAPI mode, since they are on the same PCI device which is
674 used for DMA mapping. If we set the mask we also need to set
675 the bounce limit on both ports to ensure that the block
676 layer doesn't feed addresses that cause DMA mapping to
677 choke. If either SCSI device is not allocated yet, it's OK
678 since that port will discover its correct setting when it
679 does get allocated.
680 Note: Setting 32-bit mask should not fail. */
681 if (sdev0)
682 blk_queue_bounce_limit(sdev0->request_queue,
683 ATA_DMA_MASK);
684 if (sdev1)
685 blk_queue_bounce_limit(sdev1->request_queue,
686 ATA_DMA_MASK);
687
688 pci_set_dma_mask(pdev, ATA_DMA_MASK);
689 } else {
690 /** This shouldn't fail as it was set to this value before */
691 pci_set_dma_mask(pdev, pp->adma_dma_mask);
692 if (sdev0)
693 blk_queue_bounce_limit(sdev0->request_queue,
694 pp->adma_dma_mask);
695 if (sdev1)
696 blk_queue_bounce_limit(sdev1->request_queue,
697 pp->adma_dma_mask);
698 }
699
fbbb262d
RH		 700 blk_queue_segment_boundary(sdev->request_queue, segment_boundary);
701 blk_queue_max_hw_segments(sdev->request_queue, sg_tablesize);
702 ata_port_printk(ap, KERN_INFO,
8959d300
RH		 703 "DMA mask 0x%llX, segment boundary 0x%lX, hw segs %hu\n",
704 (unsigned long long)*ap->host->dev->dma_mask,
705 segment_boundary, sg_tablesize);
706
707 spin_unlock_irqrestore(ap->lock, flags);
708
fbbb262d
RH		 709 return rc;
710}
711
2dec7555
RH		 712static int nv_adma_check_atapi_dma(struct ata_queued_cmd *qc)
713{
714 struct nv_adma_port_priv *pp = qc->ap->private_data;
715 return !(pp->flags & NV_ADMA_ATAPI_SETUP_COMPLETE);
716}
717
f2fb344b
RH		 718static void nv_adma_tf_read(struct ata_port *ap, struct ata_taskfile *tf)
719{
3f3debdb
RH		 720 /* Other than when internal or pass-through commands are executed,
721 the only time this function will be called in ADMA mode will be
722 if a command fails. In the failure case we don't care about going
723 into register mode with ADMA commands pending, as the commands will
724 all shortly be aborted anyway. We assume that NCQ commands are not
725 issued via passthrough, which is the only way that switching into
726 ADMA mode could abort outstanding commands. */
f2fb344b
RH		 727 nv_adma_register_mode(ap);
728
729 ata_tf_read(ap, tf);
730}
731
2dec7555 732static unsigned int nv_adma_tf_to_cpb(struct ata_taskfile *tf, __le16 *cpb)
fbbb262d
RH		 733{
734 unsigned int idx = 0;
735
2dcb407e 736 if (tf->flags & ATA_TFLAG_ISADDR) {
ac3d6b86
RH		 737 if (tf->flags & ATA_TFLAG_LBA48) {
738 cpb[idx++] = cpu_to_le16((ATA_REG_ERR << 8) | tf->hob_feature | WNB);
739 cpb[idx++] = cpu_to_le16((ATA_REG_NSECT << 8) | tf->hob_nsect);
740 cpb[idx++] = cpu_to_le16((ATA_REG_LBAL << 8) | tf->hob_lbal);
741 cpb[idx++] = cpu_to_le16((ATA_REG_LBAM << 8) | tf->hob_lbam);
742 cpb[idx++] = cpu_to_le16((ATA_REG_LBAH << 8) | tf->hob_lbah);
743 cpb[idx++] = cpu_to_le16((ATA_REG_ERR << 8) | tf->feature);
744 } else
745 cpb[idx++] = cpu_to_le16((ATA_REG_ERR << 8) | tf->feature | WNB);
a84471fe 746
ac3d6b86
RH		 747 cpb[idx++] = cpu_to_le16((ATA_REG_NSECT << 8) | tf->nsect);
748 cpb[idx++] = cpu_to_le16((ATA_REG_LBAL << 8) | tf->lbal);
749 cpb[idx++] = cpu_to_le16((ATA_REG_LBAM << 8) | tf->lbam);
750 cpb[idx++] = cpu_to_le16((ATA_REG_LBAH << 8) | tf->lbah);
fbbb262d 751 }
a84471fe 752
2dcb407e 753 if (tf->flags & ATA_TFLAG_DEVICE)
ac3d6b86 754 cpb[idx++] = cpu_to_le16((ATA_REG_DEVICE << 8) | tf->device);
fbbb262d
RH		 755
756 cpb[idx++] = cpu_to_le16((ATA_REG_CMD << 8) | tf->command | CMDEND);
a84471fe 757
2dcb407e 758 while (idx < 12)
ac3d6b86 759 cpb[idx++] = cpu_to_le16(IGN);
fbbb262d
RH		 760
761 return idx;
762}
763
5bd28a4b 764static int nv_adma_check_cpb(struct ata_port *ap, int cpb_num, int force_err)
fbbb262d
RH		 765{
766 struct nv_adma_port_priv *pp = ap->private_data;
2dec7555 767 u8 flags = pp->cpb[cpb_num].resp_flags;
fbbb262d
RH		 768
769 VPRINTK("CPB %d, flags=0x%x\n", cpb_num, flags);
770
5bd28a4b
RH		 771 if (unlikely((force_err ||
772 flags & (NV_CPB_RESP_ATA_ERR |
773 NV_CPB_RESP_CMD_ERR |
774 NV_CPB_RESP_CPB_ERR)))) {
9af5c9c9 775 struct ata_eh_info *ehi = &ap->link.eh_info;
5bd28a4b
RH		 776 int freeze = 0;
777
778 ata_ehi_clear_desc(ehi);
2dcb407e 779 __ata_ehi_push_desc(ehi, "CPB resp_flags 0x%x: ", flags);
5bd28a4b 780 if (flags & NV_CPB_RESP_ATA_ERR) {
b64bbc39 781 ata_ehi_push_desc(ehi, "ATA error");
5bd28a4b
RH		 782 ehi->err_mask |= AC_ERR_DEV;
783 } else if (flags & NV_CPB_RESP_CMD_ERR) {
b64bbc39 784 ata_ehi_push_desc(ehi, "CMD error");
5bd28a4b
RH		 785 ehi->err_mask |= AC_ERR_DEV;
786 } else if (flags & NV_CPB_RESP_CPB_ERR) {
b64bbc39 787 ata_ehi_push_desc(ehi, "CPB error");
5bd28a4b
RH		 788 ehi->err_mask |= AC_ERR_SYSTEM;
789 freeze = 1;
790 } else {
791 /* notifier error, but no error in CPB flags? */
b64bbc39 792 ata_ehi_push_desc(ehi, "unknown");
5bd28a4b
RH		 793 ehi->err_mask |= AC_ERR_OTHER;
794 freeze = 1;
795 }
796 /* Kill all commands. EH will determine what actually failed. */
797 if (freeze)
798 ata_port_freeze(ap);
799 else
800 ata_port_abort(ap);
801 return 1;
fbbb262d 802 }
5bd28a4b 803
f2fb344b 804 if (likely(flags & NV_CPB_RESP_DONE)) {
fbbb262d 805 struct ata_queued_cmd *qc = ata_qc_from_tag(ap, cpb_num);
5bd28a4b
RH		 806 VPRINTK("CPB flags done, flags=0x%x\n", flags);
807 if (likely(qc)) {
2dcb407e 808 DPRINTK("Completing qc from tag %d\n", cpb_num);
fbbb262d 809 ata_qc_complete(qc);
2a54cf76 810 } else {
9af5c9c9 811 struct ata_eh_info *ehi = &ap->link.eh_info;
2a54cf76
RH		 812 /* Notifier bits set without a command may indicate the drive
813 is misbehaving. Raise host state machine violation on this
814 condition. */
5796d1c4
JG		 815 ata_port_printk(ap, KERN_ERR,
816 "notifier for tag %d with no cmd?\n",
817 cpb_num);
2a54cf76 818 ehi->err_mask |= AC_ERR_HSM;
cf480626 819 ehi->action |= ATA_EH_RESET;
2a54cf76
RH		 820 ata_port_freeze(ap);
821 return 1;
fbbb262d
RH		 822 }
823 }
5bd28a4b 824 return 0;
fbbb262d
RH		 825}
826
2dec7555
RH		 827static int nv_host_intr(struct ata_port *ap, u8 irq_stat)
828{
9af5c9c9 829 struct ata_queued_cmd *qc = ata_qc_from_tag(ap, ap->link.active_tag);
2dec7555
RH		 830
831 /* freeze if hotplugged */
832 if (unlikely(irq_stat & (NV_INT_ADDED | NV_INT_REMOVED))) {
833 ata_port_freeze(ap);
834 return 1;
835 }
836
837 /* bail out if not our interrupt */
838 if (!(irq_stat & NV_INT_DEV))
839 return 0;
840
841 /* DEV interrupt w/ no active qc? */
842 if (unlikely(!qc || (qc->tf.flags & ATA_TFLAG_POLLING))) {
843 ata_check_status(ap);
844 return 1;
845 }
846
847 /* handle interrupt */
f740d168 848 return ata_host_intr(ap, qc);
2dec7555
RH		 849}
850
fbbb262d
RH		 851static irqreturn_t nv_adma_interrupt(int irq, void *dev_instance)
852{
853 struct ata_host *host = dev_instance;
854 int i, handled = 0;
2dec7555 855 u32 notifier_clears[2];
fbbb262d
RH		 856
857 spin_lock(&host->lock);
858
859 for (i = 0; i < host->n_ports; i++) {
860 struct ata_port *ap = host->ports[i];
2dec7555 861 notifier_clears[i] = 0;
fbbb262d
RH		 862
863 if (ap && !(ap->flags & ATA_FLAG_DISABLED)) {
864 struct nv_adma_port_priv *pp = ap->private_data;
cdf56bcf 865 void __iomem *mmio = pp->ctl_block;
fbbb262d
RH		 866 u16 status;
867 u32 gen_ctl;
fbbb262d 868 u32 notifier, notifier_error;
a617c09f 869
53014e25
RH		 870 /* if ADMA is disabled, use standard ata interrupt handler */
871 if (pp->flags & NV_ADMA_ATAPI_SETUP_COMPLETE) {
872 u8 irq_stat = readb(host->iomap[NV_MMIO_BAR] + NV_INT_STATUS_CK804)
873 >> (NV_INT_PORT_SHIFT * i);
874 handled += nv_host_intr(ap, irq_stat);
875 continue;
876 }
fbbb262d 877
53014e25 878 /* if in ATA register mode, check for standard interrupts */
fbbb262d 879 if (pp->flags & NV_ADMA_PORT_REGISTER_MODE) {
0d5ff566 880 u8 irq_stat = readb(host->iomap[NV_MMIO_BAR] + NV_INT_STATUS_CK804)
2dec7555 881 >> (NV_INT_PORT_SHIFT * i);
2dcb407e 882 if (ata_tag_valid(ap->link.active_tag))
f740d168
RH		 883 /** NV_INT_DEV indication seems unreliable at times
884 at least in ADMA mode. Force it on always when a
885 command is active, to prevent losing interrupts. */
886 irq_stat |= NV_INT_DEV;
2dec7555 887 handled += nv_host_intr(ap, irq_stat);
fbbb262d
RH		 888 }
889
890 notifier = readl(mmio + NV_ADMA_NOTIFIER);
891 notifier_error = readl(mmio + NV_ADMA_NOTIFIER_ERROR);
2dec7555 892 notifier_clears[i] = notifier | notifier_error;
fbbb262d 893
cdf56bcf 894 gen_ctl = readl(pp->gen_block + NV_ADMA_GEN_CTL);
fbbb262d 895
2dcb407e 896 if (!NV_ADMA_CHECK_INTR(gen_ctl, ap->port_no) && !notifier &&
fbbb262d
RH		 897 !notifier_error)
898 /* Nothing to do */
899 continue;
900
901 status = readw(mmio + NV_ADMA_STAT);
902
903 /* Clear status. Ensure the controller sees the clearing before we start
904 looking at any of the CPB statuses, so that any CPB completions after
905 this point in the handler will raise another interrupt. */
906 writew(status, mmio + NV_ADMA_STAT);
907 readw(mmio + NV_ADMA_STAT); /* flush posted write */
908 rmb();
909
5bd28a4b
RH		 910 handled++; /* irq handled if we got here */
911
912 /* freeze if hotplugged or controller error */
913 if (unlikely(status & (NV_ADMA_STAT_HOTPLUG |
914 NV_ADMA_STAT_HOTUNPLUG |
5278b50c
RH		 915 NV_ADMA_STAT_TIMEOUT |
916 NV_ADMA_STAT_SERROR))) {
9af5c9c9 917 struct ata_eh_info *ehi = &ap->link.eh_info;
5bd28a4b
RH		 918
919 ata_ehi_clear_desc(ehi);
2dcb407e 920 __ata_ehi_push_desc(ehi, "ADMA status 0x%08x: ", status);
5bd28a4b
RH		 921 if (status & NV_ADMA_STAT_TIMEOUT) {
922 ehi->err_mask |= AC_ERR_SYSTEM;
b64bbc39 923 ata_ehi_push_desc(ehi, "timeout");
5bd28a4b
RH		 924 } else if (status & NV_ADMA_STAT_HOTPLUG) {
925 ata_ehi_hotplugged(ehi);
b64bbc39 926 ata_ehi_push_desc(ehi, "hotplug");
5bd28a4b
RH		 927 } else if (status & NV_ADMA_STAT_HOTUNPLUG) {
928 ata_ehi_hotplugged(ehi);
b64bbc39 929 ata_ehi_push_desc(ehi, "hot unplug");
5278b50c
RH		 930 } else if (status & NV_ADMA_STAT_SERROR) {
931 /* let libata analyze SError and figure out the cause */
b64bbc39
TH		 932 ata_ehi_push_desc(ehi, "SError");
933 } else
934 ata_ehi_push_desc(ehi, "unknown");
fbbb262d 935 ata_port_freeze(ap);
fbbb262d
RH		 936 continue;
937 }
938
5bd28a4b 939 if (status & (NV_ADMA_STAT_DONE |
a1fe7824
RH		 940 NV_ADMA_STAT_CPBERR |
941 NV_ADMA_STAT_CMD_COMPLETE)) {
942 u32 check_commands = notifier_clears[i];
721449bf 943 int pos, error = 0;
8ba5e4cb 944
a1fe7824
RH		 945 if (status & NV_ADMA_STAT_CPBERR) {
946 /* Check all active commands */
947 if (ata_tag_valid(ap->link.active_tag))
948 check_commands = 1 <<
949 ap->link.active_tag;
950 else
951 check_commands = ap->
952 link.sactive;
953 }
8ba5e4cb 954
fbbb262d 955 /** Check CPBs for completed commands */
721449bf
RH		 956 while ((pos = ffs(check_commands)) && !error) {
957 pos--;
958 error = nv_adma_check_cpb(ap, pos,
5796d1c4
JG		 959 notifier_error & (1 << pos));
960 check_commands &= ~(1 << pos);
fbbb262d
RH		 961 }
962 }
fbbb262d
RH		 963 }
964 }
f20b16ff 965
b447916e 966 if (notifier_clears[0] || notifier_clears[1]) {
2dec7555
RH		 967 /* Note: Both notifier clear registers must be written
968 if either is set, even if one is zero, according to NVIDIA. */
cdf56bcf
RH		 969 struct nv_adma_port_priv *pp = host->ports[0]->private_data;
970 writel(notifier_clears[0], pp->notifier_clear_block);
971 pp = host->ports[1]->private_data;
972 writel(notifier_clears[1], pp->notifier_clear_block);
2dec7555 973 }
fbbb262d
RH		 974
975 spin_unlock(&host->lock);
976
977 return IRQ_RETVAL(handled);
978}
979
53014e25
RH		 980static void nv_adma_freeze(struct ata_port *ap)
981{
982 struct nv_adma_port_priv *pp = ap->private_data;
983 void __iomem *mmio = pp->ctl_block;
984 u16 tmp;
985
986 nv_ck804_freeze(ap);
987
988 if (pp->flags & NV_ADMA_ATAPI_SETUP_COMPLETE)
989 return;
990
991 /* clear any outstanding CK804 notifications */
2dcb407e 992 writeb(NV_INT_ALL << (ap->port_no * NV_INT_PORT_SHIFT),
53014e25
RH		 993 ap->host->iomap[NV_MMIO_BAR] + NV_INT_STATUS_CK804);
994
995 /* Disable interrupt */
996 tmp = readw(mmio + NV_ADMA_CTL);
2dcb407e 997 writew(tmp & ~(NV_ADMA_CTL_AIEN | NV_ADMA_CTL_HOTPLUG_IEN),
53014e25 998 mmio + NV_ADMA_CTL);
5796d1c4 999 readw(mmio + NV_ADMA_CTL); /* flush posted write */
53014e25
RH		 1000}
1001
1002static void nv_adma_thaw(struct ata_port *ap)
1003{
1004 struct nv_adma_port_priv *pp = ap->private_data;
1005 void __iomem *mmio = pp->ctl_block;
1006 u16 tmp;
1007
1008 nv_ck804_thaw(ap);
1009
1010 if (pp->flags & NV_ADMA_ATAPI_SETUP_COMPLETE)
1011 return;
1012
1013 /* Enable interrupt */
1014 tmp = readw(mmio + NV_ADMA_CTL);
2dcb407e 1015 writew(tmp | (NV_ADMA_CTL_AIEN | NV_ADMA_CTL_HOTPLUG_IEN),
53014e25 1016 mmio + NV_ADMA_CTL);
5796d1c4 1017 readw(mmio + NV_ADMA_CTL); /* flush posted write */
53014e25
RH		 1018}
1019
fbbb262d
RH		 1020static void nv_adma_irq_clear(struct ata_port *ap)
1021{
cdf56bcf
RH		 1022 struct nv_adma_port_priv *pp = ap->private_data;
1023 void __iomem *mmio = pp->ctl_block;
53014e25 1024 u32 notifier_clears[2];
fbbb262d 1025
53014e25
RH		 1026 if (pp->flags & NV_ADMA_ATAPI_SETUP_COMPLETE) {
1027 ata_bmdma_irq_clear(ap);
1028 return;
1029 }
1030
1031 /* clear any outstanding CK804 notifications */
2dcb407e 1032 writeb(NV_INT_ALL << (ap->port_no * NV_INT_PORT_SHIFT),
53014e25 1033 ap->host->iomap[NV_MMIO_BAR] + NV_INT_STATUS_CK804);
fbbb262d 1034
53014e25
RH		 1035 /* clear ADMA status */
1036 writew(0xffff, mmio + NV_ADMA_STAT);
a617c09f 1037
53014e25
RH		 1038 /* clear notifiers - note both ports need to be written with
1039 something even though we are only clearing on one */
1040 if (ap->port_no == 0) {
1041 notifier_clears[0] = 0xFFFFFFFF;
1042 notifier_clears[1] = 0;
1043 } else {
1044 notifier_clears[0] = 0;
1045 notifier_clears[1] = 0xFFFFFFFF;
1046 }
1047 pp = ap->host->ports[0]->private_data;
1048 writel(notifier_clears[0], pp->notifier_clear_block);
1049 pp = ap->host->ports[1]->private_data;
1050 writel(notifier_clears[1], pp->notifier_clear_block);
fbbb262d
RH		 1051}
1052
f5ecac2d 1053static void nv_adma_post_internal_cmd(struct ata_queued_cmd *qc)
fbbb262d 1054{
f5ecac2d 1055 struct nv_adma_port_priv *pp = qc->ap->private_data;
fbbb262d 1056
b447916e 1057 if (pp->flags & NV_ADMA_PORT_REGISTER_MODE)
f5ecac2d 1058 ata_bmdma_post_internal_cmd(qc);
fbbb262d
RH		 1059}
1060
1061static int nv_adma_port_start(struct ata_port *ap)
1062{
1063 struct device *dev = ap->host->dev;
1064 struct nv_adma_port_priv *pp;
1065 int rc;
1066 void *mem;
1067 dma_addr_t mem_dma;
cdf56bcf 1068 void __iomem *mmio;
8959d300 1069 struct pci_dev *pdev = to_pci_dev(dev);
fbbb262d
RH		 1070 u16 tmp;
1071
1072 VPRINTK("ENTER\n");
1073
8959d300
RH		 1074 /* Ensure DMA mask is set to 32-bit before allocating legacy PRD and
1075 pad buffers */
1076 rc = pci_set_dma_mask(pdev, DMA_BIT_MASK(32));
1077 if (rc)
1078 return rc;
1079 rc = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(32));
1080 if (rc)
1081 return rc;
1082
fbbb262d
RH		 1083 rc = ata_port_start(ap);
1084 if (rc)
1085 return rc;
1086
24dc5f33
TH		 1087 pp = devm_kzalloc(dev, sizeof(*pp), GFP_KERNEL);
1088 if (!pp)
1089 return -ENOMEM;
fbbb262d 1090
0d5ff566 1091 mmio = ap->host->iomap[NV_MMIO_BAR] + NV_ADMA_PORT +
cdf56bcf
RH		 1092 ap->port_no * NV_ADMA_PORT_SIZE;
1093 pp->ctl_block = mmio;
0d5ff566 1094 pp->gen_block = ap->host->iomap[NV_MMIO_BAR] + NV_ADMA_GEN;
cdf56bcf
RH		 1095 pp->notifier_clear_block = pp->gen_block +
1096 NV_ADMA_NOTIFIER_CLEAR + (4 * ap->port_no);
1097
8959d300
RH		 1098 /* Now that the legacy PRD and padding buffer are allocated we can
1099 safely raise the DMA mask to allocate the CPB/APRD table.
1100 These are allowed to fail since we store the value that ends up
1101 being used to set as the bounce limit in slave_config later if
1102 needed. */
1103 pci_set_dma_mask(pdev, DMA_BIT_MASK(64));
1104 pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64));
1105 pp->adma_dma_mask = *dev->dma_mask;
1106
24dc5f33
TH		 1107 mem = dmam_alloc_coherent(dev, NV_ADMA_PORT_PRIV_DMA_SZ,
1108 &mem_dma, GFP_KERNEL);
1109 if (!mem)
1110 return -ENOMEM;
fbbb262d
RH		 1111 memset(mem, 0, NV_ADMA_PORT_PRIV_DMA_SZ);
1112
1113 /*
1114 * First item in chunk of DMA memory:
1115 * 128-byte command parameter block (CPB)
1116 * one for each command tag
1117 */
1118 pp->cpb = mem;
1119 pp->cpb_dma = mem_dma;
1120
1121 writel(mem_dma & 0xFFFFFFFF, mmio + NV_ADMA_CPB_BASE_LOW);
5796d1c4 1122 writel((mem_dma >> 16) >> 16, mmio + NV_ADMA_CPB_BASE_HIGH);
fbbb262d
RH		 1123
1124 mem += NV_ADMA_MAX_CPBS * NV_ADMA_CPB_SZ;
1125 mem_dma += NV_ADMA_MAX_CPBS * NV_ADMA_CPB_SZ;
1126
1127 /*
1128 * Second item: block of ADMA_SGTBL_LEN s/g entries
1129 */
1130 pp->aprd = mem;
1131 pp->aprd_dma = mem_dma;
1132
1133 ap->private_data = pp;
1134
1135 /* clear any outstanding interrupt conditions */
1136 writew(0xffff, mmio + NV_ADMA_STAT);
1137
1138 /* initialize port variables */
1139 pp->flags = NV_ADMA_PORT_REGISTER_MODE;
1140
1141 /* clear CPB fetch count */
1142 writew(0, mmio + NV_ADMA_CPB_COUNT);
1143
cdf56bcf 1144 /* clear GO for register mode, enable interrupt */
fbbb262d 1145 tmp = readw(mmio + NV_ADMA_CTL);
5796d1c4
JG		 1146 writew((tmp & ~NV_ADMA_CTL_GO) | NV_ADMA_CTL_AIEN |
1147 NV_ADMA_CTL_HOTPLUG_IEN, mmio + NV_ADMA_CTL);
fbbb262d
RH		 1148
1149 tmp = readw(mmio + NV_ADMA_CTL);
1150 writew(tmp | NV_ADMA_CTL_CHANNEL_RESET, mmio + NV_ADMA_CTL);
5796d1c4 1151 readw(mmio + NV_ADMA_CTL); /* flush posted write */
fbbb262d
RH		 1152 udelay(1);
1153 writew(tmp & ~NV_ADMA_CTL_CHANNEL_RESET, mmio + NV_ADMA_CTL);
5796d1c4 1154 readw(mmio + NV_ADMA_CTL); /* flush posted write */
fbbb262d
RH		 1155
1156 return 0;
fbbb262d
RH		 1157}
1158
1159static void nv_adma_port_stop(struct ata_port *ap)
1160{
fbbb262d 1161 struct nv_adma_port_priv *pp = ap->private_data;
cdf56bcf 1162 void __iomem *mmio = pp->ctl_block;
fbbb262d
RH		 1163
1164 VPRINTK("ENTER\n");
fbbb262d 1165 writew(0, mmio + NV_ADMA_CTL);
fbbb262d
RH		 1166}
1167
438ac6d5 1168#ifdef CONFIG_PM
cdf56bcf
RH		 1169static int nv_adma_port_suspend(struct ata_port *ap, pm_message_t mesg)
1170{
1171 struct nv_adma_port_priv *pp = ap->private_data;
1172 void __iomem *mmio = pp->ctl_block;
1173
1174 /* Go to register mode - clears GO */
1175 nv_adma_register_mode(ap);
1176
1177 /* clear CPB fetch count */
1178 writew(0, mmio + NV_ADMA_CPB_COUNT);
1179
1180 /* disable interrupt, shut down port */
1181 writew(0, mmio + NV_ADMA_CTL);
1182
1183 return 0;
1184}
1185
1186static int nv_adma_port_resume(struct ata_port *ap)
1187{
1188 struct nv_adma_port_priv *pp = ap->private_data;
1189 void __iomem *mmio = pp->ctl_block;
1190 u16 tmp;
1191
1192 /* set CPB block location */
1193 writel(pp->cpb_dma & 0xFFFFFFFF, mmio + NV_ADMA_CPB_BASE_LOW);
5796d1c4 1194 writel((pp->cpb_dma >> 16) >> 16, mmio + NV_ADMA_CPB_BASE_HIGH);
cdf56bcf
RH		 1195
1196 /* clear any outstanding interrupt conditions */
1197 writew(0xffff, mmio + NV_ADMA_STAT);
1198
1199 /* initialize port variables */
1200 pp->flags |= NV_ADMA_PORT_REGISTER_MODE;
1201
1202 /* clear CPB fetch count */
1203 writew(0, mmio + NV_ADMA_CPB_COUNT);
1204
1205 /* clear GO for register mode, enable interrupt */
1206 tmp = readw(mmio + NV_ADMA_CTL);
5796d1c4
JG		 1207 writew((tmp & ~NV_ADMA_CTL_GO) | NV_ADMA_CTL_AIEN |
1208 NV_ADMA_CTL_HOTPLUG_IEN, mmio + NV_ADMA_CTL);
cdf56bcf
RH		 1209
1210 tmp = readw(mmio + NV_ADMA_CTL);
1211 writew(tmp | NV_ADMA_CTL_CHANNEL_RESET, mmio + NV_ADMA_CTL);
5796d1c4 1212 readw(mmio + NV_ADMA_CTL); /* flush posted write */
cdf56bcf
RH		 1213 udelay(1);
1214 writew(tmp & ~NV_ADMA_CTL_CHANNEL_RESET, mmio + NV_ADMA_CTL);
5796d1c4 1215 readw(mmio + NV_ADMA_CTL); /* flush posted write */
cdf56bcf
RH		 1216
1217 return 0;
1218}
438ac6d5 1219#endif
fbbb262d 1220
9a829ccf 1221static void nv_adma_setup_port(struct ata_port *ap)
fbbb262d 1222{
9a829ccf
TH		 1223 void __iomem *mmio = ap->host->iomap[NV_MMIO_BAR];
1224 struct ata_ioports *ioport = &ap->ioaddr;
fbbb262d
RH		 1225
1226 VPRINTK("ENTER\n");
1227
9a829ccf 1228 mmio += NV_ADMA_PORT + ap->port_no * NV_ADMA_PORT_SIZE;
fbbb262d 1229
0d5ff566
TH		 1230 ioport->cmd_addr = mmio;
1231 ioport->data_addr = mmio + (ATA_REG_DATA * 4);
fbbb262d 1232 ioport->error_addr =
0d5ff566
TH		 1233 ioport->feature_addr = mmio + (ATA_REG_ERR * 4);
1234 ioport->nsect_addr = mmio + (ATA_REG_NSECT * 4);
1235 ioport->lbal_addr = mmio + (ATA_REG_LBAL * 4);
1236 ioport->lbam_addr = mmio + (ATA_REG_LBAM * 4);
1237 ioport->lbah_addr = mmio + (ATA_REG_LBAH * 4);
1238 ioport->device_addr = mmio + (ATA_REG_DEVICE * 4);
fbbb262d 1239 ioport->status_addr =
0d5ff566 1240 ioport->command_addr = mmio + (ATA_REG_STATUS * 4);
fbbb262d 1241 ioport->altstatus_addr =
0d5ff566 1242 ioport->ctl_addr = mmio + 0x20;
fbbb262d
RH		 1243}
1244
9a829ccf 1245static int nv_adma_host_init(struct ata_host *host)
fbbb262d 1246{
9a829ccf 1247 struct pci_dev *pdev = to_pci_dev(host->dev);
fbbb262d
RH		 1248 unsigned int i;
1249 u32 tmp32;
1250
1251 VPRINTK("ENTER\n");
1252
1253 /* enable ADMA on the ports */
1254 pci_read_config_dword(pdev, NV_MCP_SATA_CFG_20, &tmp32);
1255 tmp32 |= NV_MCP_SATA_CFG_20_PORT0_EN |
1256 NV_MCP_SATA_CFG_20_PORT0_PWB_EN |
1257 NV_MCP_SATA_CFG_20_PORT1_EN |
1258 NV_MCP_SATA_CFG_20_PORT1_PWB_EN;
1259
1260 pci_write_config_dword(pdev, NV_MCP_SATA_CFG_20, tmp32);
1261
9a829ccf
TH		 1262 for (i = 0; i < host->n_ports; i++)
1263 nv_adma_setup_port(host->ports[i]);
fbbb262d 1264
fbbb262d
RH		 1265 return 0;
1266}
1267
1268static void nv_adma_fill_aprd(struct ata_queued_cmd *qc,
1269 struct scatterlist *sg,
1270 int idx,
1271 struct nv_adma_prd *aprd)
1272{
41949ed5 1273 u8 flags = 0;
fbbb262d
RH		 1274 if (qc->tf.flags & ATA_TFLAG_WRITE)
1275 flags |= NV_APRD_WRITE;
1276 if (idx == qc->n_elem - 1)
1277 flags |= NV_APRD_END;
1278 else if (idx != 4)
1279 flags |= NV_APRD_CONT;
1280
1281 aprd->addr = cpu_to_le64(((u64)sg_dma_address(sg)));
1282 aprd->len = cpu_to_le32(((u32)sg_dma_len(sg))); /* len in bytes */
2dec7555 1283 aprd->flags = flags;
41949ed5 1284 aprd->packet_len = 0;
fbbb262d
RH		 1285}
1286
1287static void nv_adma_fill_sg(struct ata_queued_cmd *qc, struct nv_adma_cpb *cpb)
1288{
1289 struct nv_adma_port_priv *pp = qc->ap->private_data;
fbbb262d
RH		 1290 struct nv_adma_prd *aprd;
1291 struct scatterlist *sg;
ff2aeb1e 1292 unsigned int si;
fbbb262d
RH		 1293
1294 VPRINTK("ENTER\n");
1295
ff2aeb1e
TH		 1296 for_each_sg(qc->sg, sg, qc->n_elem, si) {
1297 aprd = (si < 5) ? &cpb->aprd[si] :
1298 &pp->aprd[NV_ADMA_SGTBL_LEN * qc->tag + (si-5)];
1299 nv_adma_fill_aprd(qc, sg, si, aprd);
fbbb262d 1300 }
ff2aeb1e 1301 if (si > 5)
fbbb262d 1302 cpb->next_aprd = cpu_to_le64(((u64)(pp->aprd_dma + NV_ADMA_SGTBL_SZ * qc->tag)));
41949ed5
RH		 1303 else
1304 cpb->next_aprd = cpu_to_le64(0);
fbbb262d
RH		 1305}
1306
382a6652
RH		 1307static int nv_adma_use_reg_mode(struct ata_queued_cmd *qc)
1308{
1309 struct nv_adma_port_priv *pp = qc->ap->private_data;
1310
1311 /* ADMA engine can only be used for non-ATAPI DMA commands,
3f3debdb 1312 or interrupt-driven no-data commands. */
b447916e 1313 if ((pp->flags & NV_ADMA_ATAPI_SETUP_COMPLETE) ||
3f3debdb 1314 (qc->tf.flags & ATA_TFLAG_POLLING))
382a6652
RH		 1315 return 1;
1316
b447916e 1317 if ((qc->flags & ATA_QCFLAG_DMAMAP) ||
382a6652
RH		 1318 (qc->tf.protocol == ATA_PROT_NODATA))
1319 return 0;
1320
1321 return 1;
1322}
1323
fbbb262d
RH		 1324static void nv_adma_qc_prep(struct ata_queued_cmd *qc)
1325{
1326 struct nv_adma_port_priv *pp = qc->ap->private_data;
1327 struct nv_adma_cpb *cpb = &pp->cpb[qc->tag];
1328 u8 ctl_flags = NV_CPB_CTL_CPB_VALID |
fbbb262d
RH		 1329 NV_CPB_CTL_IEN;
1330
382a6652 1331 if (nv_adma_use_reg_mode(qc)) {
3f3debdb
RH		 1332 BUG_ON(!(pp->flags & NV_ADMA_ATAPI_SETUP_COMPLETE) &&
1333 (qc->flags & ATA_QCFLAG_DMAMAP));
2dec7555 1334 nv_adma_register_mode(qc->ap);
fbbb262d
RH		 1335 ata_qc_prep(qc);
1336 return;
1337 }
1338
41949ed5
RH		 1339 cpb->resp_flags = NV_CPB_RESP_DONE;
1340 wmb();
1341 cpb->ctl_flags = 0;
1342 wmb();
fbbb262d
RH		 1343
1344 cpb->len = 3;
1345 cpb->tag = qc->tag;
1346 cpb->next_cpb_idx = 0;
1347
1348 /* turn on NCQ flags for NCQ commands */
1349 if (qc->tf.protocol == ATA_PROT_NCQ)
1350 ctl_flags |= NV_CPB_CTL_QUEUE | NV_CPB_CTL_FPDMA;
1351
cdf56bcf
RH		 1352 VPRINTK("qc->flags = 0x%lx\n", qc->flags);
1353
fbbb262d
RH		 1354 nv_adma_tf_to_cpb(&qc->tf, cpb->tf);
1355
b447916e 1356 if (qc->flags & ATA_QCFLAG_DMAMAP) {
382a6652
RH		 1357 nv_adma_fill_sg(qc, cpb);
1358 ctl_flags |= NV_CPB_CTL_APRD_VALID;
1359 } else
1360 memset(&cpb->aprd[0], 0, sizeof(struct nv_adma_prd) * 5);
fbbb262d 1361
5796d1c4
JG		 1362 /* Be paranoid and don't let the device see NV_CPB_CTL_CPB_VALID
1363 until we are finished filling in all of the contents */
fbbb262d
RH		 1364 wmb();
1365 cpb->ctl_flags = ctl_flags;
41949ed5
RH		 1366 wmb();
1367 cpb->resp_flags = 0;
fbbb262d
RH		 1368}
1369
1370static unsigned int nv_adma_qc_issue(struct ata_queued_cmd *qc)
1371{
2dec7555 1372 struct nv_adma_port_priv *pp = qc->ap->private_data;
cdf56bcf 1373 void __iomem *mmio = pp->ctl_block;
5e5c74a5 1374 int curr_ncq = (qc->tf.protocol == ATA_PROT_NCQ);
fbbb262d
RH		 1375
1376 VPRINTK("ENTER\n");
1377
3f3debdb
RH		 1378 /* We can't handle result taskfile with NCQ commands, since
1379 retrieving the taskfile switches us out of ADMA mode and would abort
1380 existing commands. */
1381 if (unlikely(qc->tf.protocol == ATA_PROT_NCQ &&
1382 (qc->flags & ATA_QCFLAG_RESULT_TF))) {
1383 ata_dev_printk(qc->dev, KERN_ERR,
1384 "NCQ w/ RESULT_TF not allowed\n");
1385 return AC_ERR_SYSTEM;
1386 }
1387
382a6652 1388 if (nv_adma_use_reg_mode(qc)) {
fbbb262d 1389 /* use ATA register mode */
382a6652 1390 VPRINTK("using ATA register mode: 0x%lx\n", qc->flags);
3f3debdb
RH		 1391 BUG_ON(!(pp->flags & NV_ADMA_ATAPI_SETUP_COMPLETE) &&
1392 (qc->flags & ATA_QCFLAG_DMAMAP));
fbbb262d
RH		 1393 nv_adma_register_mode(qc->ap);
1394 return ata_qc_issue_prot(qc);
1395 } else
1396 nv_adma_mode(qc->ap);
1397
1398 /* write append register, command tag in lower 8 bits
1399 and (number of cpbs to append -1) in top 8 bits */
1400 wmb();
5e5c74a5 1401
b447916e 1402 if (curr_ncq != pp->last_issue_ncq) {
5796d1c4
JG		 1403 /* Seems to need some delay before switching between NCQ and
1404 non-NCQ commands, else we get command timeouts and such. */
5e5c74a5
RH		 1405 udelay(20);
1406 pp->last_issue_ncq = curr_ncq;
1407 }
1408
fbbb262d
RH		 1409 writew(qc->tag, mmio + NV_ADMA_APPEND);
1410
5796d1c4 1411 DPRINTK("Issued tag %u\n", qc->tag);
fbbb262d
RH		 1412
1413 return 0;
1414}
1415
7d12e780 1416static irqreturn_t nv_generic_interrupt(int irq, void *dev_instance)
1da177e4 1417{
cca3974e 1418 struct ata_host *host = dev_instance;
1da177e4
LT		 1419 unsigned int i;
1420 unsigned int handled = 0;
1421 unsigned long flags;
1422
cca3974e 1423 spin_lock_irqsave(&host->lock, flags);
1da177e4 1424
cca3974e 1425 for (i = 0; i < host->n_ports; i++) {
1da177e4
LT		 1426 struct ata_port *ap;
1427
cca3974e 1428 ap = host->ports[i];
c1389503 1429 if (ap &&
029f5468 1430 !(ap->flags & ATA_FLAG_DISABLED)) {
1da177e4
LT		 1431 struct ata_queued_cmd *qc;
1432
9af5c9c9 1433 qc = ata_qc_from_tag(ap, ap->link.active_tag);
e50362ec 1434 if (qc && (!(qc->tf.flags & ATA_TFLAG_POLLING)))
1da177e4 1435 handled += ata_host_intr(ap, qc);
b887030a
AC		 1436 else
1437 // No request pending? Clear interrupt status
1438 // anyway, in case there's one pending.
1439 ap->ops->check_status(ap);
1da177e4
LT		 1440 }
1441
1442 }
1443
cca3974e 1444 spin_unlock_irqrestore(&host->lock, flags);
1da177e4
LT		 1445
1446 return IRQ_RETVAL(handled);
1447}
1448
cca3974e 1449static irqreturn_t nv_do_interrupt(struct ata_host *host, u8 irq_stat)
ada364e8
TH		 1450{
1451 int i, handled = 0;
1452
cca3974e
JG		 1453 for (i = 0; i < host->n_ports; i++) {
1454 struct ata_port *ap = host->ports[i];
ada364e8
TH		 1455
1456 if (ap && !(ap->flags & ATA_FLAG_DISABLED))
1457 handled += nv_host_intr(ap, irq_stat);
1458
1459 irq_stat >>= NV_INT_PORT_SHIFT;
1460 }
1461
1462 return IRQ_RETVAL(handled);
1463}
1464
7d12e780 1465static irqreturn_t nv_nf2_interrupt(int irq, void *dev_instance)
ada364e8 1466{
cca3974e 1467 struct ata_host *host = dev_instance;
ada364e8
TH		 1468 u8 irq_stat;
1469 irqreturn_t ret;
1470
cca3974e 1471 spin_lock(&host->lock);
0d5ff566 1472 irq_stat = ioread8(host->ports[0]->ioaddr.scr_addr + NV_INT_STATUS);
cca3974e
JG		 1473 ret = nv_do_interrupt(host, irq_stat);
1474 spin_unlock(&host->lock);
ada364e8
TH		 1475
1476 return ret;
1477}
1478
7d12e780 1479static irqreturn_t nv_ck804_interrupt(int irq, void *dev_instance)
ada364e8 1480{
cca3974e 1481 struct ata_host *host = dev_instance;
ada364e8
TH		 1482 u8 irq_stat;
1483 irqreturn_t ret;
1484
cca3974e 1485 spin_lock(&host->lock);
0d5ff566 1486 irq_stat = readb(host->iomap[NV_MMIO_BAR] + NV_INT_STATUS_CK804);
cca3974e
JG		 1487 ret = nv_do_interrupt(host, irq_stat);
1488 spin_unlock(&host->lock);
ada364e8
TH		 1489
1490 return ret;
1491}
1492
da3dbb17 1493static int nv_scr_read(struct ata_port *ap, unsigned int sc_reg, u32 *val)
1da177e4 1494{
1da177e4 1495 if (sc_reg > SCR_CONTROL)
da3dbb17 1496 return -EINVAL;
1da177e4 1497
da3dbb17
TH		 1498 *val = ioread32(ap->ioaddr.scr_addr + (sc_reg * 4));
1499 return 0;
1da177e4
LT		 1500}
1501
da3dbb17 1502static int nv_scr_write(struct ata_port *ap, unsigned int sc_reg, u32 val)
1da177e4 1503{
1da177e4 1504 if (sc_reg > SCR_CONTROL)
da3dbb17 1505 return -EINVAL;
1da177e4 1506
0d5ff566 1507 iowrite32(val, ap->ioaddr.scr_addr + (sc_reg * 4));
da3dbb17 1508 return 0;
1da177e4
LT		 1509}
1510
39f87582
TH		 1511static void nv_nf2_freeze(struct ata_port *ap)
1512{
0d5ff566 1513 void __iomem *scr_addr = ap->host->ports[0]->ioaddr.scr_addr;
39f87582
TH		 1514 int shift = ap->port_no * NV_INT_PORT_SHIFT;
1515 u8 mask;
1516
0d5ff566 1517 mask = ioread8(scr_addr + NV_INT_ENABLE);
39f87582 1518 mask &= ~(NV_INT_ALL << shift);
0d5ff566 1519 iowrite8(mask, scr_addr + NV_INT_ENABLE);
39f87582
TH		 1520}
1521
1522static void nv_nf2_thaw(struct ata_port *ap)
1523{
0d5ff566 1524 void __iomem *scr_addr = ap->host->ports[0]->ioaddr.scr_addr;
39f87582
TH		 1525 int shift = ap->port_no * NV_INT_PORT_SHIFT;
1526 u8 mask;
1527
0d5ff566 1528 iowrite8(NV_INT_ALL << shift, scr_addr + NV_INT_STATUS);
39f87582 1529
0d5ff566 1530 mask = ioread8(scr_addr + NV_INT_ENABLE);
39f87582 1531 mask |= (NV_INT_MASK << shift);
0d5ff566 1532 iowrite8(mask, scr_addr + NV_INT_ENABLE);
39f87582
TH		 1533}
1534
1535static void nv_ck804_freeze(struct ata_port *ap)
1536{
0d5ff566 1537 void __iomem *mmio_base = ap->host->iomap[NV_MMIO_BAR];
39f87582
TH		 1538 int shift = ap->port_no * NV_INT_PORT_SHIFT;
1539 u8 mask;
1540
1541 mask = readb(mmio_base + NV_INT_ENABLE_CK804);
1542 mask &= ~(NV_INT_ALL << shift);
1543 writeb(mask, mmio_base + NV_INT_ENABLE_CK804);
1544}
1545
1546static void nv_ck804_thaw(struct ata_port *ap)
1547{
0d5ff566 1548 void __iomem *mmio_base = ap->host->iomap[NV_MMIO_BAR];
39f87582
TH		 1549 int shift = ap->port_no * NV_INT_PORT_SHIFT;
1550 u8 mask;
1551
1552 writeb(NV_INT_ALL << shift, mmio_base + NV_INT_STATUS_CK804);
1553
1554 mask = readb(mmio_base + NV_INT_ENABLE_CK804);
1555 mask |= (NV_INT_MASK << shift);
1556 writeb(mask, mmio_base + NV_INT_ENABLE_CK804);
1557}
1558
f140f0f1
KL		 1559static void nv_mcp55_freeze(struct ata_port *ap)
1560{
1561 void __iomem *mmio_base = ap->host->iomap[NV_MMIO_BAR];
1562 int shift = ap->port_no * NV_INT_PORT_SHIFT_MCP55;
1563 u32 mask;
1564
1565 writel(NV_INT_ALL_MCP55 << shift, mmio_base + NV_INT_STATUS_MCP55);
1566
1567 mask = readl(mmio_base + NV_INT_ENABLE_MCP55);
1568 mask &= ~(NV_INT_ALL_MCP55 << shift);
1569 writel(mask, mmio_base + NV_INT_ENABLE_MCP55);
1570 ata_bmdma_freeze(ap);
1571}
1572
1573static void nv_mcp55_thaw(struct ata_port *ap)
1574{
1575 void __iomem *mmio_base = ap->host->iomap[NV_MMIO_BAR];
1576 int shift = ap->port_no * NV_INT_PORT_SHIFT_MCP55;
1577 u32 mask;
1578
1579 writel(NV_INT_ALL_MCP55 << shift, mmio_base + NV_INT_STATUS_MCP55);
1580
1581 mask = readl(mmio_base + NV_INT_ENABLE_MCP55);
1582 mask |= (NV_INT_MASK_MCP55 << shift);
1583 writel(mask, mmio_base + NV_INT_ENABLE_MCP55);
1584 ata_bmdma_thaw(ap);
1585}
1586
cc0680a5 1587static int nv_hardreset(struct ata_link *link, unsigned int *class,
d4b2bab4 1588 unsigned long deadline)
39f87582
TH		 1589{
1590 unsigned int dummy;
1591
1592 /* SATA hardreset fails to retrieve proper device signature on
1593 * some controllers. Don't classify on hardreset. For more
647c595d 1594 * info, see http://bugzilla.kernel.org/show_bug.cgi?id=3352
39f87582 1595 */
cc0680a5 1596 return sata_std_hardreset(link, &dummy, deadline);
39f87582
TH		 1597}
1598
1599static void nv_error_handler(struct ata_port *ap)
1600{
1601 ata_bmdma_drive_eh(ap, ata_std_prereset, ata_std_softreset,
1602 nv_hardreset, ata_std_postreset);
1603}
1604
fbbb262d
RH		 1605static void nv_adma_error_handler(struct ata_port *ap)
1606{
1607 struct nv_adma_port_priv *pp = ap->private_data;
b447916e 1608 if (!(pp->flags & NV_ADMA_PORT_REGISTER_MODE)) {
cdf56bcf 1609 void __iomem *mmio = pp->ctl_block;
fbbb262d
RH		 1610 int i;
1611 u16 tmp;
a84471fe 1612
b447916e 1613 if (ata_tag_valid(ap->link.active_tag) || ap->link.sactive) {
2cb27853
RH		 1614 u32 notifier = readl(mmio + NV_ADMA_NOTIFIER);
1615 u32 notifier_error = readl(mmio + NV_ADMA_NOTIFIER_ERROR);
1616 u32 gen_ctl = readl(pp->gen_block + NV_ADMA_GEN_CTL);
1617 u32 status = readw(mmio + NV_ADMA_STAT);
08af7414
RH		 1618 u8 cpb_count = readb(mmio + NV_ADMA_CPB_COUNT);
1619 u8 next_cpb_idx = readb(mmio + NV_ADMA_NEXT_CPB_IDX);
2cb27853 1620
5796d1c4
JG		 1621 ata_port_printk(ap, KERN_ERR,
1622 "EH in ADMA mode, notifier 0x%X "
08af7414
RH		 1623 "notifier_error 0x%X gen_ctl 0x%X status 0x%X "
1624 "next cpb count 0x%X next cpb idx 0x%x\n",
1625 notifier, notifier_error, gen_ctl, status,
1626 cpb_count, next_cpb_idx);
2cb27853 1627
b447916e 1628 for (i = 0; i < NV_ADMA_MAX_CPBS; i++) {
2cb27853 1629 struct nv_adma_cpb *cpb = &pp->cpb[i];
b447916e 1630 if ((ata_tag_valid(ap->link.active_tag) && i == ap->link.active_tag) ||
5796d1c4 1631 ap->link.sactive & (1 << i))
2cb27853
RH		 1632 ata_port_printk(ap, KERN_ERR,
1633 "CPB %d: ctl_flags 0x%x, resp_flags 0x%x\n",
1634 i, cpb->ctl_flags, cpb->resp_flags);
1635 }
1636 }
fbbb262d 1637
fbbb262d
RH		 1638 /* Push us back into port register mode for error handling. */
1639 nv_adma_register_mode(ap);
1640
5796d1c4
JG		 1641 /* Mark all of the CPBs as invalid to prevent them from
1642 being executed */
b447916e 1643 for (i = 0; i < NV_ADMA_MAX_CPBS; i++)
fbbb262d
RH		 1644 pp->cpb[i].ctl_flags &= ~NV_CPB_CTL_CPB_VALID;
1645
1646 /* clear CPB fetch count */
1647 writew(0, mmio + NV_ADMA_CPB_COUNT);
1648
1649 /* Reset channel */
1650 tmp = readw(mmio + NV_ADMA_CTL);
1651 writew(tmp | NV_ADMA_CTL_CHANNEL_RESET, mmio + NV_ADMA_CTL);
b447916e 1652 readw(mmio + NV_ADMA_CTL); /* flush posted write */
fbbb262d
RH		 1653 udelay(1);
1654 writew(tmp & ~NV_ADMA_CTL_CHANNEL_RESET, mmio + NV_ADMA_CTL);
b447916e 1655 readw(mmio + NV_ADMA_CTL); /* flush posted write */
fbbb262d
RH		 1656 }
1657
1658 ata_bmdma_drive_eh(ap, ata_std_prereset, ata_std_softreset,
1659 nv_hardreset, ata_std_postreset);
1660}
1661
f140f0f1
KL		 1662static void nv_swncq_qc_to_dq(struct ata_port *ap, struct ata_queued_cmd *qc)
1663{
1664 struct nv_swncq_port_priv *pp = ap->private_data;
1665 struct defer_queue *dq = &pp->defer_queue;
1666
1667 /* queue is full */
1668 WARN_ON(dq->tail - dq->head == ATA_MAX_QUEUE);
1669 dq->defer_bits |= (1 << qc->tag);
1670 dq->tag[dq->tail++ & (ATA_MAX_QUEUE - 1)] = qc->tag;
1671}
1672
1673static struct ata_queued_cmd *nv_swncq_qc_from_dq(struct ata_port *ap)
1674{
1675 struct nv_swncq_port_priv *pp = ap->private_data;
1676 struct defer_queue *dq = &pp->defer_queue;
1677 unsigned int tag;
1678
1679 if (dq->head == dq->tail) /* null queue */
1680 return NULL;
1681
1682 tag = dq->tag[dq->head & (ATA_MAX_QUEUE - 1)];
1683 dq->tag[dq->head++ & (ATA_MAX_QUEUE - 1)] = ATA_TAG_POISON;
1684 WARN_ON(!(dq->defer_bits & (1 << tag)));
1685 dq->defer_bits &= ~(1 << tag);
1686
1687 return ata_qc_from_tag(ap, tag);
1688}
1689
1690static void nv_swncq_fis_reinit(struct ata_port *ap)
1691{
1692 struct nv_swncq_port_priv *pp = ap->private_data;
1693
1694 pp->dhfis_bits = 0;
1695 pp->dmafis_bits = 0;
1696 pp->sdbfis_bits = 0;
1697 pp->ncq_flags = 0;
1698}
1699
1700static void nv_swncq_pp_reinit(struct ata_port *ap)
1701{
1702 struct nv_swncq_port_priv *pp = ap->private_data;
1703 struct defer_queue *dq = &pp->defer_queue;
1704
1705 dq->head = 0;
1706 dq->tail = 0;
1707 dq->defer_bits = 0;
1708 pp->qc_active = 0;
1709 pp->last_issue_tag = ATA_TAG_POISON;
1710 nv_swncq_fis_reinit(ap);
1711}
1712
1713static void nv_swncq_irq_clear(struct ata_port *ap, u16 fis)
1714{
1715 struct nv_swncq_port_priv *pp = ap->private_data;
1716
1717 writew(fis, pp->irq_block);
1718}
1719
1720static void __ata_bmdma_stop(struct ata_port *ap)
1721{
1722 struct ata_queued_cmd qc;
1723
1724 qc.ap = ap;
1725 ata_bmdma_stop(&qc);
1726}
1727
1728static void nv_swncq_ncq_stop(struct ata_port *ap)
1729{
1730 struct nv_swncq_port_priv *pp = ap->private_data;
1731 unsigned int i;
1732 u32 sactive;
1733 u32 done_mask;
1734
1735 ata_port_printk(ap, KERN_ERR,
1736 "EH in SWNCQ mode,QC:qc_active 0x%X sactive 0x%X\n",
1737 ap->qc_active, ap->link.sactive);
1738 ata_port_printk(ap, KERN_ERR,
1739 "SWNCQ:qc_active 0x%X defer_bits 0x%X last_issue_tag 0x%x\n "
1740 "dhfis 0x%X dmafis 0x%X sdbfis 0x%X\n",
1741 pp->qc_active, pp->defer_queue.defer_bits, pp->last_issue_tag,
1742 pp->dhfis_bits, pp->dmafis_bits, pp->sdbfis_bits);
1743
1744 ata_port_printk(ap, KERN_ERR, "ATA_REG 0x%X ERR_REG 0x%X\n",
1745 ap->ops->check_status(ap),
1746 ioread8(ap->ioaddr.error_addr));
1747
1748 sactive = readl(pp->sactive_block);
1749 done_mask = pp->qc_active ^ sactive;
1750
1751 ata_port_printk(ap, KERN_ERR, "tag : dhfis dmafis sdbfis sacitve\n");
1752 for (i = 0; i < ATA_MAX_QUEUE; i++) {
1753 u8 err = 0;
1754 if (pp->qc_active & (1 << i))
1755 err = 0;
1756 else if (done_mask & (1 << i))
1757 err = 1;
1758 else
1759 continue;
1760
1761 ata_port_printk(ap, KERN_ERR,
1762 "tag 0x%x: %01x %01x %01x %01x %s\n", i,
1763 (pp->dhfis_bits >> i) & 0x1,
1764 (pp->dmafis_bits >> i) & 0x1,
1765 (pp->sdbfis_bits >> i) & 0x1,
1766 (sactive >> i) & 0x1,
1767 (err ? "error! tag doesn't exit" : " "));
1768 }
1769
1770 nv_swncq_pp_reinit(ap);
1771 ap->ops->irq_clear(ap);
1772 __ata_bmdma_stop(ap);
1773 nv_swncq_irq_clear(ap, 0xffff);
1774}
1775
1776static void nv_swncq_error_handler(struct ata_port *ap)
1777{
1778 struct ata_eh_context *ehc = &ap->link.eh_context;
1779
1780 if (ap->link.sactive) {
1781 nv_swncq_ncq_stop(ap);
cf480626 1782 ehc->i.action |= ATA_EH_RESET;
f140f0f1
KL		 1783 }
1784
1785 ata_bmdma_drive_eh(ap, ata_std_prereset, ata_std_softreset,
1786 nv_hardreset, ata_std_postreset);
1787}
1788
1789#ifdef CONFIG_PM
1790static int nv_swncq_port_suspend(struct ata_port *ap, pm_message_t mesg)
1791{
1792 void __iomem *mmio = ap->host->iomap[NV_MMIO_BAR];
1793 u32 tmp;
1794
1795 /* clear irq */
1796 writel(~0, mmio + NV_INT_STATUS_MCP55);
1797
1798 /* disable irq */
1799 writel(0, mmio + NV_INT_ENABLE_MCP55);
1800
1801 /* disable swncq */
1802 tmp = readl(mmio + NV_CTL_MCP55);
1803 tmp &= ~(NV_CTL_PRI_SWNCQ | NV_CTL_SEC_SWNCQ);
1804 writel(tmp, mmio + NV_CTL_MCP55);
1805
1806 return 0;
1807}
1808
1809static int nv_swncq_port_resume(struct ata_port *ap)
1810{
1811 void __iomem *mmio = ap->host->iomap[NV_MMIO_BAR];
1812 u32 tmp;
1813
1814 /* clear irq */
1815 writel(~0, mmio + NV_INT_STATUS_MCP55);
1816
1817 /* enable irq */
1818 writel(0x00fd00fd, mmio + NV_INT_ENABLE_MCP55);
1819
1820 /* enable swncq */
1821 tmp = readl(mmio + NV_CTL_MCP55);
1822 writel(tmp | NV_CTL_PRI_SWNCQ | NV_CTL_SEC_SWNCQ, mmio + NV_CTL_MCP55);
1823
1824 return 0;
1825}
1826#endif
1827
1828static void nv_swncq_host_init(struct ata_host *host)
1829{
1830 u32 tmp;
1831 void __iomem *mmio = host->iomap[NV_MMIO_BAR];
1832 struct pci_dev *pdev = to_pci_dev(host->dev);
1833 u8 regval;
1834
1835 /* disable ECO 398 */
1836 pci_read_config_byte(pdev, 0x7f, &regval);
1837 regval &= ~(1 << 7);
1838 pci_write_config_byte(pdev, 0x7f, regval);
1839
1840 /* enable swncq */
1841 tmp = readl(mmio + NV_CTL_MCP55);
1842 VPRINTK("HOST_CTL:0x%X\n", tmp);
1843 writel(tmp | NV_CTL_PRI_SWNCQ | NV_CTL_SEC_SWNCQ, mmio + NV_CTL_MCP55);
1844
1845 /* enable irq intr */
1846 tmp = readl(mmio + NV_INT_ENABLE_MCP55);
1847 VPRINTK("HOST_ENABLE:0x%X\n", tmp);
1848 writel(tmp | 0x00fd00fd, mmio + NV_INT_ENABLE_MCP55);
1849
1850 /* clear port irq */
1851 writel(~0x0, mmio + NV_INT_STATUS_MCP55);
1852}
1853
1854static int nv_swncq_slave_config(struct scsi_device *sdev)
1855{
1856 struct ata_port *ap = ata_shost_to_port(sdev->host);
1857 struct pci_dev *pdev = to_pci_dev(ap->host->dev);
1858 struct ata_device *dev;
1859 int rc;
1860 u8 rev;
1861 u8 check_maxtor = 0;
1862 unsigned char model_num[ATA_ID_PROD_LEN + 1];
1863
1864 rc = ata_scsi_slave_config(sdev);
1865 if (sdev->id >= ATA_MAX_DEVICES || sdev->channel || sdev->lun)
1866 /* Not a proper libata device, ignore */
1867 return rc;
1868
1869 dev = &ap->link.device[sdev->id];
1870 if (!(ap->flags & ATA_FLAG_NCQ) || dev->class == ATA_DEV_ATAPI)
1871 return rc;
1872
1873 /* if MCP51 and Maxtor, then disable ncq */
1874 if (pdev->device == PCI_DEVICE_ID_NVIDIA_NFORCE_MCP51_SATA ||
1875 pdev->device == PCI_DEVICE_ID_NVIDIA_NFORCE_MCP51_SATA2)
1876 check_maxtor = 1;
1877
1878 /* if MCP55 and rev <= a2 and Maxtor, then disable ncq */
1879 if (pdev->device == PCI_DEVICE_ID_NVIDIA_NFORCE_MCP55_SATA ||
1880 pdev->device == PCI_DEVICE_ID_NVIDIA_NFORCE_MCP55_SATA2) {
1881 pci_read_config_byte(pdev, 0x8, &rev);
1882 if (rev <= 0xa2)
1883 check_maxtor = 1;
1884 }
1885
1886 if (!check_maxtor)
1887 return rc;
1888
1889 ata_id_c_string(dev->id, model_num, ATA_ID_PROD, sizeof(model_num));
1890
1891 if (strncmp(model_num, "Maxtor", 6) == 0) {
1892 ata_scsi_change_queue_depth(sdev, 1);
1893 ata_dev_printk(dev, KERN_NOTICE,
1894 "Disabling SWNCQ mode (depth %x)\n", sdev->queue_depth);
1895 }
1896
1897 return rc;
1898}
1899
1900static int nv_swncq_port_start(struct ata_port *ap)
1901{
1902 struct device *dev = ap->host->dev;
1903 void __iomem *mmio = ap->host->iomap[NV_MMIO_BAR];
1904 struct nv_swncq_port_priv *pp;
1905 int rc;
1906
1907 rc = ata_port_start(ap);
1908 if (rc)
1909 return rc;
1910
1911 pp = devm_kzalloc(dev, sizeof(*pp), GFP_KERNEL);
1912 if (!pp)
1913 return -ENOMEM;
1914
1915 pp->prd = dmam_alloc_coherent(dev, ATA_PRD_TBL_SZ * ATA_MAX_QUEUE,
1916 &pp->prd_dma, GFP_KERNEL);
1917 if (!pp->prd)
1918 return -ENOMEM;
1919 memset(pp->prd, 0, ATA_PRD_TBL_SZ * ATA_MAX_QUEUE);
1920
1921 ap->private_data = pp;
1922 pp->sactive_block = ap->ioaddr.scr_addr + 4 * SCR_ACTIVE;
1923 pp->irq_block = mmio + NV_INT_STATUS_MCP55 + ap->port_no * 2;
1924 pp->tag_block = mmio + NV_NCQ_REG_MCP55 + ap->port_no * 2;
1925
1926 return 0;
1927}
1928
1929static void nv_swncq_qc_prep(struct ata_queued_cmd *qc)
1930{
1931 if (qc->tf.protocol != ATA_PROT_NCQ) {
1932 ata_qc_prep(qc);
1933 return;
1934 }
1935
1936 if (!(qc->flags & ATA_QCFLAG_DMAMAP))
1937 return;
1938
1939 nv_swncq_fill_sg(qc);
1940}
1941
1942static void nv_swncq_fill_sg(struct ata_queued_cmd *qc)
1943{
1944 struct ata_port *ap = qc->ap;
1945 struct scatterlist *sg;
f140f0f1
KL		 1946 struct nv_swncq_port_priv *pp = ap->private_data;
1947 struct ata_prd *prd;
ff2aeb1e 1948 unsigned int si, idx;
f140f0f1
KL		 1949
1950 prd = pp->prd + ATA_MAX_PRD * qc->tag;
1951
1952 idx = 0;
ff2aeb1e 1953 for_each_sg(qc->sg, sg, qc->n_elem, si) {
f140f0f1
KL		 1954 u32 addr, offset;
1955 u32 sg_len, len;
1956
1957 addr = (u32)sg_dma_address(sg);
1958 sg_len = sg_dma_len(sg);
1959
1960 while (sg_len) {
1961 offset = addr & 0xffff;
1962 len = sg_len;
1963 if ((offset + sg_len) > 0x10000)
1964 len = 0x10000 - offset;
1965
1966 prd[idx].addr = cpu_to_le32(addr);
1967 prd[idx].flags_len = cpu_to_le32(len & 0xffff);
1968
1969 idx++;
1970 sg_len -= len;
1971 addr += len;
1972 }
1973 }
1974
ff2aeb1e 1975 prd[idx - 1].flags_len |= cpu_to_le32(ATA_PRD_EOT);
f140f0f1
KL		 1976}
1977
1978static unsigned int nv_swncq_issue_atacmd(struct ata_port *ap,
1979 struct ata_queued_cmd *qc)
1980{
1981 struct nv_swncq_port_priv *pp = ap->private_data;
1982
1983 if (qc == NULL)
1984 return 0;
1985
1986 DPRINTK("Enter\n");
1987
1988 writel((1 << qc->tag), pp->sactive_block);
1989 pp->last_issue_tag = qc->tag;
1990 pp->dhfis_bits &= ~(1 << qc->tag);
1991 pp->dmafis_bits &= ~(1 << qc->tag);
1992 pp->qc_active |= (0x1 << qc->tag);
1993
1994 ap->ops->tf_load(ap, &qc->tf); /* load tf registers */
1995 ap->ops->exec_command(ap, &qc->tf);
1996
1997 DPRINTK("Issued tag %u\n", qc->tag);
1998
1999 return 0;
2000}
2001
2002static unsigned int nv_swncq_qc_issue(struct ata_queued_cmd *qc)
2003{
2004 struct ata_port *ap = qc->ap;
2005 struct nv_swncq_port_priv *pp = ap->private_data;
2006
2007 if (qc->tf.protocol != ATA_PROT_NCQ)
2008 return ata_qc_issue_prot(qc);
2009
2010 DPRINTK("Enter\n");
2011
2012 if (!pp->qc_active)
2013 nv_swncq_issue_atacmd(ap, qc);
2014 else
2015 nv_swncq_qc_to_dq(ap, qc); /* add qc to defer queue */
2016
2017 return 0;
2018}
2019
2020static void nv_swncq_hotplug(struct ata_port *ap, u32 fis)
2021{
2022 u32 serror;
2023 struct ata_eh_info *ehi = &ap->link.eh_info;
2024
2025 ata_ehi_clear_desc(ehi);
2026
2027 /* AHCI needs SError cleared; otherwise, it might lock up */
2028 sata_scr_read(&ap->link, SCR_ERROR, &serror);
2029 sata_scr_write(&ap->link, SCR_ERROR, serror);
2030
2031 /* analyze @irq_stat */
2032 if (fis & NV_SWNCQ_IRQ_ADDED)
2033 ata_ehi_push_desc(ehi, "hot plug");
2034 else if (fis & NV_SWNCQ_IRQ_REMOVED)
2035 ata_ehi_push_desc(ehi, "hot unplug");
2036
2037 ata_ehi_hotplugged(ehi);
2038
2039 /* okay, let's hand over to EH */
2040 ehi->serror |= serror;
2041
2042 ata_port_freeze(ap);
2043}
2044
2045static int nv_swncq_sdbfis(struct ata_port *ap)
2046{
2047 struct ata_queued_cmd *qc;
2048 struct nv_swncq_port_priv *pp = ap->private_data;
2049 struct ata_eh_info *ehi = &ap->link.eh_info;
2050 u32 sactive;
2051 int nr_done = 0;
2052 u32 done_mask;
2053 int i;
2054 u8 host_stat;
2055 u8 lack_dhfis = 0;
2056
2057 host_stat = ap->ops->bmdma_status(ap);
2058 if (unlikely(host_stat & ATA_DMA_ERR)) {
2059 /* error when transfering data to/from memory */
2060 ata_ehi_clear_desc(ehi);
2061 ata_ehi_push_desc(ehi, "BMDMA stat 0x%x", host_stat);
2062 ehi->err_mask |= AC_ERR_HOST_BUS;
cf480626 2063 ehi->action |= ATA_EH_RESET;
f140f0f1
KL		 2064 return -EINVAL;
2065 }
2066
2067 ap->ops->irq_clear(ap);
2068 __ata_bmdma_stop(ap);
2069
2070 sactive = readl(pp->sactive_block);
2071 done_mask = pp->qc_active ^ sactive;
2072
2073 if (unlikely(done_mask & sactive)) {
2074 ata_ehi_clear_desc(ehi);
2075 ata_ehi_push_desc(ehi, "illegal SWNCQ:qc_active transition"
2076 "(%08x->%08x)", pp->qc_active, sactive);
2077 ehi->err_mask |= AC_ERR_HSM;
cf480626 2078 ehi->action |= ATA_EH_RESET;
f140f0f1
KL		 2079 return -EINVAL;
2080 }
2081 for (i = 0; i < ATA_MAX_QUEUE; i++) {
2082 if (!(done_mask & (1 << i)))
2083 continue;
2084
2085 qc = ata_qc_from_tag(ap, i);
2086 if (qc) {
2087 ata_qc_complete(qc);
2088 pp->qc_active &= ~(1 << i);
2089 pp->dhfis_bits &= ~(1 << i);
2090 pp->dmafis_bits &= ~(1 << i);
2091 pp->sdbfis_bits |= (1 << i);
2092 nr_done++;
2093 }
2094 }
2095
2096 if (!ap->qc_active) {
2097 DPRINTK("over\n");
2098 nv_swncq_pp_reinit(ap);
2099 return nr_done;
2100 }
2101
2102 if (pp->qc_active & pp->dhfis_bits)
2103 return nr_done;
2104
2105 if ((pp->ncq_flags & ncq_saw_backout) ||
2106 (pp->qc_active ^ pp->dhfis_bits))
2107 /* if the controller cann't get a device to host register FIS,
2108 * The driver needs to reissue the new command.
2109 */
2110 lack_dhfis = 1;
2111
2112 DPRINTK("id 0x%x QC: qc_active 0x%x,"
2113 "SWNCQ:qc_active 0x%X defer_bits %X "
2114 "dhfis 0x%X dmafis 0x%X last_issue_tag %x\n",
2115 ap->print_id, ap->qc_active, pp->qc_active,
2116 pp->defer_queue.defer_bits, pp->dhfis_bits,
2117 pp->dmafis_bits, pp->last_issue_tag);
2118
2119 nv_swncq_fis_reinit(ap);
2120
2121 if (lack_dhfis) {
2122 qc = ata_qc_from_tag(ap, pp->last_issue_tag);
2123 nv_swncq_issue_atacmd(ap, qc);
2124 return nr_done;
2125 }
2126
2127 if (pp->defer_queue.defer_bits) {
2128 /* send deferral queue command */
2129 qc = nv_swncq_qc_from_dq(ap);
2130 WARN_ON(qc == NULL);
2131 nv_swncq_issue_atacmd(ap, qc);
2132 }
2133
2134 return nr_done;
2135}
2136
2137static inline u32 nv_swncq_tag(struct ata_port *ap)
2138{
2139 struct nv_swncq_port_priv *pp = ap->private_data;
2140 u32 tag;
2141
2142 tag = readb(pp->tag_block) >> 2;
2143 return (tag & 0x1f);
2144}
2145
2146static int nv_swncq_dmafis(struct ata_port *ap)
2147{
2148 struct ata_queued_cmd *qc;
2149 unsigned int rw;
2150 u8 dmactl;
2151 u32 tag;
2152 struct nv_swncq_port_priv *pp = ap->private_data;
2153
2154 __ata_bmdma_stop(ap);
2155 tag = nv_swncq_tag(ap);
2156
2157 DPRINTK("dma setup tag 0x%x\n", tag);
2158 qc = ata_qc_from_tag(ap, tag);
2159
2160 if (unlikely(!qc))
2161 return 0;
2162
2163 rw = qc->tf.flags & ATA_TFLAG_WRITE;
2164
2165 /* load PRD table addr. */
2166 iowrite32(pp->prd_dma + ATA_PRD_TBL_SZ * qc->tag,
2167 ap->ioaddr.bmdma_addr + ATA_DMA_TABLE_OFS);
2168
2169 /* specify data direction, triple-check start bit is clear */
2170 dmactl = ioread8(ap->ioaddr.bmdma_addr + ATA_DMA_CMD);
2171 dmactl &= ~ATA_DMA_WR;
2172 if (!rw)
2173 dmactl |= ATA_DMA_WR;
2174
2175 iowrite8(dmactl | ATA_DMA_START, ap->ioaddr.bmdma_addr + ATA_DMA_CMD);
2176
2177 return 1;
2178}
2179
2180static void nv_swncq_host_interrupt(struct ata_port *ap, u16 fis)
2181{
2182 struct nv_swncq_port_priv *pp = ap->private_data;
2183 struct ata_queued_cmd *qc;
2184 struct ata_eh_info *ehi = &ap->link.eh_info;
2185 u32 serror;
2186 u8 ata_stat;
2187 int rc = 0;
2188
2189 ata_stat = ap->ops->check_status(ap);
2190 nv_swncq_irq_clear(ap, fis);
2191 if (!fis)
2192 return;
2193
2194 if (ap->pflags & ATA_PFLAG_FROZEN)
2195 return;
2196
2197 if (fis & NV_SWNCQ_IRQ_HOTPLUG) {
2198 nv_swncq_hotplug(ap, fis);
2199 return;
2200 }
2201
2202 if (!pp->qc_active)
2203 return;
2204
2205 if (ap->ops->scr_read(ap, SCR_ERROR, &serror))
2206 return;
2207 ap->ops->scr_write(ap, SCR_ERROR, serror);
2208
2209 if (ata_stat & ATA_ERR) {
2210 ata_ehi_clear_desc(ehi);
2211 ata_ehi_push_desc(ehi, "Ata error. fis:0x%X", fis);
2212 ehi->err_mask |= AC_ERR_DEV;
2213 ehi->serror |= serror;
cf480626 2214 ehi->action |= ATA_EH_RESET;
f140f0f1
KL		 2215 ata_port_freeze(ap);
2216 return;
2217 }
2218
2219 if (fis & NV_SWNCQ_IRQ_BACKOUT) {
2220 /* If the IRQ is backout, driver must issue
2221 * the new command again some time later.
2222 */
2223 pp->ncq_flags |= ncq_saw_backout;
2224 }
2225
2226 if (fis & NV_SWNCQ_IRQ_SDBFIS) {
2227 pp->ncq_flags |= ncq_saw_sdb;
2228 DPRINTK("id 0x%x SWNCQ: qc_active 0x%X "
2229 "dhfis 0x%X dmafis 0x%X sactive 0x%X\n",
2230 ap->print_id, pp->qc_active, pp->dhfis_bits,
2231 pp->dmafis_bits, readl(pp->sactive_block));
2232 rc = nv_swncq_sdbfis(ap);
2233 if (rc < 0)
2234 goto irq_error;
2235 }
2236
2237 if (fis & NV_SWNCQ_IRQ_DHREGFIS) {
2238 /* The interrupt indicates the new command
2239 * was transmitted correctly to the drive.
2240 */
2241 pp->dhfis_bits |= (0x1 << pp->last_issue_tag);
2242 pp->ncq_flags |= ncq_saw_d2h;
2243 if (pp->ncq_flags & (ncq_saw_sdb | ncq_saw_backout)) {
2244 ata_ehi_push_desc(ehi, "illegal fis transaction");
2245 ehi->err_mask |= AC_ERR_HSM;
cf480626 2246 ehi->action |= ATA_EH_RESET;
f140f0f1
KL		 2247 goto irq_error;
2248 }
2249
2250 if (!(fis & NV_SWNCQ_IRQ_DMASETUP) &&
2251 !(pp->ncq_flags & ncq_saw_dmas)) {
2252 ata_stat = ap->ops->check_status(ap);
2253 if (ata_stat & ATA_BUSY)
2254 goto irq_exit;
2255
2256 if (pp->defer_queue.defer_bits) {
2257 DPRINTK("send next command\n");
2258 qc = nv_swncq_qc_from_dq(ap);
2259 nv_swncq_issue_atacmd(ap, qc);
2260 }
2261 }
2262 }
2263
2264 if (fis & NV_SWNCQ_IRQ_DMASETUP) {
2265 /* program the dma controller with appropriate PRD buffers
2266 * and start the DMA transfer for requested command.
2267 */
2268 pp->dmafis_bits |= (0x1 << nv_swncq_tag(ap));
2269 pp->ncq_flags |= ncq_saw_dmas;
2270 rc = nv_swncq_dmafis(ap);
2271 }
2272
2273irq_exit:
2274 return;
2275irq_error:
2276 ata_ehi_push_desc(ehi, "fis:0x%x", fis);
2277 ata_port_freeze(ap);
2278 return;
2279}
2280
2281static irqreturn_t nv_swncq_interrupt(int irq, void *dev_instance)
2282{
2283 struct ata_host *host = dev_instance;
2284 unsigned int i;
2285 unsigned int handled = 0;
2286 unsigned long flags;
2287 u32 irq_stat;
2288
2289 spin_lock_irqsave(&host->lock, flags);
2290
2291 irq_stat = readl(host->iomap[NV_MMIO_BAR] + NV_INT_STATUS_MCP55);
2292
2293 for (i = 0; i < host->n_ports; i++) {
2294 struct ata_port *ap = host->ports[i];
2295
2296 if (ap && !(ap->flags & ATA_FLAG_DISABLED)) {
2297 if (ap->link.sactive) {
2298 nv_swncq_host_interrupt(ap, (u16)irq_stat);
2299 handled = 1;
2300 } else {
2301 if (irq_stat) /* reserve Hotplug */
2302 nv_swncq_irq_clear(ap, 0xfff0);
2303
2304 handled += nv_host_intr(ap, (u8)irq_stat);
2305 }
2306 }
2307 irq_stat >>= NV_INT_PORT_SHIFT_MCP55;
2308 }
2309
2310 spin_unlock_irqrestore(&host->lock, flags);
2311
2312 return IRQ_RETVAL(handled);
2313}
2314
5796d1c4 2315static int nv_init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
1da177e4 2316{
5796d1c4 2317 static int printed_version;
1626aeb8 2318 const struct ata_port_info *ppi[] = { NULL, NULL };
9a829ccf 2319 struct ata_host *host;
cdf56bcf 2320 struct nv_host_priv *hpriv;
1da177e4
LT		 2321 int rc;
2322 u32 bar;
0d5ff566 2323 void __iomem *base;
fbbb262d 2324 unsigned long type = ent->driver_data;
1da177e4
LT		 2325
2326 // Make sure this is a SATA controller by counting the number of bars
2327 // (NVIDIA SATA controllers will always have six bars). Otherwise,
2328 // it's an IDE controller and we ignore it.
5796d1c4 2329 for (bar = 0; bar < 6; bar++)
1da177e4
LT		 2330 if (pci_resource_start(pdev, bar) == 0)
2331 return -ENODEV;
2332
cdf56bcf 2333 if (!printed_version++)
a9524a76 2334 dev_printk(KERN_DEBUG, &pdev->dev, "version " DRV_VERSION "\n");
1da177e4 2335
24dc5f33 2336 rc = pcim_enable_device(pdev);
1da177e4 2337 if (rc)
24dc5f33 2338 return rc;
1da177e4 2339
9a829ccf 2340 /* determine type and allocate host */
f140f0f1 2341 if (type == CK804 && adma_enabled) {
fbbb262d
RH		 2342 dev_printk(KERN_NOTICE, &pdev->dev, "Using ADMA mode\n");
2343 type = ADMA;
fbbb262d
RH		 2344 }
2345
360737a9
JG		 2346 if (type == SWNCQ) {
2347 if (swncq_enabled)
2348 dev_printk(KERN_NOTICE, &pdev->dev,
2349 "Using SWNCQ mode\n");
2350 else
2351 type = GENERIC;
2352 }
2353
1626aeb8 2354 ppi[0] = &nv_port_info[type];
d583bc18 2355 rc = ata_pci_prepare_sff_host(pdev, ppi, &host);
9a829ccf
TH		 2356 if (rc)
2357 return rc;
1da177e4 2358
24dc5f33 2359 hpriv = devm_kzalloc(&pdev->dev, sizeof(*hpriv), GFP_KERNEL);
cdf56bcf 2360 if (!hpriv)
24dc5f33 2361 return -ENOMEM;
9a829ccf
TH		 2362 hpriv->type = type;
2363 host->private_data = hpriv;
cdf56bcf 2364
9a829ccf
TH		 2365 /* request and iomap NV_MMIO_BAR */
2366 rc = pcim_iomap_regions(pdev, 1 << NV_MMIO_BAR, DRV_NAME);
2367 if (rc)
2368 return rc;
1da177e4 2369
9a829ccf
TH		 2370 /* configure SCR access */
2371 base = host->iomap[NV_MMIO_BAR];
2372 host->ports[0]->ioaddr.scr_addr = base + NV_PORT0_SCR_REG_OFFSET;
2373 host->ports[1]->ioaddr.scr_addr = base + NV_PORT1_SCR_REG_OFFSET;
1da177e4 2374
ada364e8 2375 /* enable SATA space for CK804 */
fbbb262d 2376 if (type >= CK804) {
ada364e8
TH		 2377 u8 regval;
2378
2379 pci_read_config_byte(pdev, NV_MCP_SATA_CFG_20, &regval);
2380 regval |= NV_MCP_SATA_CFG_20_SATA_SPACE_EN;
2381 pci_write_config_byte(pdev, NV_MCP_SATA_CFG_20, regval);
2382 }
2383
9a829ccf 2384 /* init ADMA */
fbbb262d 2385 if (type == ADMA) {
9a829ccf 2386 rc = nv_adma_host_init(host);
fbbb262d 2387 if (rc)
24dc5f33 2388 return rc;
360737a9 2389 } else if (type == SWNCQ)
f140f0f1 2390 nv_swncq_host_init(host);
fbbb262d 2391
9a829ccf
TH		 2392 pci_set_master(pdev);
2393 return ata_host_activate(host, pdev->irq, ppi[0]->irq_handler,
2394 IRQF_SHARED, ppi[0]->sht);
1da177e4
LT		 2395}
2396
438ac6d5 2397#ifdef CONFIG_PM
cdf56bcf
RH		 2398static int nv_pci_device_resume(struct pci_dev *pdev)
2399{
2400 struct ata_host *host = dev_get_drvdata(&pdev->dev);
2401 struct nv_host_priv *hpriv = host->private_data;
ce053fa8 2402 int rc;
cdf56bcf 2403
ce053fa8 2404 rc = ata_pci_device_do_resume(pdev);
b447916e 2405 if (rc)
ce053fa8 2406 return rc;
cdf56bcf
RH		 2407
2408 if (pdev->dev.power.power_state.event == PM_EVENT_SUSPEND) {
b447916e 2409 if (hpriv->type >= CK804) {
cdf56bcf
RH		 2410 u8 regval;
2411
2412 pci_read_config_byte(pdev, NV_MCP_SATA_CFG_20, &regval);
2413 regval |= NV_MCP_SATA_CFG_20_SATA_SPACE_EN;
2414 pci_write_config_byte(pdev, NV_MCP_SATA_CFG_20, regval);
2415 }
b447916e 2416 if (hpriv->type == ADMA) {
cdf56bcf
RH		 2417 u32 tmp32;
2418 struct nv_adma_port_priv *pp;
2419 /* enable/disable ADMA on the ports appropriately */
2420 pci_read_config_dword(pdev, NV_MCP_SATA_CFG_20, &tmp32);
2421
2422 pp = host->ports[0]->private_data;
b447916e 2423 if (pp->flags & NV_ADMA_ATAPI_SETUP_COMPLETE)
cdf56bcf 2424 tmp32 &= ~(NV_MCP_SATA_CFG_20_PORT0_EN |
5796d1c4 2425 NV_MCP_SATA_CFG_20_PORT0_PWB_EN);
cdf56bcf
RH		 2426 else
2427 tmp32 |= (NV_MCP_SATA_CFG_20_PORT0_EN |
5796d1c4 2428 NV_MCP_SATA_CFG_20_PORT0_PWB_EN);
cdf56bcf 2429 pp = host->ports[1]->private_data;
b447916e 2430 if (pp->flags & NV_ADMA_ATAPI_SETUP_COMPLETE)
cdf56bcf 2431 tmp32 &= ~(NV_MCP_SATA_CFG_20_PORT1_EN |
5796d1c4 2432 NV_MCP_SATA_CFG_20_PORT1_PWB_EN);
cdf56bcf
RH		 2433 else
2434 tmp32 |= (NV_MCP_SATA_CFG_20_PORT1_EN |
5796d1c4 2435 NV_MCP_SATA_CFG_20_PORT1_PWB_EN);
cdf56bcf
RH		 2436
2437 pci_write_config_dword(pdev, NV_MCP_SATA_CFG_20, tmp32);
2438 }
2439 }
2440
2441 ata_host_resume(host);
2442
2443 return 0;
2444}
438ac6d5 2445#endif
cdf56bcf 2446
cca3974e 2447static void nv_ck804_host_stop(struct ata_host *host)
ada364e8 2448{
cca3974e 2449 struct pci_dev *pdev = to_pci_dev(host->dev);
ada364e8
TH		 2450 u8 regval;
2451
2452 /* disable SATA space for CK804 */
2453 pci_read_config_byte(pdev, NV_MCP_SATA_CFG_20, &regval);
2454 regval &= ~NV_MCP_SATA_CFG_20_SATA_SPACE_EN;
2455 pci_write_config_byte(pdev, NV_MCP_SATA_CFG_20, regval);
ada364e8
TH		 2456}
2457
fbbb262d
RH		 2458static void nv_adma_host_stop(struct ata_host *host)
2459{
2460 struct pci_dev *pdev = to_pci_dev(host->dev);
fbbb262d
RH		 2461 u32 tmp32;
2462
fbbb262d
RH		 2463 /* disable ADMA on the ports */
2464 pci_read_config_dword(pdev, NV_MCP_SATA_CFG_20, &tmp32);
2465 tmp32 &= ~(NV_MCP_SATA_CFG_20_PORT0_EN |
2466 NV_MCP_SATA_CFG_20_PORT0_PWB_EN |
2467 NV_MCP_SATA_CFG_20_PORT1_EN |
2468 NV_MCP_SATA_CFG_20_PORT1_PWB_EN);
2469
2470 pci_write_config_dword(pdev, NV_MCP_SATA_CFG_20, tmp32);
2471
2472 nv_ck804_host_stop(host);
2473}
2474
1da177e4
LT		 2475static int __init nv_init(void)
2476{
b7887196 2477 return pci_register_driver(&nv_pci_driver);
1da177e4
LT		 2478}
2479
2480static void __exit nv_exit(void)
2481{
2482 pci_unregister_driver(&nv_pci_driver);
2483}
2484
2485module_init(nv_init);
2486module_exit(nv_exit);
fbbb262d
RH		 2487module_param_named(adma, adma_enabled, bool, 0444);
2488MODULE_PARM_DESC(adma, "Enable use of ADMA (Default: true)");
f140f0f1
KL		 2489module_param_named(swncq, swncq_enabled, bool, 0444);
2490MODULE_PARM_DESC(swncq, "Enable use of SWNCQ (Default: false)");
2491
Linux kernel - Deliverables
This page took 0.410959 seconds and 5 git commands to generate.