projects / deliverable / linux.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
[libata] fix 'if(' and similar areas that lack whitespace
[deliverable/linux.git] / drivers / ata / sata_nv.c
CommitLineData
1da177e4
LT		 1/*
2 * sata_nv.c - NVIDIA nForce SATA
3 *
4 * Copyright 2004 NVIDIA Corp. All rights reserved.
5 * Copyright 2004 Andrew Chew
6 *
aa7e16d6
JG		 7 *
8 * This program is free software; you can redistribute it and/or modify
9 * it under the terms of the GNU General Public License as published by
10 * the Free Software Foundation; either version 2, or (at your option)
11 * any later version.
12 *
13 * This program is distributed in the hope that it will be useful,
14 * but WITHOUT ANY WARRANTY; without even the implied warranty of
15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 * GNU General Public License for more details.
17 *
18 * You should have received a copy of the GNU General Public License
19 * along with this program; see the file COPYING. If not, write to
20 * the Free Software Foundation, 675 Mass Ave, Cambridge, MA 02139, USA.
1da177e4 21 *
af36d7f0
JG		 22 *
23 * libata documentation is available via 'make {ps|pdf}docs',
24 * as Documentation/DocBook/libata.*
25 *
26 * No hardware documentation available outside of NVIDIA.
27 * This driver programs the NVIDIA SATA controller in a similar
28 * fashion as with other PCI IDE BMDMA controllers, with a few
29 * NV-specific details such as register offsets, SATA phy location,
30 * hotplug info, etc.
31 *
fbbb262d
RH		 32 * CK804/MCP04 controllers support an alternate programming interface
33 * similar to the ADMA specification (with some modifications).
34 * This allows the use of NCQ. Non-DMA-mapped ATA commands are still
35 * sent through the legacy interface.
36 *
1da177e4
LT		 37 */
38
1da177e4
LT		 39#include <linux/kernel.h>
40#include <linux/module.h>
41#include <linux/pci.h>
42#include <linux/init.h>
43#include <linux/blkdev.h>
44#include <linux/delay.h>
45#include <linux/interrupt.h>
a9524a76 46#include <linux/device.h>
1da177e4 47#include <scsi/scsi_host.h>
fbbb262d 48#include <scsi/scsi_device.h>
1da177e4
LT		 49#include <linux/libata.h>
50
51#define DRV_NAME "sata_nv"
2a3103ce 52#define DRV_VERSION "3.5"
fbbb262d
RH		 53
54#define NV_ADMA_DMA_BOUNDARY 0xffffffffUL
1da177e4 55
10ad05df 56enum {
0d5ff566
TH		 57 NV_MMIO_BAR = 5,
58
10ad05df
JG		 59 NV_PORTS = 2,
60 NV_PIO_MASK = 0x1f,
61 NV_MWDMA_MASK = 0x07,
62 NV_UDMA_MASK = 0x7f,
63 NV_PORT0_SCR_REG_OFFSET = 0x00,
64 NV_PORT1_SCR_REG_OFFSET = 0x40,
1da177e4 65
27e4b274 66 /* INT_STATUS/ENABLE */
10ad05df 67 NV_INT_STATUS = 0x10,
10ad05df 68 NV_INT_ENABLE = 0x11,
27e4b274 69 NV_INT_STATUS_CK804 = 0x440,
10ad05df 70 NV_INT_ENABLE_CK804 = 0x441,
1da177e4 71
27e4b274
TH		 72 /* INT_STATUS/ENABLE bits */
73 NV_INT_DEV = 0x01,
74 NV_INT_PM = 0x02,
75 NV_INT_ADDED = 0x04,
76 NV_INT_REMOVED = 0x08,
77
78 NV_INT_PORT_SHIFT = 4, /* each port occupies 4 bits */
79
39f87582 80 NV_INT_ALL = 0x0f,
5a44efff
TH		 81 NV_INT_MASK = NV_INT_DEV |
82 NV_INT_ADDED | NV_INT_REMOVED,
39f87582 83
27e4b274 84 /* INT_CONFIG */
10ad05df
JG		 85 NV_INT_CONFIG = 0x12,
86 NV_INT_CONFIG_METHD = 0x01, // 0 = INT, 1 = SMI
1da177e4 87
10ad05df
JG		 88 // For PCI config register 20
89 NV_MCP_SATA_CFG_20 = 0x50,
90 NV_MCP_SATA_CFG_20_SATA_SPACE_EN = 0x04,
fbbb262d
RH		 91 NV_MCP_SATA_CFG_20_PORT0_EN = (1 << 17),
92 NV_MCP_SATA_CFG_20_PORT1_EN = (1 << 16),
93 NV_MCP_SATA_CFG_20_PORT0_PWB_EN = (1 << 14),
94 NV_MCP_SATA_CFG_20_PORT1_PWB_EN = (1 << 12),
95
96 NV_ADMA_MAX_CPBS = 32,
97 NV_ADMA_CPB_SZ = 128,
98 NV_ADMA_APRD_SZ = 16,
99 NV_ADMA_SGTBL_LEN = (1024 - NV_ADMA_CPB_SZ) /
100 NV_ADMA_APRD_SZ,
101 NV_ADMA_SGTBL_TOTAL_LEN = NV_ADMA_SGTBL_LEN + 5,
102 NV_ADMA_SGTBL_SZ = NV_ADMA_SGTBL_LEN * NV_ADMA_APRD_SZ,
103 NV_ADMA_PORT_PRIV_DMA_SZ = NV_ADMA_MAX_CPBS *
104 (NV_ADMA_CPB_SZ + NV_ADMA_SGTBL_SZ),
105
106 /* BAR5 offset to ADMA general registers */
107 NV_ADMA_GEN = 0x400,
108 NV_ADMA_GEN_CTL = 0x00,
109 NV_ADMA_NOTIFIER_CLEAR = 0x30,
110
111 /* BAR5 offset to ADMA ports */
112 NV_ADMA_PORT = 0x480,
113
114 /* size of ADMA port register space */
115 NV_ADMA_PORT_SIZE = 0x100,
116
117 /* ADMA port registers */
118 NV_ADMA_CTL = 0x40,
119 NV_ADMA_CPB_COUNT = 0x42,
120 NV_ADMA_NEXT_CPB_IDX = 0x43,
121 NV_ADMA_STAT = 0x44,
122 NV_ADMA_CPB_BASE_LOW = 0x48,
123 NV_ADMA_CPB_BASE_HIGH = 0x4C,
124 NV_ADMA_APPEND = 0x50,
125 NV_ADMA_NOTIFIER = 0x68,
126 NV_ADMA_NOTIFIER_ERROR = 0x6C,
127
128 /* NV_ADMA_CTL register bits */
129 NV_ADMA_CTL_HOTPLUG_IEN = (1 << 0),
130 NV_ADMA_CTL_CHANNEL_RESET = (1 << 5),
131 NV_ADMA_CTL_GO = (1 << 7),
132 NV_ADMA_CTL_AIEN = (1 << 8),
133 NV_ADMA_CTL_READ_NON_COHERENT = (1 << 11),
134 NV_ADMA_CTL_WRITE_NON_COHERENT = (1 << 12),
135
136 /* CPB response flag bits */
137 NV_CPB_RESP_DONE = (1 << 0),
138 NV_CPB_RESP_ATA_ERR = (1 << 3),
139 NV_CPB_RESP_CMD_ERR = (1 << 4),
140 NV_CPB_RESP_CPB_ERR = (1 << 7),
141
142 /* CPB control flag bits */
143 NV_CPB_CTL_CPB_VALID = (1 << 0),
144 NV_CPB_CTL_QUEUE = (1 << 1),
145 NV_CPB_CTL_APRD_VALID = (1 << 2),
146 NV_CPB_CTL_IEN = (1 << 3),
147 NV_CPB_CTL_FPDMA = (1 << 4),
148
149 /* APRD flags */
150 NV_APRD_WRITE = (1 << 1),
151 NV_APRD_END = (1 << 2),
152 NV_APRD_CONT = (1 << 3),
153
154 /* NV_ADMA_STAT flags */
155 NV_ADMA_STAT_TIMEOUT = (1 << 0),
156 NV_ADMA_STAT_HOTUNPLUG = (1 << 1),
157 NV_ADMA_STAT_HOTPLUG = (1 << 2),
158 NV_ADMA_STAT_CPBERR = (1 << 4),
159 NV_ADMA_STAT_SERROR = (1 << 5),
160 NV_ADMA_STAT_CMD_COMPLETE = (1 << 6),
161 NV_ADMA_STAT_IDLE = (1 << 8),
162 NV_ADMA_STAT_LEGACY = (1 << 9),
163 NV_ADMA_STAT_STOPPED = (1 << 10),
164 NV_ADMA_STAT_DONE = (1 << 12),
165 NV_ADMA_STAT_ERR = NV_ADMA_STAT_CPBERR |
2dcb407e 166 NV_ADMA_STAT_TIMEOUT,
fbbb262d
RH		 167
168 /* port flags */
169 NV_ADMA_PORT_REGISTER_MODE = (1 << 0),
2dec7555 170 NV_ADMA_ATAPI_SETUP_COMPLETE = (1 << 1),
fbbb262d 171
f140f0f1
KL		 172 /* MCP55 reg offset */
173 NV_CTL_MCP55 = 0x400,
174 NV_INT_STATUS_MCP55 = 0x440,
175 NV_INT_ENABLE_MCP55 = 0x444,
176 NV_NCQ_REG_MCP55 = 0x448,
177
178 /* MCP55 */
179 NV_INT_ALL_MCP55 = 0xffff,
180 NV_INT_PORT_SHIFT_MCP55 = 16, /* each port occupies 16 bits */
181 NV_INT_MASK_MCP55 = NV_INT_ALL_MCP55 & 0xfffd,
182
183 /* SWNCQ ENABLE BITS*/
184 NV_CTL_PRI_SWNCQ = 0x02,
185 NV_CTL_SEC_SWNCQ = 0x04,
186
187 /* SW NCQ status bits*/
188 NV_SWNCQ_IRQ_DEV = (1 << 0),
189 NV_SWNCQ_IRQ_PM = (1 << 1),
190 NV_SWNCQ_IRQ_ADDED = (1 << 2),
191 NV_SWNCQ_IRQ_REMOVED = (1 << 3),
192
193 NV_SWNCQ_IRQ_BACKOUT = (1 << 4),
194 NV_SWNCQ_IRQ_SDBFIS = (1 << 5),
195 NV_SWNCQ_IRQ_DHREGFIS = (1 << 6),
196 NV_SWNCQ_IRQ_DMASETUP = (1 << 7),
197
198 NV_SWNCQ_IRQ_HOTPLUG = NV_SWNCQ_IRQ_ADDED |
199 NV_SWNCQ_IRQ_REMOVED,
200
fbbb262d
RH		 201};
202
203/* ADMA Physical Region Descriptor - one SG segment */
204struct nv_adma_prd {
205 __le64 addr;
206 __le32 len;
207 u8 flags;
208 u8 packet_len;
209 __le16 reserved;
210};
211
212enum nv_adma_regbits {
213 CMDEND = (1 << 15), /* end of command list */
214 WNB = (1 << 14), /* wait-not-BSY */
215 IGN = (1 << 13), /* ignore this entry */
216 CS1n = (1 << (4 + 8)), /* std. PATA signals follow... */
217 DA2 = (1 << (2 + 8)),
218 DA1 = (1 << (1 + 8)),
219 DA0 = (1 << (0 + 8)),
220};
221
222/* ADMA Command Parameter Block
223 The first 5 SG segments are stored inside the Command Parameter Block itself.
224 If there are more than 5 segments the remainder are stored in a separate
225 memory area indicated by next_aprd. */
226struct nv_adma_cpb {
227 u8 resp_flags; /* 0 */
228 u8 reserved1; /* 1 */
229 u8 ctl_flags; /* 2 */
230 /* len is length of taskfile in 64 bit words */
2dcb407e 231 u8 len; /* 3 */
fbbb262d
RH		 232 u8 tag; /* 4 */
233 u8 next_cpb_idx; /* 5 */
234 __le16 reserved2; /* 6-7 */
235 __le16 tf[12]; /* 8-31 */
236 struct nv_adma_prd aprd[5]; /* 32-111 */
237 __le64 next_aprd; /* 112-119 */
238 __le64 reserved3; /* 120-127 */
10ad05df 239};
1da177e4 240
fbbb262d
RH		 241
242struct nv_adma_port_priv {
243 struct nv_adma_cpb *cpb;
244 dma_addr_t cpb_dma;
245 struct nv_adma_prd *aprd;
246 dma_addr_t aprd_dma;
2dcb407e
JG		 247 void __iomem *ctl_block;
248 void __iomem *gen_block;
249 void __iomem *notifier_clear_block;
fbbb262d 250 u8 flags;
5e5c74a5 251 int last_issue_ncq;
fbbb262d
RH		 252};
253
cdf56bcf
RH		 254struct nv_host_priv {
255 unsigned long type;
256};
257
f140f0f1
KL		 258struct defer_queue {
259 u32 defer_bits;
260 unsigned int head;
261 unsigned int tail;
262 unsigned int tag[ATA_MAX_QUEUE];
263};
264
265enum ncq_saw_flag_list {
266 ncq_saw_d2h = (1U << 0),
267 ncq_saw_dmas = (1U << 1),
268 ncq_saw_sdb = (1U << 2),
269 ncq_saw_backout = (1U << 3),
270};
271
272struct nv_swncq_port_priv {
273 struct ata_prd *prd; /* our SG list */
274 dma_addr_t prd_dma; /* and its DMA mapping */
275 void __iomem *sactive_block;
276 void __iomem *irq_block;
277 void __iomem *tag_block;
278 u32 qc_active;
279
280 unsigned int last_issue_tag;
281
282 /* fifo circular queue to store deferral command */
283 struct defer_queue defer_queue;
284
285 /* for NCQ interrupt analysis */
286 u32 dhfis_bits;
287 u32 dmafis_bits;
288 u32 sdbfis_bits;
289
290 unsigned int ncq_flags;
291};
292
293
fbbb262d
RH		 294#define NV_ADMA_CHECK_INTR(GCTL, PORT) ((GCTL) & ( 1 << (19 + (12 * (PORT)))))
295
2dcb407e 296static int nv_init_one(struct pci_dev *pdev, const struct pci_device_id *ent);
438ac6d5 297#ifdef CONFIG_PM
cdf56bcf 298static int nv_pci_device_resume(struct pci_dev *pdev);
438ac6d5 299#endif
cca3974e 300static void nv_ck804_host_stop(struct ata_host *host);
7d12e780
DH		 301static irqreturn_t nv_generic_interrupt(int irq, void *dev_instance);
302static irqreturn_t nv_nf2_interrupt(int irq, void *dev_instance);
303static irqreturn_t nv_ck804_interrupt(int irq, void *dev_instance);
2dcb407e
JG		 304static int nv_scr_read(struct ata_port *ap, unsigned int sc_reg, u32 *val);
305static int nv_scr_write(struct ata_port *ap, unsigned int sc_reg, u32 val);
1da177e4 306
39f87582
TH		 307static void nv_nf2_freeze(struct ata_port *ap);
308static void nv_nf2_thaw(struct ata_port *ap);
309static void nv_ck804_freeze(struct ata_port *ap);
310static void nv_ck804_thaw(struct ata_port *ap);
311static void nv_error_handler(struct ata_port *ap);
fbbb262d 312static int nv_adma_slave_config(struct scsi_device *sdev);
2dec7555 313static int nv_adma_check_atapi_dma(struct ata_queued_cmd *qc);
fbbb262d
RH		 314static void nv_adma_qc_prep(struct ata_queued_cmd *qc);
315static unsigned int nv_adma_qc_issue(struct ata_queued_cmd *qc);
316static irqreturn_t nv_adma_interrupt(int irq, void *dev_instance);
317static void nv_adma_irq_clear(struct ata_port *ap);
318static int nv_adma_port_start(struct ata_port *ap);
319static void nv_adma_port_stop(struct ata_port *ap);
438ac6d5 320#ifdef CONFIG_PM
cdf56bcf
RH		 321static int nv_adma_port_suspend(struct ata_port *ap, pm_message_t mesg);
322static int nv_adma_port_resume(struct ata_port *ap);
438ac6d5 323#endif
53014e25
RH		 324static void nv_adma_freeze(struct ata_port *ap);
325static void nv_adma_thaw(struct ata_port *ap);
fbbb262d
RH		 326static void nv_adma_error_handler(struct ata_port *ap);
327static void nv_adma_host_stop(struct ata_host *host);
f5ecac2d 328static void nv_adma_post_internal_cmd(struct ata_queued_cmd *qc);
f2fb344b 329static void nv_adma_tf_read(struct ata_port *ap, struct ata_taskfile *tf);
39f87582 330
f140f0f1
KL		 331static void nv_mcp55_thaw(struct ata_port *ap);
332static void nv_mcp55_freeze(struct ata_port *ap);
333static void nv_swncq_error_handler(struct ata_port *ap);
334static int nv_swncq_slave_config(struct scsi_device *sdev);
335static int nv_swncq_port_start(struct ata_port *ap);
336static void nv_swncq_qc_prep(struct ata_queued_cmd *qc);
337static void nv_swncq_fill_sg(struct ata_queued_cmd *qc);
338static unsigned int nv_swncq_qc_issue(struct ata_queued_cmd *qc);
339static void nv_swncq_irq_clear(struct ata_port *ap, u16 fis);
340static irqreturn_t nv_swncq_interrupt(int irq, void *dev_instance);
341#ifdef CONFIG_PM
342static int nv_swncq_port_suspend(struct ata_port *ap, pm_message_t mesg);
343static int nv_swncq_port_resume(struct ata_port *ap);
344#endif
345
1da177e4
LT		 346enum nv_host_type
347{
348 GENERIC,
349 NFORCE2,
27e4b274 350 NFORCE3 = NFORCE2, /* NF2 == NF3 as far as sata_nv is concerned */
fbbb262d 351 CK804,
f140f0f1
KL		 352 ADMA,
353 SWNCQ,
1da177e4
LT		 354};
355
3b7d697d 356static const struct pci_device_id nv_pci_tbl[] = {
54bb3a94
JG		 357 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE2S_SATA), NFORCE2 },
358 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE3S_SATA), NFORCE3 },
359 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE3S_SATA2), NFORCE3 },
360 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_CK804_SATA), CK804 },
361 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_CK804_SATA2), CK804 },
362 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_MCP04_SATA), CK804 },
363 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_MCP04_SATA2), CK804 },
f140f0f1
KL		 364 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_MCP51_SATA), SWNCQ },
365 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_MCP51_SATA2), SWNCQ },
366 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_MCP55_SATA), SWNCQ },
367 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_MCP55_SATA2), SWNCQ },
e2e031eb
KL		 368 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_MCP61_SATA), GENERIC },
369 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_MCP61_SATA2), GENERIC },
370 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_MCP61_SATA3), GENERIC },
2d2744fc
JG		 371
372 { } /* terminate list */
1da177e4
LT		 373};
374
1da177e4
LT		 375static struct pci_driver nv_pci_driver = {
376 .name = DRV_NAME,
377 .id_table = nv_pci_tbl,
378 .probe = nv_init_one,
438ac6d5 379#ifdef CONFIG_PM
cdf56bcf
RH		 380 .suspend = ata_pci_device_suspend,
381 .resume = nv_pci_device_resume,
438ac6d5 382#endif
1daf9ce7 383 .remove = ata_pci_remove_one,
1da177e4
LT		 384};
385
193515d5 386static struct scsi_host_template nv_sht = {
1da177e4
LT		 387 .module = THIS_MODULE,
388 .name = DRV_NAME,
389 .ioctl = ata_scsi_ioctl,
390 .queuecommand = ata_scsi_queuecmd,
1da177e4
LT		 391 .can_queue = ATA_DEF_QUEUE,
392 .this_id = ATA_SHT_THIS_ID,
393 .sg_tablesize = LIBATA_MAX_PRD,
1da177e4
LT		 394 .cmd_per_lun = ATA_SHT_CMD_PER_LUN,
395 .emulated = ATA_SHT_EMULATED,
396 .use_clustering = ATA_SHT_USE_CLUSTERING,
397 .proc_name = DRV_NAME,
398 .dma_boundary = ATA_DMA_BOUNDARY,
399 .slave_configure = ata_scsi_slave_config,
ccf68c34 400 .slave_destroy = ata_scsi_slave_destroy,
1da177e4 401 .bios_param = ata_std_bios_param,
1da177e4
LT		 402};
403
fbbb262d
RH		 404static struct scsi_host_template nv_adma_sht = {
405 .module = THIS_MODULE,
406 .name = DRV_NAME,
407 .ioctl = ata_scsi_ioctl,
408 .queuecommand = ata_scsi_queuecmd,
1e0b5ab8 409 .change_queue_depth = ata_scsi_change_queue_depth,
fbbb262d
RH		 410 .can_queue = NV_ADMA_MAX_CPBS,
411 .this_id = ATA_SHT_THIS_ID,
412 .sg_tablesize = NV_ADMA_SGTBL_TOTAL_LEN,
fbbb262d
RH		 413 .cmd_per_lun = ATA_SHT_CMD_PER_LUN,
414 .emulated = ATA_SHT_EMULATED,
415 .use_clustering = ATA_SHT_USE_CLUSTERING,
416 .proc_name = DRV_NAME,
417 .dma_boundary = NV_ADMA_DMA_BOUNDARY,
418 .slave_configure = nv_adma_slave_config,
419 .slave_destroy = ata_scsi_slave_destroy,
420 .bios_param = ata_std_bios_param,
421};
422
f140f0f1
KL		 423static struct scsi_host_template nv_swncq_sht = {
424 .module = THIS_MODULE,
425 .name = DRV_NAME,
426 .ioctl = ata_scsi_ioctl,
427 .queuecommand = ata_scsi_queuecmd,
428 .change_queue_depth = ata_scsi_change_queue_depth,
429 .can_queue = ATA_MAX_QUEUE,
430 .this_id = ATA_SHT_THIS_ID,
431 .sg_tablesize = LIBATA_MAX_PRD,
432 .cmd_per_lun = ATA_SHT_CMD_PER_LUN,
433 .emulated = ATA_SHT_EMULATED,
434 .use_clustering = ATA_SHT_USE_CLUSTERING,
435 .proc_name = DRV_NAME,
436 .dma_boundary = ATA_DMA_BOUNDARY,
437 .slave_configure = nv_swncq_slave_config,
438 .slave_destroy = ata_scsi_slave_destroy,
439 .bios_param = ata_std_bios_param,
440};
441
ada364e8 442static const struct ata_port_operations nv_generic_ops = {
1da177e4
LT		 443 .tf_load = ata_tf_load,
444 .tf_read = ata_tf_read,
445 .exec_command = ata_exec_command,
446 .check_status = ata_check_status,
447 .dev_select = ata_std_dev_select,
1da177e4
LT		 448 .bmdma_setup = ata_bmdma_setup,
449 .bmdma_start = ata_bmdma_start,
450 .bmdma_stop = ata_bmdma_stop,
451 .bmdma_status = ata_bmdma_status,
452 .qc_prep = ata_qc_prep,
453 .qc_issue = ata_qc_issue_prot,
39f87582
TH		 454 .freeze = ata_bmdma_freeze,
455 .thaw = ata_bmdma_thaw,
456 .error_handler = nv_error_handler,
457 .post_internal_cmd = ata_bmdma_post_internal_cmd,
0d5ff566 458 .data_xfer = ata_data_xfer,
1da177e4 459 .irq_clear = ata_bmdma_irq_clear,
246ce3b6 460 .irq_on = ata_irq_on,
1da177e4
LT		 461 .scr_read = nv_scr_read,
462 .scr_write = nv_scr_write,
463 .port_start = ata_port_start,
1da177e4
LT		 464};
465
ada364e8 466static const struct ata_port_operations nv_nf2_ops = {
ada364e8
TH		 467 .tf_load = ata_tf_load,
468 .tf_read = ata_tf_read,
469 .exec_command = ata_exec_command,
470 .check_status = ata_check_status,
471 .dev_select = ata_std_dev_select,
ada364e8
TH		 472 .bmdma_setup = ata_bmdma_setup,
473 .bmdma_start = ata_bmdma_start,
474 .bmdma_stop = ata_bmdma_stop,
475 .bmdma_status = ata_bmdma_status,
476 .qc_prep = ata_qc_prep,
477 .qc_issue = ata_qc_issue_prot,
39f87582
TH		 478 .freeze = nv_nf2_freeze,
479 .thaw = nv_nf2_thaw,
480 .error_handler = nv_error_handler,
481 .post_internal_cmd = ata_bmdma_post_internal_cmd,
0d5ff566 482 .data_xfer = ata_data_xfer,
ada364e8 483 .irq_clear = ata_bmdma_irq_clear,
246ce3b6 484 .irq_on = ata_irq_on,
ada364e8
TH		 485 .scr_read = nv_scr_read,
486 .scr_write = nv_scr_write,
487 .port_start = ata_port_start,
ada364e8
TH		 488};
489
490static const struct ata_port_operations nv_ck804_ops = {
ada364e8
TH		 491 .tf_load = ata_tf_load,
492 .tf_read = ata_tf_read,
493 .exec_command = ata_exec_command,
494 .check_status = ata_check_status,
495 .dev_select = ata_std_dev_select,
ada364e8
TH		 496 .bmdma_setup = ata_bmdma_setup,
497 .bmdma_start = ata_bmdma_start,
498 .bmdma_stop = ata_bmdma_stop,
499 .bmdma_status = ata_bmdma_status,
500 .qc_prep = ata_qc_prep,
501 .qc_issue = ata_qc_issue_prot,
39f87582
TH		 502 .freeze = nv_ck804_freeze,
503 .thaw = nv_ck804_thaw,
504 .error_handler = nv_error_handler,
505 .post_internal_cmd = ata_bmdma_post_internal_cmd,
0d5ff566 506 .data_xfer = ata_data_xfer,
ada364e8 507 .irq_clear = ata_bmdma_irq_clear,
246ce3b6 508 .irq_on = ata_irq_on,
ada364e8
TH		 509 .scr_read = nv_scr_read,
510 .scr_write = nv_scr_write,
511 .port_start = ata_port_start,
ada364e8
TH		 512 .host_stop = nv_ck804_host_stop,
513};
514
fbbb262d 515static const struct ata_port_operations nv_adma_ops = {
fbbb262d 516 .tf_load = ata_tf_load,
f2fb344b 517 .tf_read = nv_adma_tf_read,
2dec7555 518 .check_atapi_dma = nv_adma_check_atapi_dma,
fbbb262d
RH		 519 .exec_command = ata_exec_command,
520 .check_status = ata_check_status,
521 .dev_select = ata_std_dev_select,
f5ecac2d
RH		 522 .bmdma_setup = ata_bmdma_setup,
523 .bmdma_start = ata_bmdma_start,
524 .bmdma_stop = ata_bmdma_stop,
525 .bmdma_status = ata_bmdma_status,
31cc23b3 526 .qc_defer = ata_std_qc_defer,
fbbb262d
RH		 527 .qc_prep = nv_adma_qc_prep,
528 .qc_issue = nv_adma_qc_issue,
53014e25
RH		 529 .freeze = nv_adma_freeze,
530 .thaw = nv_adma_thaw,
fbbb262d 531 .error_handler = nv_adma_error_handler,
f5ecac2d 532 .post_internal_cmd = nv_adma_post_internal_cmd,
0d5ff566 533 .data_xfer = ata_data_xfer,
fbbb262d 534 .irq_clear = nv_adma_irq_clear,
246ce3b6 535 .irq_on = ata_irq_on,
fbbb262d
RH		 536 .scr_read = nv_scr_read,
537 .scr_write = nv_scr_write,
538 .port_start = nv_adma_port_start,
539 .port_stop = nv_adma_port_stop,
438ac6d5 540#ifdef CONFIG_PM
cdf56bcf
RH		 541 .port_suspend = nv_adma_port_suspend,
542 .port_resume = nv_adma_port_resume,
438ac6d5 543#endif
fbbb262d
RH		 544 .host_stop = nv_adma_host_stop,
545};
546
f140f0f1
KL		 547static const struct ata_port_operations nv_swncq_ops = {
548 .tf_load = ata_tf_load,
549 .tf_read = ata_tf_read,
550 .exec_command = ata_exec_command,
551 .check_status = ata_check_status,
552 .dev_select = ata_std_dev_select,
553 .bmdma_setup = ata_bmdma_setup,
554 .bmdma_start = ata_bmdma_start,
555 .bmdma_stop = ata_bmdma_stop,
556 .bmdma_status = ata_bmdma_status,
557 .qc_defer = ata_std_qc_defer,
558 .qc_prep = nv_swncq_qc_prep,
559 .qc_issue = nv_swncq_qc_issue,
560 .freeze = nv_mcp55_freeze,
561 .thaw = nv_mcp55_thaw,
562 .error_handler = nv_swncq_error_handler,
563 .post_internal_cmd = ata_bmdma_post_internal_cmd,
564 .data_xfer = ata_data_xfer,
565 .irq_clear = ata_bmdma_irq_clear,
566 .irq_on = ata_irq_on,
567 .scr_read = nv_scr_read,
568 .scr_write = nv_scr_write,
569#ifdef CONFIG_PM
570 .port_suspend = nv_swncq_port_suspend,
571 .port_resume = nv_swncq_port_resume,
572#endif
573 .port_start = nv_swncq_port_start,
574};
575
1626aeb8 576static const struct ata_port_info nv_port_info[] = {
ada364e8
TH		 577 /* generic */
578 {
579 .sht = &nv_sht,
0c88758b
TH		 580 .flags = ATA_FLAG_SATA | ATA_FLAG_NO_LEGACY,
581 .link_flags = ATA_LFLAG_HRST_TO_RESUME,
ada364e8
TH		 582 .pio_mask = NV_PIO_MASK,
583 .mwdma_mask = NV_MWDMA_MASK,
584 .udma_mask = NV_UDMA_MASK,
585 .port_ops = &nv_generic_ops,
9a829ccf 586 .irq_handler = nv_generic_interrupt,
ada364e8
TH		 587 },
588 /* nforce2/3 */
589 {
590 .sht = &nv_sht,
0c88758b
TH		 591 .flags = ATA_FLAG_SATA | ATA_FLAG_NO_LEGACY,
592 .link_flags = ATA_LFLAG_HRST_TO_RESUME,
ada364e8
TH		 593 .pio_mask = NV_PIO_MASK,
594 .mwdma_mask = NV_MWDMA_MASK,
595 .udma_mask = NV_UDMA_MASK,
596 .port_ops = &nv_nf2_ops,
9a829ccf 597 .irq_handler = nv_nf2_interrupt,
ada364e8
TH		 598 },
599 /* ck804 */
600 {
601 .sht = &nv_sht,
0c88758b
TH		 602 .flags = ATA_FLAG_SATA | ATA_FLAG_NO_LEGACY,
603 .link_flags = ATA_LFLAG_HRST_TO_RESUME,
ada364e8
TH		 604 .pio_mask = NV_PIO_MASK,
605 .mwdma_mask = NV_MWDMA_MASK,
606 .udma_mask = NV_UDMA_MASK,
607 .port_ops = &nv_ck804_ops,
9a829ccf 608 .irq_handler = nv_ck804_interrupt,
ada364e8 609 },
fbbb262d
RH		 610 /* ADMA */
611 {
612 .sht = &nv_adma_sht,
613 .flags = ATA_FLAG_SATA | ATA_FLAG_NO_LEGACY |
614 ATA_FLAG_MMIO | ATA_FLAG_NCQ,
0c88758b 615 .link_flags = ATA_LFLAG_HRST_TO_RESUME,
fbbb262d
RH		 616 .pio_mask = NV_PIO_MASK,
617 .mwdma_mask = NV_MWDMA_MASK,
618 .udma_mask = NV_UDMA_MASK,
619 .port_ops = &nv_adma_ops,
9a829ccf 620 .irq_handler = nv_adma_interrupt,
fbbb262d 621 },
f140f0f1
KL		 622 /* SWNCQ */
623 {
624 .sht = &nv_swncq_sht,
625 .flags = ATA_FLAG_SATA | ATA_FLAG_NO_LEGACY |
626 ATA_FLAG_NCQ,
627 .link_flags = ATA_LFLAG_HRST_TO_RESUME,
628 .pio_mask = NV_PIO_MASK,
629 .mwdma_mask = NV_MWDMA_MASK,
630 .udma_mask = NV_UDMA_MASK,
631 .port_ops = &nv_swncq_ops,
632 .irq_handler = nv_swncq_interrupt,
633 },
1da177e4
LT		 634};
635
636MODULE_AUTHOR("NVIDIA");
637MODULE_DESCRIPTION("low-level driver for NVIDIA nForce SATA controller");
638MODULE_LICENSE("GPL");
639MODULE_DEVICE_TABLE(pci, nv_pci_tbl);
640MODULE_VERSION(DRV_VERSION);
641
fbbb262d 642static int adma_enabled = 1;
f140f0f1 643static int swncq_enabled;
fbbb262d 644
2dec7555
RH		 645static void nv_adma_register_mode(struct ata_port *ap)
646{
2dec7555 647 struct nv_adma_port_priv *pp = ap->private_data;
cdf56bcf 648 void __iomem *mmio = pp->ctl_block;
a2cfe81a
RH		 649 u16 tmp, status;
650 int count = 0;
2dec7555
RH		 651
652 if (pp->flags & NV_ADMA_PORT_REGISTER_MODE)
653 return;
654
a2cfe81a 655 status = readw(mmio + NV_ADMA_STAT);
2dcb407e 656 while (!(status & NV_ADMA_STAT_IDLE) && count < 20) {
a2cfe81a
RH		 657 ndelay(50);
658 status = readw(mmio + NV_ADMA_STAT);
659 count++;
660 }
2dcb407e 661 if (count == 20)
a2cfe81a
RH		 662 ata_port_printk(ap, KERN_WARNING,
663 "timeout waiting for ADMA IDLE, stat=0x%hx\n",
664 status);
665
2dec7555
RH		 666 tmp = readw(mmio + NV_ADMA_CTL);
667 writew(tmp & ~NV_ADMA_CTL_GO, mmio + NV_ADMA_CTL);
668
a2cfe81a
RH		 669 count = 0;
670 status = readw(mmio + NV_ADMA_STAT);
2dcb407e 671 while (!(status & NV_ADMA_STAT_LEGACY) && count < 20) {
a2cfe81a
RH		 672 ndelay(50);
673 status = readw(mmio + NV_ADMA_STAT);
674 count++;
675 }
2dcb407e 676 if (count == 20)
a2cfe81a
RH		 677 ata_port_printk(ap, KERN_WARNING,
678 "timeout waiting for ADMA LEGACY, stat=0x%hx\n",
679 status);
680
2dec7555
RH		 681 pp->flags |= NV_ADMA_PORT_REGISTER_MODE;
682}
683
684static void nv_adma_mode(struct ata_port *ap)
685{
2dec7555 686 struct nv_adma_port_priv *pp = ap->private_data;
cdf56bcf 687 void __iomem *mmio = pp->ctl_block;
a2cfe81a
RH		 688 u16 tmp, status;
689 int count = 0;
2dec7555
RH		 690
691 if (!(pp->flags & NV_ADMA_PORT_REGISTER_MODE))
692 return;
f20b16ff 693
2dec7555
RH		 694 WARN_ON(pp->flags & NV_ADMA_ATAPI_SETUP_COMPLETE);
695
696 tmp = readw(mmio + NV_ADMA_CTL);
697 writew(tmp | NV_ADMA_CTL_GO, mmio + NV_ADMA_CTL);
698
a2cfe81a 699 status = readw(mmio + NV_ADMA_STAT);
2dcb407e 700 while (((status & NV_ADMA_STAT_LEGACY) ||
a2cfe81a
RH		 701 !(status & NV_ADMA_STAT_IDLE)) && count < 20) {
702 ndelay(50);
703 status = readw(mmio + NV_ADMA_STAT);
704 count++;
705 }
2dcb407e 706 if (count == 20)
a2cfe81a
RH		 707 ata_port_printk(ap, KERN_WARNING,
708 "timeout waiting for ADMA LEGACY clear and IDLE, stat=0x%hx\n",
709 status);
710
2dec7555
RH		 711 pp->flags &= ~NV_ADMA_PORT_REGISTER_MODE;
712}
713
fbbb262d
RH		 714static int nv_adma_slave_config(struct scsi_device *sdev)
715{
716 struct ata_port *ap = ata_shost_to_port(sdev->host);
2dec7555
RH		 717 struct nv_adma_port_priv *pp = ap->private_data;
718 struct pci_dev *pdev = to_pci_dev(ap->host->dev);
fbbb262d
RH		 719 u64 bounce_limit;
720 unsigned long segment_boundary;
721 unsigned short sg_tablesize;
722 int rc;
2dec7555
RH		 723 int adma_enable;
724 u32 current_reg, new_reg, config_mask;
fbbb262d
RH		 725
726 rc = ata_scsi_slave_config(sdev);
727
728 if (sdev->id >= ATA_MAX_DEVICES || sdev->channel || sdev->lun)
729 /* Not a proper libata device, ignore */
730 return rc;
731
9af5c9c9 732 if (ap->link.device[sdev->id].class == ATA_DEV_ATAPI) {
fbbb262d
RH		 733 /*
734 * NVIDIA reports that ADMA mode does not support ATAPI commands.
735 * Therefore ATAPI commands are sent through the legacy interface.
736 * However, the legacy interface only supports 32-bit DMA.
737 * Restrict DMA parameters as required by the legacy interface
738 * when an ATAPI device is connected.
739 */
740 bounce_limit = ATA_DMA_MASK;
741 segment_boundary = ATA_DMA_BOUNDARY;
742 /* Subtract 1 since an extra entry may be needed for padding, see
743 libata-scsi.c */
744 sg_tablesize = LIBATA_MAX_PRD - 1;
f20b16ff 745
2dec7555
RH		 746 /* Since the legacy DMA engine is in use, we need to disable ADMA
747 on the port. */
748 adma_enable = 0;
749 nv_adma_register_mode(ap);
2dcb407e 750 } else {
fbbb262d
RH		 751 bounce_limit = *ap->dev->dma_mask;
752 segment_boundary = NV_ADMA_DMA_BOUNDARY;
753 sg_tablesize = NV_ADMA_SGTBL_TOTAL_LEN;
2dec7555 754 adma_enable = 1;
fbbb262d 755 }
f20b16ff 756
2dec7555
RH		 757 pci_read_config_dword(pdev, NV_MCP_SATA_CFG_20, &current_reg);
758
2dcb407e 759 if (ap->port_no == 1)
2dec7555
RH		 760 config_mask = NV_MCP_SATA_CFG_20_PORT1_EN |
761 NV_MCP_SATA_CFG_20_PORT1_PWB_EN;
762 else
763 config_mask = NV_MCP_SATA_CFG_20_PORT0_EN |
764 NV_MCP_SATA_CFG_20_PORT0_PWB_EN;
f20b16ff 765
2dcb407e 766 if (adma_enable) {
2dec7555
RH		 767 new_reg = current_reg | config_mask;
768 pp->flags &= ~NV_ADMA_ATAPI_SETUP_COMPLETE;
2dcb407e 769 } else {
2dec7555
RH		 770 new_reg = current_reg & ~config_mask;
771 pp->flags |= NV_ADMA_ATAPI_SETUP_COMPLETE;
772 }
f20b16ff 773
2dcb407e 774 if (current_reg != new_reg)
2dec7555 775 pci_write_config_dword(pdev, NV_MCP_SATA_CFG_20, new_reg);
f20b16ff 776
fbbb262d
RH		 777 blk_queue_bounce_limit(sdev->request_queue, bounce_limit);
778 blk_queue_segment_boundary(sdev->request_queue, segment_boundary);
779 blk_queue_max_hw_segments(sdev->request_queue, sg_tablesize);
780 ata_port_printk(ap, KERN_INFO,
781 "bounce limit 0x%llX, segment boundary 0x%lX, hw segs %hu\n",
782 (unsigned long long)bounce_limit, segment_boundary, sg_tablesize);
783 return rc;
784}
785
2dec7555
RH		 786static int nv_adma_check_atapi_dma(struct ata_queued_cmd *qc)
787{
788 struct nv_adma_port_priv *pp = qc->ap->private_data;
789 return !(pp->flags & NV_ADMA_ATAPI_SETUP_COMPLETE);
790}
791
f2fb344b
RH		 792static void nv_adma_tf_read(struct ata_port *ap, struct ata_taskfile *tf)
793{
794 /* Since commands where a result TF is requested are not
795 executed in ADMA mode, the only time this function will be called
796 in ADMA mode will be if a command fails. In this case we
797 don't care about going into register mode with ADMA commands
798 pending, as the commands will all shortly be aborted anyway. */
799 nv_adma_register_mode(ap);
800
801 ata_tf_read(ap, tf);
802}
803
2dec7555 804static unsigned int nv_adma_tf_to_cpb(struct ata_taskfile *tf, __le16 *cpb)
fbbb262d
RH		 805{
806 unsigned int idx = 0;
807
2dcb407e 808 if (tf->flags & ATA_TFLAG_ISADDR) {
ac3d6b86
RH		 809 if (tf->flags & ATA_TFLAG_LBA48) {
810 cpb[idx++] = cpu_to_le16((ATA_REG_ERR << 8) | tf->hob_feature | WNB);
811 cpb[idx++] = cpu_to_le16((ATA_REG_NSECT << 8) | tf->hob_nsect);
812 cpb[idx++] = cpu_to_le16((ATA_REG_LBAL << 8) | tf->hob_lbal);
813 cpb[idx++] = cpu_to_le16((ATA_REG_LBAM << 8) | tf->hob_lbam);
814 cpb[idx++] = cpu_to_le16((ATA_REG_LBAH << 8) | tf->hob_lbah);
815 cpb[idx++] = cpu_to_le16((ATA_REG_ERR << 8) | tf->feature);
816 } else
817 cpb[idx++] = cpu_to_le16((ATA_REG_ERR << 8) | tf->feature | WNB);
a84471fe 818
ac3d6b86
RH		 819 cpb[idx++] = cpu_to_le16((ATA_REG_NSECT << 8) | tf->nsect);
820 cpb[idx++] = cpu_to_le16((ATA_REG_LBAL << 8) | tf->lbal);
821 cpb[idx++] = cpu_to_le16((ATA_REG_LBAM << 8) | tf->lbam);
822 cpb[idx++] = cpu_to_le16((ATA_REG_LBAH << 8) | tf->lbah);
fbbb262d 823 }
a84471fe 824
2dcb407e 825 if (tf->flags & ATA_TFLAG_DEVICE)
ac3d6b86 826 cpb[idx++] = cpu_to_le16((ATA_REG_DEVICE << 8) | tf->device);
fbbb262d
RH		 827
828 cpb[idx++] = cpu_to_le16((ATA_REG_CMD << 8) | tf->command | CMDEND);
a84471fe 829
2dcb407e 830 while (idx < 12)
ac3d6b86 831 cpb[idx++] = cpu_to_le16(IGN);
fbbb262d
RH		 832
833 return idx;
834}
835
5bd28a4b 836static int nv_adma_check_cpb(struct ata_port *ap, int cpb_num, int force_err)
fbbb262d
RH		 837{
838 struct nv_adma_port_priv *pp = ap->private_data;
2dec7555 839 u8 flags = pp->cpb[cpb_num].resp_flags;
fbbb262d
RH		 840
841 VPRINTK("CPB %d, flags=0x%x\n", cpb_num, flags);
842
5bd28a4b
RH		 843 if (unlikely((force_err ||
844 flags & (NV_CPB_RESP_ATA_ERR |
845 NV_CPB_RESP_CMD_ERR |
846 NV_CPB_RESP_CPB_ERR)))) {
9af5c9c9 847 struct ata_eh_info *ehi = &ap->link.eh_info;
5bd28a4b
RH		 848 int freeze = 0;
849
850 ata_ehi_clear_desc(ehi);
2dcb407e 851 __ata_ehi_push_desc(ehi, "CPB resp_flags 0x%x: ", flags);
5bd28a4b 852 if (flags & NV_CPB_RESP_ATA_ERR) {
b64bbc39 853 ata_ehi_push_desc(ehi, "ATA error");
5bd28a4b
RH		 854 ehi->err_mask |= AC_ERR_DEV;
855 } else if (flags & NV_CPB_RESP_CMD_ERR) {
b64bbc39 856 ata_ehi_push_desc(ehi, "CMD error");
5bd28a4b
RH		 857 ehi->err_mask |= AC_ERR_DEV;
858 } else if (flags & NV_CPB_RESP_CPB_ERR) {
b64bbc39 859 ata_ehi_push_desc(ehi, "CPB error");
5bd28a4b
RH		 860 ehi->err_mask |= AC_ERR_SYSTEM;
861 freeze = 1;
862 } else {
863 /* notifier error, but no error in CPB flags? */
b64bbc39 864 ata_ehi_push_desc(ehi, "unknown");
5bd28a4b
RH		 865 ehi->err_mask |= AC_ERR_OTHER;
866 freeze = 1;
867 }
868 /* Kill all commands. EH will determine what actually failed. */
869 if (freeze)
870 ata_port_freeze(ap);
871 else
872 ata_port_abort(ap);
873 return 1;
fbbb262d 874 }
5bd28a4b 875
f2fb344b 876 if (likely(flags & NV_CPB_RESP_DONE)) {
fbbb262d 877 struct ata_queued_cmd *qc = ata_qc_from_tag(ap, cpb_num);
5bd28a4b
RH		 878 VPRINTK("CPB flags done, flags=0x%x\n", flags);
879 if (likely(qc)) {
2dcb407e 880 DPRINTK("Completing qc from tag %d\n", cpb_num);
fbbb262d 881 ata_qc_complete(qc);
2a54cf76 882 } else {
9af5c9c9 883 struct ata_eh_info *ehi = &ap->link.eh_info;
2a54cf76
RH		 884 /* Notifier bits set without a command may indicate the drive
885 is misbehaving. Raise host state machine violation on this
886 condition. */
887 ata_port_printk(ap, KERN_ERR, "notifier for tag %d with no command?\n",
888 cpb_num);
889 ehi->err_mask |= AC_ERR_HSM;
890 ehi->action |= ATA_EH_SOFTRESET;
891 ata_port_freeze(ap);
892 return 1;
fbbb262d
RH		 893 }
894 }
5bd28a4b 895 return 0;
fbbb262d
RH		 896}
897
2dec7555
RH		 898static int nv_host_intr(struct ata_port *ap, u8 irq_stat)
899{
9af5c9c9 900 struct ata_queued_cmd *qc = ata_qc_from_tag(ap, ap->link.active_tag);
2dec7555
RH		 901
902 /* freeze if hotplugged */
903 if (unlikely(irq_stat & (NV_INT_ADDED | NV_INT_REMOVED))) {
904 ata_port_freeze(ap);
905 return 1;
906 }
907
908 /* bail out if not our interrupt */
909 if (!(irq_stat & NV_INT_DEV))
910 return 0;
911
912 /* DEV interrupt w/ no active qc? */
913 if (unlikely(!qc || (qc->tf.flags & ATA_TFLAG_POLLING))) {
914 ata_check_status(ap);
915 return 1;
916 }
917
918 /* handle interrupt */
f740d168 919 return ata_host_intr(ap, qc);
2dec7555
RH		 920}
921
fbbb262d
RH		 922static irqreturn_t nv_adma_interrupt(int irq, void *dev_instance)
923{
924 struct ata_host *host = dev_instance;
925 int i, handled = 0;
2dec7555 926 u32 notifier_clears[2];
fbbb262d
RH		 927
928 spin_lock(&host->lock);
929
930 for (i = 0; i < host->n_ports; i++) {
931 struct ata_port *ap = host->ports[i];
2dec7555 932 notifier_clears[i] = 0;
fbbb262d
RH		 933
934 if (ap && !(ap->flags & ATA_FLAG_DISABLED)) {
935 struct nv_adma_port_priv *pp = ap->private_data;
cdf56bcf 936 void __iomem *mmio = pp->ctl_block;
fbbb262d
RH		 937 u16 status;
938 u32 gen_ctl;
fbbb262d 939 u32 notifier, notifier_error;
a617c09f 940
53014e25
RH		 941 /* if ADMA is disabled, use standard ata interrupt handler */
942 if (pp->flags & NV_ADMA_ATAPI_SETUP_COMPLETE) {
943 u8 irq_stat = readb(host->iomap[NV_MMIO_BAR] + NV_INT_STATUS_CK804)
944 >> (NV_INT_PORT_SHIFT * i);
945 handled += nv_host_intr(ap, irq_stat);
946 continue;
947 }
fbbb262d 948
53014e25 949 /* if in ATA register mode, check for standard interrupts */
fbbb262d 950 if (pp->flags & NV_ADMA_PORT_REGISTER_MODE) {
0d5ff566 951 u8 irq_stat = readb(host->iomap[NV_MMIO_BAR] + NV_INT_STATUS_CK804)
2dec7555 952 >> (NV_INT_PORT_SHIFT * i);
2dcb407e 953 if (ata_tag_valid(ap->link.active_tag))
f740d168
RH		 954 /** NV_INT_DEV indication seems unreliable at times
955 at least in ADMA mode. Force it on always when a
956 command is active, to prevent losing interrupts. */
957 irq_stat |= NV_INT_DEV;
2dec7555 958 handled += nv_host_intr(ap, irq_stat);
fbbb262d
RH		 959 }
960
961 notifier = readl(mmio + NV_ADMA_NOTIFIER);
962 notifier_error = readl(mmio + NV_ADMA_NOTIFIER_ERROR);
2dec7555 963 notifier_clears[i] = notifier | notifier_error;
fbbb262d 964
cdf56bcf 965 gen_ctl = readl(pp->gen_block + NV_ADMA_GEN_CTL);
fbbb262d 966
2dcb407e 967 if (!NV_ADMA_CHECK_INTR(gen_ctl, ap->port_no) && !notifier &&
fbbb262d
RH		 968 !notifier_error)
969 /* Nothing to do */
970 continue;
971
972 status = readw(mmio + NV_ADMA_STAT);
973
974 /* Clear status. Ensure the controller sees the clearing before we start
975 looking at any of the CPB statuses, so that any CPB completions after
976 this point in the handler will raise another interrupt. */
977 writew(status, mmio + NV_ADMA_STAT);
978 readw(mmio + NV_ADMA_STAT); /* flush posted write */
979 rmb();
980
5bd28a4b
RH		 981 handled++; /* irq handled if we got here */
982
983 /* freeze if hotplugged or controller error */
984 if (unlikely(status & (NV_ADMA_STAT_HOTPLUG |
985 NV_ADMA_STAT_HOTUNPLUG |
5278b50c
RH		 986 NV_ADMA_STAT_TIMEOUT |
987 NV_ADMA_STAT_SERROR))) {
9af5c9c9 988 struct ata_eh_info *ehi = &ap->link.eh_info;
5bd28a4b
RH		 989
990 ata_ehi_clear_desc(ehi);
2dcb407e 991 __ata_ehi_push_desc(ehi, "ADMA status 0x%08x: ", status);
5bd28a4b
RH		 992 if (status & NV_ADMA_STAT_TIMEOUT) {
993 ehi->err_mask |= AC_ERR_SYSTEM;
b64bbc39 994 ata_ehi_push_desc(ehi, "timeout");
5bd28a4b
RH		 995 } else if (status & NV_ADMA_STAT_HOTPLUG) {
996 ata_ehi_hotplugged(ehi);
b64bbc39 997 ata_ehi_push_desc(ehi, "hotplug");
5bd28a4b
RH		 998 } else if (status & NV_ADMA_STAT_HOTUNPLUG) {
999 ata_ehi_hotplugged(ehi);
b64bbc39 1000 ata_ehi_push_desc(ehi, "hot unplug");
5278b50c
RH		 1001 } else if (status & NV_ADMA_STAT_SERROR) {
1002 /* let libata analyze SError and figure out the cause */
b64bbc39
TH		 1003 ata_ehi_push_desc(ehi, "SError");
1004 } else
1005 ata_ehi_push_desc(ehi, "unknown");
fbbb262d 1006 ata_port_freeze(ap);
fbbb262d
RH		 1007 continue;
1008 }
1009
5bd28a4b
RH		 1010 if (status & (NV_ADMA_STAT_DONE |
1011 NV_ADMA_STAT_CPBERR)) {
8ba5e4cb 1012 u32 check_commands;
721449bf 1013 int pos, error = 0;
8ba5e4cb 1014
b447916e 1015 if (ata_tag_valid(ap->link.active_tag))
9af5c9c9 1016 check_commands = 1 << ap->link.active_tag;
8ba5e4cb 1017 else
9af5c9c9 1018 check_commands = ap->link.sactive;
8ba5e4cb 1019
fbbb262d 1020 /** Check CPBs for completed commands */
721449bf
RH		 1021 while ((pos = ffs(check_commands)) && !error) {
1022 pos--;
1023 error = nv_adma_check_cpb(ap, pos,
1024 notifier_error & (1 << pos) );
1025 check_commands &= ~(1 << pos );
fbbb262d
RH		 1026 }
1027 }
fbbb262d
RH		 1028 }
1029 }
f20b16ff 1030
b447916e 1031 if (notifier_clears[0] || notifier_clears[1]) {
2dec7555
RH		 1032 /* Note: Both notifier clear registers must be written
1033 if either is set, even if one is zero, according to NVIDIA. */
cdf56bcf
RH		 1034 struct nv_adma_port_priv *pp = host->ports[0]->private_data;
1035 writel(notifier_clears[0], pp->notifier_clear_block);
1036 pp = host->ports[1]->private_data;
1037 writel(notifier_clears[1], pp->notifier_clear_block);
2dec7555 1038 }
fbbb262d
RH		 1039
1040 spin_unlock(&host->lock);
1041
1042 return IRQ_RETVAL(handled);
1043}
1044
53014e25
RH		 1045static void nv_adma_freeze(struct ata_port *ap)
1046{
1047 struct nv_adma_port_priv *pp = ap->private_data;
1048 void __iomem *mmio = pp->ctl_block;
1049 u16 tmp;
1050
1051 nv_ck804_freeze(ap);
1052
1053 if (pp->flags & NV_ADMA_ATAPI_SETUP_COMPLETE)
1054 return;
1055
1056 /* clear any outstanding CK804 notifications */
2dcb407e 1057 writeb(NV_INT_ALL << (ap->port_no * NV_INT_PORT_SHIFT),
53014e25
RH		 1058 ap->host->iomap[NV_MMIO_BAR] + NV_INT_STATUS_CK804);
1059
1060 /* Disable interrupt */
1061 tmp = readw(mmio + NV_ADMA_CTL);
2dcb407e 1062 writew(tmp & ~(NV_ADMA_CTL_AIEN | NV_ADMA_CTL_HOTPLUG_IEN),
53014e25 1063 mmio + NV_ADMA_CTL);
2dcb407e 1064 readw(mmio + NV_ADMA_CTL ); /* flush posted write */
53014e25
RH		 1065}
1066
1067static void nv_adma_thaw(struct ata_port *ap)
1068{
1069 struct nv_adma_port_priv *pp = ap->private_data;
1070 void __iomem *mmio = pp->ctl_block;
1071 u16 tmp;
1072
1073 nv_ck804_thaw(ap);
1074
1075 if (pp->flags & NV_ADMA_ATAPI_SETUP_COMPLETE)
1076 return;
1077
1078 /* Enable interrupt */
1079 tmp = readw(mmio + NV_ADMA_CTL);
2dcb407e 1080 writew(tmp | (NV_ADMA_CTL_AIEN | NV_ADMA_CTL_HOTPLUG_IEN),
53014e25 1081 mmio + NV_ADMA_CTL);
2dcb407e 1082 readw(mmio + NV_ADMA_CTL ); /* flush posted write */
53014e25
RH		 1083}
1084
fbbb262d
RH		 1085static void nv_adma_irq_clear(struct ata_port *ap)
1086{
cdf56bcf
RH		 1087 struct nv_adma_port_priv *pp = ap->private_data;
1088 void __iomem *mmio = pp->ctl_block;
53014e25 1089 u32 notifier_clears[2];
fbbb262d 1090
53014e25
RH		 1091 if (pp->flags & NV_ADMA_ATAPI_SETUP_COMPLETE) {
1092 ata_bmdma_irq_clear(ap);
1093 return;
1094 }
1095
1096 /* clear any outstanding CK804 notifications */
2dcb407e 1097 writeb(NV_INT_ALL << (ap->port_no * NV_INT_PORT_SHIFT),
53014e25 1098 ap->host->iomap[NV_MMIO_BAR] + NV_INT_STATUS_CK804);
fbbb262d 1099
53014e25
RH		 1100 /* clear ADMA status */
1101 writew(0xffff, mmio + NV_ADMA_STAT);
a617c09f 1102
53014e25
RH		 1103 /* clear notifiers - note both ports need to be written with
1104 something even though we are only clearing on one */
1105 if (ap->port_no == 0) {
1106 notifier_clears[0] = 0xFFFFFFFF;
1107 notifier_clears[1] = 0;
1108 } else {
1109 notifier_clears[0] = 0;
1110 notifier_clears[1] = 0xFFFFFFFF;
1111 }
1112 pp = ap->host->ports[0]->private_data;
1113 writel(notifier_clears[0], pp->notifier_clear_block);
1114 pp = ap->host->ports[1]->private_data;
1115 writel(notifier_clears[1], pp->notifier_clear_block);
fbbb262d
RH		 1116}
1117
f5ecac2d 1118static void nv_adma_post_internal_cmd(struct ata_queued_cmd *qc)
fbbb262d 1119{
f5ecac2d 1120 struct nv_adma_port_priv *pp = qc->ap->private_data;
fbbb262d 1121
b447916e 1122 if (pp->flags & NV_ADMA_PORT_REGISTER_MODE)
f5ecac2d 1123 ata_bmdma_post_internal_cmd(qc);
fbbb262d
RH		 1124}
1125
1126static int nv_adma_port_start(struct ata_port *ap)
1127{
1128 struct device *dev = ap->host->dev;
1129 struct nv_adma_port_priv *pp;
1130 int rc;
1131 void *mem;
1132 dma_addr_t mem_dma;
cdf56bcf 1133 void __iomem *mmio;
fbbb262d
RH		 1134 u16 tmp;
1135
1136 VPRINTK("ENTER\n");
1137
1138 rc = ata_port_start(ap);
1139 if (rc)
1140 return rc;
1141
24dc5f33
TH		 1142 pp = devm_kzalloc(dev, sizeof(*pp), GFP_KERNEL);
1143 if (!pp)
1144 return -ENOMEM;
fbbb262d 1145
0d5ff566 1146 mmio = ap->host->iomap[NV_MMIO_BAR] + NV_ADMA_PORT +
cdf56bcf
RH		 1147 ap->port_no * NV_ADMA_PORT_SIZE;
1148 pp->ctl_block = mmio;
0d5ff566 1149 pp->gen_block = ap->host->iomap[NV_MMIO_BAR] + NV_ADMA_GEN;
cdf56bcf
RH		 1150 pp->notifier_clear_block = pp->gen_block +
1151 NV_ADMA_NOTIFIER_CLEAR + (4 * ap->port_no);
1152
24dc5f33
TH		 1153 mem = dmam_alloc_coherent(dev, NV_ADMA_PORT_PRIV_DMA_SZ,
1154 &mem_dma, GFP_KERNEL);
1155 if (!mem)
1156 return -ENOMEM;
fbbb262d
RH		 1157 memset(mem, 0, NV_ADMA_PORT_PRIV_DMA_SZ);
1158
1159 /*
1160 * First item in chunk of DMA memory:
1161 * 128-byte command parameter block (CPB)
1162 * one for each command tag
1163 */
1164 pp->cpb = mem;
1165 pp->cpb_dma = mem_dma;
1166
1167 writel(mem_dma & 0xFFFFFFFF, mmio + NV_ADMA_CPB_BASE_LOW);
1168 writel((mem_dma >> 16 ) >> 16, mmio + NV_ADMA_CPB_BASE_HIGH);
1169
1170 mem += NV_ADMA_MAX_CPBS * NV_ADMA_CPB_SZ;
1171 mem_dma += NV_ADMA_MAX_CPBS * NV_ADMA_CPB_SZ;
1172
1173 /*
1174 * Second item: block of ADMA_SGTBL_LEN s/g entries
1175 */
1176 pp->aprd = mem;
1177 pp->aprd_dma = mem_dma;
1178
1179 ap->private_data = pp;
1180
1181 /* clear any outstanding interrupt conditions */
1182 writew(0xffff, mmio + NV_ADMA_STAT);
1183
1184 /* initialize port variables */
1185 pp->flags = NV_ADMA_PORT_REGISTER_MODE;
1186
1187 /* clear CPB fetch count */
1188 writew(0, mmio + NV_ADMA_CPB_COUNT);
1189
cdf56bcf 1190 /* clear GO for register mode, enable interrupt */
fbbb262d 1191 tmp = readw(mmio + NV_ADMA_CTL);
5ce0cf6f
RH		 1192 writew( (tmp & ~NV_ADMA_CTL_GO) | NV_ADMA_CTL_AIEN |
1193 NV_ADMA_CTL_HOTPLUG_IEN, mmio + NV_ADMA_CTL);
fbbb262d
RH		 1194
1195 tmp = readw(mmio + NV_ADMA_CTL);
1196 writew(tmp | NV_ADMA_CTL_CHANNEL_RESET, mmio + NV_ADMA_CTL);
b447916e 1197 readw(mmio + NV_ADMA_CTL ); /* flush posted write */
fbbb262d
RH		 1198 udelay(1);
1199 writew(tmp & ~NV_ADMA_CTL_CHANNEL_RESET, mmio + NV_ADMA_CTL);
b447916e 1200 readw(mmio + NV_ADMA_CTL ); /* flush posted write */
fbbb262d
RH		 1201
1202 return 0;
fbbb262d
RH		 1203}
1204
1205static void nv_adma_port_stop(struct ata_port *ap)
1206{
fbbb262d 1207 struct nv_adma_port_priv *pp = ap->private_data;
cdf56bcf 1208 void __iomem *mmio = pp->ctl_block;
fbbb262d
RH		 1209
1210 VPRINTK("ENTER\n");
fbbb262d 1211 writew(0, mmio + NV_ADMA_CTL);
fbbb262d
RH		 1212}
1213
438ac6d5 1214#ifdef CONFIG_PM
cdf56bcf
RH		 1215static int nv_adma_port_suspend(struct ata_port *ap, pm_message_t mesg)
1216{
1217 struct nv_adma_port_priv *pp = ap->private_data;
1218 void __iomem *mmio = pp->ctl_block;
1219
1220 /* Go to register mode - clears GO */
1221 nv_adma_register_mode(ap);
1222
1223 /* clear CPB fetch count */
1224 writew(0, mmio + NV_ADMA_CPB_COUNT);
1225
1226 /* disable interrupt, shut down port */
1227 writew(0, mmio + NV_ADMA_CTL);
1228
1229 return 0;
1230}
1231
1232static int nv_adma_port_resume(struct ata_port *ap)
1233{
1234 struct nv_adma_port_priv *pp = ap->private_data;
1235 void __iomem *mmio = pp->ctl_block;
1236 u16 tmp;
1237
1238 /* set CPB block location */
1239 writel(pp->cpb_dma & 0xFFFFFFFF, mmio + NV_ADMA_CPB_BASE_LOW);
1240 writel((pp->cpb_dma >> 16 ) >> 16, mmio + NV_ADMA_CPB_BASE_HIGH);
1241
1242 /* clear any outstanding interrupt conditions */
1243 writew(0xffff, mmio + NV_ADMA_STAT);
1244
1245 /* initialize port variables */
1246 pp->flags |= NV_ADMA_PORT_REGISTER_MODE;
1247
1248 /* clear CPB fetch count */
1249 writew(0, mmio + NV_ADMA_CPB_COUNT);
1250
1251 /* clear GO for register mode, enable interrupt */
1252 tmp = readw(mmio + NV_ADMA_CTL);
5ce0cf6f
RH		 1253 writew( (tmp & ~NV_ADMA_CTL_GO) | NV_ADMA_CTL_AIEN |
1254 NV_ADMA_CTL_HOTPLUG_IEN, mmio + NV_ADMA_CTL);
cdf56bcf
RH		 1255
1256 tmp = readw(mmio + NV_ADMA_CTL);
1257 writew(tmp | NV_ADMA_CTL_CHANNEL_RESET, mmio + NV_ADMA_CTL);
b447916e 1258 readw(mmio + NV_ADMA_CTL ); /* flush posted write */
cdf56bcf
RH		 1259 udelay(1);
1260 writew(tmp & ~NV_ADMA_CTL_CHANNEL_RESET, mmio + NV_ADMA_CTL);
b447916e 1261 readw(mmio + NV_ADMA_CTL ); /* flush posted write */
cdf56bcf
RH		 1262
1263 return 0;
1264}
438ac6d5 1265#endif
fbbb262d 1266
9a829ccf 1267static void nv_adma_setup_port(struct ata_port *ap)
fbbb262d 1268{
9a829ccf
TH		 1269 void __iomem *mmio = ap->host->iomap[NV_MMIO_BAR];
1270 struct ata_ioports *ioport = &ap->ioaddr;
fbbb262d
RH		 1271
1272 VPRINTK("ENTER\n");
1273
9a829ccf 1274 mmio += NV_ADMA_PORT + ap->port_no * NV_ADMA_PORT_SIZE;
fbbb262d 1275
0d5ff566
TH		 1276 ioport->cmd_addr = mmio;
1277 ioport->data_addr = mmio + (ATA_REG_DATA * 4);
fbbb262d 1278 ioport->error_addr =
0d5ff566
TH		 1279 ioport->feature_addr = mmio + (ATA_REG_ERR * 4);
1280 ioport->nsect_addr = mmio + (ATA_REG_NSECT * 4);
1281 ioport->lbal_addr = mmio + (ATA_REG_LBAL * 4);
1282 ioport->lbam_addr = mmio + (ATA_REG_LBAM * 4);
1283 ioport->lbah_addr = mmio + (ATA_REG_LBAH * 4);
1284 ioport->device_addr = mmio + (ATA_REG_DEVICE * 4);
fbbb262d 1285 ioport->status_addr =
0d5ff566 1286 ioport->command_addr = mmio + (ATA_REG_STATUS * 4);
fbbb262d 1287 ioport->altstatus_addr =
0d5ff566 1288 ioport->ctl_addr = mmio + 0x20;
fbbb262d
RH		 1289}
1290
9a829ccf 1291static int nv_adma_host_init(struct ata_host *host)
fbbb262d 1292{
9a829ccf 1293 struct pci_dev *pdev = to_pci_dev(host->dev);
fbbb262d
RH		 1294 unsigned int i;
1295 u32 tmp32;
1296
1297 VPRINTK("ENTER\n");
1298
1299 /* enable ADMA on the ports */
1300 pci_read_config_dword(pdev, NV_MCP_SATA_CFG_20, &tmp32);
1301 tmp32 |= NV_MCP_SATA_CFG_20_PORT0_EN |
1302 NV_MCP_SATA_CFG_20_PORT0_PWB_EN |
1303 NV_MCP_SATA_CFG_20_PORT1_EN |
1304 NV_MCP_SATA_CFG_20_PORT1_PWB_EN;
1305
1306 pci_write_config_dword(pdev, NV_MCP_SATA_CFG_20, tmp32);
1307
9a829ccf
TH		 1308 for (i = 0; i < host->n_ports; i++)
1309 nv_adma_setup_port(host->ports[i]);
fbbb262d 1310
fbbb262d
RH		 1311 return 0;
1312}
1313
1314static void nv_adma_fill_aprd(struct ata_queued_cmd *qc,
1315 struct scatterlist *sg,
1316 int idx,
1317 struct nv_adma_prd *aprd)
1318{
41949ed5 1319 u8 flags = 0;
fbbb262d
RH		 1320 if (qc->tf.flags & ATA_TFLAG_WRITE)
1321 flags |= NV_APRD_WRITE;
1322 if (idx == qc->n_elem - 1)
1323 flags |= NV_APRD_END;
1324 else if (idx != 4)
1325 flags |= NV_APRD_CONT;
1326
1327 aprd->addr = cpu_to_le64(((u64)sg_dma_address(sg)));
1328 aprd->len = cpu_to_le32(((u32)sg_dma_len(sg))); /* len in bytes */
2dec7555 1329 aprd->flags = flags;
41949ed5 1330 aprd->packet_len = 0;
fbbb262d
RH		 1331}
1332
1333static void nv_adma_fill_sg(struct ata_queued_cmd *qc, struct nv_adma_cpb *cpb)
1334{
1335 struct nv_adma_port_priv *pp = qc->ap->private_data;
1336 unsigned int idx;
1337 struct nv_adma_prd *aprd;
1338 struct scatterlist *sg;
1339
1340 VPRINTK("ENTER\n");
1341
1342 idx = 0;
1343
1344 ata_for_each_sg(sg, qc) {
1345 aprd = (idx < 5) ? &cpb->aprd[idx] : &pp->aprd[NV_ADMA_SGTBL_LEN * qc->tag + (idx-5)];
1346 nv_adma_fill_aprd(qc, sg, idx, aprd);
1347 idx++;
1348 }
1349 if (idx > 5)
1350 cpb->next_aprd = cpu_to_le64(((u64)(pp->aprd_dma + NV_ADMA_SGTBL_SZ * qc->tag)));
41949ed5
RH		 1351 else
1352 cpb->next_aprd = cpu_to_le64(0);
fbbb262d
RH		 1353}
1354
382a6652
RH		 1355static int nv_adma_use_reg_mode(struct ata_queued_cmd *qc)
1356{
1357 struct nv_adma_port_priv *pp = qc->ap->private_data;
1358
1359 /* ADMA engine can only be used for non-ATAPI DMA commands,
f2fb344b
RH		 1360 or interrupt-driven no-data commands, where a result taskfile
1361 is not required. */
b447916e 1362 if ((pp->flags & NV_ADMA_ATAPI_SETUP_COMPLETE) ||
f2fb344b
RH		 1363 (qc->tf.flags & ATA_TFLAG_POLLING) ||
1364 (qc->flags & ATA_QCFLAG_RESULT_TF))
382a6652
RH		 1365 return 1;
1366
b447916e 1367 if ((qc->flags & ATA_QCFLAG_DMAMAP) ||
382a6652
RH		 1368 (qc->tf.protocol == ATA_PROT_NODATA))
1369 return 0;
1370
1371 return 1;
1372}
1373
fbbb262d
RH		 1374static void nv_adma_qc_prep(struct ata_queued_cmd *qc)
1375{
1376 struct nv_adma_port_priv *pp = qc->ap->private_data;
1377 struct nv_adma_cpb *cpb = &pp->cpb[qc->tag];
1378 u8 ctl_flags = NV_CPB_CTL_CPB_VALID |
fbbb262d
RH		 1379 NV_CPB_CTL_IEN;
1380
382a6652 1381 if (nv_adma_use_reg_mode(qc)) {
2dec7555 1382 nv_adma_register_mode(qc->ap);
fbbb262d
RH		 1383 ata_qc_prep(qc);
1384 return;
1385 }
1386
41949ed5
RH		 1387 cpb->resp_flags = NV_CPB_RESP_DONE;
1388 wmb();
1389 cpb->ctl_flags = 0;
1390 wmb();
fbbb262d
RH		 1391
1392 cpb->len = 3;
1393 cpb->tag = qc->tag;
1394 cpb->next_cpb_idx = 0;
1395
1396 /* turn on NCQ flags for NCQ commands */
1397 if (qc->tf.protocol == ATA_PROT_NCQ)
1398 ctl_flags |= NV_CPB_CTL_QUEUE | NV_CPB_CTL_FPDMA;
1399
cdf56bcf
RH		 1400 VPRINTK("qc->flags = 0x%lx\n", qc->flags);
1401
fbbb262d
RH		 1402 nv_adma_tf_to_cpb(&qc->tf, cpb->tf);
1403
b447916e 1404 if (qc->flags & ATA_QCFLAG_DMAMAP) {
382a6652
RH		 1405 nv_adma_fill_sg(qc, cpb);
1406 ctl_flags |= NV_CPB_CTL_APRD_VALID;
1407 } else
1408 memset(&cpb->aprd[0], 0, sizeof(struct nv_adma_prd) * 5);
fbbb262d
RH		 1409
1410 /* Be paranoid and don't let the device see NV_CPB_CTL_CPB_VALID until we are
1411 finished filling in all of the contents */
1412 wmb();
1413 cpb->ctl_flags = ctl_flags;
41949ed5
RH		 1414 wmb();
1415 cpb->resp_flags = 0;
fbbb262d
RH		 1416}
1417
1418static unsigned int nv_adma_qc_issue(struct ata_queued_cmd *qc)
1419{
2dec7555 1420 struct nv_adma_port_priv *pp = qc->ap->private_data;
cdf56bcf 1421 void __iomem *mmio = pp->ctl_block;
5e5c74a5 1422 int curr_ncq = (qc->tf.protocol == ATA_PROT_NCQ);
fbbb262d
RH		 1423
1424 VPRINTK("ENTER\n");
1425
382a6652 1426 if (nv_adma_use_reg_mode(qc)) {
fbbb262d 1427 /* use ATA register mode */
382a6652 1428 VPRINTK("using ATA register mode: 0x%lx\n", qc->flags);
fbbb262d
RH		 1429 nv_adma_register_mode(qc->ap);
1430 return ata_qc_issue_prot(qc);
1431 } else
1432 nv_adma_mode(qc->ap);
1433
1434 /* write append register, command tag in lower 8 bits
1435 and (number of cpbs to append -1) in top 8 bits */
1436 wmb();
5e5c74a5 1437
b447916e 1438 if (curr_ncq != pp->last_issue_ncq) {
5e5c74a5
RH		 1439 /* Seems to need some delay before switching between NCQ and non-NCQ
1440 commands, else we get command timeouts and such. */
1441 udelay(20);
1442 pp->last_issue_ncq = curr_ncq;
1443 }
1444
fbbb262d
RH		 1445 writew(qc->tag, mmio + NV_ADMA_APPEND);
1446
1447 DPRINTK("Issued tag %u\n",qc->tag);
1448
1449 return 0;
1450}
1451
7d12e780 1452static irqreturn_t nv_generic_interrupt(int irq, void *dev_instance)
1da177e4 1453{
cca3974e 1454 struct ata_host *host = dev_instance;
1da177e4
LT		 1455 unsigned int i;
1456 unsigned int handled = 0;
1457 unsigned long flags;
1458
cca3974e 1459 spin_lock_irqsave(&host->lock, flags);
1da177e4 1460
cca3974e 1461 for (i = 0; i < host->n_ports; i++) {
1da177e4
LT		 1462 struct ata_port *ap;
1463
cca3974e 1464 ap = host->ports[i];
c1389503 1465 if (ap &&
029f5468 1466 !(ap->flags & ATA_FLAG_DISABLED)) {
1da177e4
LT		 1467 struct ata_queued_cmd *qc;
1468
9af5c9c9 1469 qc = ata_qc_from_tag(ap, ap->link.active_tag);
e50362ec 1470 if (qc && (!(qc->tf.flags & ATA_TFLAG_POLLING)))
1da177e4 1471 handled += ata_host_intr(ap, qc);
b887030a
AC		 1472 else
1473 // No request pending? Clear interrupt status
1474 // anyway, in case there's one pending.
1475 ap->ops->check_status(ap);
1da177e4
LT		 1476 }
1477
1478 }
1479
cca3974e 1480 spin_unlock_irqrestore(&host->lock, flags);
1da177e4
LT		 1481
1482 return IRQ_RETVAL(handled);
1483}
1484
cca3974e 1485static irqreturn_t nv_do_interrupt(struct ata_host *host, u8 irq_stat)
ada364e8
TH		 1486{
1487 int i, handled = 0;
1488
cca3974e
JG		 1489 for (i = 0; i < host->n_ports; i++) {
1490 struct ata_port *ap = host->ports[i];
ada364e8
TH		 1491
1492 if (ap && !(ap->flags & ATA_FLAG_DISABLED))
1493 handled += nv_host_intr(ap, irq_stat);
1494
1495 irq_stat >>= NV_INT_PORT_SHIFT;
1496 }
1497
1498 return IRQ_RETVAL(handled);
1499}
1500
7d12e780 1501static irqreturn_t nv_nf2_interrupt(int irq, void *dev_instance)
ada364e8 1502{
cca3974e 1503 struct ata_host *host = dev_instance;
ada364e8
TH		 1504 u8 irq_stat;
1505 irqreturn_t ret;
1506
cca3974e 1507 spin_lock(&host->lock);
0d5ff566 1508 irq_stat = ioread8(host->ports[0]->ioaddr.scr_addr + NV_INT_STATUS);
cca3974e
JG		 1509 ret = nv_do_interrupt(host, irq_stat);
1510 spin_unlock(&host->lock);
ada364e8
TH		 1511
1512 return ret;
1513}
1514
7d12e780 1515static irqreturn_t nv_ck804_interrupt(int irq, void *dev_instance)
ada364e8 1516{
cca3974e 1517 struct ata_host *host = dev_instance;
ada364e8
TH		 1518 u8 irq_stat;
1519 irqreturn_t ret;
1520
cca3974e 1521 spin_lock(&host->lock);
0d5ff566 1522 irq_stat = readb(host->iomap[NV_MMIO_BAR] + NV_INT_STATUS_CK804);
cca3974e
JG		 1523 ret = nv_do_interrupt(host, irq_stat);
1524 spin_unlock(&host->lock);
ada364e8
TH		 1525
1526 return ret;
1527}
1528
da3dbb17 1529static int nv_scr_read(struct ata_port *ap, unsigned int sc_reg, u32 *val)
1da177e4 1530{
1da177e4 1531 if (sc_reg > SCR_CONTROL)
da3dbb17 1532 return -EINVAL;
1da177e4 1533
da3dbb17
TH		 1534 *val = ioread32(ap->ioaddr.scr_addr + (sc_reg * 4));
1535 return 0;
1da177e4
LT		 1536}
1537
da3dbb17 1538static int nv_scr_write(struct ata_port *ap, unsigned int sc_reg, u32 val)
1da177e4 1539{
1da177e4 1540 if (sc_reg > SCR_CONTROL)
da3dbb17 1541 return -EINVAL;
1da177e4 1542
0d5ff566 1543 iowrite32(val, ap->ioaddr.scr_addr + (sc_reg * 4));
da3dbb17 1544 return 0;
1da177e4
LT		 1545}
1546
39f87582
TH		 1547static void nv_nf2_freeze(struct ata_port *ap)
1548{
0d5ff566 1549 void __iomem *scr_addr = ap->host->ports[0]->ioaddr.scr_addr;
39f87582
TH		 1550 int shift = ap->port_no * NV_INT_PORT_SHIFT;
1551 u8 mask;
1552
0d5ff566 1553 mask = ioread8(scr_addr + NV_INT_ENABLE);
39f87582 1554 mask &= ~(NV_INT_ALL << shift);
0d5ff566 1555 iowrite8(mask, scr_addr + NV_INT_ENABLE);
39f87582
TH		 1556}
1557
1558static void nv_nf2_thaw(struct ata_port *ap)
1559{
0d5ff566 1560 void __iomem *scr_addr = ap->host->ports[0]->ioaddr.scr_addr;
39f87582
TH		 1561 int shift = ap->port_no * NV_INT_PORT_SHIFT;
1562 u8 mask;
1563
0d5ff566 1564 iowrite8(NV_INT_ALL << shift, scr_addr + NV_INT_STATUS);
39f87582 1565
0d5ff566 1566 mask = ioread8(scr_addr + NV_INT_ENABLE);
39f87582 1567 mask |= (NV_INT_MASK << shift);
0d5ff566 1568 iowrite8(mask, scr_addr + NV_INT_ENABLE);
39f87582
TH		 1569}
1570
1571static void nv_ck804_freeze(struct ata_port *ap)
1572{
0d5ff566 1573 void __iomem *mmio_base = ap->host->iomap[NV_MMIO_BAR];
39f87582
TH		 1574 int shift = ap->port_no * NV_INT_PORT_SHIFT;
1575 u8 mask;
1576
1577 mask = readb(mmio_base + NV_INT_ENABLE_CK804);
1578 mask &= ~(NV_INT_ALL << shift);
1579 writeb(mask, mmio_base + NV_INT_ENABLE_CK804);
1580}
1581
1582static void nv_ck804_thaw(struct ata_port *ap)
1583{
0d5ff566 1584 void __iomem *mmio_base = ap->host->iomap[NV_MMIO_BAR];
39f87582
TH		 1585 int shift = ap->port_no * NV_INT_PORT_SHIFT;
1586 u8 mask;
1587
1588 writeb(NV_INT_ALL << shift, mmio_base + NV_INT_STATUS_CK804);
1589
1590 mask = readb(mmio_base + NV_INT_ENABLE_CK804);
1591 mask |= (NV_INT_MASK << shift);
1592 writeb(mask, mmio_base + NV_INT_ENABLE_CK804);
1593}
1594
f140f0f1
KL		 1595static void nv_mcp55_freeze(struct ata_port *ap)
1596{
1597 void __iomem *mmio_base = ap->host->iomap[NV_MMIO_BAR];
1598 int shift = ap->port_no * NV_INT_PORT_SHIFT_MCP55;
1599 u32 mask;
1600
1601 writel(NV_INT_ALL_MCP55 << shift, mmio_base + NV_INT_STATUS_MCP55);
1602
1603 mask = readl(mmio_base + NV_INT_ENABLE_MCP55);
1604 mask &= ~(NV_INT_ALL_MCP55 << shift);
1605 writel(mask, mmio_base + NV_INT_ENABLE_MCP55);
1606 ata_bmdma_freeze(ap);
1607}
1608
1609static void nv_mcp55_thaw(struct ata_port *ap)
1610{
1611 void __iomem *mmio_base = ap->host->iomap[NV_MMIO_BAR];
1612 int shift = ap->port_no * NV_INT_PORT_SHIFT_MCP55;
1613 u32 mask;
1614
1615 writel(NV_INT_ALL_MCP55 << shift, mmio_base + NV_INT_STATUS_MCP55);
1616
1617 mask = readl(mmio_base + NV_INT_ENABLE_MCP55);
1618 mask |= (NV_INT_MASK_MCP55 << shift);
1619 writel(mask, mmio_base + NV_INT_ENABLE_MCP55);
1620 ata_bmdma_thaw(ap);
1621}
1622
cc0680a5 1623static int nv_hardreset(struct ata_link *link, unsigned int *class,
d4b2bab4 1624 unsigned long deadline)
39f87582
TH		 1625{
1626 unsigned int dummy;
1627
1628 /* SATA hardreset fails to retrieve proper device signature on
1629 * some controllers. Don't classify on hardreset. For more
1630 * info, see http://bugme.osdl.org/show_bug.cgi?id=3352
1631 */
cc0680a5 1632 return sata_std_hardreset(link, &dummy, deadline);
39f87582
TH		 1633}
1634
1635static void nv_error_handler(struct ata_port *ap)
1636{
1637 ata_bmdma_drive_eh(ap, ata_std_prereset, ata_std_softreset,
1638 nv_hardreset, ata_std_postreset);
1639}
1640
fbbb262d
RH		 1641static void nv_adma_error_handler(struct ata_port *ap)
1642{
1643 struct nv_adma_port_priv *pp = ap->private_data;
b447916e 1644 if (!(pp->flags & NV_ADMA_PORT_REGISTER_MODE)) {
cdf56bcf 1645 void __iomem *mmio = pp->ctl_block;
fbbb262d
RH		 1646 int i;
1647 u16 tmp;
a84471fe 1648
b447916e 1649 if (ata_tag_valid(ap->link.active_tag) || ap->link.sactive) {
2cb27853
RH		 1650 u32 notifier = readl(mmio + NV_ADMA_NOTIFIER);
1651 u32 notifier_error = readl(mmio + NV_ADMA_NOTIFIER_ERROR);
1652 u32 gen_ctl = readl(pp->gen_block + NV_ADMA_GEN_CTL);
1653 u32 status = readw(mmio + NV_ADMA_STAT);
08af7414
RH		 1654 u8 cpb_count = readb(mmio + NV_ADMA_CPB_COUNT);
1655 u8 next_cpb_idx = readb(mmio + NV_ADMA_NEXT_CPB_IDX);
2cb27853
RH		 1656
1657 ata_port_printk(ap, KERN_ERR, "EH in ADMA mode, notifier 0x%X "
08af7414
RH		 1658 "notifier_error 0x%X gen_ctl 0x%X status 0x%X "
1659 "next cpb count 0x%X next cpb idx 0x%x\n",
1660 notifier, notifier_error, gen_ctl, status,
1661 cpb_count, next_cpb_idx);
2cb27853 1662
b447916e 1663 for (i = 0; i < NV_ADMA_MAX_CPBS; i++) {
2cb27853 1664 struct nv_adma_cpb *cpb = &pp->cpb[i];
b447916e 1665 if ((ata_tag_valid(ap->link.active_tag) && i == ap->link.active_tag) ||
9af5c9c9 1666 ap->link.sactive & (1 << i) )
2cb27853
RH		 1667 ata_port_printk(ap, KERN_ERR,
1668 "CPB %d: ctl_flags 0x%x, resp_flags 0x%x\n",
1669 i, cpb->ctl_flags, cpb->resp_flags);
1670 }
1671 }
fbbb262d 1672
fbbb262d
RH		 1673 /* Push us back into port register mode for error handling. */
1674 nv_adma_register_mode(ap);
1675
fbbb262d 1676 /* Mark all of the CPBs as invalid to prevent them from being executed */
b447916e 1677 for (i = 0; i < NV_ADMA_MAX_CPBS; i++)
fbbb262d
RH		 1678 pp->cpb[i].ctl_flags &= ~NV_CPB_CTL_CPB_VALID;
1679
1680 /* clear CPB fetch count */
1681 writew(0, mmio + NV_ADMA_CPB_COUNT);
1682
1683 /* Reset channel */
1684 tmp = readw(mmio + NV_ADMA_CTL);
1685 writew(tmp | NV_ADMA_CTL_CHANNEL_RESET, mmio + NV_ADMA_CTL);
b447916e 1686 readw(mmio + NV_ADMA_CTL); /* flush posted write */
fbbb262d
RH		 1687 udelay(1);
1688 writew(tmp & ~NV_ADMA_CTL_CHANNEL_RESET, mmio + NV_ADMA_CTL);
b447916e 1689 readw(mmio + NV_ADMA_CTL); /* flush posted write */
fbbb262d
RH		 1690 }
1691
1692 ata_bmdma_drive_eh(ap, ata_std_prereset, ata_std_softreset,
1693 nv_hardreset, ata_std_postreset);
1694}
1695
f140f0f1
KL		 1696static void nv_swncq_qc_to_dq(struct ata_port *ap, struct ata_queued_cmd *qc)
1697{
1698 struct nv_swncq_port_priv *pp = ap->private_data;
1699 struct defer_queue *dq = &pp->defer_queue;
1700
1701 /* queue is full */
1702 WARN_ON(dq->tail - dq->head == ATA_MAX_QUEUE);
1703 dq->defer_bits |= (1 << qc->tag);
1704 dq->tag[dq->tail++ & (ATA_MAX_QUEUE - 1)] = qc->tag;
1705}
1706
1707static struct ata_queued_cmd *nv_swncq_qc_from_dq(struct ata_port *ap)
1708{
1709 struct nv_swncq_port_priv *pp = ap->private_data;
1710 struct defer_queue *dq = &pp->defer_queue;
1711 unsigned int tag;
1712
1713 if (dq->head == dq->tail) /* null queue */
1714 return NULL;
1715
1716 tag = dq->tag[dq->head & (ATA_MAX_QUEUE - 1)];
1717 dq->tag[dq->head++ & (ATA_MAX_QUEUE - 1)] = ATA_TAG_POISON;
1718 WARN_ON(!(dq->defer_bits & (1 << tag)));
1719 dq->defer_bits &= ~(1 << tag);
1720
1721 return ata_qc_from_tag(ap, tag);
1722}
1723
1724static void nv_swncq_fis_reinit(struct ata_port *ap)
1725{
1726 struct nv_swncq_port_priv *pp = ap->private_data;
1727
1728 pp->dhfis_bits = 0;
1729 pp->dmafis_bits = 0;
1730 pp->sdbfis_bits = 0;
1731 pp->ncq_flags = 0;
1732}
1733
1734static void nv_swncq_pp_reinit(struct ata_port *ap)
1735{
1736 struct nv_swncq_port_priv *pp = ap->private_data;
1737 struct defer_queue *dq = &pp->defer_queue;
1738
1739 dq->head = 0;
1740 dq->tail = 0;
1741 dq->defer_bits = 0;
1742 pp->qc_active = 0;
1743 pp->last_issue_tag = ATA_TAG_POISON;
1744 nv_swncq_fis_reinit(ap);
1745}
1746
1747static void nv_swncq_irq_clear(struct ata_port *ap, u16 fis)
1748{
1749 struct nv_swncq_port_priv *pp = ap->private_data;
1750
1751 writew(fis, pp->irq_block);
1752}
1753
1754static void __ata_bmdma_stop(struct ata_port *ap)
1755{
1756 struct ata_queued_cmd qc;
1757
1758 qc.ap = ap;
1759 ata_bmdma_stop(&qc);
1760}
1761
1762static void nv_swncq_ncq_stop(struct ata_port *ap)
1763{
1764 struct nv_swncq_port_priv *pp = ap->private_data;
1765 unsigned int i;
1766 u32 sactive;
1767 u32 done_mask;
1768
1769 ata_port_printk(ap, KERN_ERR,
1770 "EH in SWNCQ mode,QC:qc_active 0x%X sactive 0x%X\n",
1771 ap->qc_active, ap->link.sactive);
1772 ata_port_printk(ap, KERN_ERR,
1773 "SWNCQ:qc_active 0x%X defer_bits 0x%X last_issue_tag 0x%x\n "
1774 "dhfis 0x%X dmafis 0x%X sdbfis 0x%X\n",
1775 pp->qc_active, pp->defer_queue.defer_bits, pp->last_issue_tag,
1776 pp->dhfis_bits, pp->dmafis_bits, pp->sdbfis_bits);
1777
1778 ata_port_printk(ap, KERN_ERR, "ATA_REG 0x%X ERR_REG 0x%X\n",
1779 ap->ops->check_status(ap),
1780 ioread8(ap->ioaddr.error_addr));
1781
1782 sactive = readl(pp->sactive_block);
1783 done_mask = pp->qc_active ^ sactive;
1784
1785 ata_port_printk(ap, KERN_ERR, "tag : dhfis dmafis sdbfis sacitve\n");
1786 for (i = 0; i < ATA_MAX_QUEUE; i++) {
1787 u8 err = 0;
1788 if (pp->qc_active & (1 << i))
1789 err = 0;
1790 else if (done_mask & (1 << i))
1791 err = 1;
1792 else
1793 continue;
1794
1795 ata_port_printk(ap, KERN_ERR,
1796 "tag 0x%x: %01x %01x %01x %01x %s\n", i,
1797 (pp->dhfis_bits >> i) & 0x1,
1798 (pp->dmafis_bits >> i) & 0x1,
1799 (pp->sdbfis_bits >> i) & 0x1,
1800 (sactive >> i) & 0x1,
1801 (err ? "error! tag doesn't exit" : " "));
1802 }
1803
1804 nv_swncq_pp_reinit(ap);
1805 ap->ops->irq_clear(ap);
1806 __ata_bmdma_stop(ap);
1807 nv_swncq_irq_clear(ap, 0xffff);
1808}
1809
1810static void nv_swncq_error_handler(struct ata_port *ap)
1811{
1812 struct ata_eh_context *ehc = &ap->link.eh_context;
1813
1814 if (ap->link.sactive) {
1815 nv_swncq_ncq_stop(ap);
1816 ehc->i.action |= ATA_EH_HARDRESET;
1817 }
1818
1819 ata_bmdma_drive_eh(ap, ata_std_prereset, ata_std_softreset,
1820 nv_hardreset, ata_std_postreset);
1821}
1822
1823#ifdef CONFIG_PM
1824static int nv_swncq_port_suspend(struct ata_port *ap, pm_message_t mesg)
1825{
1826 void __iomem *mmio = ap->host->iomap[NV_MMIO_BAR];
1827 u32 tmp;
1828
1829 /* clear irq */
1830 writel(~0, mmio + NV_INT_STATUS_MCP55);
1831
1832 /* disable irq */
1833 writel(0, mmio + NV_INT_ENABLE_MCP55);
1834
1835 /* disable swncq */
1836 tmp = readl(mmio + NV_CTL_MCP55);
1837 tmp &= ~(NV_CTL_PRI_SWNCQ | NV_CTL_SEC_SWNCQ);
1838 writel(tmp, mmio + NV_CTL_MCP55);
1839
1840 return 0;
1841}
1842
1843static int nv_swncq_port_resume(struct ata_port *ap)
1844{
1845 void __iomem *mmio = ap->host->iomap[NV_MMIO_BAR];
1846 u32 tmp;
1847
1848 /* clear irq */
1849 writel(~0, mmio + NV_INT_STATUS_MCP55);
1850
1851 /* enable irq */
1852 writel(0x00fd00fd, mmio + NV_INT_ENABLE_MCP55);
1853
1854 /* enable swncq */
1855 tmp = readl(mmio + NV_CTL_MCP55);
1856 writel(tmp | NV_CTL_PRI_SWNCQ | NV_CTL_SEC_SWNCQ, mmio + NV_CTL_MCP55);
1857
1858 return 0;
1859}
1860#endif
1861
1862static void nv_swncq_host_init(struct ata_host *host)
1863{
1864 u32 tmp;
1865 void __iomem *mmio = host->iomap[NV_MMIO_BAR];
1866 struct pci_dev *pdev = to_pci_dev(host->dev);
1867 u8 regval;
1868
1869 /* disable ECO 398 */
1870 pci_read_config_byte(pdev, 0x7f, &regval);
1871 regval &= ~(1 << 7);
1872 pci_write_config_byte(pdev, 0x7f, regval);
1873
1874 /* enable swncq */
1875 tmp = readl(mmio + NV_CTL_MCP55);
1876 VPRINTK("HOST_CTL:0x%X\n", tmp);
1877 writel(tmp | NV_CTL_PRI_SWNCQ | NV_CTL_SEC_SWNCQ, mmio + NV_CTL_MCP55);
1878
1879 /* enable irq intr */
1880 tmp = readl(mmio + NV_INT_ENABLE_MCP55);
1881 VPRINTK("HOST_ENABLE:0x%X\n", tmp);
1882 writel(tmp | 0x00fd00fd, mmio + NV_INT_ENABLE_MCP55);
1883
1884 /* clear port irq */
1885 writel(~0x0, mmio + NV_INT_STATUS_MCP55);
1886}
1887
1888static int nv_swncq_slave_config(struct scsi_device *sdev)
1889{
1890 struct ata_port *ap = ata_shost_to_port(sdev->host);
1891 struct pci_dev *pdev = to_pci_dev(ap->host->dev);
1892 struct ata_device *dev;
1893 int rc;
1894 u8 rev;
1895 u8 check_maxtor = 0;
1896 unsigned char model_num[ATA_ID_PROD_LEN + 1];
1897
1898 rc = ata_scsi_slave_config(sdev);
1899 if (sdev->id >= ATA_MAX_DEVICES || sdev->channel || sdev->lun)
1900 /* Not a proper libata device, ignore */
1901 return rc;
1902
1903 dev = &ap->link.device[sdev->id];
1904 if (!(ap->flags & ATA_FLAG_NCQ) || dev->class == ATA_DEV_ATAPI)
1905 return rc;
1906
1907 /* if MCP51 and Maxtor, then disable ncq */
1908 if (pdev->device == PCI_DEVICE_ID_NVIDIA_NFORCE_MCP51_SATA ||
1909 pdev->device == PCI_DEVICE_ID_NVIDIA_NFORCE_MCP51_SATA2)
1910 check_maxtor = 1;
1911
1912 /* if MCP55 and rev <= a2 and Maxtor, then disable ncq */
1913 if (pdev->device == PCI_DEVICE_ID_NVIDIA_NFORCE_MCP55_SATA ||
1914 pdev->device == PCI_DEVICE_ID_NVIDIA_NFORCE_MCP55_SATA2) {
1915 pci_read_config_byte(pdev, 0x8, &rev);
1916 if (rev <= 0xa2)
1917 check_maxtor = 1;
1918 }
1919
1920 if (!check_maxtor)
1921 return rc;
1922
1923 ata_id_c_string(dev->id, model_num, ATA_ID_PROD, sizeof(model_num));
1924
1925 if (strncmp(model_num, "Maxtor", 6) == 0) {
1926 ata_scsi_change_queue_depth(sdev, 1);
1927 ata_dev_printk(dev, KERN_NOTICE,
1928 "Disabling SWNCQ mode (depth %x)\n", sdev->queue_depth);
1929 }
1930
1931 return rc;
1932}
1933
1934static int nv_swncq_port_start(struct ata_port *ap)
1935{
1936 struct device *dev = ap->host->dev;
1937 void __iomem *mmio = ap->host->iomap[NV_MMIO_BAR];
1938 struct nv_swncq_port_priv *pp;
1939 int rc;
1940
1941 rc = ata_port_start(ap);
1942 if (rc)
1943 return rc;
1944
1945 pp = devm_kzalloc(dev, sizeof(*pp), GFP_KERNEL);
1946 if (!pp)
1947 return -ENOMEM;
1948
1949 pp->prd = dmam_alloc_coherent(dev, ATA_PRD_TBL_SZ * ATA_MAX_QUEUE,
1950 &pp->prd_dma, GFP_KERNEL);
1951 if (!pp->prd)
1952 return -ENOMEM;
1953 memset(pp->prd, 0, ATA_PRD_TBL_SZ * ATA_MAX_QUEUE);
1954
1955 ap->private_data = pp;
1956 pp->sactive_block = ap->ioaddr.scr_addr + 4 * SCR_ACTIVE;
1957 pp->irq_block = mmio + NV_INT_STATUS_MCP55 + ap->port_no * 2;
1958 pp->tag_block = mmio + NV_NCQ_REG_MCP55 + ap->port_no * 2;
1959
1960 return 0;
1961}
1962
1963static void nv_swncq_qc_prep(struct ata_queued_cmd *qc)
1964{
1965 if (qc->tf.protocol != ATA_PROT_NCQ) {
1966 ata_qc_prep(qc);
1967 return;
1968 }
1969
1970 if (!(qc->flags & ATA_QCFLAG_DMAMAP))
1971 return;
1972
1973 nv_swncq_fill_sg(qc);
1974}
1975
1976static void nv_swncq_fill_sg(struct ata_queued_cmd *qc)
1977{
1978 struct ata_port *ap = qc->ap;
1979 struct scatterlist *sg;
1980 unsigned int idx;
1981 struct nv_swncq_port_priv *pp = ap->private_data;
1982 struct ata_prd *prd;
1983
1984 WARN_ON(qc->__sg == NULL);
1985 WARN_ON(qc->n_elem == 0 && qc->pad_len == 0);
1986
1987 prd = pp->prd + ATA_MAX_PRD * qc->tag;
1988
1989 idx = 0;
1990 ata_for_each_sg(sg, qc) {
1991 u32 addr, offset;
1992 u32 sg_len, len;
1993
1994 addr = (u32)sg_dma_address(sg);
1995 sg_len = sg_dma_len(sg);
1996
1997 while (sg_len) {
1998 offset = addr & 0xffff;
1999 len = sg_len;
2000 if ((offset + sg_len) > 0x10000)
2001 len = 0x10000 - offset;
2002
2003 prd[idx].addr = cpu_to_le32(addr);
2004 prd[idx].flags_len = cpu_to_le32(len & 0xffff);
2005
2006 idx++;
2007 sg_len -= len;
2008 addr += len;
2009 }
2010 }
2011
2012 if (idx)
2013 prd[idx - 1].flags_len |= cpu_to_le32(ATA_PRD_EOT);
2014}
2015
2016static unsigned int nv_swncq_issue_atacmd(struct ata_port *ap,
2017 struct ata_queued_cmd *qc)
2018{
2019 struct nv_swncq_port_priv *pp = ap->private_data;
2020
2021 if (qc == NULL)
2022 return 0;
2023
2024 DPRINTK("Enter\n");
2025
2026 writel((1 << qc->tag), pp->sactive_block);
2027 pp->last_issue_tag = qc->tag;
2028 pp->dhfis_bits &= ~(1 << qc->tag);
2029 pp->dmafis_bits &= ~(1 << qc->tag);
2030 pp->qc_active |= (0x1 << qc->tag);
2031
2032 ap->ops->tf_load(ap, &qc->tf); /* load tf registers */
2033 ap->ops->exec_command(ap, &qc->tf);
2034
2035 DPRINTK("Issued tag %u\n", qc->tag);
2036
2037 return 0;
2038}
2039
2040static unsigned int nv_swncq_qc_issue(struct ata_queued_cmd *qc)
2041{
2042 struct ata_port *ap = qc->ap;
2043 struct nv_swncq_port_priv *pp = ap->private_data;
2044
2045 if (qc->tf.protocol != ATA_PROT_NCQ)
2046 return ata_qc_issue_prot(qc);
2047
2048 DPRINTK("Enter\n");
2049
2050 if (!pp->qc_active)
2051 nv_swncq_issue_atacmd(ap, qc);
2052 else
2053 nv_swncq_qc_to_dq(ap, qc); /* add qc to defer queue */
2054
2055 return 0;
2056}
2057
2058static void nv_swncq_hotplug(struct ata_port *ap, u32 fis)
2059{
2060 u32 serror;
2061 struct ata_eh_info *ehi = &ap->link.eh_info;
2062
2063 ata_ehi_clear_desc(ehi);
2064
2065 /* AHCI needs SError cleared; otherwise, it might lock up */
2066 sata_scr_read(&ap->link, SCR_ERROR, &serror);
2067 sata_scr_write(&ap->link, SCR_ERROR, serror);
2068
2069 /* analyze @irq_stat */
2070 if (fis & NV_SWNCQ_IRQ_ADDED)
2071 ata_ehi_push_desc(ehi, "hot plug");
2072 else if (fis & NV_SWNCQ_IRQ_REMOVED)
2073 ata_ehi_push_desc(ehi, "hot unplug");
2074
2075 ata_ehi_hotplugged(ehi);
2076
2077 /* okay, let's hand over to EH */
2078 ehi->serror |= serror;
2079
2080 ata_port_freeze(ap);
2081}
2082
2083static int nv_swncq_sdbfis(struct ata_port *ap)
2084{
2085 struct ata_queued_cmd *qc;
2086 struct nv_swncq_port_priv *pp = ap->private_data;
2087 struct ata_eh_info *ehi = &ap->link.eh_info;
2088 u32 sactive;
2089 int nr_done = 0;
2090 u32 done_mask;
2091 int i;
2092 u8 host_stat;
2093 u8 lack_dhfis = 0;
2094
2095 host_stat = ap->ops->bmdma_status(ap);
2096 if (unlikely(host_stat & ATA_DMA_ERR)) {
2097 /* error when transfering data to/from memory */
2098 ata_ehi_clear_desc(ehi);
2099 ata_ehi_push_desc(ehi, "BMDMA stat 0x%x", host_stat);
2100 ehi->err_mask |= AC_ERR_HOST_BUS;
2101 ehi->action |= ATA_EH_SOFTRESET;
2102 return -EINVAL;
2103 }
2104
2105 ap->ops->irq_clear(ap);
2106 __ata_bmdma_stop(ap);
2107
2108 sactive = readl(pp->sactive_block);
2109 done_mask = pp->qc_active ^ sactive;
2110
2111 if (unlikely(done_mask & sactive)) {
2112 ata_ehi_clear_desc(ehi);
2113 ata_ehi_push_desc(ehi, "illegal SWNCQ:qc_active transition"
2114 "(%08x->%08x)", pp->qc_active, sactive);
2115 ehi->err_mask |= AC_ERR_HSM;
2116 ehi->action |= ATA_EH_HARDRESET;
2117 return -EINVAL;
2118 }
2119 for (i = 0; i < ATA_MAX_QUEUE; i++) {
2120 if (!(done_mask & (1 << i)))
2121 continue;
2122
2123 qc = ata_qc_from_tag(ap, i);
2124 if (qc) {
2125 ata_qc_complete(qc);
2126 pp->qc_active &= ~(1 << i);
2127 pp->dhfis_bits &= ~(1 << i);
2128 pp->dmafis_bits &= ~(1 << i);
2129 pp->sdbfis_bits |= (1 << i);
2130 nr_done++;
2131 }
2132 }
2133
2134 if (!ap->qc_active) {
2135 DPRINTK("over\n");
2136 nv_swncq_pp_reinit(ap);
2137 return nr_done;
2138 }
2139
2140 if (pp->qc_active & pp->dhfis_bits)
2141 return nr_done;
2142
2143 if ((pp->ncq_flags & ncq_saw_backout) ||
2144 (pp->qc_active ^ pp->dhfis_bits))
2145 /* if the controller cann't get a device to host register FIS,
2146 * The driver needs to reissue the new command.
2147 */
2148 lack_dhfis = 1;
2149
2150 DPRINTK("id 0x%x QC: qc_active 0x%x,"
2151 "SWNCQ:qc_active 0x%X defer_bits %X "
2152 "dhfis 0x%X dmafis 0x%X last_issue_tag %x\n",
2153 ap->print_id, ap->qc_active, pp->qc_active,
2154 pp->defer_queue.defer_bits, pp->dhfis_bits,
2155 pp->dmafis_bits, pp->last_issue_tag);
2156
2157 nv_swncq_fis_reinit(ap);
2158
2159 if (lack_dhfis) {
2160 qc = ata_qc_from_tag(ap, pp->last_issue_tag);
2161 nv_swncq_issue_atacmd(ap, qc);
2162 return nr_done;
2163 }
2164
2165 if (pp->defer_queue.defer_bits) {
2166 /* send deferral queue command */
2167 qc = nv_swncq_qc_from_dq(ap);
2168 WARN_ON(qc == NULL);
2169 nv_swncq_issue_atacmd(ap, qc);
2170 }
2171
2172 return nr_done;
2173}
2174
2175static inline u32 nv_swncq_tag(struct ata_port *ap)
2176{
2177 struct nv_swncq_port_priv *pp = ap->private_data;
2178 u32 tag;
2179
2180 tag = readb(pp->tag_block) >> 2;
2181 return (tag & 0x1f);
2182}
2183
2184static int nv_swncq_dmafis(struct ata_port *ap)
2185{
2186 struct ata_queued_cmd *qc;
2187 unsigned int rw;
2188 u8 dmactl;
2189 u32 tag;
2190 struct nv_swncq_port_priv *pp = ap->private_data;
2191
2192 __ata_bmdma_stop(ap);
2193 tag = nv_swncq_tag(ap);
2194
2195 DPRINTK("dma setup tag 0x%x\n", tag);
2196 qc = ata_qc_from_tag(ap, tag);
2197
2198 if (unlikely(!qc))
2199 return 0;
2200
2201 rw = qc->tf.flags & ATA_TFLAG_WRITE;
2202
2203 /* load PRD table addr. */
2204 iowrite32(pp->prd_dma + ATA_PRD_TBL_SZ * qc->tag,
2205 ap->ioaddr.bmdma_addr + ATA_DMA_TABLE_OFS);
2206
2207 /* specify data direction, triple-check start bit is clear */
2208 dmactl = ioread8(ap->ioaddr.bmdma_addr + ATA_DMA_CMD);
2209 dmactl &= ~ATA_DMA_WR;
2210 if (!rw)
2211 dmactl |= ATA_DMA_WR;
2212
2213 iowrite8(dmactl | ATA_DMA_START, ap->ioaddr.bmdma_addr + ATA_DMA_CMD);
2214
2215 return 1;
2216}
2217
2218static void nv_swncq_host_interrupt(struct ata_port *ap, u16 fis)
2219{
2220 struct nv_swncq_port_priv *pp = ap->private_data;
2221 struct ata_queued_cmd *qc;
2222 struct ata_eh_info *ehi = &ap->link.eh_info;
2223 u32 serror;
2224 u8 ata_stat;
2225 int rc = 0;
2226
2227 ata_stat = ap->ops->check_status(ap);
2228 nv_swncq_irq_clear(ap, fis);
2229 if (!fis)
2230 return;
2231
2232 if (ap->pflags & ATA_PFLAG_FROZEN)
2233 return;
2234
2235 if (fis & NV_SWNCQ_IRQ_HOTPLUG) {
2236 nv_swncq_hotplug(ap, fis);
2237 return;
2238 }
2239
2240 if (!pp->qc_active)
2241 return;
2242
2243 if (ap->ops->scr_read(ap, SCR_ERROR, &serror))
2244 return;
2245 ap->ops->scr_write(ap, SCR_ERROR, serror);
2246
2247 if (ata_stat & ATA_ERR) {
2248 ata_ehi_clear_desc(ehi);
2249 ata_ehi_push_desc(ehi, "Ata error. fis:0x%X", fis);
2250 ehi->err_mask |= AC_ERR_DEV;
2251 ehi->serror |= serror;
2252 ehi->action |= ATA_EH_SOFTRESET;
2253 ata_port_freeze(ap);
2254 return;
2255 }
2256
2257 if (fis & NV_SWNCQ_IRQ_BACKOUT) {
2258 /* If the IRQ is backout, driver must issue
2259 * the new command again some time later.
2260 */
2261 pp->ncq_flags |= ncq_saw_backout;
2262 }
2263
2264 if (fis & NV_SWNCQ_IRQ_SDBFIS) {
2265 pp->ncq_flags |= ncq_saw_sdb;
2266 DPRINTK("id 0x%x SWNCQ: qc_active 0x%X "
2267 "dhfis 0x%X dmafis 0x%X sactive 0x%X\n",
2268 ap->print_id, pp->qc_active, pp->dhfis_bits,
2269 pp->dmafis_bits, readl(pp->sactive_block));
2270 rc = nv_swncq_sdbfis(ap);
2271 if (rc < 0)
2272 goto irq_error;
2273 }
2274
2275 if (fis & NV_SWNCQ_IRQ_DHREGFIS) {
2276 /* The interrupt indicates the new command
2277 * was transmitted correctly to the drive.
2278 */
2279 pp->dhfis_bits |= (0x1 << pp->last_issue_tag);
2280 pp->ncq_flags |= ncq_saw_d2h;
2281 if (pp->ncq_flags & (ncq_saw_sdb | ncq_saw_backout)) {
2282 ata_ehi_push_desc(ehi, "illegal fis transaction");
2283 ehi->err_mask |= AC_ERR_HSM;
2284 ehi->action |= ATA_EH_HARDRESET;
2285 goto irq_error;
2286 }
2287
2288 if (!(fis & NV_SWNCQ_IRQ_DMASETUP) &&
2289 !(pp->ncq_flags & ncq_saw_dmas)) {
2290 ata_stat = ap->ops->check_status(ap);
2291 if (ata_stat & ATA_BUSY)
2292 goto irq_exit;
2293
2294 if (pp->defer_queue.defer_bits) {
2295 DPRINTK("send next command\n");
2296 qc = nv_swncq_qc_from_dq(ap);
2297 nv_swncq_issue_atacmd(ap, qc);
2298 }
2299 }
2300 }
2301
2302 if (fis & NV_SWNCQ_IRQ_DMASETUP) {
2303 /* program the dma controller with appropriate PRD buffers
2304 * and start the DMA transfer for requested command.
2305 */
2306 pp->dmafis_bits |= (0x1 << nv_swncq_tag(ap));
2307 pp->ncq_flags |= ncq_saw_dmas;
2308 rc = nv_swncq_dmafis(ap);
2309 }
2310
2311irq_exit:
2312 return;
2313irq_error:
2314 ata_ehi_push_desc(ehi, "fis:0x%x", fis);
2315 ata_port_freeze(ap);
2316 return;
2317}
2318
2319static irqreturn_t nv_swncq_interrupt(int irq, void *dev_instance)
2320{
2321 struct ata_host *host = dev_instance;
2322 unsigned int i;
2323 unsigned int handled = 0;
2324 unsigned long flags;
2325 u32 irq_stat;
2326
2327 spin_lock_irqsave(&host->lock, flags);
2328
2329 irq_stat = readl(host->iomap[NV_MMIO_BAR] + NV_INT_STATUS_MCP55);
2330
2331 for (i = 0; i < host->n_ports; i++) {
2332 struct ata_port *ap = host->ports[i];
2333
2334 if (ap && !(ap->flags & ATA_FLAG_DISABLED)) {
2335 if (ap->link.sactive) {
2336 nv_swncq_host_interrupt(ap, (u16)irq_stat);
2337 handled = 1;
2338 } else {
2339 if (irq_stat) /* reserve Hotplug */
2340 nv_swncq_irq_clear(ap, 0xfff0);
2341
2342 handled += nv_host_intr(ap, (u8)irq_stat);
2343 }
2344 }
2345 irq_stat >>= NV_INT_PORT_SHIFT_MCP55;
2346 }
2347
2348 spin_unlock_irqrestore(&host->lock, flags);
2349
2350 return IRQ_RETVAL(handled);
2351}
2352
1da177e4
LT		 2353static int nv_init_one (struct pci_dev *pdev, const struct pci_device_id *ent)
2354{
2355 static int printed_version = 0;
1626aeb8 2356 const struct ata_port_info *ppi[] = { NULL, NULL };
9a829ccf 2357 struct ata_host *host;
cdf56bcf 2358 struct nv_host_priv *hpriv;
1da177e4
LT		 2359 int rc;
2360 u32 bar;
0d5ff566 2361 void __iomem *base;
fbbb262d 2362 unsigned long type = ent->driver_data;
1da177e4
LT		 2363
2364 // Make sure this is a SATA controller by counting the number of bars
2365 // (NVIDIA SATA controllers will always have six bars). Otherwise,
2366 // it's an IDE controller and we ignore it.
2367 for (bar=0; bar<6; bar++)
2368 if (pci_resource_start(pdev, bar) == 0)
2369 return -ENODEV;
2370
cdf56bcf 2371 if (!printed_version++)
a9524a76 2372 dev_printk(KERN_DEBUG, &pdev->dev, "version " DRV_VERSION "\n");
1da177e4 2373
24dc5f33 2374 rc = pcim_enable_device(pdev);
1da177e4 2375 if (rc)
24dc5f33 2376 return rc;
1da177e4 2377
9a829ccf 2378 /* determine type and allocate host */
f140f0f1 2379 if (type == CK804 && adma_enabled) {
fbbb262d
RH		 2380 dev_printk(KERN_NOTICE, &pdev->dev, "Using ADMA mode\n");
2381 type = ADMA;
fbbb262d
RH		 2382 }
2383
1626aeb8 2384 ppi[0] = &nv_port_info[type];
d583bc18 2385 rc = ata_pci_prepare_sff_host(pdev, ppi, &host);
9a829ccf
TH		 2386 if (rc)
2387 return rc;
1da177e4 2388
24dc5f33 2389 hpriv = devm_kzalloc(&pdev->dev, sizeof(*hpriv), GFP_KERNEL);
cdf56bcf 2390 if (!hpriv)
24dc5f33 2391 return -ENOMEM;
9a829ccf
TH		 2392 hpriv->type = type;
2393 host->private_data = hpriv;
cdf56bcf 2394
9a829ccf
TH		 2395 /* set 64bit dma masks, may fail */
2396 if (type == ADMA) {
2397 if (pci_set_dma_mask(pdev, DMA_64BIT_MASK) == 0)
2398 pci_set_consistent_dma_mask(pdev, DMA_64BIT_MASK);
2399 }
1da177e4 2400
9a829ccf
TH		 2401 /* request and iomap NV_MMIO_BAR */
2402 rc = pcim_iomap_regions(pdev, 1 << NV_MMIO_BAR, DRV_NAME);
2403 if (rc)
2404 return rc;
1da177e4 2405
9a829ccf
TH		 2406 /* configure SCR access */
2407 base = host->iomap[NV_MMIO_BAR];
2408 host->ports[0]->ioaddr.scr_addr = base + NV_PORT0_SCR_REG_OFFSET;
2409 host->ports[1]->ioaddr.scr_addr = base + NV_PORT1_SCR_REG_OFFSET;
1da177e4 2410
ada364e8 2411 /* enable SATA space for CK804 */
fbbb262d 2412 if (type >= CK804) {
ada364e8
TH		 2413 u8 regval;
2414
2415 pci_read_config_byte(pdev, NV_MCP_SATA_CFG_20, &regval);
2416 regval |= NV_MCP_SATA_CFG_20_SATA_SPACE_EN;
2417 pci_write_config_byte(pdev, NV_MCP_SATA_CFG_20, regval);
2418 }
2419
9a829ccf 2420 /* init ADMA */
fbbb262d 2421 if (type == ADMA) {
9a829ccf 2422 rc = nv_adma_host_init(host);
fbbb262d 2423 if (rc)
24dc5f33 2424 return rc;
f140f0f1
KL		 2425 } else if (type == SWNCQ && swncq_enabled) {
2426 dev_printk(KERN_NOTICE, &pdev->dev, "Using SWNCQ mode\n");
2427 nv_swncq_host_init(host);
fbbb262d
RH		 2428 }
2429
9a829ccf
TH		 2430 pci_set_master(pdev);
2431 return ata_host_activate(host, pdev->irq, ppi[0]->irq_handler,
2432 IRQF_SHARED, ppi[0]->sht);
1da177e4
LT		 2433}
2434
438ac6d5 2435#ifdef CONFIG_PM
cdf56bcf
RH		 2436static int nv_pci_device_resume(struct pci_dev *pdev)
2437{
2438 struct ata_host *host = dev_get_drvdata(&pdev->dev);
2439 struct nv_host_priv *hpriv = host->private_data;
ce053fa8 2440 int rc;
cdf56bcf 2441
ce053fa8 2442 rc = ata_pci_device_do_resume(pdev);
b447916e 2443 if (rc)
ce053fa8 2444 return rc;
cdf56bcf
RH		 2445
2446 if (pdev->dev.power.power_state.event == PM_EVENT_SUSPEND) {
b447916e 2447 if (hpriv->type >= CK804) {
cdf56bcf
RH		 2448 u8 regval;
2449
2450 pci_read_config_byte(pdev, NV_MCP_SATA_CFG_20, &regval);
2451 regval |= NV_MCP_SATA_CFG_20_SATA_SPACE_EN;
2452 pci_write_config_byte(pdev, NV_MCP_SATA_CFG_20, regval);
2453 }
b447916e 2454 if (hpriv->type == ADMA) {
cdf56bcf
RH		 2455 u32 tmp32;
2456 struct nv_adma_port_priv *pp;
2457 /* enable/disable ADMA on the ports appropriately */
2458 pci_read_config_dword(pdev, NV_MCP_SATA_CFG_20, &tmp32);
2459
2460 pp = host->ports[0]->private_data;
b447916e 2461 if (pp->flags & NV_ADMA_ATAPI_SETUP_COMPLETE)
cdf56bcf
RH		 2462 tmp32 &= ~(NV_MCP_SATA_CFG_20_PORT0_EN |
2463 NV_MCP_SATA_CFG_20_PORT0_PWB_EN);
2464 else
2465 tmp32 |= (NV_MCP_SATA_CFG_20_PORT0_EN |
2466 NV_MCP_SATA_CFG_20_PORT0_PWB_EN);
2467 pp = host->ports[1]->private_data;
b447916e 2468 if (pp->flags & NV_ADMA_ATAPI_SETUP_COMPLETE)
cdf56bcf
RH		 2469 tmp32 &= ~(NV_MCP_SATA_CFG_20_PORT1_EN |
2470 NV_MCP_SATA_CFG_20_PORT1_PWB_EN);
2471 else
2472 tmp32 |= (NV_MCP_SATA_CFG_20_PORT1_EN |
2473 NV_MCP_SATA_CFG_20_PORT1_PWB_EN);
2474
2475 pci_write_config_dword(pdev, NV_MCP_SATA_CFG_20, tmp32);
2476 }
2477 }
2478
2479 ata_host_resume(host);
2480
2481 return 0;
2482}
438ac6d5 2483#endif
cdf56bcf 2484
cca3974e 2485static void nv_ck804_host_stop(struct ata_host *host)
ada364e8 2486{
cca3974e 2487 struct pci_dev *pdev = to_pci_dev(host->dev);
ada364e8
TH		 2488 u8 regval;
2489
2490 /* disable SATA space for CK804 */
2491 pci_read_config_byte(pdev, NV_MCP_SATA_CFG_20, &regval);
2492 regval &= ~NV_MCP_SATA_CFG_20_SATA_SPACE_EN;
2493 pci_write_config_byte(pdev, NV_MCP_SATA_CFG_20, regval);
ada364e8
TH		 2494}
2495
fbbb262d
RH		 2496static void nv_adma_host_stop(struct ata_host *host)
2497{
2498 struct pci_dev *pdev = to_pci_dev(host->dev);
fbbb262d
RH		 2499 u32 tmp32;
2500
fbbb262d
RH		 2501 /* disable ADMA on the ports */
2502 pci_read_config_dword(pdev, NV_MCP_SATA_CFG_20, &tmp32);
2503 tmp32 &= ~(NV_MCP_SATA_CFG_20_PORT0_EN |
2504 NV_MCP_SATA_CFG_20_PORT0_PWB_EN |
2505 NV_MCP_SATA_CFG_20_PORT1_EN |
2506 NV_MCP_SATA_CFG_20_PORT1_PWB_EN);
2507
2508 pci_write_config_dword(pdev, NV_MCP_SATA_CFG_20, tmp32);
2509
2510 nv_ck804_host_stop(host);
2511}
2512
1da177e4
LT		 2513static int __init nv_init(void)
2514{
b7887196 2515 return pci_register_driver(&nv_pci_driver);
1da177e4
LT		 2516}
2517
2518static void __exit nv_exit(void)
2519{
2520 pci_unregister_driver(&nv_pci_driver);
2521}
2522
2523module_init(nv_init);
2524module_exit(nv_exit);
fbbb262d
RH		 2525module_param_named(adma, adma_enabled, bool, 0444);
2526MODULE_PARM_DESC(adma, "Enable use of ADMA (Default: true)");
f140f0f1
KL		 2527module_param_named(swncq, swncq_enabled, bool, 0444);
2528MODULE_PARM_DESC(swncq, "Enable use of SWNCQ (Default: false)");
2529
Linux kernel - Deliverables
This page took 0.361077 seconds and 5 git commands to generate.