projects / deliverable / linux.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
rapidio/tsi721: fix locking in OB_MSG processing
[deliverable/linux.git] / drivers / rapidio / devices / tsi721.c
CommitLineData
48618fb4
AB		 1/*
2 * RapidIO mport driver for Tsi721 PCIExpress-to-SRIO bridge
3 *
4 * Copyright 2011 Integrated Device Technology, Inc.
5 * Alexandre Bounine <alexandre.bounine@idt.com>
6 * Chul Kim <chul.kim@idt.com>
7 *
8 * This program is free software; you can redistribute it and/or modify it
9 * under the terms of the GNU General Public License as published by the Free
10 * Software Foundation; either version 2 of the License, or (at your option)
11 * any later version.
12 *
13 * This program is distributed in the hope that it will be useful, but WITHOUT
14 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
15 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
16 * more details.
17 *
18 * You should have received a copy of the GNU General Public License along with
19 * this program; if not, write to the Free Software Foundation, Inc., 59
20 * Temple Place - Suite 330, Boston, MA 02111-1307, USA.
21 */
22
23#include <linux/io.h>
24#include <linux/errno.h>
25#include <linux/init.h>
26#include <linux/ioport.h>
27#include <linux/kernel.h>
28#include <linux/module.h>
29#include <linux/pci.h>
30#include <linux/rio.h>
31#include <linux/rio_drv.h>
32#include <linux/dma-mapping.h>
33#include <linux/interrupt.h>
34#include <linux/kfifo.h>
35#include <linux/delay.h>
36
37#include "tsi721.h"
38
48618fb4
AB		 39static void tsi721_omsg_handler(struct tsi721_device *priv, int ch);
40static void tsi721_imsg_handler(struct tsi721_device *priv, int ch);
41
42/**
43 * tsi721_lcread - read from local SREP config space
44 * @mport: RapidIO master port info
45 * @index: ID of RapdiIO interface
46 * @offset: Offset into configuration space
47 * @len: Length (in bytes) of the maintenance transaction
48 * @data: Value to be read into
49 *
50 * Generates a local SREP space read. Returns %0 on
51 * success or %-EINVAL on failure.
52 */
53static int tsi721_lcread(struct rio_mport *mport, int index, u32 offset,
54 int len, u32 *data)
55{
56 struct tsi721_device *priv = mport->priv;
57
58 if (len != sizeof(u32))
59 return -EINVAL; /* only 32-bit access is supported */
60
61 *data = ioread32(priv->regs + offset);
62
63 return 0;
64}
65
66/**
67 * tsi721_lcwrite - write into local SREP config space
68 * @mport: RapidIO master port info
69 * @index: ID of RapdiIO interface
70 * @offset: Offset into configuration space
71 * @len: Length (in bytes) of the maintenance transaction
72 * @data: Value to be written
73 *
74 * Generates a local write into SREP configuration space. Returns %0 on
75 * success or %-EINVAL on failure.
76 */
77static int tsi721_lcwrite(struct rio_mport *mport, int index, u32 offset,
78 int len, u32 data)
79{
80 struct tsi721_device *priv = mport->priv;
81
82 if (len != sizeof(u32))
83 return -EINVAL; /* only 32-bit access is supported */
84
85 iowrite32(data, priv->regs + offset);
86
87 return 0;
88}
89
90/**
91 * tsi721_maint_dma - Helper function to generate RapidIO maintenance
92 * transactions using designated Tsi721 DMA channel.
93 * @priv: pointer to tsi721 private data
94 * @sys_size: RapdiIO transport system size
95 * @destid: Destination ID of transaction
96 * @hopcount: Number of hops to target device
97 * @offset: Offset into configuration space
98 * @len: Length (in bytes) of the maintenance transaction
99 * @data: Location to be read from or write into
100 * @do_wr: Operation flag (1 == MAINT_WR)
101 *
102 * Generates a RapidIO maintenance transaction (Read or Write).
103 * Returns %0 on success and %-EINVAL or %-EFAULT on failure.
104 */
105static int tsi721_maint_dma(struct tsi721_device *priv, u32 sys_size,
106 u16 destid, u8 hopcount, u32 offset, int len,
107 u32 *data, int do_wr)
108{
9eaa3d9b 109 void __iomem *regs = priv->regs + TSI721_DMAC_BASE(priv->mdma.ch_id);
48618fb4
AB		 110 struct tsi721_dma_desc *bd_ptr;
111 u32 rd_count, swr_ptr, ch_stat;
112 int i, err = 0;
113 u32 op = do_wr ? MAINT_WR : MAINT_RD;
114
115 if (offset > (RIO_MAINT_SPACE_SZ - len) || (len != sizeof(u32)))
116 return -EINVAL;
117
9eaa3d9b 118 bd_ptr = priv->mdma.bd_base;
48618fb4 119
9eaa3d9b 120 rd_count = ioread32(regs + TSI721_DMAC_DRDCNT);
48618fb4
AB		 121
122 /* Initialize DMA descriptor */
123 bd_ptr[0].type_id = cpu_to_le32((DTYPE2 << 29) | (op << 19) | destid);
124 bd_ptr[0].bcount = cpu_to_le32((sys_size << 26) | 0x04);
125 bd_ptr[0].raddr_lo = cpu_to_le32((hopcount << 24) | offset);
126 bd_ptr[0].raddr_hi = 0;
127 if (do_wr)
128 bd_ptr[0].data[0] = cpu_to_be32p(data);
129 else
130 bd_ptr[0].data[0] = 0xffffffff;
131
132 mb();
133
134 /* Start DMA operation */
9eaa3d9b
AB		 135 iowrite32(rd_count + 2, regs + TSI721_DMAC_DWRCNT);
136 ioread32(regs + TSI721_DMAC_DWRCNT);
48618fb4
AB		 137 i = 0;
138
139 /* Wait until DMA transfer is finished */
9eaa3d9b
AB		 140 while ((ch_stat = ioread32(regs + TSI721_DMAC_STS))
141 & TSI721_DMAC_STS_RUN) {
48618fb4
AB		 142 udelay(1);
143 if (++i >= 5000000) {
144 dev_dbg(&priv->pdev->dev,
145 "%s : DMA[%d] read timeout ch_status=%x\n",
9eaa3d9b 146 __func__, priv->mdma.ch_id, ch_stat);
48618fb4
AB		 147 if (!do_wr)
148 *data = 0xffffffff;
149 err = -EIO;
150 goto err_out;
151 }
152 }
153
154 if (ch_stat & TSI721_DMAC_STS_ABORT) {
155 /* If DMA operation aborted due to error,
156 * reinitialize DMA channel
157 */
158 dev_dbg(&priv->pdev->dev, "%s : DMA ABORT ch_stat=%x\n",
159 __func__, ch_stat);
160 dev_dbg(&priv->pdev->dev, "OP=%d : destid=%x hc=%x off=%x\n",
161 do_wr ? MAINT_WR : MAINT_RD, destid, hopcount, offset);
9eaa3d9b
AB		 162 iowrite32(TSI721_DMAC_INT_ALL, regs + TSI721_DMAC_INT);
163 iowrite32(TSI721_DMAC_CTL_INIT, regs + TSI721_DMAC_CTL);
48618fb4 164 udelay(10);
9eaa3d9b 165 iowrite32(0, regs + TSI721_DMAC_DWRCNT);
48618fb4
AB		 166 udelay(1);
167 if (!do_wr)
168 *data = 0xffffffff;
169 err = -EIO;
170 goto err_out;
171 }
172
173 if (!do_wr)
174 *data = be32_to_cpu(bd_ptr[0].data[0]);
175
176 /*
177 * Update descriptor status FIFO RD pointer.
178 * NOTE: Skipping check and clear FIFO entries because we are waiting
179 * for transfer to be completed.
180 */
9eaa3d9b
AB		 181 swr_ptr = ioread32(regs + TSI721_DMAC_DSWP);
182 iowrite32(swr_ptr, regs + TSI721_DMAC_DSRP);
48618fb4
AB		 183err_out:
184
185 return err;
186}
187
188/**
189 * tsi721_cread_dma - Generate a RapidIO maintenance read transaction
190 * using Tsi721 BDMA engine.
191 * @mport: RapidIO master port control structure
192 * @index: ID of RapdiIO interface
193 * @destid: Destination ID of transaction
194 * @hopcount: Number of hops to target device
195 * @offset: Offset into configuration space
196 * @len: Length (in bytes) of the maintenance transaction
197 * @val: Location to be read into
198 *
199 * Generates a RapidIO maintenance read transaction.
200 * Returns %0 on success and %-EINVAL or %-EFAULT on failure.
201 */
202static int tsi721_cread_dma(struct rio_mport *mport, int index, u16 destid,
203 u8 hopcount, u32 offset, int len, u32 *data)
204{
205 struct tsi721_device *priv = mport->priv;
206
207 return tsi721_maint_dma(priv, mport->sys_size, destid, hopcount,
208 offset, len, data, 0);
209}
210
211/**
212 * tsi721_cwrite_dma - Generate a RapidIO maintenance write transaction
213 * using Tsi721 BDMA engine
214 * @mport: RapidIO master port control structure
215 * @index: ID of RapdiIO interface
216 * @destid: Destination ID of transaction
217 * @hopcount: Number of hops to target device
218 * @offset: Offset into configuration space
219 * @len: Length (in bytes) of the maintenance transaction
220 * @val: Value to be written
221 *
222 * Generates a RapidIO maintenance write transaction.
223 * Returns %0 on success and %-EINVAL or %-EFAULT on failure.
224 */
225static int tsi721_cwrite_dma(struct rio_mport *mport, int index, u16 destid,
226 u8 hopcount, u32 offset, int len, u32 data)
227{
228 struct tsi721_device *priv = mport->priv;
229 u32 temp = data;
230
231 return tsi721_maint_dma(priv, mport->sys_size, destid, hopcount,
232 offset, len, &temp, 1);
233}
234
235/**
236 * tsi721_pw_handler - Tsi721 inbound port-write interrupt handler
748353cc 237 * @priv: tsi721 device private structure
48618fb4
AB		 238 *
239 * Handles inbound port-write interrupts. Copies PW message from an internal
240 * buffer into PW message FIFO and schedules deferred routine to process
241 * queued messages.
242 */
243static int
748353cc 244tsi721_pw_handler(struct tsi721_device *priv)
48618fb4 245{
48618fb4
AB		 246 u32 pw_stat;
247 u32 pw_buf[TSI721_RIO_PW_MSG_SIZE/sizeof(u32)];
248
249
250 pw_stat = ioread32(priv->regs + TSI721_RIO_PW_RX_STAT);
251
252 if (pw_stat & TSI721_RIO_PW_RX_STAT_PW_VAL) {
253 pw_buf[0] = ioread32(priv->regs + TSI721_RIO_PW_RX_CAPT(0));
254 pw_buf[1] = ioread32(priv->regs + TSI721_RIO_PW_RX_CAPT(1));
255 pw_buf[2] = ioread32(priv->regs + TSI721_RIO_PW_RX_CAPT(2));
256 pw_buf[3] = ioread32(priv->regs + TSI721_RIO_PW_RX_CAPT(3));
257
258 /* Queue PW message (if there is room in FIFO),
259 * otherwise discard it.
260 */
261 spin_lock(&priv->pw_fifo_lock);
262 if (kfifo_avail(&priv->pw_fifo) >= TSI721_RIO_PW_MSG_SIZE)
263 kfifo_in(&priv->pw_fifo, pw_buf,
264 TSI721_RIO_PW_MSG_SIZE);
265 else
266 priv->pw_discard_count++;
267 spin_unlock(&priv->pw_fifo_lock);
268 }
269
270 /* Clear pending PW interrupts */
271 iowrite32(TSI721_RIO_PW_RX_STAT_PW_DISC | TSI721_RIO_PW_RX_STAT_PW_VAL,
272 priv->regs + TSI721_RIO_PW_RX_STAT);
273
274 schedule_work(&priv->pw_work);
275
276 return 0;
277}
278
279static void tsi721_pw_dpc(struct work_struct *work)
280{
281 struct tsi721_device *priv = container_of(work, struct tsi721_device,
282 pw_work);
9a0b0627 283 union rio_pw_msg pwmsg;
48618fb4
AB		 284
285 /*
286 * Process port-write messages
287 */
9a0b0627 288 while (kfifo_out_spinlocked(&priv->pw_fifo, (unsigned char *)&pwmsg,
48618fb4 289 TSI721_RIO_PW_MSG_SIZE, &priv->pw_fifo_lock)) {
48618fb4 290 /* Pass the port-write message to RIO core for processing */
9a0b0627 291 rio_inb_pwrite_handler(&priv->mport, &pwmsg);
48618fb4
AB		 292 }
293}
294
295/**
296 * tsi721_pw_enable - enable/disable port-write interface init
297 * @mport: Master port implementing the port write unit
298 * @enable: 1=enable; 0=disable port-write message handling
299 */
300static int tsi721_pw_enable(struct rio_mport *mport, int enable)
301{
302 struct tsi721_device *priv = mport->priv;
303 u32 rval;
304
305 rval = ioread32(priv->regs + TSI721_RIO_EM_INT_ENABLE);
306
307 if (enable)
308 rval |= TSI721_RIO_EM_INT_ENABLE_PW_RX;
309 else
310 rval &= ~TSI721_RIO_EM_INT_ENABLE_PW_RX;
311
312 /* Clear pending PW interrupts */
313 iowrite32(TSI721_RIO_PW_RX_STAT_PW_DISC | TSI721_RIO_PW_RX_STAT_PW_VAL,
314 priv->regs + TSI721_RIO_PW_RX_STAT);
315 /* Update enable bits */
316 iowrite32(rval, priv->regs + TSI721_RIO_EM_INT_ENABLE);
317
318 return 0;
319}
320
321/**
322 * tsi721_dsend - Send a RapidIO doorbell
323 * @mport: RapidIO master port info
324 * @index: ID of RapidIO interface
325 * @destid: Destination ID of target device
326 * @data: 16-bit info field of RapidIO doorbell
327 *
328 * Sends a RapidIO doorbell message. Always returns %0.
329 */
330static int tsi721_dsend(struct rio_mport *mport, int index,
331 u16 destid, u16 data)
332{
333 struct tsi721_device *priv = mport->priv;
334 u32 offset;
335
336 offset = (((mport->sys_size) ? RIO_TT_CODE_16 : RIO_TT_CODE_8) << 18) |
337 (destid << 2);
338
339 dev_dbg(&priv->pdev->dev,
340 "Send Doorbell 0x%04x to destID 0x%x\n", data, destid);
341 iowrite16be(data, priv->odb_base + offset);
342
343 return 0;
344}
345
346/**
347 * tsi721_dbell_handler - Tsi721 doorbell interrupt handler
748353cc 348 * @priv: tsi721 device-specific data structure
48618fb4
AB		 349 *
350 * Handles inbound doorbell interrupts. Copies doorbell entry from an internal
351 * buffer into DB message FIFO and schedules deferred routine to process
352 * queued DBs.
353 */
354static int
748353cc 355tsi721_dbell_handler(struct tsi721_device *priv)
48618fb4 356{
48618fb4
AB		 357 u32 regval;
358
359 /* Disable IDB interrupts */
360 regval = ioread32(priv->regs + TSI721_SR_CHINTE(IDB_QUEUE));
361 regval &= ~TSI721_SR_CHINT_IDBQRCV;
362 iowrite32(regval,
363 priv->regs + TSI721_SR_CHINTE(IDB_QUEUE));
364
365 schedule_work(&priv->idb_work);
366
367 return 0;
368}
369
370static void tsi721_db_dpc(struct work_struct *work)
371{
372 struct tsi721_device *priv = container_of(work, struct tsi721_device,
373 idb_work);
374 struct rio_mport *mport;
375 struct rio_dbell *dbell;
376 int found = 0;
377 u32 wr_ptr, rd_ptr;
378 u64 *idb_entry;
379 u32 regval;
380 union {
381 u64 msg;
382 u8 bytes[8];
383 } idb;
384
385 /*
386 * Process queued inbound doorbells
387 */
748353cc 388 mport = &priv->mport;
48618fb4 389
b24823e6
AB		 390 wr_ptr = ioread32(priv->regs + TSI721_IDQ_WP(IDB_QUEUE)) % IDB_QSIZE;
391 rd_ptr = ioread32(priv->regs + TSI721_IDQ_RP(IDB_QUEUE)) % IDB_QSIZE;
48618fb4
AB		 392
393 while (wr_ptr != rd_ptr) {
394 idb_entry = (u64 *)(priv->idb_base +
395 (TSI721_IDB_ENTRY_SIZE * rd_ptr));
396 rd_ptr++;
b24823e6 397 rd_ptr %= IDB_QSIZE;
48618fb4
AB		 398 idb.msg = *idb_entry;
399 *idb_entry = 0;
400
401 /* Process one doorbell */
402 list_for_each_entry(dbell, &mport->dbells, node) {
403 if ((dbell->res->start <= DBELL_INF(idb.bytes)) &&
404 (dbell->res->end >= DBELL_INF(idb.bytes))) {
405 found = 1;
406 break;
407 }
408 }
409
410 if (found) {
411 dbell->dinb(mport, dbell->dev_id, DBELL_SID(idb.bytes),
412 DBELL_TID(idb.bytes), DBELL_INF(idb.bytes));
413 } else {
414 dev_dbg(&priv->pdev->dev,
415 "spurious inb doorbell, sid %2.2x tid %2.2x"
416 " info %4.4x\n", DBELL_SID(idb.bytes),
417 DBELL_TID(idb.bytes), DBELL_INF(idb.bytes));
418 }
3670e7e1
AB		 419
420 wr_ptr = ioread32(priv->regs +
421 TSI721_IDQ_WP(IDB_QUEUE)) % IDB_QSIZE;
48618fb4
AB		 422 }
423
424 iowrite32(rd_ptr & (IDB_QSIZE - 1),
425 priv->regs + TSI721_IDQ_RP(IDB_QUEUE));
426
427 /* Re-enable IDB interrupts */
428 regval = ioread32(priv->regs + TSI721_SR_CHINTE(IDB_QUEUE));
429 regval |= TSI721_SR_CHINT_IDBQRCV;
430 iowrite32(regval,
431 priv->regs + TSI721_SR_CHINTE(IDB_QUEUE));
3670e7e1
AB		 432
433 wr_ptr = ioread32(priv->regs + TSI721_IDQ_WP(IDB_QUEUE)) % IDB_QSIZE;
434 if (wr_ptr != rd_ptr)
435 schedule_work(&priv->idb_work);
48618fb4
AB		 436}
437
438/**
439 * tsi721_irqhandler - Tsi721 interrupt handler
440 * @irq: Linux interrupt number
748353cc 441 * @ptr: Pointer to interrupt-specific data (tsi721_device structure)
48618fb4
AB		 442 *
443 * Handles Tsi721 interrupts signaled using MSI and INTA. Checks reported
444 * interrupt events and calls an event-specific handler(s).
445 */
446static irqreturn_t tsi721_irqhandler(int irq, void *ptr)
447{
748353cc 448 struct tsi721_device *priv = (struct tsi721_device *)ptr;
48618fb4
AB		 449 u32 dev_int;
450 u32 dev_ch_int;
451 u32 intval;
452 u32 ch_inte;
453
1ccc819d
AB		 454 /* For MSI mode disable all device-level interrupts */
455 if (priv->flags & TSI721_USING_MSI)
456 iowrite32(0, priv->regs + TSI721_DEV_INTE);
457
48618fb4
AB		 458 dev_int = ioread32(priv->regs + TSI721_DEV_INT);
459 if (!dev_int)
460 return IRQ_NONE;
461
462 dev_ch_int = ioread32(priv->regs + TSI721_DEV_CHAN_INT);
463
464 if (dev_int & TSI721_DEV_INT_SR2PC_CH) {
465 /* Service SR2PC Channel interrupts */
466 if (dev_ch_int & TSI721_INT_SR2PC_CHAN(IDB_QUEUE)) {
467 /* Service Inbound Doorbell interrupt */
468 intval = ioread32(priv->regs +
469 TSI721_SR_CHINT(IDB_QUEUE));
470 if (intval & TSI721_SR_CHINT_IDBQRCV)
748353cc 471 tsi721_dbell_handler(priv);
48618fb4
AB		 472 else
473 dev_info(&priv->pdev->dev,
474 "Unsupported SR_CH_INT %x\n", intval);
475
476 /* Clear interrupts */
477 iowrite32(intval,
478 priv->regs + TSI721_SR_CHINT(IDB_QUEUE));
479 ioread32(priv->regs + TSI721_SR_CHINT(IDB_QUEUE));
480 }
481 }
482
483 if (dev_int & TSI721_DEV_INT_SMSG_CH) {
484 int ch;
485
486 /*
487 * Service channel interrupts from Messaging Engine
488 */
489
490 if (dev_ch_int & TSI721_INT_IMSG_CHAN_M) { /* Inbound Msg */
491 /* Disable signaled OB MSG Channel interrupts */
492 ch_inte = ioread32(priv->regs + TSI721_DEV_CHAN_INTE);
493 ch_inte &= ~(dev_ch_int & TSI721_INT_IMSG_CHAN_M);
494 iowrite32(ch_inte, priv->regs + TSI721_DEV_CHAN_INTE);
495
496 /*
497 * Process Inbound Message interrupt for each MBOX
498 */
499 for (ch = 4; ch < RIO_MAX_MBOX + 4; ch++) {
500 if (!(dev_ch_int & TSI721_INT_IMSG_CHAN(ch)))
501 continue;
502 tsi721_imsg_handler(priv, ch);
503 }
504 }
505
506 if (dev_ch_int & TSI721_INT_OMSG_CHAN_M) { /* Outbound Msg */
507 /* Disable signaled OB MSG Channel interrupts */
508 ch_inte = ioread32(priv->regs + TSI721_DEV_CHAN_INTE);
509 ch_inte &= ~(dev_ch_int & TSI721_INT_OMSG_CHAN_M);
510 iowrite32(ch_inte, priv->regs + TSI721_DEV_CHAN_INTE);
511
512 /*
513 * Process Outbound Message interrupts for each MBOX
514 */
515
516 for (ch = 0; ch < RIO_MAX_MBOX; ch++) {
517 if (!(dev_ch_int & TSI721_INT_OMSG_CHAN(ch)))
518 continue;
519 tsi721_omsg_handler(priv, ch);
520 }
521 }
522 }
523
524 if (dev_int & TSI721_DEV_INT_SRIO) {
525 /* Service SRIO MAC interrupts */
526 intval = ioread32(priv->regs + TSI721_RIO_EM_INT_STAT);
527 if (intval & TSI721_RIO_EM_INT_STAT_PW_RX)
748353cc 528 tsi721_pw_handler(priv);
48618fb4
AB		 529 }
530
9eaa3d9b
AB		 531#ifdef CONFIG_RAPIDIO_DMA_ENGINE
532 if (dev_int & TSI721_DEV_INT_BDMA_CH) {
533 int ch;
534
535 if (dev_ch_int & TSI721_INT_BDMA_CHAN_M) {
536 dev_dbg(&priv->pdev->dev,
537 "IRQ from DMA channel 0x%08x\n", dev_ch_int);
538
539 for (ch = 0; ch < TSI721_DMA_MAXCH; ch++) {
540 if (!(dev_ch_int & TSI721_INT_BDMA_CHAN(ch)))
541 continue;
542 tsi721_bdma_handler(&priv->bdma[ch]);
543 }
544 }
545 }
546#endif
1ccc819d
AB		 547
548 /* For MSI mode re-enable device-level interrupts */
549 if (priv->flags & TSI721_USING_MSI) {
550 dev_int = TSI721_DEV_INT_SR2PC_CH | TSI721_DEV_INT_SRIO |
551 TSI721_DEV_INT_SMSG_CH | TSI721_DEV_INT_BDMA_CH;
552 iowrite32(dev_int, priv->regs + TSI721_DEV_INTE);
553 }
554
48618fb4
AB		 555 return IRQ_HANDLED;
556}
557
558static void tsi721_interrupts_init(struct tsi721_device *priv)
559{
560 u32 intr;
561
562 /* Enable IDB interrupts */
563 iowrite32(TSI721_SR_CHINT_ALL,
564 priv->regs + TSI721_SR_CHINT(IDB_QUEUE));
565 iowrite32(TSI721_SR_CHINT_IDBQRCV,
566 priv->regs + TSI721_SR_CHINTE(IDB_QUEUE));
48618fb4
AB		 567
568 /* Enable SRIO MAC interrupts */
569 iowrite32(TSI721_RIO_EM_DEV_INT_EN_INT,
570 priv->regs + TSI721_RIO_EM_DEV_INT_EN);
571
9eaa3d9b
AB		 572 /* Enable interrupts from channels in use */
573#ifdef CONFIG_RAPIDIO_DMA_ENGINE
574 intr = TSI721_INT_SR2PC_CHAN(IDB_QUEUE) |
575 (TSI721_INT_BDMA_CHAN_M &
576 ~TSI721_INT_BDMA_CHAN(TSI721_DMACH_MAINT));
577#else
578 intr = TSI721_INT_SR2PC_CHAN(IDB_QUEUE);
579#endif
580 iowrite32(intr, priv->regs + TSI721_DEV_CHAN_INTE);
581
48618fb4
AB		 582 if (priv->flags & TSI721_USING_MSIX)
583 intr = TSI721_DEV_INT_SRIO;
584 else
585 intr = TSI721_DEV_INT_SR2PC_CH | TSI721_DEV_INT_SRIO |
9eaa3d9b 586 TSI721_DEV_INT_SMSG_CH | TSI721_DEV_INT_BDMA_CH;
48618fb4
AB		 587
588 iowrite32(intr, priv->regs + TSI721_DEV_INTE);
589 ioread32(priv->regs + TSI721_DEV_INTE);
590}
591
592#ifdef CONFIG_PCI_MSI
593/**
594 * tsi721_omsg_msix - MSI-X interrupt handler for outbound messaging
595 * @irq: Linux interrupt number
748353cc 596 * @ptr: Pointer to interrupt-specific data (tsi721_device structure)
48618fb4
AB		 597 *
598 * Handles outbound messaging interrupts signaled using MSI-X.
599 */
600static irqreturn_t tsi721_omsg_msix(int irq, void *ptr)
601{
748353cc 602 struct tsi721_device *priv = (struct tsi721_device *)ptr;
48618fb4
AB		 603 int mbox;
604
605 mbox = (irq - priv->msix[TSI721_VECT_OMB0_DONE].vector) % RIO_MAX_MBOX;
606 tsi721_omsg_handler(priv, mbox);
607 return IRQ_HANDLED;
608}
609
610/**
611 * tsi721_imsg_msix - MSI-X interrupt handler for inbound messaging
612 * @irq: Linux interrupt number
748353cc 613 * @ptr: Pointer to interrupt-specific data (tsi721_device structure)
48618fb4
AB		 614 *
615 * Handles inbound messaging interrupts signaled using MSI-X.
616 */
617static irqreturn_t tsi721_imsg_msix(int irq, void *ptr)
618{
748353cc 619 struct tsi721_device *priv = (struct tsi721_device *)ptr;
48618fb4
AB		 620 int mbox;
621
622 mbox = (irq - priv->msix[TSI721_VECT_IMB0_RCV].vector) % RIO_MAX_MBOX;
623 tsi721_imsg_handler(priv, mbox + 4);
624 return IRQ_HANDLED;
625}
626
627/**
628 * tsi721_srio_msix - Tsi721 MSI-X SRIO MAC interrupt handler
629 * @irq: Linux interrupt number
748353cc 630 * @ptr: Pointer to interrupt-specific data (tsi721_device structure)
48618fb4
AB		 631 *
632 * Handles Tsi721 interrupts from SRIO MAC.
633 */
634static irqreturn_t tsi721_srio_msix(int irq, void *ptr)
635{
748353cc 636 struct tsi721_device *priv = (struct tsi721_device *)ptr;
48618fb4
AB		 637 u32 srio_int;
638
639 /* Service SRIO MAC interrupts */
640 srio_int = ioread32(priv->regs + TSI721_RIO_EM_INT_STAT);
641 if (srio_int & TSI721_RIO_EM_INT_STAT_PW_RX)
748353cc 642 tsi721_pw_handler(priv);
48618fb4
AB		 643
644 return IRQ_HANDLED;
645}
646
647/**
648 * tsi721_sr2pc_ch_msix - Tsi721 MSI-X SR2PC Channel interrupt handler
649 * @irq: Linux interrupt number
748353cc 650 * @ptr: Pointer to interrupt-specific data (tsi721_device structure)
48618fb4
AB		 651 *
652 * Handles Tsi721 interrupts from SR2PC Channel.
653 * NOTE: At this moment services only one SR2PC channel associated with inbound
654 * doorbells.
655 */
656static irqreturn_t tsi721_sr2pc_ch_msix(int irq, void *ptr)
657{
748353cc 658 struct tsi721_device *priv = (struct tsi721_device *)ptr;
48618fb4
AB		 659 u32 sr_ch_int;
660
661 /* Service Inbound DB interrupt from SR2PC channel */
662 sr_ch_int = ioread32(priv->regs + TSI721_SR_CHINT(IDB_QUEUE));
663 if (sr_ch_int & TSI721_SR_CHINT_IDBQRCV)
748353cc 664 tsi721_dbell_handler(priv);
48618fb4
AB		 665
666 /* Clear interrupts */
667 iowrite32(sr_ch_int, priv->regs + TSI721_SR_CHINT(IDB_QUEUE));
668 /* Read back to ensure that interrupt was cleared */
669 sr_ch_int = ioread32(priv->regs + TSI721_SR_CHINT(IDB_QUEUE));
670
671 return IRQ_HANDLED;
672}
673
674/**
675 * tsi721_request_msix - register interrupt service for MSI-X mode.
748353cc 676 * @priv: tsi721 device-specific data structure
48618fb4
AB		 677 *
678 * Registers MSI-X interrupt service routines for interrupts that are active
679 * immediately after mport initialization. Messaging interrupt service routines
680 * should be registered during corresponding open requests.
681 */
748353cc 682static int tsi721_request_msix(struct tsi721_device *priv)
48618fb4 683{
48618fb4
AB		 684 int err = 0;
685
686 err = request_irq(priv->msix[TSI721_VECT_IDB].vector,
687 tsi721_sr2pc_ch_msix, 0,
748353cc 688 priv->msix[TSI721_VECT_IDB].irq_name, (void *)priv);
48618fb4 689 if (err)
748353cc 690 return err;
48618fb4
AB		 691
692 err = request_irq(priv->msix[TSI721_VECT_PWRX].vector,
693 tsi721_srio_msix, 0,
748353cc
AB		 694 priv->msix[TSI721_VECT_PWRX].irq_name, (void *)priv);
695 if (err) {
696 free_irq(priv->msix[TSI721_VECT_IDB].vector, (void *)priv);
697 return err;
698 }
699
700 return 0;
48618fb4
AB		 701}
702
703/**
704 * tsi721_enable_msix - Attempts to enable MSI-X support for Tsi721.
705 * @priv: pointer to tsi721 private data
706 *
707 * Configures MSI-X support for Tsi721. Supports only an exact number
708 * of requested vectors.
709 */
710static int tsi721_enable_msix(struct tsi721_device *priv)
711{
712 struct msix_entry entries[TSI721_VECT_MAX];
713 int err;
714 int i;
715
716 entries[TSI721_VECT_IDB].entry = TSI721_MSIX_SR2PC_IDBQ_RCV(IDB_QUEUE);
717 entries[TSI721_VECT_PWRX].entry = TSI721_MSIX_SRIO_MAC_INT;
718
719 /*
720 * Initialize MSI-X entries for Messaging Engine:
721 * this driver supports four RIO mailboxes (inbound and outbound)
722 * NOTE: Inbound message MBOX 0...4 use IB channels 4...7. Therefore
723 * offset +4 is added to IB MBOX number.
724 */
725 for (i = 0; i < RIO_MAX_MBOX; i++) {
726 entries[TSI721_VECT_IMB0_RCV + i].entry =
727 TSI721_MSIX_IMSG_DQ_RCV(i + 4);
728 entries[TSI721_VECT_IMB0_INT + i].entry =
729 TSI721_MSIX_IMSG_INT(i + 4);
730 entries[TSI721_VECT_OMB0_DONE + i].entry =
731 TSI721_MSIX_OMSG_DONE(i);
732 entries[TSI721_VECT_OMB0_INT + i].entry =
733 TSI721_MSIX_OMSG_INT(i);
734 }
735
9eaa3d9b
AB		 736#ifdef CONFIG_RAPIDIO_DMA_ENGINE
737 /*
738 * Initialize MSI-X entries for Block DMA Engine:
739 * this driver supports XXX DMA channels
740 * (one is reserved for SRIO maintenance transactions)
741 */
742 for (i = 0; i < TSI721_DMA_CHNUM; i++) {
743 entries[TSI721_VECT_DMA0_DONE + i].entry =
744 TSI721_MSIX_DMACH_DONE(i);
745 entries[TSI721_VECT_DMA0_INT + i].entry =
746 TSI721_MSIX_DMACH_INT(i);
747 }
748#endif /* CONFIG_RAPIDIO_DMA_ENGINE */
749
1c92ab1e 750 err = pci_enable_msix_exact(priv->pdev, entries, ARRAY_SIZE(entries));
48618fb4 751 if (err) {
1c92ab1e
AG		 752 dev_err(&priv->pdev->dev,
753 "Failed to enable MSI-X (err=%d)\n", err);
48618fb4
AB		 754 return err;
755 }
756
757 /*
758 * Copy MSI-X vector information into tsi721 private structure
759 */
760 priv->msix[TSI721_VECT_IDB].vector = entries[TSI721_VECT_IDB].vector;
761 snprintf(priv->msix[TSI721_VECT_IDB].irq_name, IRQ_DEVICE_NAME_MAX,
762 DRV_NAME "-idb@pci:%s", pci_name(priv->pdev));
763 priv->msix[TSI721_VECT_PWRX].vector = entries[TSI721_VECT_PWRX].vector;
764 snprintf(priv->msix[TSI721_VECT_PWRX].irq_name, IRQ_DEVICE_NAME_MAX,
765 DRV_NAME "-pwrx@pci:%s", pci_name(priv->pdev));
766
767 for (i = 0; i < RIO_MAX_MBOX; i++) {
768 priv->msix[TSI721_VECT_IMB0_RCV + i].vector =
769 entries[TSI721_VECT_IMB0_RCV + i].vector;
770 snprintf(priv->msix[TSI721_VECT_IMB0_RCV + i].irq_name,
771 IRQ_DEVICE_NAME_MAX, DRV_NAME "-imbr%d@pci:%s",
772 i, pci_name(priv->pdev));
773
774 priv->msix[TSI721_VECT_IMB0_INT + i].vector =
775 entries[TSI721_VECT_IMB0_INT + i].vector;
776 snprintf(priv->msix[TSI721_VECT_IMB0_INT + i].irq_name,
777 IRQ_DEVICE_NAME_MAX, DRV_NAME "-imbi%d@pci:%s",
778 i, pci_name(priv->pdev));
779
780 priv->msix[TSI721_VECT_OMB0_DONE + i].vector =
781 entries[TSI721_VECT_OMB0_DONE + i].vector;
782 snprintf(priv->msix[TSI721_VECT_OMB0_DONE + i].irq_name,
783 IRQ_DEVICE_NAME_MAX, DRV_NAME "-ombd%d@pci:%s",
784 i, pci_name(priv->pdev));
785
786 priv->msix[TSI721_VECT_OMB0_INT + i].vector =
787 entries[TSI721_VECT_OMB0_INT + i].vector;
788 snprintf(priv->msix[TSI721_VECT_OMB0_INT + i].irq_name,
789 IRQ_DEVICE_NAME_MAX, DRV_NAME "-ombi%d@pci:%s",
790 i, pci_name(priv->pdev));
791 }
792
9eaa3d9b
AB		 793#ifdef CONFIG_RAPIDIO_DMA_ENGINE
794 for (i = 0; i < TSI721_DMA_CHNUM; i++) {
795 priv->msix[TSI721_VECT_DMA0_DONE + i].vector =
796 entries[TSI721_VECT_DMA0_DONE + i].vector;
797 snprintf(priv->msix[TSI721_VECT_DMA0_DONE + i].irq_name,
798 IRQ_DEVICE_NAME_MAX, DRV_NAME "-dmad%d@pci:%s",
799 i, pci_name(priv->pdev));
800
801 priv->msix[TSI721_VECT_DMA0_INT + i].vector =
802 entries[TSI721_VECT_DMA0_INT + i].vector;
803 snprintf(priv->msix[TSI721_VECT_DMA0_INT + i].irq_name,
804 IRQ_DEVICE_NAME_MAX, DRV_NAME "-dmai%d@pci:%s",
805 i, pci_name(priv->pdev));
806 }
807#endif /* CONFIG_RAPIDIO_DMA_ENGINE */
808
48618fb4
AB		 809 return 0;
810}
811#endif /* CONFIG_PCI_MSI */
812
748353cc 813static int tsi721_request_irq(struct tsi721_device *priv)
48618fb4 814{
48618fb4
AB		 815 int err;
816
817#ifdef CONFIG_PCI_MSI
818 if (priv->flags & TSI721_USING_MSIX)
748353cc 819 err = tsi721_request_msix(priv);
48618fb4
AB		 820 else
821#endif
822 err = request_irq(priv->pdev->irq, tsi721_irqhandler,
823 (priv->flags & TSI721_USING_MSI) ? 0 : IRQF_SHARED,
748353cc 824 DRV_NAME, (void *)priv);
48618fb4
AB		 825
826 if (err)
827 dev_err(&priv->pdev->dev,
828 "Unable to allocate interrupt, Error: %d\n", err);
829
830 return err;
831}
832
748353cc
AB		 833static void tsi721_free_irq(struct tsi721_device *priv)
834{
835#ifdef CONFIG_PCI_MSI
836 if (priv->flags & TSI721_USING_MSIX) {
837 free_irq(priv->msix[TSI721_VECT_IDB].vector, (void *)priv);
838 free_irq(priv->msix[TSI721_VECT_PWRX].vector, (void *)priv);
839 } else
840#endif
841 free_irq(priv->pdev->irq, (void *)priv);
842}
843
48618fb4
AB		 844/**
845 * tsi721_init_pc2sr_mapping - initializes outbound (PCIe->SRIO)
846 * translation regions.
847 * @priv: pointer to tsi721 private data
848 *
849 * Disables SREP translation regions.
850 */
851static void tsi721_init_pc2sr_mapping(struct tsi721_device *priv)
852{
853 int i;
854
855 /* Disable all PC2SR translation windows */
856 for (i = 0; i < TSI721_OBWIN_NUM; i++)
857 iowrite32(0, priv->regs + TSI721_OBWINLB(i));
858}
859
71afe341
AB		 860/**
861 * tsi721_rio_map_inb_mem -- Mapping inbound memory region.
862 * @mport: RapidIO master port
863 * @lstart: Local memory space start address.
864 * @rstart: RapidIO space start address.
865 * @size: The mapping region size.
866 * @flags: Flags for mapping. 0 for using default flags.
867 *
868 * Return: 0 -- Success.
869 *
870 * This function will create the inbound mapping
871 * from rstart to lstart.
872 */
873static int tsi721_rio_map_inb_mem(struct rio_mport *mport, dma_addr_t lstart,
874 u64 rstart, u32 size, u32 flags)
875{
876 struct tsi721_device *priv = mport->priv;
ba5d141b 877 int i, avail = -1;
71afe341 878 u32 regval;
ba5d141b 879 struct tsi721_ib_win *ib_win;
9673b883
AB		 880 bool direct = (lstart == rstart);
881 u64 ibw_size;
882 dma_addr_t loc_start;
883 u64 ibw_start;
884 struct tsi721_ib_win_mapping *map = NULL;
ba5d141b 885 int ret = -EBUSY;
71afe341 886
9673b883
AB		 887 if (direct) {
888 dev_dbg(&priv->pdev->dev,
889 "Direct (RIO_0x%llx -> PCIe_0x%pad), size=0x%x",
890 rstart, &lstart, size);
891
892 /* Calculate minimal acceptable window size and base address */
893
894 ibw_size = roundup_pow_of_two(size);
895 ibw_start = lstart & ~(ibw_size - 1);
896
897 while ((lstart + size) > (ibw_start + ibw_size)) {
898 ibw_size *= 2;
899 ibw_start = lstart & ~(ibw_size - 1);
900 if (ibw_size > 0x80000000) { /* Limit max size to 2GB */
901 return -EBUSY;
902 }
903 }
904
905 loc_start = ibw_start;
906
907 map = kzalloc(sizeof(struct tsi721_ib_win_mapping), GFP_ATOMIC);
908 if (map == NULL)
909 return -ENOMEM;
910
911 } else {
912 dev_dbg(&priv->pdev->dev,
913 "Translated (RIO_0x%llx -> PCIe_0x%pad), size=0x%x",
914 rstart, &lstart, size);
915
916 if (!is_power_of_2(size) || size < 0x1000 ||
917 ((u64)lstart & (size - 1)) || (rstart & (size - 1)))
918 return -EINVAL;
919 if (priv->ibwin_cnt == 0)
920 return -EBUSY;
921 ibw_start = rstart;
922 ibw_size = size;
923 loc_start = lstart;
924 }
71afe341 925
ba5d141b
AB		 926 /*
927 * Scan for overlapping with active regions and mark the first available
928 * IB window at the same time.
929 */
71afe341 930 for (i = 0; i < TSI721_IBWIN_NUM; i++) {
ba5d141b 931 ib_win = &priv->ib_win[i];
9673b883 932
ba5d141b
AB		 933 if (!ib_win->active) {
934 if (avail == -1) {
935 avail = i;
936 ret = 0;
937 }
9673b883
AB		 938 } else if (ibw_start < (ib_win->rstart + ib_win->size) &&
939 (ibw_start + ibw_size) > ib_win->rstart) {
940 /* Return error if address translation involved */
941 if (direct && ib_win->xlat) {
942 ret = -EFAULT;
943 break;
944 }
945
946 /*
947 * Direct mappings usually are larger than originally
948 * requested fragments - check if this new request fits
949 * into it.
950 */
951 if (rstart >= ib_win->rstart &&
952 (rstart + size) <= (ib_win->rstart +
953 ib_win->size)) {
954 /* We are in - no further mapping required */
955 map->lstart = lstart;
956 list_add_tail(&map->node, &ib_win->mappings);
957 return 0;
958 }
959
ba5d141b 960 ret = -EFAULT;
71afe341 961 break;
ba5d141b 962 }
71afe341
AB		 963 }
964
ba5d141b 965 if (ret)
9673b883 966 goto out;
ba5d141b
AB		 967 i = avail;
968
969 /* Sanity check: available IB window must be disabled at this point */
970 regval = ioread32(priv->regs + TSI721_IBWIN_LB(i));
971 if (WARN_ON(regval & TSI721_IBWIN_LB_WEN)) {
972 ret = -EIO;
9673b883 973 goto out;
71afe341
AB		 974 }
975
ba5d141b
AB		 976 ib_win = &priv->ib_win[i];
977 ib_win->active = true;
9673b883
AB		 978 ib_win->rstart = ibw_start;
979 ib_win->lstart = loc_start;
980 ib_win->size = ibw_size;
981 ib_win->xlat = (lstart != rstart);
982 INIT_LIST_HEAD(&ib_win->mappings);
ba5d141b 983
9673b883
AB		 984 /*
985 * When using direct IBW mapping and have larger than requested IBW size
986 * we can have multiple local memory blocks mapped through the same IBW
987 * To handle this situation we maintain list of "clients" for such IBWs.
988 */
989 if (direct) {
990 map->lstart = lstart;
991 list_add_tail(&map->node, &ib_win->mappings);
992 }
993
994 iowrite32(TSI721_IBWIN_SIZE(ibw_size) << 8,
71afe341
AB		 995 priv->regs + TSI721_IBWIN_SZ(i));
996
9673b883
AB		 997 iowrite32(((u64)loc_start >> 32), priv->regs + TSI721_IBWIN_TUA(i));
998 iowrite32(((u64)loc_start & TSI721_IBWIN_TLA_ADD),
71afe341
AB		 999 priv->regs + TSI721_IBWIN_TLA(i));
1000
9673b883
AB		 1001 iowrite32(ibw_start >> 32, priv->regs + TSI721_IBWIN_UB(i));
1002 iowrite32((ibw_start & TSI721_IBWIN_LB_BA) | TSI721_IBWIN_LB_WEN,
71afe341 1003 priv->regs + TSI721_IBWIN_LB(i));
9673b883
AB		 1004
1005 priv->ibwin_cnt--;
1006
71afe341 1007 dev_dbg(&priv->pdev->dev,
9673b883
AB		 1008 "Configured IBWIN%d (RIO_0x%llx -> PCIe_0x%llx), size=0x%llx\n",
1009 i, ibw_start, (unsigned long long)loc_start, ibw_size);
71afe341
AB		 1010
1011 return 0;
9673b883
AB		 1012out:
1013 kfree(map);
ba5d141b 1014 return ret;
71afe341
AB		 1015}
1016
1017/**
9673b883 1018 * tsi721_rio_unmap_inb_mem -- Unmapping inbound memory region.
71afe341
AB		 1019 * @mport: RapidIO master port
1020 * @lstart: Local memory space start address.
1021 */
1022static void tsi721_rio_unmap_inb_mem(struct rio_mport *mport,
1023 dma_addr_t lstart)
1024{
1025 struct tsi721_device *priv = mport->priv;
ba5d141b 1026 struct tsi721_ib_win *ib_win;
71afe341 1027 int i;
71afe341 1028
9673b883
AB		 1029 dev_dbg(&priv->pdev->dev,
1030 "Unmap IBW mapped to PCIe_0x%pad", &lstart);
1031
71afe341
AB		 1032 /* Search for matching active inbound translation window */
1033 for (i = 0; i < TSI721_IBWIN_NUM; i++) {
ba5d141b 1034 ib_win = &priv->ib_win[i];
9673b883
AB		 1035
1036 /* Address translating IBWs must to be an exact march */
1037 if (!ib_win->active ||
1038 (ib_win->xlat && lstart != ib_win->lstart))
1039 continue;
1040
1041 if (lstart >= ib_win->lstart &&
1042 lstart < (ib_win->lstart + ib_win->size)) {
1043
1044 if (!ib_win->xlat) {
1045 struct tsi721_ib_win_mapping *map;
1046 int found = 0;
1047
1048 list_for_each_entry(map,
1049 &ib_win->mappings, node) {
1050 if (map->lstart == lstart) {
1051 list_del(&map->node);
1052 kfree(map);
1053 found = 1;
1054 break;
1055 }
1056 }
1057
1058 if (!found)
1059 continue;
1060
1061 if (!list_empty(&ib_win->mappings))
1062 break;
1063 }
1064
1065 dev_dbg(&priv->pdev->dev, "Disable IBWIN_%d", i);
ba5d141b
AB		 1066 iowrite32(0, priv->regs + TSI721_IBWIN_LB(i));
1067 ib_win->active = false;
9673b883 1068 priv->ibwin_cnt++;
ba5d141b 1069 break;
71afe341
AB		 1070 }
1071 }
ba5d141b
AB		 1072
1073 if (i == TSI721_IBWIN_NUM)
9673b883
AB		 1074 dev_dbg(&priv->pdev->dev,
1075 "IB window mapped to %pad not found", &lstart);
71afe341
AB		 1076}
1077
48618fb4
AB		 1078/**
1079 * tsi721_init_sr2pc_mapping - initializes inbound (SRIO->PCIe)
1080 * translation regions.
1081 * @priv: pointer to tsi721 private data
1082 *
1083 * Disables inbound windows.
1084 */
1085static void tsi721_init_sr2pc_mapping(struct tsi721_device *priv)
1086{
1087 int i;
1088
1089 /* Disable all SR2PC inbound windows */
1090 for (i = 0; i < TSI721_IBWIN_NUM; i++)
71afe341 1091 iowrite32(0, priv->regs + TSI721_IBWIN_LB(i));
9673b883 1092 priv->ibwin_cnt = TSI721_IBWIN_NUM;
48618fb4
AB		 1093}
1094
748353cc
AB		 1095/*
1096 * tsi721_close_sr2pc_mapping - closes all active inbound (SRIO->PCIe)
1097 * translation regions.
1098 * @priv: pointer to tsi721 device private data
1099 */
1100static void tsi721_close_sr2pc_mapping(struct tsi721_device *priv)
1101{
1102 struct tsi721_ib_win *ib_win;
1103 int i;
1104
1105 /* Disable all active SR2PC inbound windows */
1106 for (i = 0; i < TSI721_IBWIN_NUM; i++) {
1107 ib_win = &priv->ib_win[i];
1108 if (ib_win->active) {
1109 iowrite32(0, priv->regs + TSI721_IBWIN_LB(i));
1110 ib_win->active = false;
1111 }
1112 }
1113}
1114
48618fb4
AB		 1115/**
1116 * tsi721_port_write_init - Inbound port write interface init
1117 * @priv: pointer to tsi721 private data
1118 *
1119 * Initializes inbound port write handler.
1120 * Returns %0 on success or %-ENOMEM on failure.
1121 */
1122static int tsi721_port_write_init(struct tsi721_device *priv)
1123{
1124 priv->pw_discard_count = 0;
1125 INIT_WORK(&priv->pw_work, tsi721_pw_dpc);
1126 spin_lock_init(&priv->pw_fifo_lock);
1127 if (kfifo_alloc(&priv->pw_fifo,
1128 TSI721_RIO_PW_MSG_SIZE * 32, GFP_KERNEL)) {
1129 dev_err(&priv->pdev->dev, "PW FIFO allocation failed\n");
1130 return -ENOMEM;
1131 }
1132
1133 /* Use reliable port-write capture mode */
1134 iowrite32(TSI721_RIO_PW_CTL_PWC_REL, priv->regs + TSI721_RIO_PW_CTL);
1135 return 0;
1136}
1137
748353cc
AB		 1138static void tsi721_port_write_free(struct tsi721_device *priv)
1139{
1140 kfifo_free(&priv->pw_fifo);
1141}
1142
48618fb4
AB		 1143static int tsi721_doorbell_init(struct tsi721_device *priv)
1144{
1145 /* Outbound Doorbells do not require any setup.
1146 * Tsi721 uses dedicated PCI BAR1 to generate doorbells.
1147 * That BAR1 was mapped during the probe routine.
1148 */
1149
1150 /* Initialize Inbound Doorbell processing DPC and queue */
1151 priv->db_discard_count = 0;
1152 INIT_WORK(&priv->idb_work, tsi721_db_dpc);
1153
1154 /* Allocate buffer for inbound doorbells queue */
ceb96398 1155 priv->idb_base = dma_zalloc_coherent(&priv->pdev->dev,
48618fb4
AB		 1156 IDB_QSIZE * TSI721_IDB_ENTRY_SIZE,
1157 &priv->idb_dma, GFP_KERNEL);
1158 if (!priv->idb_base)
1159 return -ENOMEM;
1160
48618fb4
AB		 1161 dev_dbg(&priv->pdev->dev, "Allocated IDB buffer @ %p (phys = %llx)\n",
1162 priv->idb_base, (unsigned long long)priv->idb_dma);
1163
1164 iowrite32(TSI721_IDQ_SIZE_VAL(IDB_QSIZE),
1165 priv->regs + TSI721_IDQ_SIZE(IDB_QUEUE));
1166 iowrite32(((u64)priv->idb_dma >> 32),
1167 priv->regs + TSI721_IDQ_BASEU(IDB_QUEUE));
1168 iowrite32(((u64)priv->idb_dma & TSI721_IDQ_BASEL_ADDR),
1169 priv->regs + TSI721_IDQ_BASEL(IDB_QUEUE));
1170 /* Enable accepting all inbound doorbells */
1171 iowrite32(0, priv->regs + TSI721_IDQ_MASK(IDB_QUEUE));
1172
1173 iowrite32(TSI721_IDQ_INIT, priv->regs + TSI721_IDQ_CTL(IDB_QUEUE));
1174
1175 iowrite32(0, priv->regs + TSI721_IDQ_RP(IDB_QUEUE));
1176
1177 return 0;
1178}
1179
1180static void tsi721_doorbell_free(struct tsi721_device *priv)
1181{
1182 if (priv->idb_base == NULL)
1183 return;
1184
1185 /* Free buffer allocated for inbound doorbell queue */
1186 dma_free_coherent(&priv->pdev->dev, IDB_QSIZE * TSI721_IDB_ENTRY_SIZE,
1187 priv->idb_base, priv->idb_dma);
1188 priv->idb_base = NULL;
1189}
1190
9eaa3d9b
AB		 1191/**
1192 * tsi721_bdma_maint_init - Initialize maintenance request BDMA channel.
1193 * @priv: pointer to tsi721 private data
1194 *
1195 * Initialize BDMA channel allocated for RapidIO maintenance read/write
1196 * request generation
1197 * Returns %0 on success or %-ENOMEM on failure.
1198 */
1199static int tsi721_bdma_maint_init(struct tsi721_device *priv)
48618fb4
AB		 1200{
1201 struct tsi721_dma_desc *bd_ptr;
1202 u64 *sts_ptr;
1203 dma_addr_t bd_phys, sts_phys;
1204 int sts_size;
9eaa3d9b
AB		 1205 int bd_num = 2;
1206 void __iomem *regs;
48618fb4 1207
9eaa3d9b
AB		 1208 dev_dbg(&priv->pdev->dev,
1209 "Init Block DMA Engine for Maintenance requests, CH%d\n",
1210 TSI721_DMACH_MAINT);
48618fb4
AB		 1211
1212 /*
1213 * Initialize DMA channel for maintenance requests
1214 */
1215
9eaa3d9b
AB		 1216 priv->mdma.ch_id = TSI721_DMACH_MAINT;
1217 regs = priv->regs + TSI721_DMAC_BASE(TSI721_DMACH_MAINT);
1218
48618fb4 1219 /* Allocate space for DMA descriptors */
ceb96398 1220 bd_ptr = dma_zalloc_coherent(&priv->pdev->dev,
48618fb4
AB		 1221 bd_num * sizeof(struct tsi721_dma_desc),
1222 &bd_phys, GFP_KERNEL);
1223 if (!bd_ptr)
1224 return -ENOMEM;
1225
9eaa3d9b
AB		 1226 priv->mdma.bd_num = bd_num;
1227 priv->mdma.bd_phys = bd_phys;
1228 priv->mdma.bd_base = bd_ptr;
48618fb4 1229
48618fb4
AB		 1230 dev_dbg(&priv->pdev->dev, "DMA descriptors @ %p (phys = %llx)\n",
1231 bd_ptr, (unsigned long long)bd_phys);
1232
1233 /* Allocate space for descriptor status FIFO */
1234 sts_size = (bd_num >= TSI721_DMA_MINSTSSZ) ?
1235 bd_num : TSI721_DMA_MINSTSSZ;
1236 sts_size = roundup_pow_of_two(sts_size);
ceb96398 1237 sts_ptr = dma_zalloc_coherent(&priv->pdev->dev,
48618fb4
AB		 1238 sts_size * sizeof(struct tsi721_dma_sts),
1239 &sts_phys, GFP_KERNEL);
1240 if (!sts_ptr) {
1241 /* Free space allocated for DMA descriptors */
1242 dma_free_coherent(&priv->pdev->dev,
1243 bd_num * sizeof(struct tsi721_dma_desc),
1244 bd_ptr, bd_phys);
9eaa3d9b 1245 priv->mdma.bd_base = NULL;
48618fb4
AB		 1246 return -ENOMEM;
1247 }
1248
9eaa3d9b
AB		 1249 priv->mdma.sts_phys = sts_phys;
1250 priv->mdma.sts_base = sts_ptr;
1251 priv->mdma.sts_size = sts_size;
48618fb4 1252
48618fb4
AB		 1253 dev_dbg(&priv->pdev->dev,
1254 "desc status FIFO @ %p (phys = %llx) size=0x%x\n",
1255 sts_ptr, (unsigned long long)sts_phys, sts_size);
1256
1257 /* Initialize DMA descriptors ring */
1258 bd_ptr[bd_num - 1].type_id = cpu_to_le32(DTYPE3 << 29);
1259 bd_ptr[bd_num - 1].next_lo = cpu_to_le32((u64)bd_phys &
1260 TSI721_DMAC_DPTRL_MASK);
1261 bd_ptr[bd_num - 1].next_hi = cpu_to_le32((u64)bd_phys >> 32);
1262
1263 /* Setup DMA descriptor pointers */
9eaa3d9b 1264 iowrite32(((u64)bd_phys >> 32), regs + TSI721_DMAC_DPTRH);
48618fb4 1265 iowrite32(((u64)bd_phys & TSI721_DMAC_DPTRL_MASK),
9eaa3d9b 1266 regs + TSI721_DMAC_DPTRL);
48618fb4
AB		 1267
1268 /* Setup descriptor status FIFO */
9eaa3d9b 1269 iowrite32(((u64)sts_phys >> 32), regs + TSI721_DMAC_DSBH);
48618fb4 1270 iowrite32(((u64)sts_phys & TSI721_DMAC_DSBL_MASK),
9eaa3d9b 1271 regs + TSI721_DMAC_DSBL);
48618fb4 1272 iowrite32(TSI721_DMAC_DSSZ_SIZE(sts_size),
9eaa3d9b 1273 regs + TSI721_DMAC_DSSZ);
48618fb4
AB		 1274
1275 /* Clear interrupt bits */
9eaa3d9b 1276 iowrite32(TSI721_DMAC_INT_ALL, regs + TSI721_DMAC_INT);
48618fb4 1277
9eaa3d9b 1278 ioread32(regs + TSI721_DMAC_INT);
48618fb4
AB		 1279
1280 /* Toggle DMA channel initialization */
9eaa3d9b
AB		 1281 iowrite32(TSI721_DMAC_CTL_INIT, regs + TSI721_DMAC_CTL);
1282 ioread32(regs + TSI721_DMAC_CTL);
48618fb4
AB		 1283 udelay(10);
1284
1285 return 0;
1286}
1287
9eaa3d9b 1288static int tsi721_bdma_maint_free(struct tsi721_device *priv)
48618fb4
AB		 1289{
1290 u32 ch_stat;
9eaa3d9b
AB		 1291 struct tsi721_bdma_maint *mdma = &priv->mdma;
1292 void __iomem *regs = priv->regs + TSI721_DMAC_BASE(mdma->ch_id);
48618fb4 1293
9eaa3d9b 1294 if (mdma->bd_base == NULL)
48618fb4
AB		 1295 return 0;
1296
1297 /* Check if DMA channel still running */
9eaa3d9b 1298 ch_stat = ioread32(regs + TSI721_DMAC_STS);
48618fb4
AB		 1299 if (ch_stat & TSI721_DMAC_STS_RUN)
1300 return -EFAULT;
1301
1302 /* Put DMA channel into init state */
9eaa3d9b 1303 iowrite32(TSI721_DMAC_CTL_INIT, regs + TSI721_DMAC_CTL);
48618fb4
AB		 1304
1305 /* Free space allocated for DMA descriptors */
1306 dma_free_coherent(&priv->pdev->dev,
9eaa3d9b
AB		 1307 mdma->bd_num * sizeof(struct tsi721_dma_desc),
1308 mdma->bd_base, mdma->bd_phys);
1309 mdma->bd_base = NULL;
48618fb4
AB		 1310
1311 /* Free space allocated for status FIFO */
1312 dma_free_coherent(&priv->pdev->dev,
9eaa3d9b
AB		 1313 mdma->sts_size * sizeof(struct tsi721_dma_sts),
1314 mdma->sts_base, mdma->sts_phys);
1315 mdma->sts_base = NULL;
48618fb4
AB		 1316 return 0;
1317}
1318
48618fb4
AB		 1319/* Enable Inbound Messaging Interrupts */
1320static void
1321tsi721_imsg_interrupt_enable(struct tsi721_device *priv, int ch,
1322 u32 inte_mask)
1323{
1324 u32 rval;
1325
1326 if (!inte_mask)
1327 return;
1328
1329 /* Clear pending Inbound Messaging interrupts */
1330 iowrite32(inte_mask, priv->regs + TSI721_IBDMAC_INT(ch));
1331
1332 /* Enable Inbound Messaging interrupts */
1333 rval = ioread32(priv->regs + TSI721_IBDMAC_INTE(ch));
1334 iowrite32(rval | inte_mask, priv->regs + TSI721_IBDMAC_INTE(ch));
1335
1336 if (priv->flags & TSI721_USING_MSIX)
1337 return; /* Finished if we are in MSI-X mode */
1338
1339 /*
1340 * For MSI and INTA interrupt signalling we need to enable next levels
1341 */
1342
1343 /* Enable Device Channel Interrupt */
1344 rval = ioread32(priv->regs + TSI721_DEV_CHAN_INTE);
1345 iowrite32(rval | TSI721_INT_IMSG_CHAN(ch),
1346 priv->regs + TSI721_DEV_CHAN_INTE);
1347}
1348
1349/* Disable Inbound Messaging Interrupts */
1350static void
1351tsi721_imsg_interrupt_disable(struct tsi721_device *priv, int ch,
1352 u32 inte_mask)
1353{
1354 u32 rval;
1355
1356 if (!inte_mask)
1357 return;
1358
1359 /* Clear pending Inbound Messaging interrupts */
1360 iowrite32(inte_mask, priv->regs + TSI721_IBDMAC_INT(ch));
1361
1362 /* Disable Inbound Messaging interrupts */
1363 rval = ioread32(priv->regs + TSI721_IBDMAC_INTE(ch));
1364 rval &= ~inte_mask;
1365 iowrite32(rval, priv->regs + TSI721_IBDMAC_INTE(ch));
1366
1367 if (priv->flags & TSI721_USING_MSIX)
1368 return; /* Finished if we are in MSI-X mode */
1369
1370 /*
1371 * For MSI and INTA interrupt signalling we need to disable next levels
1372 */
1373
1374 /* Disable Device Channel Interrupt */
1375 rval = ioread32(priv->regs + TSI721_DEV_CHAN_INTE);
1376 rval &= ~TSI721_INT_IMSG_CHAN(ch);
1377 iowrite32(rval, priv->regs + TSI721_DEV_CHAN_INTE);
1378}
1379
1380/* Enable Outbound Messaging interrupts */
1381static void
1382tsi721_omsg_interrupt_enable(struct tsi721_device *priv, int ch,
1383 u32 inte_mask)
1384{
1385 u32 rval;
1386
1387 if (!inte_mask)
1388 return;
1389
1390 /* Clear pending Outbound Messaging interrupts */
1391 iowrite32(inte_mask, priv->regs + TSI721_OBDMAC_INT(ch));
1392
1393 /* Enable Outbound Messaging channel interrupts */
1394 rval = ioread32(priv->regs + TSI721_OBDMAC_INTE(ch));
1395 iowrite32(rval | inte_mask, priv->regs + TSI721_OBDMAC_INTE(ch));
1396
1397 if (priv->flags & TSI721_USING_MSIX)
1398 return; /* Finished if we are in MSI-X mode */
1399
1400 /*
1401 * For MSI and INTA interrupt signalling we need to enable next levels
1402 */
1403
1404 /* Enable Device Channel Interrupt */
1405 rval = ioread32(priv->regs + TSI721_DEV_CHAN_INTE);
1406 iowrite32(rval | TSI721_INT_OMSG_CHAN(ch),
1407 priv->regs + TSI721_DEV_CHAN_INTE);
1408}
1409
1410/* Disable Outbound Messaging interrupts */
1411static void
1412tsi721_omsg_interrupt_disable(struct tsi721_device *priv, int ch,
1413 u32 inte_mask)
1414{
1415 u32 rval;
1416
1417 if (!inte_mask)
1418 return;
1419
1420 /* Clear pending Outbound Messaging interrupts */
1421 iowrite32(inte_mask, priv->regs + TSI721_OBDMAC_INT(ch));
1422
1423 /* Disable Outbound Messaging interrupts */
1424 rval = ioread32(priv->regs + TSI721_OBDMAC_INTE(ch));
1425 rval &= ~inte_mask;
1426 iowrite32(rval, priv->regs + TSI721_OBDMAC_INTE(ch));
1427
1428 if (priv->flags & TSI721_USING_MSIX)
1429 return; /* Finished if we are in MSI-X mode */
1430
1431 /*
1432 * For MSI and INTA interrupt signalling we need to disable next levels
1433 */
1434
1435 /* Disable Device Channel Interrupt */
1436 rval = ioread32(priv->regs + TSI721_DEV_CHAN_INTE);
1437 rval &= ~TSI721_INT_OMSG_CHAN(ch);
1438 iowrite32(rval, priv->regs + TSI721_DEV_CHAN_INTE);
1439}
1440
1441/**
1442 * tsi721_add_outb_message - Add message to the Tsi721 outbound message queue
1443 * @mport: Master port with outbound message queue
1444 * @rdev: Target of outbound message
1445 * @mbox: Outbound mailbox
1446 * @buffer: Message to add to outbound queue
1447 * @len: Length of message
1448 */
1449static int
1450tsi721_add_outb_message(struct rio_mport *mport, struct rio_dev *rdev, int mbox,
1451 void *buffer, size_t len)
1452{
1453 struct tsi721_device *priv = mport->priv;
1454 struct tsi721_omsg_desc *desc;
1455 u32 tx_slot;
2ece1caf 1456 unsigned long flags;
48618fb4
AB		 1457
1458 if (!priv->omsg_init[mbox] ||
1459 len > TSI721_MSG_MAX_SIZE || len < 8)
1460 return -EINVAL;
1461
2ece1caf
AB		 1462 spin_lock_irqsave(&priv->omsg_ring[mbox].lock, flags);
1463
48618fb4
AB		 1464 tx_slot = priv->omsg_ring[mbox].tx_slot;
1465
1466 /* Copy copy message into transfer buffer */
1467 memcpy(priv->omsg_ring[mbox].omq_base[tx_slot], buffer, len);
1468
1469 if (len & 0x7)
1470 len += 8;
1471
1472 /* Build descriptor associated with buffer */
1473 desc = priv->omsg_ring[mbox].omd_base;
1474 desc[tx_slot].type_id = cpu_to_le32((DTYPE4 << 29) | rdev->destid);
2ece1caf
AB		 1475#ifdef TSI721_OMSG_DESC_INT
1476 /* Request IOF_DONE interrupt generation for each N-th frame in queue */
48618fb4
AB		 1477 if (tx_slot % 4 == 0)
1478 desc[tx_slot].type_id |= cpu_to_le32(TSI721_OMD_IOF);
2ece1caf 1479#endif
48618fb4
AB		 1480 desc[tx_slot].msg_info =
1481 cpu_to_le32((mport->sys_size << 26) | (mbox << 22) |
1482 (0xe << 12) | (len & 0xff8));
1483 desc[tx_slot].bufptr_lo =
1484 cpu_to_le32((u64)priv->omsg_ring[mbox].omq_phys[tx_slot] &
1485 0xffffffff);
1486 desc[tx_slot].bufptr_hi =
1487 cpu_to_le32((u64)priv->omsg_ring[mbox].omq_phys[tx_slot] >> 32);
1488
1489 priv->omsg_ring[mbox].wr_count++;
1490
1491 /* Go to next descriptor */
1492 if (++priv->omsg_ring[mbox].tx_slot == priv->omsg_ring[mbox].size) {
1493 priv->omsg_ring[mbox].tx_slot = 0;
1494 /* Move through the ring link descriptor at the end */
1495 priv->omsg_ring[mbox].wr_count++;
1496 }
1497
1498 mb();
1499
1500 /* Set new write count value */
1501 iowrite32(priv->omsg_ring[mbox].wr_count,
1502 priv->regs + TSI721_OBDMAC_DWRCNT(mbox));
1503 ioread32(priv->regs + TSI721_OBDMAC_DWRCNT(mbox));
1504
2ece1caf
AB		 1505 spin_unlock_irqrestore(&priv->omsg_ring[mbox].lock, flags);
1506
48618fb4
AB		 1507 return 0;
1508}
1509
1510/**
1511 * tsi721_omsg_handler - Outbound Message Interrupt Handler
1512 * @priv: pointer to tsi721 private data
1513 * @ch: number of OB MSG channel to service
1514 *
1515 * Services channel interrupts from outbound messaging engine.
1516 */
1517static void tsi721_omsg_handler(struct tsi721_device *priv, int ch)
1518{
1519 u32 omsg_int;
748353cc 1520 struct rio_mport *mport = &priv->mport;
2ece1caf
AB		 1521 void *dev_id = NULL;
1522 u32 tx_slot = 0xffffffff;
1523 int do_callback = 0;
48618fb4
AB		 1524
1525 spin_lock(&priv->omsg_ring[ch].lock);
1526
1527 omsg_int = ioread32(priv->regs + TSI721_OBDMAC_INT(ch));
1528
1529 if (omsg_int & TSI721_OBDMAC_INT_ST_FULL)
1530 dev_info(&priv->pdev->dev,
1531 "OB MBOX%d: Status FIFO is full\n", ch);
1532
1533 if (omsg_int & (TSI721_OBDMAC_INT_DONE | TSI721_OBDMAC_INT_IOF_DONE)) {
1534 u32 srd_ptr;
1535 u64 *sts_ptr, last_ptr = 0, prev_ptr = 0;
1536 int i, j;
48618fb4
AB		 1537
1538 /*
1539 * Find last successfully processed descriptor
1540 */
1541
1542 /* Check and clear descriptor status FIFO entries */
1543 srd_ptr = priv->omsg_ring[ch].sts_rdptr;
1544 sts_ptr = priv->omsg_ring[ch].sts_base;
1545 j = srd_ptr * 8;
1546 while (sts_ptr[j]) {
1547 for (i = 0; i < 8 && sts_ptr[j]; i++, j++) {
1548 prev_ptr = last_ptr;
1549 last_ptr = le64_to_cpu(sts_ptr[j]);
1550 sts_ptr[j] = 0;
1551 }
1552
1553 ++srd_ptr;
1554 srd_ptr %= priv->omsg_ring[ch].sts_size;
1555 j = srd_ptr * 8;
1556 }
1557
1558 if (last_ptr == 0)
1559 goto no_sts_update;
1560
1561 priv->omsg_ring[ch].sts_rdptr = srd_ptr;
1562 iowrite32(srd_ptr, priv->regs + TSI721_OBDMAC_DSRP(ch));
1563
748353cc 1564 if (!mport->outb_msg[ch].mcback)
48618fb4
AB		 1565 goto no_sts_update;
1566
1567 /* Inform upper layer about transfer completion */
1568
1569 tx_slot = (last_ptr - (u64)priv->omsg_ring[ch].omd_phys)/
1570 sizeof(struct tsi721_omsg_desc);
1571
1572 /*
1573 * Check if this is a Link Descriptor (LD).
1574 * If yes, ignore LD and use descriptor processed
1575 * before LD.
1576 */
1577 if (tx_slot == priv->omsg_ring[ch].size) {
1578 if (prev_ptr)
1579 tx_slot = (prev_ptr -
1580 (u64)priv->omsg_ring[ch].omd_phys)/
1581 sizeof(struct tsi721_omsg_desc);
1582 else
1583 goto no_sts_update;
1584 }
1585
2ece1caf
AB		 1586 if (tx_slot >= priv->omsg_ring[ch].size)
1587 dev_dbg(&priv->pdev->dev,
1588 "OB_MSG tx_slot=%x > size=%x",
1589 tx_slot, priv->omsg_ring[ch].size);
1590 WARN_ON(tx_slot >= priv->omsg_ring[ch].size);
1591
48618fb4
AB		 1592 /* Move slot index to the next message to be sent */
1593 ++tx_slot;
1594 if (tx_slot == priv->omsg_ring[ch].size)
1595 tx_slot = 0;
2ece1caf
AB		 1596
1597 dev_id = priv->omsg_ring[ch].dev_id;
1598 do_callback = 1;
48618fb4
AB		 1599 }
1600
1601no_sts_update:
1602
1603 if (omsg_int & TSI721_OBDMAC_INT_ERROR) {
1604 /*
1605 * Outbound message operation aborted due to error,
1606 * reinitialize OB MSG channel
1607 */
1608
1609 dev_dbg(&priv->pdev->dev, "OB MSG ABORT ch_stat=%x\n",
1610 ioread32(priv->regs + TSI721_OBDMAC_STS(ch)));
1611
1612 iowrite32(TSI721_OBDMAC_INT_ERROR,
1613 priv->regs + TSI721_OBDMAC_INT(ch));
2ece1caf 1614 iowrite32(TSI721_OBDMAC_CTL_RETRY_THR | TSI721_OBDMAC_CTL_INIT,
48618fb4
AB		 1615 priv->regs + TSI721_OBDMAC_CTL(ch));
1616 ioread32(priv->regs + TSI721_OBDMAC_CTL(ch));
1617
1618 /* Inform upper level to clear all pending tx slots */
2ece1caf
AB		 1619 dev_id = priv->omsg_ring[ch].dev_id;
1620 tx_slot = priv->omsg_ring[ch].tx_slot;
1621 do_callback = 1;
1622
48618fb4
AB		 1623 /* Synch tx_slot tracking */
1624 iowrite32(priv->omsg_ring[ch].tx_slot,
1625 priv->regs + TSI721_OBDMAC_DRDCNT(ch));
1626 ioread32(priv->regs + TSI721_OBDMAC_DRDCNT(ch));
1627 priv->omsg_ring[ch].wr_count = priv->omsg_ring[ch].tx_slot;
1628 priv->omsg_ring[ch].sts_rdptr = 0;
1629 }
1630
1631 /* Clear channel interrupts */
1632 iowrite32(omsg_int, priv->regs + TSI721_OBDMAC_INT(ch));
1633
1634 if (!(priv->flags & TSI721_USING_MSIX)) {
1635 u32 ch_inte;
1636
1637 /* Re-enable channel interrupts */
1638 ch_inte = ioread32(priv->regs + TSI721_DEV_CHAN_INTE);
1639 ch_inte |= TSI721_INT_OMSG_CHAN(ch);
1640 iowrite32(ch_inte, priv->regs + TSI721_DEV_CHAN_INTE);
1641 }
1642
1643 spin_unlock(&priv->omsg_ring[ch].lock);
2ece1caf
AB		 1644
1645 if (mport->outb_msg[ch].mcback && do_callback)
1646 mport->outb_msg[ch].mcback(mport, dev_id, ch, tx_slot);
48618fb4
AB		 1647}
1648
1649/**
1650 * tsi721_open_outb_mbox - Initialize Tsi721 outbound mailbox
1651 * @mport: Master port implementing Outbound Messaging Engine
1652 * @dev_id: Device specific pointer to pass on event
1653 * @mbox: Mailbox to open
1654 * @entries: Number of entries in the outbound mailbox ring
1655 */
1656static int tsi721_open_outb_mbox(struct rio_mport *mport, void *dev_id,
1657 int mbox, int entries)
1658{
1659 struct tsi721_device *priv = mport->priv;
1660 struct tsi721_omsg_desc *bd_ptr;
1661 int i, rc = 0;
1662
1663 if ((entries < TSI721_OMSGD_MIN_RING_SIZE) ||
1664 (entries > (TSI721_OMSGD_RING_SIZE)) ||
1665 (!is_power_of_2(entries)) || mbox >= RIO_MAX_MBOX) {
1666 rc = -EINVAL;
1667 goto out;
1668 }
1669
1670 priv->omsg_ring[mbox].dev_id = dev_id;
1671 priv->omsg_ring[mbox].size = entries;
1672 priv->omsg_ring[mbox].sts_rdptr = 0;
1673 spin_lock_init(&priv->omsg_ring[mbox].lock);
1674
1675 /* Outbound Msg Buffer allocation based on
1676 the number of maximum descriptor entries */
1677 for (i = 0; i < entries; i++) {
1678 priv->omsg_ring[mbox].omq_base[i] =
1679 dma_alloc_coherent(
1680 &priv->pdev->dev, TSI721_MSG_BUFFER_SIZE,
1681 &priv->omsg_ring[mbox].omq_phys[i],
1682 GFP_KERNEL);
1683 if (priv->omsg_ring[mbox].omq_base[i] == NULL) {
1684 dev_dbg(&priv->pdev->dev,
1685 "Unable to allocate OB MSG data buffer for"
1686 " MBOX%d\n", mbox);
1687 rc = -ENOMEM;
1688 goto out_buf;
1689 }
1690 }
1691
1692 /* Outbound message descriptor allocation */
1693 priv->omsg_ring[mbox].omd_base = dma_alloc_coherent(
1694 &priv->pdev->dev,
1695 (entries + 1) * sizeof(struct tsi721_omsg_desc),
1696 &priv->omsg_ring[mbox].omd_phys, GFP_KERNEL);
1697 if (priv->omsg_ring[mbox].omd_base == NULL) {
1698 dev_dbg(&priv->pdev->dev,
1699 "Unable to allocate OB MSG descriptor memory "
1700 "for MBOX%d\n", mbox);
1701 rc = -ENOMEM;
1702 goto out_buf;
1703 }
1704
1705 priv->omsg_ring[mbox].tx_slot = 0;
1706
1707 /* Outbound message descriptor status FIFO allocation */
1708 priv->omsg_ring[mbox].sts_size = roundup_pow_of_two(entries + 1);
ceb96398 1709 priv->omsg_ring[mbox].sts_base = dma_zalloc_coherent(&priv->pdev->dev,
48618fb4
AB		 1710 priv->omsg_ring[mbox].sts_size *
1711 sizeof(struct tsi721_dma_sts),
1712 &priv->omsg_ring[mbox].sts_phys, GFP_KERNEL);
1713 if (priv->omsg_ring[mbox].sts_base == NULL) {
1714 dev_dbg(&priv->pdev->dev,
1715 "Unable to allocate OB MSG descriptor status FIFO "
1716 "for MBOX%d\n", mbox);
1717 rc = -ENOMEM;
1718 goto out_desc;
1719 }
1720
48618fb4
AB		 1721 /*
1722 * Configure Outbound Messaging Engine
1723 */
1724
1725 /* Setup Outbound Message descriptor pointer */
1726 iowrite32(((u64)priv->omsg_ring[mbox].omd_phys >> 32),
1727 priv->regs + TSI721_OBDMAC_DPTRH(mbox));
1728 iowrite32(((u64)priv->omsg_ring[mbox].omd_phys &
1729 TSI721_OBDMAC_DPTRL_MASK),
1730 priv->regs + TSI721_OBDMAC_DPTRL(mbox));
1731
1732 /* Setup Outbound Message descriptor status FIFO */
1733 iowrite32(((u64)priv->omsg_ring[mbox].sts_phys >> 32),
1734 priv->regs + TSI721_OBDMAC_DSBH(mbox));
1735 iowrite32(((u64)priv->omsg_ring[mbox].sts_phys &
1736 TSI721_OBDMAC_DSBL_MASK),
1737 priv->regs + TSI721_OBDMAC_DSBL(mbox));
1738 iowrite32(TSI721_DMAC_DSSZ_SIZE(priv->omsg_ring[mbox].sts_size),
1739 priv->regs + (u32)TSI721_OBDMAC_DSSZ(mbox));
1740
1741 /* Enable interrupts */
1742
1743#ifdef CONFIG_PCI_MSI
1744 if (priv->flags & TSI721_USING_MSIX) {
748353cc
AB		 1745 int idx = TSI721_VECT_OMB0_DONE + mbox;
1746
48618fb4 1747 /* Request interrupt service if we are in MSI-X mode */
748353cc
AB		 1748 rc = request_irq(priv->msix[idx].vector, tsi721_omsg_msix, 0,
1749 priv->msix[idx].irq_name, (void *)priv);
48618fb4
AB		 1750
1751 if (rc) {
1752 dev_dbg(&priv->pdev->dev,
1753 "Unable to allocate MSI-X interrupt for "
1754 "OBOX%d-DONE\n", mbox);
1755 goto out_stat;
1756 }
1757
748353cc
AB		 1758 idx = TSI721_VECT_OMB0_INT + mbox;
1759 rc = request_irq(priv->msix[idx].vector, tsi721_omsg_msix, 0,
1760 priv->msix[idx].irq_name, (void *)priv);
48618fb4
AB		 1761
1762 if (rc) {
1763 dev_dbg(&priv->pdev->dev,
1764 "Unable to allocate MSI-X interrupt for "
1765 "MBOX%d-INT\n", mbox);
748353cc
AB		 1766 idx = TSI721_VECT_OMB0_DONE + mbox;
1767 free_irq(priv->msix[idx].vector, (void *)priv);
48618fb4
AB		 1768 goto out_stat;
1769 }
1770 }
1771#endif /* CONFIG_PCI_MSI */
1772
1773 tsi721_omsg_interrupt_enable(priv, mbox, TSI721_OBDMAC_INT_ALL);
1774
1775 /* Initialize Outbound Message descriptors ring */
1776 bd_ptr = priv->omsg_ring[mbox].omd_base;
1777 bd_ptr[entries].type_id = cpu_to_le32(DTYPE5 << 29);
1778 bd_ptr[entries].msg_info = 0;
1779 bd_ptr[entries].next_lo =
1780 cpu_to_le32((u64)priv->omsg_ring[mbox].omd_phys &
1781 TSI721_OBDMAC_DPTRL_MASK);
1782 bd_ptr[entries].next_hi =
1783 cpu_to_le32((u64)priv->omsg_ring[mbox].omd_phys >> 32);
1784 priv->omsg_ring[mbox].wr_count = 0;
1785 mb();
1786
1787 /* Initialize Outbound Message engine */
2ece1caf
AB		 1788 iowrite32(TSI721_OBDMAC_CTL_RETRY_THR | TSI721_OBDMAC_CTL_INIT,
1789 priv->regs + TSI721_OBDMAC_CTL(mbox));
48618fb4
AB		 1790 ioread32(priv->regs + TSI721_OBDMAC_DWRCNT(mbox));
1791 udelay(10);
1792
1793 priv->omsg_init[mbox] = 1;
1794
1795 return 0;
1796
1797#ifdef CONFIG_PCI_MSI
1798out_stat:
1799 dma_free_coherent(&priv->pdev->dev,
1800 priv->omsg_ring[mbox].sts_size * sizeof(struct tsi721_dma_sts),
1801 priv->omsg_ring[mbox].sts_base,
1802 priv->omsg_ring[mbox].sts_phys);
1803
1804 priv->omsg_ring[mbox].sts_base = NULL;
1805#endif /* CONFIG_PCI_MSI */
1806
1807out_desc:
1808 dma_free_coherent(&priv->pdev->dev,
1809 (entries + 1) * sizeof(struct tsi721_omsg_desc),
1810 priv->omsg_ring[mbox].omd_base,
1811 priv->omsg_ring[mbox].omd_phys);
1812
1813 priv->omsg_ring[mbox].omd_base = NULL;
1814
1815out_buf:
1816 for (i = 0; i < priv->omsg_ring[mbox].size; i++) {
1817 if (priv->omsg_ring[mbox].omq_base[i]) {
1818 dma_free_coherent(&priv->pdev->dev,
1819 TSI721_MSG_BUFFER_SIZE,
1820 priv->omsg_ring[mbox].omq_base[i],
1821 priv->omsg_ring[mbox].omq_phys[i]);
1822
1823 priv->omsg_ring[mbox].omq_base[i] = NULL;
1824 }
1825 }
1826
1827out:
1828 return rc;
1829}
1830
1831/**
1832 * tsi721_close_outb_mbox - Close Tsi721 outbound mailbox
1833 * @mport: Master port implementing the outbound message unit
1834 * @mbox: Mailbox to close
1835 */
1836static void tsi721_close_outb_mbox(struct rio_mport *mport, int mbox)
1837{
1838 struct tsi721_device *priv = mport->priv;
1839 u32 i;
1840
1841 if (!priv->omsg_init[mbox])
1842 return;
1843 priv->omsg_init[mbox] = 0;
1844
1845 /* Disable Interrupts */
1846
1847 tsi721_omsg_interrupt_disable(priv, mbox, TSI721_OBDMAC_INT_ALL);
1848
1849#ifdef CONFIG_PCI_MSI
1850 if (priv->flags & TSI721_USING_MSIX) {
1851 free_irq(priv->msix[TSI721_VECT_OMB0_DONE + mbox].vector,
748353cc 1852 (void *)priv);
48618fb4 1853 free_irq(priv->msix[TSI721_VECT_OMB0_INT + mbox].vector,
748353cc 1854 (void *)priv);
48618fb4
AB		 1855 }
1856#endif /* CONFIG_PCI_MSI */
1857
1858 /* Free OMSG Descriptor Status FIFO */
1859 dma_free_coherent(&priv->pdev->dev,
1860 priv->omsg_ring[mbox].sts_size * sizeof(struct tsi721_dma_sts),
1861 priv->omsg_ring[mbox].sts_base,
1862 priv->omsg_ring[mbox].sts_phys);
1863
1864 priv->omsg_ring[mbox].sts_base = NULL;
1865
1866 /* Free OMSG descriptors */
1867 dma_free_coherent(&priv->pdev->dev,
1868 (priv->omsg_ring[mbox].size + 1) *
1869 sizeof(struct tsi721_omsg_desc),
1870 priv->omsg_ring[mbox].omd_base,
1871 priv->omsg_ring[mbox].omd_phys);
1872
1873 priv->omsg_ring[mbox].omd_base = NULL;
1874
1875 /* Free message buffers */
1876 for (i = 0; i < priv->omsg_ring[mbox].size; i++) {
1877 if (priv->omsg_ring[mbox].omq_base[i]) {
1878 dma_free_coherent(&priv->pdev->dev,
1879 TSI721_MSG_BUFFER_SIZE,
1880 priv->omsg_ring[mbox].omq_base[i],
1881 priv->omsg_ring[mbox].omq_phys[i]);
1882
1883 priv->omsg_ring[mbox].omq_base[i] = NULL;
1884 }
1885 }
1886}
1887
1888/**
1889 * tsi721_imsg_handler - Inbound Message Interrupt Handler
1890 * @priv: pointer to tsi721 private data
1891 * @ch: inbound message channel number to service
1892 *
1893 * Services channel interrupts from inbound messaging engine.
1894 */
1895static void tsi721_imsg_handler(struct tsi721_device *priv, int ch)
1896{
1897 u32 mbox = ch - 4;
1898 u32 imsg_int;
748353cc 1899 struct rio_mport *mport = &priv->mport;
48618fb4
AB		 1900
1901 spin_lock(&priv->imsg_ring[mbox].lock);
1902
1903 imsg_int = ioread32(priv->regs + TSI721_IBDMAC_INT(ch));
1904
1905 if (imsg_int & TSI721_IBDMAC_INT_SRTO)
1906 dev_info(&priv->pdev->dev, "IB MBOX%d SRIO timeout\n",
1907 mbox);
1908
1909 if (imsg_int & TSI721_IBDMAC_INT_PC_ERROR)
1910 dev_info(&priv->pdev->dev, "IB MBOX%d PCIe error\n",
1911 mbox);
1912
1913 if (imsg_int & TSI721_IBDMAC_INT_FQ_LOW)
1914 dev_info(&priv->pdev->dev,
1915 "IB MBOX%d IB free queue low\n", mbox);
1916
1917 /* Clear IB channel interrupts */
1918 iowrite32(imsg_int, priv->regs + TSI721_IBDMAC_INT(ch));
1919
1920 /* If an IB Msg is received notify the upper layer */
1921 if (imsg_int & TSI721_IBDMAC_INT_DQ_RCV &&
748353cc
AB		 1922 mport->inb_msg[mbox].mcback)
1923 mport->inb_msg[mbox].mcback(mport,
48618fb4
AB		 1924 priv->imsg_ring[mbox].dev_id, mbox, -1);
1925
1926 if (!(priv->flags & TSI721_USING_MSIX)) {
1927 u32 ch_inte;
1928
1929 /* Re-enable channel interrupts */
1930 ch_inte = ioread32(priv->regs + TSI721_DEV_CHAN_INTE);
1931 ch_inte |= TSI721_INT_IMSG_CHAN(ch);
1932 iowrite32(ch_inte, priv->regs + TSI721_DEV_CHAN_INTE);
1933 }
1934
1935 spin_unlock(&priv->imsg_ring[mbox].lock);
1936}
1937
1938/**
1939 * tsi721_open_inb_mbox - Initialize Tsi721 inbound mailbox
1940 * @mport: Master port implementing the Inbound Messaging Engine
1941 * @dev_id: Device specific pointer to pass on event
1942 * @mbox: Mailbox to open
1943 * @entries: Number of entries in the inbound mailbox ring
1944 */
1945static int tsi721_open_inb_mbox(struct rio_mport *mport, void *dev_id,
1946 int mbox, int entries)
1947{
1948 struct tsi721_device *priv = mport->priv;
1949 int ch = mbox + 4;
1950 int i;
1951 u64 *free_ptr;
1952 int rc = 0;
1953
1954 if ((entries < TSI721_IMSGD_MIN_RING_SIZE) ||
1955 (entries > TSI721_IMSGD_RING_SIZE) ||
1956 (!is_power_of_2(entries)) || mbox >= RIO_MAX_MBOX) {
1957 rc = -EINVAL;
1958 goto out;
1959 }
1960
1961 /* Initialize IB Messaging Ring */
1962 priv->imsg_ring[mbox].dev_id = dev_id;
1963 priv->imsg_ring[mbox].size = entries;
1964 priv->imsg_ring[mbox].rx_slot = 0;
1965 priv->imsg_ring[mbox].desc_rdptr = 0;
1966 priv->imsg_ring[mbox].fq_wrptr = 0;
1967 for (i = 0; i < priv->imsg_ring[mbox].size; i++)
1968 priv->imsg_ring[mbox].imq_base[i] = NULL;
1969 spin_lock_init(&priv->imsg_ring[mbox].lock);
1970
1971 /* Allocate buffers for incoming messages */
1972 priv->imsg_ring[mbox].buf_base =
1973 dma_alloc_coherent(&priv->pdev->dev,
1974 entries * TSI721_MSG_BUFFER_SIZE,
1975 &priv->imsg_ring[mbox].buf_phys,
1976 GFP_KERNEL);
1977
1978 if (priv->imsg_ring[mbox].buf_base == NULL) {
1979 dev_err(&priv->pdev->dev,
1980 "Failed to allocate buffers for IB MBOX%d\n", mbox);
1981 rc = -ENOMEM;
1982 goto out;
1983 }
1984
1985 /* Allocate memory for circular free list */
1986 priv->imsg_ring[mbox].imfq_base =
1987 dma_alloc_coherent(&priv->pdev->dev,
1988 entries * 8,
1989 &priv->imsg_ring[mbox].imfq_phys,
1990 GFP_KERNEL);
1991
1992 if (priv->imsg_ring[mbox].imfq_base == NULL) {
1993 dev_err(&priv->pdev->dev,
1994 "Failed to allocate free queue for IB MBOX%d\n", mbox);
1995 rc = -ENOMEM;
1996 goto out_buf;
1997 }
1998
1999 /* Allocate memory for Inbound message descriptors */
2000 priv->imsg_ring[mbox].imd_base =
2001 dma_alloc_coherent(&priv->pdev->dev,
2002 entries * sizeof(struct tsi721_imsg_desc),
2003 &priv->imsg_ring[mbox].imd_phys, GFP_KERNEL);
2004
2005 if (priv->imsg_ring[mbox].imd_base == NULL) {
2006 dev_err(&priv->pdev->dev,
2007 "Failed to allocate descriptor memory for IB MBOX%d\n",
2008 mbox);
2009 rc = -ENOMEM;
2010 goto out_dma;
2011 }
2012
2013 /* Fill free buffer pointer list */
2014 free_ptr = priv->imsg_ring[mbox].imfq_base;
2015 for (i = 0; i < entries; i++)
2016 free_ptr[i] = cpu_to_le64(
2017 (u64)(priv->imsg_ring[mbox].buf_phys) +
2018 i * 0x1000);
2019
2020 mb();
2021
2022 /*
2023 * For mapping of inbound SRIO Messages into appropriate queues we need
2024 * to set Inbound Device ID register in the messaging engine. We do it
2025 * once when first inbound mailbox is requested.
2026 */
2027 if (!(priv->flags & TSI721_IMSGID_SET)) {
748353cc 2028 iowrite32((u32)priv->mport.host_deviceid,
48618fb4
AB		 2029 priv->regs + TSI721_IB_DEVID);
2030 priv->flags |= TSI721_IMSGID_SET;
2031 }
2032
2033 /*
2034 * Configure Inbound Messaging channel (ch = mbox + 4)
2035 */
2036
2037 /* Setup Inbound Message free queue */
2038 iowrite32(((u64)priv->imsg_ring[mbox].imfq_phys >> 32),
2039 priv->regs + TSI721_IBDMAC_FQBH(ch));
2040 iowrite32(((u64)priv->imsg_ring[mbox].imfq_phys &
2041 TSI721_IBDMAC_FQBL_MASK),
2042 priv->regs+TSI721_IBDMAC_FQBL(ch));
2043 iowrite32(TSI721_DMAC_DSSZ_SIZE(entries),
2044 priv->regs + TSI721_IBDMAC_FQSZ(ch));
2045
2046 /* Setup Inbound Message descriptor queue */
2047 iowrite32(((u64)priv->imsg_ring[mbox].imd_phys >> 32),
2048 priv->regs + TSI721_IBDMAC_DQBH(ch));
2049 iowrite32(((u32)priv->imsg_ring[mbox].imd_phys &
2050 (u32)TSI721_IBDMAC_DQBL_MASK),
2051 priv->regs+TSI721_IBDMAC_DQBL(ch));
2052 iowrite32(TSI721_DMAC_DSSZ_SIZE(entries),
2053 priv->regs + TSI721_IBDMAC_DQSZ(ch));
2054
2055 /* Enable interrupts */
2056
2057#ifdef CONFIG_PCI_MSI
2058 if (priv->flags & TSI721_USING_MSIX) {
748353cc
AB		 2059 int idx = TSI721_VECT_IMB0_RCV + mbox;
2060
48618fb4 2061 /* Request interrupt service if we are in MSI-X mode */
748353cc
AB		 2062 rc = request_irq(priv->msix[idx].vector, tsi721_imsg_msix, 0,
2063 priv->msix[idx].irq_name, (void *)priv);
48618fb4
AB		 2064
2065 if (rc) {
2066 dev_dbg(&priv->pdev->dev,
2067 "Unable to allocate MSI-X interrupt for "
2068 "IBOX%d-DONE\n", mbox);
2069 goto out_desc;
2070 }
2071
748353cc
AB		 2072 idx = TSI721_VECT_IMB0_INT + mbox;
2073 rc = request_irq(priv->msix[idx].vector, tsi721_imsg_msix, 0,
2074 priv->msix[idx].irq_name, (void *)priv);
48618fb4
AB		 2075
2076 if (rc) {
2077 dev_dbg(&priv->pdev->dev,
2078 "Unable to allocate MSI-X interrupt for "
2079 "IBOX%d-INT\n", mbox);
2080 free_irq(
2081 priv->msix[TSI721_VECT_IMB0_RCV + mbox].vector,
748353cc 2082 (void *)priv);
48618fb4
AB		 2083 goto out_desc;
2084 }
2085 }
2086#endif /* CONFIG_PCI_MSI */
2087
2088 tsi721_imsg_interrupt_enable(priv, ch, TSI721_IBDMAC_INT_ALL);
2089
2090 /* Initialize Inbound Message Engine */
2091 iowrite32(TSI721_IBDMAC_CTL_INIT, priv->regs + TSI721_IBDMAC_CTL(ch));
2092 ioread32(priv->regs + TSI721_IBDMAC_CTL(ch));
2093 udelay(10);
2094 priv->imsg_ring[mbox].fq_wrptr = entries - 1;
2095 iowrite32(entries - 1, priv->regs + TSI721_IBDMAC_FQWP(ch));
2096
2097 priv->imsg_init[mbox] = 1;
2098 return 0;
2099
2100#ifdef CONFIG_PCI_MSI
2101out_desc:
2102 dma_free_coherent(&priv->pdev->dev,
2103 priv->imsg_ring[mbox].size * sizeof(struct tsi721_imsg_desc),
2104 priv->imsg_ring[mbox].imd_base,
2105 priv->imsg_ring[mbox].imd_phys);
2106
2107 priv->imsg_ring[mbox].imd_base = NULL;
2108#endif /* CONFIG_PCI_MSI */
2109
2110out_dma:
2111 dma_free_coherent(&priv->pdev->dev,
2112 priv->imsg_ring[mbox].size * 8,
2113 priv->imsg_ring[mbox].imfq_base,
2114 priv->imsg_ring[mbox].imfq_phys);
2115
2116 priv->imsg_ring[mbox].imfq_base = NULL;
2117
2118out_buf:
2119 dma_free_coherent(&priv->pdev->dev,
2120 priv->imsg_ring[mbox].size * TSI721_MSG_BUFFER_SIZE,
2121 priv->imsg_ring[mbox].buf_base,
2122 priv->imsg_ring[mbox].buf_phys);
2123
2124 priv->imsg_ring[mbox].buf_base = NULL;
2125
2126out:
2127 return rc;
2128}
2129
2130/**
2131 * tsi721_close_inb_mbox - Shut down Tsi721 inbound mailbox
2132 * @mport: Master port implementing the Inbound Messaging Engine
2133 * @mbox: Mailbox to close
2134 */
2135static void tsi721_close_inb_mbox(struct rio_mport *mport, int mbox)
2136{
2137 struct tsi721_device *priv = mport->priv;
2138 u32 rx_slot;
2139 int ch = mbox + 4;
2140
2141 if (!priv->imsg_init[mbox]) /* mbox isn't initialized yet */
2142 return;
2143 priv->imsg_init[mbox] = 0;
2144
2145 /* Disable Inbound Messaging Engine */
2146
2147 /* Disable Interrupts */
2148 tsi721_imsg_interrupt_disable(priv, ch, TSI721_OBDMAC_INT_MASK);
2149
2150#ifdef CONFIG_PCI_MSI
2151 if (priv->flags & TSI721_USING_MSIX) {
2152 free_irq(priv->msix[TSI721_VECT_IMB0_RCV + mbox].vector,
748353cc 2153 (void *)priv);
48618fb4 2154 free_irq(priv->msix[TSI721_VECT_IMB0_INT + mbox].vector,
748353cc 2155 (void *)priv);
48618fb4
AB		 2156 }
2157#endif /* CONFIG_PCI_MSI */
2158
2159 /* Clear Inbound Buffer Queue */
2160 for (rx_slot = 0; rx_slot < priv->imsg_ring[mbox].size; rx_slot++)
2161 priv->imsg_ring[mbox].imq_base[rx_slot] = NULL;
2162
2163 /* Free memory allocated for message buffers */
2164 dma_free_coherent(&priv->pdev->dev,
2165 priv->imsg_ring[mbox].size * TSI721_MSG_BUFFER_SIZE,
2166 priv->imsg_ring[mbox].buf_base,
2167 priv->imsg_ring[mbox].buf_phys);
2168
2169 priv->imsg_ring[mbox].buf_base = NULL;
2170
2171 /* Free memory allocated for free pointr list */
2172 dma_free_coherent(&priv->pdev->dev,
2173 priv->imsg_ring[mbox].size * 8,
2174 priv->imsg_ring[mbox].imfq_base,
2175 priv->imsg_ring[mbox].imfq_phys);
2176
2177 priv->imsg_ring[mbox].imfq_base = NULL;
2178
2179 /* Free memory allocated for RX descriptors */
2180 dma_free_coherent(&priv->pdev->dev,
2181 priv->imsg_ring[mbox].size * sizeof(struct tsi721_imsg_desc),
2182 priv->imsg_ring[mbox].imd_base,
2183 priv->imsg_ring[mbox].imd_phys);
2184
2185 priv->imsg_ring[mbox].imd_base = NULL;
2186}
2187
2188/**
2189 * tsi721_add_inb_buffer - Add buffer to the Tsi721 inbound message queue
2190 * @mport: Master port implementing the Inbound Messaging Engine
2191 * @mbox: Inbound mailbox number
2192 * @buf: Buffer to add to inbound queue
2193 */
2194static int tsi721_add_inb_buffer(struct rio_mport *mport, int mbox, void *buf)
2195{
2196 struct tsi721_device *priv = mport->priv;
2197 u32 rx_slot;
2198 int rc = 0;
2199
2200 rx_slot = priv->imsg_ring[mbox].rx_slot;
2201 if (priv->imsg_ring[mbox].imq_base[rx_slot]) {
2202 dev_err(&priv->pdev->dev,
2203 "Error adding inbound buffer %d, buffer exists\n",
2204 rx_slot);
2205 rc = -EINVAL;
2206 goto out;
2207 }
2208
2209 priv->imsg_ring[mbox].imq_base[rx_slot] = buf;
2210
2211 if (++priv->imsg_ring[mbox].rx_slot == priv->imsg_ring[mbox].size)
2212 priv->imsg_ring[mbox].rx_slot = 0;
2213
2214out:
2215 return rc;
2216}
2217
2218/**
2219 * tsi721_get_inb_message - Fetch inbound message from the Tsi721 MSG Queue
2220 * @mport: Master port implementing the Inbound Messaging Engine
2221 * @mbox: Inbound mailbox number
2222 *
2223 * Returns pointer to the message on success or NULL on failure.
2224 */
2225static void *tsi721_get_inb_message(struct rio_mport *mport, int mbox)
2226{
2227 struct tsi721_device *priv = mport->priv;
2228 struct tsi721_imsg_desc *desc;
2229 u32 rx_slot;
2230 void *rx_virt = NULL;
2231 u64 rx_phys;
2232 void *buf = NULL;
2233 u64 *free_ptr;
2234 int ch = mbox + 4;
2235 int msg_size;
2236
2237 if (!priv->imsg_init[mbox])
2238 return NULL;
2239
2240 desc = priv->imsg_ring[mbox].imd_base;
2241 desc += priv->imsg_ring[mbox].desc_rdptr;
2242
2243 if (!(le32_to_cpu(desc->msg_info) & TSI721_IMD_HO))
2244 goto out;
2245
2246 rx_slot = priv->imsg_ring[mbox].rx_slot;
2247 while (priv->imsg_ring[mbox].imq_base[rx_slot] == NULL) {
2248 if (++rx_slot == priv->imsg_ring[mbox].size)
2249 rx_slot = 0;
2250 }
2251
2252 rx_phys = ((u64)le32_to_cpu(desc->bufptr_hi) << 32) |
2253 le32_to_cpu(desc->bufptr_lo);
2254
2255 rx_virt = priv->imsg_ring[mbox].buf_base +
2256 (rx_phys - (u64)priv->imsg_ring[mbox].buf_phys);
2257
2258 buf = priv->imsg_ring[mbox].imq_base[rx_slot];
2259 msg_size = le32_to_cpu(desc->msg_info) & TSI721_IMD_BCOUNT;
2260 if (msg_size == 0)
2261 msg_size = RIO_MAX_MSG_SIZE;
2262
2263 memcpy(buf, rx_virt, msg_size);
2264 priv->imsg_ring[mbox].imq_base[rx_slot] = NULL;
2265
2266 desc->msg_info &= cpu_to_le32(~TSI721_IMD_HO);
2267 if (++priv->imsg_ring[mbox].desc_rdptr == priv->imsg_ring[mbox].size)
2268 priv->imsg_ring[mbox].desc_rdptr = 0;
2269
2270 iowrite32(priv->imsg_ring[mbox].desc_rdptr,
2271 priv->regs + TSI721_IBDMAC_DQRP(ch));
2272
2273 /* Return free buffer into the pointer list */
2274 free_ptr = priv->imsg_ring[mbox].imfq_base;
2275 free_ptr[priv->imsg_ring[mbox].fq_wrptr] = cpu_to_le64(rx_phys);
2276
2277 if (++priv->imsg_ring[mbox].fq_wrptr == priv->imsg_ring[mbox].size)
2278 priv->imsg_ring[mbox].fq_wrptr = 0;
2279
2280 iowrite32(priv->imsg_ring[mbox].fq_wrptr,
2281 priv->regs + TSI721_IBDMAC_FQWP(ch));
2282out:
2283 return buf;
2284}
2285
2286/**
2287 * tsi721_messages_init - Initialization of Messaging Engine
2288 * @priv: pointer to tsi721 private data
2289 *
2290 * Configures Tsi721 messaging engine.
2291 */
2292static int tsi721_messages_init(struct tsi721_device *priv)
2293{
2294 int ch;
2295
2296 iowrite32(0, priv->regs + TSI721_SMSG_ECC_LOG);
2297 iowrite32(0, priv->regs + TSI721_RETRY_GEN_CNT);
2298 iowrite32(0, priv->regs + TSI721_RETRY_RX_CNT);
2299
2300 /* Set SRIO Message Request/Response Timeout */
2301 iowrite32(TSI721_RQRPTO_VAL, priv->regs + TSI721_RQRPTO);
2302
2303 /* Initialize Inbound Messaging Engine Registers */
2304 for (ch = 0; ch < TSI721_IMSG_CHNUM; ch++) {
2305 /* Clear interrupt bits */
2306 iowrite32(TSI721_IBDMAC_INT_MASK,
2307 priv->regs + TSI721_IBDMAC_INT(ch));
2308 /* Clear Status */
2309 iowrite32(0, priv->regs + TSI721_IBDMAC_STS(ch));
2310
2311 iowrite32(TSI721_SMSG_ECC_COR_LOG_MASK,
2312 priv->regs + TSI721_SMSG_ECC_COR_LOG(ch));
2313 iowrite32(TSI721_SMSG_ECC_NCOR_MASK,
2314 priv->regs + TSI721_SMSG_ECC_NCOR(ch));
2315 }
2316
2317 return 0;
2318}
2319
dbe74afe
AB		 2320/**
2321 * tsi721_query_mport - Fetch inbound message from the Tsi721 MSG Queue
2322 * @mport: Master port implementing the Inbound Messaging Engine
2323 * @mbox: Inbound mailbox number
2324 *
2325 * Returns pointer to the message on success or NULL on failure.
2326 */
2327static int tsi721_query_mport(struct rio_mport *mport,
2328 struct rio_mport_attr *attr)
2329{
2330 struct tsi721_device *priv = mport->priv;
2331 u32 rval;
2332
2333 rval = ioread32(priv->regs + (0x100 + RIO_PORT_N_ERR_STS_CSR(0)));
2334 if (rval & RIO_PORT_N_ERR_STS_PORT_OK) {
2335 rval = ioread32(priv->regs + (0x100 + RIO_PORT_N_CTL2_CSR(0)));
2336 attr->link_speed = (rval & RIO_PORT_N_CTL2_SEL_BAUD) >> 28;
2337 rval = ioread32(priv->regs + (0x100 + RIO_PORT_N_CTL_CSR(0)));
2338 attr->link_width = (rval & RIO_PORT_N_CTL_IPW) >> 27;
2339 } else
2340 attr->link_speed = RIO_LINK_DOWN;
2341
2342#ifdef CONFIG_RAPIDIO_DMA_ENGINE
2343 attr->flags = RIO_MPORT_DMA | RIO_MPORT_DMA_SG;
2344 attr->dma_max_sge = 0;
2345 attr->dma_max_size = TSI721_BDMA_MAX_BCOUNT;
2346 attr->dma_align = 0;
2347#else
2348 attr->flags = 0;
2349#endif
2350 return 0;
2351}
2352
48618fb4
AB		 2353/**
2354 * tsi721_disable_ints - disables all device interrupts
2355 * @priv: pointer to tsi721 private data
2356 */
2357static void tsi721_disable_ints(struct tsi721_device *priv)
2358{
2359 int ch;
2360
2361 /* Disable all device level interrupts */
2362 iowrite32(0, priv->regs + TSI721_DEV_INTE);
2363
2364 /* Disable all Device Channel interrupts */
2365 iowrite32(0, priv->regs + TSI721_DEV_CHAN_INTE);
2366
2367 /* Disable all Inbound Msg Channel interrupts */
2368 for (ch = 0; ch < TSI721_IMSG_CHNUM; ch++)
2369 iowrite32(0, priv->regs + TSI721_IBDMAC_INTE(ch));
2370
2371 /* Disable all Outbound Msg Channel interrupts */
2372 for (ch = 0; ch < TSI721_OMSG_CHNUM; ch++)
2373 iowrite32(0, priv->regs + TSI721_OBDMAC_INTE(ch));
2374
2375 /* Disable all general messaging interrupts */
2376 iowrite32(0, priv->regs + TSI721_SMSG_INTE);
2377
2378 /* Disable all BDMA Channel interrupts */
2379 for (ch = 0; ch < TSI721_DMA_MAXCH; ch++)
9eaa3d9b
AB		 2380 iowrite32(0,
2381 priv->regs + TSI721_DMAC_BASE(ch) + TSI721_DMAC_INTE);
48618fb4
AB		 2382
2383 /* Disable all general BDMA interrupts */
2384 iowrite32(0, priv->regs + TSI721_BDMA_INTE);
2385
2386 /* Disable all SRIO Channel interrupts */
2387 for (ch = 0; ch < TSI721_SRIO_MAXCH; ch++)
2388 iowrite32(0, priv->regs + TSI721_SR_CHINTE(ch));
2389
2390 /* Disable all general SR2PC interrupts */
2391 iowrite32(0, priv->regs + TSI721_SR2PC_GEN_INTE);
2392
2393 /* Disable all PC2SR interrupts */
2394 iowrite32(0, priv->regs + TSI721_PC2SR_INTE);
2395
2396 /* Disable all I2C interrupts */
2397 iowrite32(0, priv->regs + TSI721_I2C_INT_ENABLE);
2398
2399 /* Disable SRIO MAC interrupts */
2400 iowrite32(0, priv->regs + TSI721_RIO_EM_INT_ENABLE);
2401 iowrite32(0, priv->regs + TSI721_RIO_EM_DEV_INT_EN);
2402}
2403
748353cc
AB		 2404static struct rio_ops tsi721_rio_ops = {
2405 .lcread = tsi721_lcread,
2406 .lcwrite = tsi721_lcwrite,
2407 .cread = tsi721_cread_dma,
2408 .cwrite = tsi721_cwrite_dma,
2409 .dsend = tsi721_dsend,
2410 .open_inb_mbox = tsi721_open_inb_mbox,
2411 .close_inb_mbox = tsi721_close_inb_mbox,
2412 .open_outb_mbox = tsi721_open_outb_mbox,
2413 .close_outb_mbox = tsi721_close_outb_mbox,
2414 .add_outb_message = tsi721_add_outb_message,
2415 .add_inb_buffer = tsi721_add_inb_buffer,
2416 .get_inb_message = tsi721_get_inb_message,
2417 .map_inb = tsi721_rio_map_inb_mem,
2418 .unmap_inb = tsi721_rio_unmap_inb_mem,
2419 .pwenable = tsi721_pw_enable,
2420 .query_mport = tsi721_query_mport,
2421};
2422
2423static void tsi721_mport_release(struct device *dev)
2424{
2425 struct rio_mport *mport = to_rio_mport(dev);
2426
2427 dev_dbg(dev, "RIO: %s %s id=%d\n", __func__, mport->name, mport->id);
2428}
2429
48618fb4
AB		 2430/**
2431 * tsi721_setup_mport - Setup Tsi721 as RapidIO subsystem master port
2432 * @priv: pointer to tsi721 private data
2433 *
2434 * Configures Tsi721 as RapidIO master port.
2435 */
305c891e 2436static int tsi721_setup_mport(struct tsi721_device *priv)
48618fb4
AB		 2437{
2438 struct pci_dev *pdev = priv->pdev;
2439 int err = 0;
748353cc 2440 struct rio_mport *mport = &priv->mport;
48618fb4 2441
748353cc
AB		 2442 err = rio_mport_initialize(mport);
2443 if (err)
2444 return err;
48618fb4 2445
748353cc 2446 mport->ops = &tsi721_rio_ops;
48618fb4
AB		 2447 mport->index = 0;
2448 mport->sys_size = 0; /* small system */
2449 mport->phy_type = RIO_PHY_SERIAL;
2450 mport->priv = (void *)priv;
2451 mport->phys_efptr = 0x100;
2aaf308b 2452 mport->dev.parent = &pdev->dev;
748353cc 2453 mport->dev.release = tsi721_mport_release;
48618fb4
AB		 2454
2455 INIT_LIST_HEAD(&mport->dbells);
2456
2457 rio_init_dbell_res(&mport->riores[RIO_DOORBELL_RESOURCE], 0, 0xffff);
b439e66f
AB		 2458 rio_init_mbox_res(&mport->riores[RIO_INB_MBOX_RESOURCE], 0, 3);
2459 rio_init_mbox_res(&mport->riores[RIO_OUTB_MBOX_RESOURCE], 0, 3);
ed43f44f
AB		 2460 snprintf(mport->name, RIO_MAX_MPORT_NAME, "%s(%s)",
2461 dev_driver_string(&pdev->dev), dev_name(&pdev->dev));
48618fb4
AB		 2462
2463 /* Hook up interrupt handler */
2464
2465#ifdef CONFIG_PCI_MSI
2466 if (!tsi721_enable_msix(priv))
2467 priv->flags |= TSI721_USING_MSIX;
2468 else if (!pci_enable_msi(pdev))
2469 priv->flags |= TSI721_USING_MSI;
2470 else
2471 dev_info(&pdev->dev,
2472 "MSI/MSI-X is not available. Using legacy INTx.\n");
2473#endif /* CONFIG_PCI_MSI */
2474
748353cc 2475 err = tsi721_request_irq(priv);
48618fb4 2476
748353cc 2477 if (err) {
48618fb4
AB		 2478 dev_err(&pdev->dev, "Unable to get assigned PCI IRQ "
2479 "vector %02X err=0x%x\n", pdev->irq, err);
748353cc 2480 return err;
9eaa3d9b 2481 }
48618fb4 2482
9eaa3d9b 2483#ifdef CONFIG_RAPIDIO_DMA_ENGINE
748353cc
AB		 2484 err = tsi721_register_dma(priv);
2485 if (err)
2486 goto err_exit;
9eaa3d9b 2487#endif
48618fb4
AB		 2488 /* Enable SRIO link */
2489 iowrite32(ioread32(priv->regs + TSI721_DEVCTL) |
2490 TSI721_DEVCTL_SRBOOT_CMPL,
2491 priv->regs + TSI721_DEVCTL);
2492
48618fb4
AB		 2493 if (mport->host_deviceid >= 0)
2494 iowrite32(RIO_PORT_GEN_HOST | RIO_PORT_GEN_MASTER |
2495 RIO_PORT_GEN_DISCOVERED,
2496 priv->regs + (0x100 + RIO_PORT_GEN_CTL_CSR));
2497 else
2498 iowrite32(0, priv->regs + (0x100 + RIO_PORT_GEN_CTL_CSR));
2499
748353cc
AB		 2500 err = rio_register_mport(mport);
2501 if (err) {
2502 tsi721_unregister_dma(priv);
2503 goto err_exit;
2504 }
2505
48618fb4 2506 return 0;
9eaa3d9b
AB		 2507
2508err_exit:
748353cc 2509 tsi721_free_irq(priv);
9eaa3d9b 2510 return err;
48618fb4
AB		 2511}
2512
305c891e 2513static int tsi721_probe(struct pci_dev *pdev,
48618fb4
AB		 2514 const struct pci_device_id *id)
2515{
2516 struct tsi721_device *priv;
48618fb4 2517 int err;
48618fb4
AB		 2518
2519 priv = kzalloc(sizeof(struct tsi721_device), GFP_KERNEL);
2520 if (priv == NULL) {
2521 dev_err(&pdev->dev, "Failed to allocate memory for device\n");
2522 err = -ENOMEM;
2523 goto err_exit;
2524 }
2525
2526 err = pci_enable_device(pdev);
2527 if (err) {
2528 dev_err(&pdev->dev, "Failed to enable PCI device\n");
2529 goto err_clean;
2530 }
2531
2532 priv->pdev = pdev;
2533
2534#ifdef DEBUG
9a9a9a7a
AB		 2535 {
2536 int i;
48618fb4
AB		 2537 for (i = 0; i <= PCI_STD_RESOURCE_END; i++) {
2538 dev_dbg(&pdev->dev, "res[%d] @ 0x%llx (0x%lx, 0x%lx)\n",
2539 i, (unsigned long long)pci_resource_start(pdev, i),
2540 (unsigned long)pci_resource_len(pdev, i),
2541 pci_resource_flags(pdev, i));
2542 }
9a9a9a7a 2543 }
48618fb4
AB		 2544#endif
2545 /*
2546 * Verify BAR configuration
2547 */
2548
2549 /* BAR_0 (registers) must be 512KB+ in 32-bit address space */
2550 if (!(pci_resource_flags(pdev, BAR_0) & IORESOURCE_MEM) ||
2551 pci_resource_flags(pdev, BAR_0) & IORESOURCE_MEM_64 ||
2552 pci_resource_len(pdev, BAR_0) < TSI721_REG_SPACE_SIZE) {
2553 dev_err(&pdev->dev,
2554 "Missing or misconfigured CSR BAR0, aborting.\n");
2555 err = -ENODEV;
2556 goto err_disable_pdev;
2557 }
2558
2559 /* BAR_1 (outbound doorbells) must be 16MB+ in 32-bit address space */
2560 if (!(pci_resource_flags(pdev, BAR_1) & IORESOURCE_MEM) ||
2561 pci_resource_flags(pdev, BAR_1) & IORESOURCE_MEM_64 ||
2562 pci_resource_len(pdev, BAR_1) < TSI721_DB_WIN_SIZE) {
2563 dev_err(&pdev->dev,
2564 "Missing or misconfigured Doorbell BAR1, aborting.\n");
2565 err = -ENODEV;
2566 goto err_disable_pdev;
2567 }
2568
2569 /*
2570 * BAR_2 and BAR_4 (outbound translation) must be in 64-bit PCIe address
2571 * space.
2572 * NOTE: BAR_2 and BAR_4 are not used by this version of driver.
2573 * It may be a good idea to keep them disabled using HW configuration
2574 * to save PCI memory space.
2575 */
2576 if ((pci_resource_flags(pdev, BAR_2) & IORESOURCE_MEM) &&
2577 (pci_resource_flags(pdev, BAR_2) & IORESOURCE_MEM_64)) {
2578 dev_info(&pdev->dev, "Outbound BAR2 is not used but enabled.\n");
2579 }
2580
2581 if ((pci_resource_flags(pdev, BAR_4) & IORESOURCE_MEM) &&
2582 (pci_resource_flags(pdev, BAR_4) & IORESOURCE_MEM_64)) {
2583 dev_info(&pdev->dev, "Outbound BAR4 is not used but enabled.\n");
2584 }
2585
2586 err = pci_request_regions(pdev, DRV_NAME);
2587 if (err) {
2588 dev_err(&pdev->dev, "Cannot obtain PCI resources, "
2589 "aborting.\n");
2590 goto err_disable_pdev;
2591 }
2592
2593 pci_set_master(pdev);
2594
2595 priv->regs = pci_ioremap_bar(pdev, BAR_0);
2596 if (!priv->regs) {
2597 dev_err(&pdev->dev,
2598 "Unable to map device registers space, aborting\n");
2599 err = -ENOMEM;
2600 goto err_free_res;
2601 }
2602
2603 priv->odb_base = pci_ioremap_bar(pdev, BAR_1);
2604 if (!priv->odb_base) {
2605 dev_err(&pdev->dev,
2606 "Unable to map outbound doorbells space, aborting\n");
2607 err = -ENOMEM;
2608 goto err_unmap_bars;
2609 }
2610
2611 /* Configure DMA attributes. */
2612 if (pci_set_dma_mask(pdev, DMA_BIT_MASK(64))) {
18f6287f
PST		 2613 err = pci_set_dma_mask(pdev, DMA_BIT_MASK(32));
2614 if (err) {
48618fb4
AB		 2615 dev_info(&pdev->dev, "Unable to set DMA mask\n");
2616 goto err_unmap_bars;
2617 }
2618
2619 if (pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(32)))
2620 dev_info(&pdev->dev, "Unable to set consistent DMA mask\n");
2621 } else {
2622 err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64));
2623 if (err)
2624 dev_info(&pdev->dev, "Unable to set consistent DMA mask\n");
2625 }
2626
5cdaaf8a 2627 BUG_ON(!pci_is_pcie(pdev));
1cee22b7 2628
174f1a71 2629 /* Clear "no snoop" and "relaxed ordering" bits. */
5cdaaf8a 2630 pcie_capability_clear_and_set_word(pdev, PCI_EXP_DEVCTL,
174f1a71 2631 PCI_EXP_DEVCTL_RELAX_EN | PCI_EXP_DEVCTL_NOSNOOP_EN, 0);
1cee22b7
AB		 2632
2633 /* Adjust PCIe completion timeout. */
5cdaaf8a 2634 pcie_capability_clear_and_set_word(pdev, PCI_EXP_DEVCTL2, 0xf, 0x2);
48618fb4
AB		 2635
2636 /*
2637 * FIXUP: correct offsets of MSI-X tables in the MSI-X Capability Block
2638 */
2639 pci_write_config_dword(pdev, TSI721_PCIECFG_EPCTL, 0x01);
2640 pci_write_config_dword(pdev, TSI721_PCIECFG_MSIXTBL,
2641 TSI721_MSIXTBL_OFFSET);
2642 pci_write_config_dword(pdev, TSI721_PCIECFG_MSIXPBA,
2643 TSI721_MSIXPBA_OFFSET);
2644 pci_write_config_dword(pdev, TSI721_PCIECFG_EPCTL, 0);
2645 /* End of FIXUP */
2646
2647 tsi721_disable_ints(priv);
2648
2649 tsi721_init_pc2sr_mapping(priv);
2650 tsi721_init_sr2pc_mapping(priv);
2651
9eaa3d9b 2652 if (tsi721_bdma_maint_init(priv)) {
48618fb4
AB		 2653 dev_err(&pdev->dev, "BDMA initialization failed, aborting\n");
2654 err = -ENOMEM;
2655 goto err_unmap_bars;
2656 }
2657
2658 err = tsi721_doorbell_init(priv);
2659 if (err)
2660 goto err_free_bdma;
2661
2662 tsi721_port_write_init(priv);
2663
2664 err = tsi721_messages_init(priv);
2665 if (err)
2666 goto err_free_consistent;
2667
2668 err = tsi721_setup_mport(priv);
2669 if (err)
2670 goto err_free_consistent;
2671
e3dd8cd4 2672 pci_set_drvdata(pdev, priv);
748353cc 2673 tsi721_interrupts_init(priv);
e3dd8cd4 2674
48618fb4
AB		 2675 return 0;
2676
2677err_free_consistent:
748353cc 2678 tsi721_port_write_free(priv);
48618fb4
AB		 2679 tsi721_doorbell_free(priv);
2680err_free_bdma:
9eaa3d9b 2681 tsi721_bdma_maint_free(priv);
48618fb4
AB		 2682err_unmap_bars:
2683 if (priv->regs)
2684 iounmap(priv->regs);
2685 if (priv->odb_base)
2686 iounmap(priv->odb_base);
2687err_free_res:
2688 pci_release_regions(pdev);
2689 pci_clear_master(pdev);
2690err_disable_pdev:
2691 pci_disable_device(pdev);
2692err_clean:
2693 kfree(priv);
2694err_exit:
2695 return err;
2696}
2697
748353cc
AB		 2698static void tsi721_remove(struct pci_dev *pdev)
2699{
2700 struct tsi721_device *priv = pci_get_drvdata(pdev);
2701
2702 dev_dbg(&pdev->dev, "%s enter\n", __func__);
2703
2704 tsi721_disable_ints(priv);
2705 tsi721_free_irq(priv);
9a0b0627 2706 flush_scheduled_work();
748353cc
AB		 2707 rio_unregister_mport(&priv->mport);
2708
2709 tsi721_unregister_dma(priv);
2710 tsi721_bdma_maint_free(priv);
2711 tsi721_doorbell_free(priv);
2712 tsi721_port_write_free(priv);
2713 tsi721_close_sr2pc_mapping(priv);
2714
2715 if (priv->regs)
2716 iounmap(priv->regs);
2717 if (priv->odb_base)
2718 iounmap(priv->odb_base);
2719#ifdef CONFIG_PCI_MSI
2720 if (priv->flags & TSI721_USING_MSIX)
2721 pci_disable_msix(priv->pdev);
2722 else if (priv->flags & TSI721_USING_MSI)
2723 pci_disable_msi(priv->pdev);
2724#endif
2725 pci_release_regions(pdev);
2726 pci_clear_master(pdev);
2727 pci_disable_device(pdev);
2728 pci_set_drvdata(pdev, NULL);
2729 kfree(priv);
2730 dev_dbg(&pdev->dev, "%s exit\n", __func__);
2731}
2732
e3dd8cd4
AB		 2733static void tsi721_shutdown(struct pci_dev *pdev)
2734{
2735 struct tsi721_device *priv = pci_get_drvdata(pdev);
2736
2737 dev_dbg(&pdev->dev, "RIO: %s\n", __func__);
2738
2739 tsi721_disable_ints(priv);
2740 tsi721_dma_stop_all(priv);
2741 pci_clear_master(pdev);
2742 pci_disable_device(pdev);
2743}
2744
9baa3c34 2745static const struct pci_device_id tsi721_pci_tbl[] = {
48618fb4
AB		 2746 { PCI_DEVICE(PCI_VENDOR_ID_IDT, PCI_DEVICE_ID_TSI721) },
2747 { 0, } /* terminate list */
2748};
2749
2750MODULE_DEVICE_TABLE(pci, tsi721_pci_tbl);
2751
2752static struct pci_driver tsi721_driver = {
2753 .name = "tsi721",
2754 .id_table = tsi721_pci_tbl,
2755 .probe = tsi721_probe,
748353cc 2756 .remove = tsi721_remove,
e3dd8cd4 2757 .shutdown = tsi721_shutdown,
48618fb4
AB		 2758};
2759
748353cc 2760module_pci_driver(tsi721_driver);
94d9bd45
AB		 2761
2762MODULE_DESCRIPTION("IDT Tsi721 PCIExpress-to-SRIO bridge driver");
2763MODULE_AUTHOR("Integrated Device Technology, Inc.");
2764MODULE_LICENSE("GPL");
Linux kernel - Deliverables
This page took 0.779074 seconds and 5 git commands to generate.