Commit | Line | Data |
---|---|---|
1da177e4 LT |
1 | /* |
2 | * This file is subject to the terms and conditions of the GNU General Public | |
3 | * License. See the file "COPYING" in the main directory of this archive | |
4 | * for more details. | |
5 | * | |
5b53ed1f | 6 | * Copyright (C) 2001-2005 Silicon Graphics, Inc. All rights reserved. |
1da177e4 LT |
7 | */ |
8 | ||
9 | #include <linux/types.h> | |
10 | #include <linux/pci.h> | |
c13cf371 | 11 | #include <asm/sn/addrs.h> |
1da177e4 | 12 | #include <asm/sn/geo.h> |
c13cf371 | 13 | #include <asm/sn/pcibr_provider.h> |
9b08ebd1 MM |
14 | #include <asm/sn/pcibus_provider_defs.h> |
15 | #include <asm/sn/pcidev.h> | |
c13cf371 PB |
16 | #include <asm/sn/pic.h> |
17 | #include <asm/sn/sn_sal.h> | |
18 | #include <asm/sn/tiocp.h> | |
1da177e4 | 19 | #include "tio.h" |
c13cf371 PB |
20 | #include "xtalk/xwidgetdev.h" |
21 | #include "xtalk/hubdev.h" | |
1da177e4 LT |
22 | |
23 | extern int sn_ioif_inited; | |
24 | ||
25 | /* ===================================================================== | |
26 | * DMA MANAGEMENT | |
27 | * | |
28 | * The Bridge ASIC provides three methods of doing DMA: via a "direct map" | |
29 | * register available in 32-bit PCI space (which selects a contiguous 2G | |
30 | * address space on some other widget), via "direct" addressing via 64-bit | |
31 | * PCI space (all destination information comes from the PCI address, | |
32 | * including transfer attributes), and via a "mapped" region that allows | |
33 | * a bunch of different small mappings to be established with the PMU. | |
34 | * | |
35 | * For efficiency, we most prefer to use the 32bit direct mapping facility, | |
36 | * since it requires no resource allocations. The advantage of using the | |
37 | * PMU over the 64-bit direct is that single-cycle PCI addressing can be | |
38 | * used; the advantage of using 64-bit direct over PMU addressing is that | |
39 | * we do not have to allocate entries in the PMU. | |
40 | */ | |
41 | ||
e955d825 | 42 | static dma_addr_t |
1da177e4 | 43 | pcibr_dmamap_ate32(struct pcidev_info *info, |
53493dcf | 44 | u64 paddr, size_t req_size, u64 flags) |
1da177e4 LT |
45 | { |
46 | ||
47 | struct pcidev_info *pcidev_info = info->pdi_host_pcidev_info; | |
48 | struct pcibus_info *pcibus_info = (struct pcibus_info *)pcidev_info-> | |
49 | pdi_pcibus_info; | |
53493dcf | 50 | u8 internal_device = (PCI_SLOT(pcidev_info->pdi_host_pcidev_info-> |
1da177e4 LT |
51 | pdi_linux_pcidev->devfn)) - 1; |
52 | int ate_count; | |
53 | int ate_index; | |
53493dcf PB |
54 | u64 ate_flags = flags | PCI32_ATE_V; |
55 | u64 ate; | |
56 | u64 pci_addr; | |
57 | u64 xio_addr; | |
58 | u64 offset; | |
1da177e4 LT |
59 | |
60 | /* PIC in PCI-X mode does not supports 32bit PageMap mode */ | |
61 | if (IS_PIC_SOFT(pcibus_info) && IS_PCIX(pcibus_info)) { | |
62 | return 0; | |
63 | } | |
64 | ||
65 | /* Calculate the number of ATEs needed. */ | |
66 | if (!(MINIMAL_ATE_FLAG(paddr, req_size))) { | |
67 | ate_count = IOPG((IOPGSIZE - 1) /* worst case start offset */ | |
68 | +req_size /* max mapping bytes */ | |
69 | - 1) + 1; /* round UP */ | |
70 | } else { /* assume requested target is page aligned */ | |
71 | ate_count = IOPG(req_size /* max mapping bytes */ | |
72 | - 1) + 1; /* round UP */ | |
73 | } | |
74 | ||
75 | /* Get the number of ATEs required. */ | |
76 | ate_index = pcibr_ate_alloc(pcibus_info, ate_count); | |
77 | if (ate_index < 0) | |
78 | return 0; | |
79 | ||
80 | /* In PCI-X mode, Prefetch not supported */ | |
81 | if (IS_PCIX(pcibus_info)) | |
82 | ate_flags &= ~(PCI32_ATE_PREF); | |
83 | ||
84 | xio_addr = | |
85 | IS_PIC_SOFT(pcibus_info) ? PHYS_TO_DMA(paddr) : | |
86 | PHYS_TO_TIODMA(paddr); | |
87 | offset = IOPGOFF(xio_addr); | |
88 | ate = ate_flags | (xio_addr - offset); | |
89 | ||
90 | /* If PIC, put the targetid in the ATE */ | |
91 | if (IS_PIC_SOFT(pcibus_info)) { | |
92 | ate |= (pcibus_info->pbi_hub_xid << PIC_ATE_TARGETID_SHFT); | |
93 | } | |
94 | ate_write(pcibus_info, ate_index, ate_count, ate); | |
95 | ||
96 | /* | |
97 | * Set up the DMA mapped Address. | |
98 | */ | |
99 | pci_addr = PCI32_MAPPED_BASE + offset + IOPGSIZE * ate_index; | |
100 | ||
101 | /* | |
102 | * If swap was set in device in pcibr_endian_set() | |
103 | * we need to turn swapping on. | |
104 | */ | |
105 | if (pcibus_info->pbi_devreg[internal_device] & PCIBR_DEV_SWAP_DIR) | |
106 | ATE_SWAP_ON(pci_addr); | |
107 | ||
108 | return pci_addr; | |
109 | } | |
110 | ||
e955d825 | 111 | static dma_addr_t |
53493dcf PB |
112 | pcibr_dmatrans_direct64(struct pcidev_info * info, u64 paddr, |
113 | u64 dma_attributes) | |
1da177e4 LT |
114 | { |
115 | struct pcibus_info *pcibus_info = (struct pcibus_info *) | |
116 | ((info->pdi_host_pcidev_info)->pdi_pcibus_info); | |
53493dcf | 117 | u64 pci_addr; |
1da177e4 LT |
118 | |
119 | /* Translate to Crosstalk View of Physical Address */ | |
120 | pci_addr = (IS_PIC_SOFT(pcibus_info) ? PHYS_TO_DMA(paddr) : | |
121 | PHYS_TO_TIODMA(paddr)) | dma_attributes; | |
122 | ||
123 | /* Handle Bus mode */ | |
124 | if (IS_PCIX(pcibus_info)) | |
125 | pci_addr &= ~PCI64_ATTR_PREF; | |
126 | ||
127 | /* Handle Bridge Chipset differences */ | |
128 | if (IS_PIC_SOFT(pcibus_info)) { | |
129 | pci_addr |= | |
53493dcf | 130 | ((u64) pcibus_info-> |
1da177e4 LT |
131 | pbi_hub_xid << PIC_PCI64_ATTR_TARG_SHFT); |
132 | } else | |
133 | pci_addr |= TIOCP_PCI64_CMDTYPE_MEM; | |
134 | ||
135 | /* If PCI mode, func zero uses VCHAN0, every other func uses VCHAN1 */ | |
136 | if (!IS_PCIX(pcibus_info) && PCI_FUNC(info->pdi_linux_pcidev->devfn)) | |
137 | pci_addr |= PCI64_ATTR_VIRTUAL; | |
138 | ||
139 | return pci_addr; | |
1da177e4 LT |
140 | } |
141 | ||
e955d825 | 142 | static dma_addr_t |
1da177e4 | 143 | pcibr_dmatrans_direct32(struct pcidev_info * info, |
53493dcf | 144 | u64 paddr, size_t req_size, u64 flags) |
1da177e4 | 145 | { |
1da177e4 LT |
146 | struct pcidev_info *pcidev_info = info->pdi_host_pcidev_info; |
147 | struct pcibus_info *pcibus_info = (struct pcibus_info *)pcidev_info-> | |
148 | pdi_pcibus_info; | |
53493dcf | 149 | u64 xio_addr; |
1da177e4 | 150 | |
53493dcf PB |
151 | u64 xio_base; |
152 | u64 offset; | |
153 | u64 endoff; | |
1da177e4 LT |
154 | |
155 | if (IS_PCIX(pcibus_info)) { | |
156 | return 0; | |
157 | } | |
158 | ||
159 | xio_addr = IS_PIC_SOFT(pcibus_info) ? PHYS_TO_DMA(paddr) : | |
160 | PHYS_TO_TIODMA(paddr); | |
161 | ||
162 | xio_base = pcibus_info->pbi_dir_xbase; | |
163 | offset = xio_addr - xio_base; | |
164 | endoff = req_size + offset; | |
165 | if ((req_size > (1ULL << 31)) || /* Too Big */ | |
166 | (xio_addr < xio_base) || /* Out of range for mappings */ | |
167 | (endoff > (1ULL << 31))) { /* Too Big */ | |
168 | return 0; | |
169 | } | |
170 | ||
171 | return PCI32_DIRECT_BASE | offset; | |
1da177e4 LT |
172 | } |
173 | ||
174 | /* | |
175 | * Wrapper routine for free'ing DMA maps | |
176 | * DMA mappings for Direct 64 and 32 do not have any DMA maps. | |
177 | */ | |
178 | void | |
e955d825 | 179 | pcibr_dma_unmap(struct pci_dev *hwdev, dma_addr_t dma_handle, int direction) |
1da177e4 | 180 | { |
e955d825 MM |
181 | struct pcidev_info *pcidev_info = SN_PCIDEV_INFO(hwdev); |
182 | struct pcibus_info *pcibus_info = | |
183 | (struct pcibus_info *)pcidev_info->pdi_pcibus_info; | |
1da177e4 LT |
184 | |
185 | if (IS_PCI32_MAPPED(dma_handle)) { | |
186 | int ate_index; | |
187 | ||
188 | ate_index = | |
189 | IOPG((ATE_SWAP_OFF(dma_handle) - PCI32_MAPPED_BASE)); | |
190 | pcibr_ate_free(pcibus_info, ate_index); | |
191 | } | |
192 | } | |
193 | ||
194 | /* | |
195 | * On SN systems there is a race condition between a PIO read response and | |
196 | * DMA's. In rare cases, the read response may beat the DMA, causing the | |
197 | * driver to think that data in memory is complete and meaningful. This code | |
198 | * eliminates that race. This routine is called by the PIO read routines | |
199 | * after doing the read. For PIC this routine then forces a fake interrupt | |
200 | * on another line, which is logically associated with the slot that the PIO | |
201 | * is addressed to. It then spins while watching the memory location that | |
202 | * the interrupt is targetted to. When the interrupt response arrives, we | |
203 | * are sure that the DMA has landed in memory and it is safe for the driver | |
204 | * to proceed. For TIOCP use the Device(x) Write Request Buffer Flush | |
205 | * Bridge register since it ensures the data has entered the coherence domain, | |
206 | * unlike the PIC Device(x) Write Request Buffer Flush register. | |
207 | */ | |
208 | ||
53493dcf | 209 | void sn_dma_flush(u64 addr) |
1da177e4 LT |
210 | { |
211 | nasid_t nasid; | |
212 | int is_tio; | |
213 | int wid_num; | |
214 | int i, j; | |
53493dcf PB |
215 | u64 flags; |
216 | u64 itte; | |
1da177e4 | 217 | struct hubdev_info *hubinfo; |
8ed9b2c7 JS |
218 | struct sn_flush_device_kernel *p; |
219 | struct sn_flush_device_common *common; | |
1da177e4 LT |
220 | struct sn_flush_nasid_entry *flush_nasid_list; |
221 | ||
222 | if (!sn_ioif_inited) | |
223 | return; | |
224 | ||
225 | nasid = NASID_GET(addr); | |
226 | if (-1 == nasid_to_cnodeid(nasid)) | |
227 | return; | |
228 | ||
229 | hubinfo = (NODEPDA(nasid_to_cnodeid(nasid)))->pdinfo; | |
230 | ||
231 | if (!hubinfo) { | |
232 | BUG(); | |
233 | } | |
1da177e4 LT |
234 | |
235 | flush_nasid_list = &hubinfo->hdi_flush_nasid_list; | |
236 | if (flush_nasid_list->widget_p == NULL) | |
237 | return; | |
5b53ed1f MM |
238 | |
239 | is_tio = (nasid & 1); | |
240 | if (is_tio) { | |
241 | int itte_index; | |
242 | ||
243 | if (TIO_HWIN(addr)) | |
244 | itte_index = 0; | |
245 | else if (TIO_BWIN_WINDOWNUM(addr)) | |
246 | itte_index = TIO_BWIN_WINDOWNUM(addr); | |
247 | else | |
248 | itte_index = -1; | |
249 | ||
250 | if (itte_index >= 0) { | |
251 | itte = flush_nasid_list->iio_itte[itte_index]; | |
252 | if (! TIO_ITTE_VALID(itte)) | |
253 | return; | |
254 | wid_num = TIO_ITTE_WIDGET(itte); | |
255 | } else | |
256 | wid_num = TIO_SWIN_WIDGETNUM(addr); | |
257 | } else { | |
258 | if (BWIN_WINDOWNUM(addr)) { | |
259 | itte = flush_nasid_list->iio_itte[BWIN_WINDOWNUM(addr)]; | |
260 | wid_num = IIO_ITTE_WIDGET(itte); | |
261 | } else | |
262 | wid_num = SWIN_WIDGETNUM(addr); | |
1da177e4 | 263 | } |
1da177e4 LT |
264 | if (flush_nasid_list->widget_p[wid_num] == NULL) |
265 | return; | |
266 | p = &flush_nasid_list->widget_p[wid_num][0]; | |
267 | ||
268 | /* find a matching BAR */ | |
6d6e4200 PB |
269 | for (i = 0; i < DEV_PER_WIDGET; i++,p++) { |
270 | common = p->common; | |
1da177e4 | 271 | for (j = 0; j < PCI_ROM_RESOURCE; j++) { |
6d6e4200 | 272 | if (common->sfdl_bar_list[j].start == 0) |
1da177e4 | 273 | break; |
6d6e4200 PB |
274 | if (addr >= common->sfdl_bar_list[j].start |
275 | && addr <= common->sfdl_bar_list[j].end) | |
1da177e4 LT |
276 | break; |
277 | } | |
6d6e4200 | 278 | if (j < PCI_ROM_RESOURCE && common->sfdl_bar_list[j].start != 0) |
1da177e4 | 279 | break; |
1da177e4 LT |
280 | } |
281 | ||
282 | /* if no matching BAR, return without doing anything. */ | |
283 | if (i == DEV_PER_WIDGET) | |
284 | return; | |
285 | ||
286 | /* | |
287 | * For TIOCP use the Device(x) Write Request Buffer Flush Bridge | |
288 | * register since it ensures the data has entered the coherence | |
5b53ed1f | 289 | * domain, unlike PIC. |
1da177e4 LT |
290 | */ |
291 | if (is_tio) { | |
5b53ed1f MM |
292 | /* |
293 | * Note: devices behind TIOCE should never be matched in the | |
294 | * above code, and so the following code is PIC/CP centric. | |
295 | * If CE ever needs the sn_dma_flush mechanism, we will have | |
296 | * to account for that here and in tioce_bus_fixup(). | |
297 | */ | |
53493dcf PB |
298 | u32 tio_id = HUB_L(TIO_IOSPACE_ADDR(nasid, TIO_NODE_ID)); |
299 | u32 revnum = XWIDGET_PART_REV_NUM(tio_id); | |
1da177e4 LT |
300 | |
301 | /* TIOCP BRINGUP WAR (PV907516): Don't write buffer flush reg */ | |
302 | if ((1 << XWIDGET_PART_REV_NUM_REV(revnum)) & PV907516) { | |
303 | return; | |
304 | } else { | |
6d6e4200 PB |
305 | pcireg_wrb_flush_get(common->sfdl_pcibus_info, |
306 | (common->sfdl_slot - 1)); | |
1da177e4 LT |
307 | } |
308 | } else { | |
8ed9b2c7 | 309 | spin_lock_irqsave(&p->sfdl_flush_lock, flags); |
6d6e4200 | 310 | *common->sfdl_flush_addr = 0; |
1da177e4 LT |
311 | |
312 | /* force an interrupt. */ | |
53493dcf | 313 | *(volatile u32 *)(common->sfdl_force_int_addr) = 1; |
1da177e4 LT |
314 | |
315 | /* wait for the interrupt to come back. */ | |
6d6e4200 | 316 | while (*(common->sfdl_flush_addr) != 0x10f) |
68b9753f | 317 | cpu_relax(); |
1da177e4 LT |
318 | |
319 | /* okay, everything is synched up. */ | |
8ed9b2c7 | 320 | spin_unlock_irqrestore(&p->sfdl_flush_lock, flags); |
1da177e4 LT |
321 | } |
322 | return; | |
323 | } | |
324 | ||
325 | /* | |
e955d825 | 326 | * DMA interfaces. Called from pci_dma.c routines. |
1da177e4 LT |
327 | */ |
328 | ||
e955d825 MM |
329 | dma_addr_t |
330 | pcibr_dma_map(struct pci_dev * hwdev, unsigned long phys_addr, size_t size) | |
1da177e4 LT |
331 | { |
332 | dma_addr_t dma_handle; | |
e955d825 | 333 | struct pcidev_info *pcidev_info = SN_PCIDEV_INFO(hwdev); |
1da177e4 | 334 | |
e955d825 MM |
335 | /* SN cannot support DMA addresses smaller than 32 bits. */ |
336 | if (hwdev->dma_mask < 0x7fffffff) { | |
337 | return 0; | |
338 | } | |
1da177e4 | 339 | |
e955d825 MM |
340 | if (hwdev->dma_mask == ~0UL) { |
341 | /* | |
342 | * Handle the most common case: 64 bit cards. This | |
343 | * call should always succeed. | |
344 | */ | |
345 | ||
346 | dma_handle = pcibr_dmatrans_direct64(pcidev_info, phys_addr, | |
347 | PCI64_ATTR_PREF); | |
348 | } else { | |
349 | /* Handle 32-63 bit cards via direct mapping */ | |
350 | dma_handle = pcibr_dmatrans_direct32(pcidev_info, phys_addr, | |
351 | size, 0); | |
352 | if (!dma_handle) { | |
1da177e4 | 353 | /* |
e955d825 MM |
354 | * It is a 32 bit card and we cannot do direct mapping, |
355 | * so we use an ATE. | |
1da177e4 LT |
356 | */ |
357 | ||
e955d825 MM |
358 | dma_handle = pcibr_dmamap_ate32(pcidev_info, phys_addr, |
359 | size, PCI32_ATE_PREF); | |
1da177e4 LT |
360 | } |
361 | } | |
362 | ||
363 | return dma_handle; | |
364 | } | |
365 | ||
e955d825 MM |
366 | dma_addr_t |
367 | pcibr_dma_map_consistent(struct pci_dev * hwdev, unsigned long phys_addr, | |
368 | size_t size) | |
369 | { | |
370 | dma_addr_t dma_handle; | |
371 | struct pcidev_info *pcidev_info = SN_PCIDEV_INFO(hwdev); | |
372 | ||
373 | if (hwdev->dev.coherent_dma_mask == ~0UL) { | |
374 | dma_handle = pcibr_dmatrans_direct64(pcidev_info, phys_addr, | |
375 | PCI64_ATTR_BAR); | |
376 | } else { | |
377 | dma_handle = (dma_addr_t) pcibr_dmamap_ate32(pcidev_info, | |
378 | phys_addr, size, | |
379 | PCI32_ATE_BAR); | |
380 | } | |
381 | ||
382 | return dma_handle; | |
383 | } | |
384 | ||
1da177e4 | 385 | EXPORT_SYMBOL(sn_dma_flush); |