Commit | Line | Data |
---|---|---|
1da177e4 LT |
1 | /* |
2 | * This file is subject to the terms and conditions of the GNU General Public | |
3 | * License. See the file "COPYING" in the main directory of this archive | |
4 | * for more details. | |
5 | * | |
5b53ed1f | 6 | * Copyright (C) 2001-2005 Silicon Graphics, Inc. All rights reserved. |
1da177e4 LT |
7 | */ |
8 | ||
9 | #include <linux/types.h> | |
10 | #include <linux/pci.h> | |
c13cf371 | 11 | #include <asm/sn/addrs.h> |
1da177e4 | 12 | #include <asm/sn/geo.h> |
c13cf371 | 13 | #include <asm/sn/pcibr_provider.h> |
9b08ebd1 MM |
14 | #include <asm/sn/pcibus_provider_defs.h> |
15 | #include <asm/sn/pcidev.h> | |
c13cf371 PB |
16 | #include <asm/sn/pic.h> |
17 | #include <asm/sn/sn_sal.h> | |
18 | #include <asm/sn/tiocp.h> | |
1da177e4 | 19 | #include "tio.h" |
c13cf371 PB |
20 | #include "xtalk/xwidgetdev.h" |
21 | #include "xtalk/hubdev.h" | |
1da177e4 LT |
22 | |
23 | extern int sn_ioif_inited; | |
24 | ||
25 | /* ===================================================================== | |
26 | * DMA MANAGEMENT | |
27 | * | |
28 | * The Bridge ASIC provides three methods of doing DMA: via a "direct map" | |
29 | * register available in 32-bit PCI space (which selects a contiguous 2G | |
30 | * address space on some other widget), via "direct" addressing via 64-bit | |
31 | * PCI space (all destination information comes from the PCI address, | |
32 | * including transfer attributes), and via a "mapped" region that allows | |
33 | * a bunch of different small mappings to be established with the PMU. | |
34 | * | |
35 | * For efficiency, we most prefer to use the 32bit direct mapping facility, | |
36 | * since it requires no resource allocations. The advantage of using the | |
37 | * PMU over the 64-bit direct is that single-cycle PCI addressing can be | |
38 | * used; the advantage of using 64-bit direct over PMU addressing is that | |
39 | * we do not have to allocate entries in the PMU. | |
40 | */ | |
41 | ||
e955d825 | 42 | static dma_addr_t |
1da177e4 | 43 | pcibr_dmamap_ate32(struct pcidev_info *info, |
83821d3f | 44 | u64 paddr, size_t req_size, u64 flags, int dma_flags) |
1da177e4 LT |
45 | { |
46 | ||
47 | struct pcidev_info *pcidev_info = info->pdi_host_pcidev_info; | |
48 | struct pcibus_info *pcibus_info = (struct pcibus_info *)pcidev_info-> | |
49 | pdi_pcibus_info; | |
53493dcf | 50 | u8 internal_device = (PCI_SLOT(pcidev_info->pdi_host_pcidev_info-> |
1da177e4 LT |
51 | pdi_linux_pcidev->devfn)) - 1; |
52 | int ate_count; | |
53 | int ate_index; | |
53493dcf PB |
54 | u64 ate_flags = flags | PCI32_ATE_V; |
55 | u64 ate; | |
56 | u64 pci_addr; | |
57 | u64 xio_addr; | |
58 | u64 offset; | |
1da177e4 LT |
59 | |
60 | /* PIC in PCI-X mode does not supports 32bit PageMap mode */ | |
61 | if (IS_PIC_SOFT(pcibus_info) && IS_PCIX(pcibus_info)) { | |
62 | return 0; | |
63 | } | |
64 | ||
65 | /* Calculate the number of ATEs needed. */ | |
66 | if (!(MINIMAL_ATE_FLAG(paddr, req_size))) { | |
67 | ate_count = IOPG((IOPGSIZE - 1) /* worst case start offset */ | |
68 | +req_size /* max mapping bytes */ | |
69 | - 1) + 1; /* round UP */ | |
70 | } else { /* assume requested target is page aligned */ | |
71 | ate_count = IOPG(req_size /* max mapping bytes */ | |
72 | - 1) + 1; /* round UP */ | |
73 | } | |
74 | ||
75 | /* Get the number of ATEs required. */ | |
76 | ate_index = pcibr_ate_alloc(pcibus_info, ate_count); | |
77 | if (ate_index < 0) | |
78 | return 0; | |
79 | ||
80 | /* In PCI-X mode, Prefetch not supported */ | |
81 | if (IS_PCIX(pcibus_info)) | |
82 | ate_flags &= ~(PCI32_ATE_PREF); | |
83 | ||
83821d3f MM |
84 | if (SN_DMA_ADDRTYPE(dma_flags == SN_DMA_ADDR_PHYS)) |
85 | xio_addr = IS_PIC_SOFT(pcibus_info) ? PHYS_TO_DMA(paddr) : | |
86 | PHYS_TO_TIODMA(paddr); | |
87 | else | |
88 | xio_addr = paddr; | |
89 | ||
1da177e4 LT |
90 | offset = IOPGOFF(xio_addr); |
91 | ate = ate_flags | (xio_addr - offset); | |
92 | ||
93 | /* If PIC, put the targetid in the ATE */ | |
94 | if (IS_PIC_SOFT(pcibus_info)) { | |
95 | ate |= (pcibus_info->pbi_hub_xid << PIC_ATE_TARGETID_SHFT); | |
96 | } | |
83821d3f MM |
97 | |
98 | /* | |
2e0d232b MH |
99 | * If we're mapping for MSI, set the MSI bit in the ATE. If it's a |
100 | * TIOCP based pci bus, we also need to set the PIO bit in the ATE. | |
83821d3f | 101 | */ |
2e0d232b | 102 | if (dma_flags & SN_DMA_MSI) { |
83821d3f | 103 | ate |= PCI32_ATE_MSI; |
2e0d232b MH |
104 | if (IS_TIOCP_SOFT(pcibus_info)) |
105 | ate |= PCI32_ATE_PIO; | |
106 | } | |
83821d3f | 107 | |
1da177e4 LT |
108 | ate_write(pcibus_info, ate_index, ate_count, ate); |
109 | ||
110 | /* | |
111 | * Set up the DMA mapped Address. | |
112 | */ | |
113 | pci_addr = PCI32_MAPPED_BASE + offset + IOPGSIZE * ate_index; | |
114 | ||
115 | /* | |
116 | * If swap was set in device in pcibr_endian_set() | |
117 | * we need to turn swapping on. | |
118 | */ | |
119 | if (pcibus_info->pbi_devreg[internal_device] & PCIBR_DEV_SWAP_DIR) | |
120 | ATE_SWAP_ON(pci_addr); | |
121 | ||
83821d3f | 122 | |
1da177e4 LT |
123 | return pci_addr; |
124 | } | |
125 | ||
e955d825 | 126 | static dma_addr_t |
53493dcf | 127 | pcibr_dmatrans_direct64(struct pcidev_info * info, u64 paddr, |
83821d3f | 128 | u64 dma_attributes, int dma_flags) |
1da177e4 LT |
129 | { |
130 | struct pcibus_info *pcibus_info = (struct pcibus_info *) | |
131 | ((info->pdi_host_pcidev_info)->pdi_pcibus_info); | |
53493dcf | 132 | u64 pci_addr; |
1da177e4 LT |
133 | |
134 | /* Translate to Crosstalk View of Physical Address */ | |
83821d3f MM |
135 | if (SN_DMA_ADDRTYPE(dma_flags) == SN_DMA_ADDR_PHYS) |
136 | pci_addr = IS_PIC_SOFT(pcibus_info) ? | |
137 | PHYS_TO_DMA(paddr) : | |
c63c5805 | 138 | PHYS_TO_TIODMA(paddr); |
83821d3f | 139 | else |
c63c5805 JH |
140 | pci_addr = paddr; |
141 | pci_addr |= dma_attributes; | |
1da177e4 LT |
142 | |
143 | /* Handle Bus mode */ | |
144 | if (IS_PCIX(pcibus_info)) | |
145 | pci_addr &= ~PCI64_ATTR_PREF; | |
146 | ||
147 | /* Handle Bridge Chipset differences */ | |
148 | if (IS_PIC_SOFT(pcibus_info)) { | |
149 | pci_addr |= | |
53493dcf | 150 | ((u64) pcibus_info-> |
1da177e4 LT |
151 | pbi_hub_xid << PIC_PCI64_ATTR_TARG_SHFT); |
152 | } else | |
83821d3f MM |
153 | pci_addr |= (dma_flags & SN_DMA_MSI) ? |
154 | TIOCP_PCI64_CMDTYPE_MSI : | |
155 | TIOCP_PCI64_CMDTYPE_MEM; | |
1da177e4 LT |
156 | |
157 | /* If PCI mode, func zero uses VCHAN0, every other func uses VCHAN1 */ | |
158 | if (!IS_PCIX(pcibus_info) && PCI_FUNC(info->pdi_linux_pcidev->devfn)) | |
159 | pci_addr |= PCI64_ATTR_VIRTUAL; | |
160 | ||
161 | return pci_addr; | |
1da177e4 LT |
162 | } |
163 | ||
e955d825 | 164 | static dma_addr_t |
1da177e4 | 165 | pcibr_dmatrans_direct32(struct pcidev_info * info, |
83821d3f | 166 | u64 paddr, size_t req_size, u64 flags, int dma_flags) |
1da177e4 | 167 | { |
1da177e4 LT |
168 | struct pcidev_info *pcidev_info = info->pdi_host_pcidev_info; |
169 | struct pcibus_info *pcibus_info = (struct pcibus_info *)pcidev_info-> | |
170 | pdi_pcibus_info; | |
53493dcf | 171 | u64 xio_addr; |
1da177e4 | 172 | |
53493dcf PB |
173 | u64 xio_base; |
174 | u64 offset; | |
175 | u64 endoff; | |
1da177e4 LT |
176 | |
177 | if (IS_PCIX(pcibus_info)) { | |
178 | return 0; | |
179 | } | |
180 | ||
83821d3f MM |
181 | if (dma_flags & SN_DMA_MSI) |
182 | return 0; | |
183 | ||
184 | if (SN_DMA_ADDRTYPE(dma_flags) == SN_DMA_ADDR_PHYS) | |
185 | xio_addr = IS_PIC_SOFT(pcibus_info) ? PHYS_TO_DMA(paddr) : | |
186 | PHYS_TO_TIODMA(paddr); | |
187 | else | |
188 | xio_addr = paddr; | |
1da177e4 LT |
189 | |
190 | xio_base = pcibus_info->pbi_dir_xbase; | |
191 | offset = xio_addr - xio_base; | |
192 | endoff = req_size + offset; | |
193 | if ((req_size > (1ULL << 31)) || /* Too Big */ | |
194 | (xio_addr < xio_base) || /* Out of range for mappings */ | |
195 | (endoff > (1ULL << 31))) { /* Too Big */ | |
196 | return 0; | |
197 | } | |
198 | ||
199 | return PCI32_DIRECT_BASE | offset; | |
1da177e4 LT |
200 | } |
201 | ||
202 | /* | |
72fdbdce | 203 | * Wrapper routine for freeing DMA maps |
1da177e4 LT |
204 | * DMA mappings for Direct 64 and 32 do not have any DMA maps. |
205 | */ | |
206 | void | |
e955d825 | 207 | pcibr_dma_unmap(struct pci_dev *hwdev, dma_addr_t dma_handle, int direction) |
1da177e4 | 208 | { |
e955d825 MM |
209 | struct pcidev_info *pcidev_info = SN_PCIDEV_INFO(hwdev); |
210 | struct pcibus_info *pcibus_info = | |
211 | (struct pcibus_info *)pcidev_info->pdi_pcibus_info; | |
1da177e4 LT |
212 | |
213 | if (IS_PCI32_MAPPED(dma_handle)) { | |
214 | int ate_index; | |
215 | ||
216 | ate_index = | |
217 | IOPG((ATE_SWAP_OFF(dma_handle) - PCI32_MAPPED_BASE)); | |
218 | pcibr_ate_free(pcibus_info, ate_index); | |
219 | } | |
220 | } | |
221 | ||
222 | /* | |
223 | * On SN systems there is a race condition between a PIO read response and | |
224 | * DMA's. In rare cases, the read response may beat the DMA, causing the | |
225 | * driver to think that data in memory is complete and meaningful. This code | |
226 | * eliminates that race. This routine is called by the PIO read routines | |
227 | * after doing the read. For PIC this routine then forces a fake interrupt | |
228 | * on another line, which is logically associated with the slot that the PIO | |
229 | * is addressed to. It then spins while watching the memory location that | |
230 | * the interrupt is targetted to. When the interrupt response arrives, we | |
231 | * are sure that the DMA has landed in memory and it is safe for the driver | |
232 | * to proceed. For TIOCP use the Device(x) Write Request Buffer Flush | |
233 | * Bridge register since it ensures the data has entered the coherence domain, | |
234 | * unlike the PIC Device(x) Write Request Buffer Flush register. | |
235 | */ | |
236 | ||
53493dcf | 237 | void sn_dma_flush(u64 addr) |
1da177e4 LT |
238 | { |
239 | nasid_t nasid; | |
240 | int is_tio; | |
241 | int wid_num; | |
242 | int i, j; | |
c53421b1 | 243 | unsigned long flags; |
53493dcf | 244 | u64 itte; |
1da177e4 | 245 | struct hubdev_info *hubinfo; |
8ed9b2c7 JS |
246 | struct sn_flush_device_kernel *p; |
247 | struct sn_flush_device_common *common; | |
1da177e4 LT |
248 | struct sn_flush_nasid_entry *flush_nasid_list; |
249 | ||
250 | if (!sn_ioif_inited) | |
251 | return; | |
252 | ||
253 | nasid = NASID_GET(addr); | |
254 | if (-1 == nasid_to_cnodeid(nasid)) | |
255 | return; | |
256 | ||
257 | hubinfo = (NODEPDA(nasid_to_cnodeid(nasid)))->pdinfo; | |
258 | ||
259 | if (!hubinfo) { | |
260 | BUG(); | |
261 | } | |
1da177e4 LT |
262 | |
263 | flush_nasid_list = &hubinfo->hdi_flush_nasid_list; | |
264 | if (flush_nasid_list->widget_p == NULL) | |
265 | return; | |
5b53ed1f MM |
266 | |
267 | is_tio = (nasid & 1); | |
268 | if (is_tio) { | |
269 | int itte_index; | |
270 | ||
271 | if (TIO_HWIN(addr)) | |
272 | itte_index = 0; | |
273 | else if (TIO_BWIN_WINDOWNUM(addr)) | |
274 | itte_index = TIO_BWIN_WINDOWNUM(addr); | |
275 | else | |
276 | itte_index = -1; | |
277 | ||
278 | if (itte_index >= 0) { | |
279 | itte = flush_nasid_list->iio_itte[itte_index]; | |
280 | if (! TIO_ITTE_VALID(itte)) | |
281 | return; | |
282 | wid_num = TIO_ITTE_WIDGET(itte); | |
283 | } else | |
284 | wid_num = TIO_SWIN_WIDGETNUM(addr); | |
285 | } else { | |
286 | if (BWIN_WINDOWNUM(addr)) { | |
287 | itte = flush_nasid_list->iio_itte[BWIN_WINDOWNUM(addr)]; | |
288 | wid_num = IIO_ITTE_WIDGET(itte); | |
289 | } else | |
290 | wid_num = SWIN_WIDGETNUM(addr); | |
1da177e4 | 291 | } |
1da177e4 LT |
292 | if (flush_nasid_list->widget_p[wid_num] == NULL) |
293 | return; | |
294 | p = &flush_nasid_list->widget_p[wid_num][0]; | |
295 | ||
296 | /* find a matching BAR */ | |
6d6e4200 PB |
297 | for (i = 0; i < DEV_PER_WIDGET; i++,p++) { |
298 | common = p->common; | |
1da177e4 | 299 | for (j = 0; j < PCI_ROM_RESOURCE; j++) { |
6d6e4200 | 300 | if (common->sfdl_bar_list[j].start == 0) |
1da177e4 | 301 | break; |
6d6e4200 PB |
302 | if (addr >= common->sfdl_bar_list[j].start |
303 | && addr <= common->sfdl_bar_list[j].end) | |
1da177e4 LT |
304 | break; |
305 | } | |
6d6e4200 | 306 | if (j < PCI_ROM_RESOURCE && common->sfdl_bar_list[j].start != 0) |
1da177e4 | 307 | break; |
1da177e4 LT |
308 | } |
309 | ||
310 | /* if no matching BAR, return without doing anything. */ | |
311 | if (i == DEV_PER_WIDGET) | |
312 | return; | |
313 | ||
314 | /* | |
315 | * For TIOCP use the Device(x) Write Request Buffer Flush Bridge | |
316 | * register since it ensures the data has entered the coherence | |
5b53ed1f | 317 | * domain, unlike PIC. |
1da177e4 LT |
318 | */ |
319 | if (is_tio) { | |
5b53ed1f MM |
320 | /* |
321 | * Note: devices behind TIOCE should never be matched in the | |
322 | * above code, and so the following code is PIC/CP centric. | |
323 | * If CE ever needs the sn_dma_flush mechanism, we will have | |
324 | * to account for that here and in tioce_bus_fixup(). | |
325 | */ | |
53493dcf PB |
326 | u32 tio_id = HUB_L(TIO_IOSPACE_ADDR(nasid, TIO_NODE_ID)); |
327 | u32 revnum = XWIDGET_PART_REV_NUM(tio_id); | |
1da177e4 LT |
328 | |
329 | /* TIOCP BRINGUP WAR (PV907516): Don't write buffer flush reg */ | |
330 | if ((1 << XWIDGET_PART_REV_NUM_REV(revnum)) & PV907516) { | |
331 | return; | |
332 | } else { | |
6d6e4200 PB |
333 | pcireg_wrb_flush_get(common->sfdl_pcibus_info, |
334 | (common->sfdl_slot - 1)); | |
1da177e4 LT |
335 | } |
336 | } else { | |
8ed9b2c7 | 337 | spin_lock_irqsave(&p->sfdl_flush_lock, flags); |
6d6e4200 | 338 | *common->sfdl_flush_addr = 0; |
1da177e4 LT |
339 | |
340 | /* force an interrupt. */ | |
53493dcf | 341 | *(volatile u32 *)(common->sfdl_force_int_addr) = 1; |
1da177e4 LT |
342 | |
343 | /* wait for the interrupt to come back. */ | |
6d6e4200 | 344 | while (*(common->sfdl_flush_addr) != 0x10f) |
68b9753f | 345 | cpu_relax(); |
1da177e4 LT |
346 | |
347 | /* okay, everything is synched up. */ | |
8ed9b2c7 | 348 | spin_unlock_irqrestore(&p->sfdl_flush_lock, flags); |
1da177e4 LT |
349 | } |
350 | return; | |
351 | } | |
352 | ||
353 | /* | |
e955d825 | 354 | * DMA interfaces. Called from pci_dma.c routines. |
1da177e4 LT |
355 | */ |
356 | ||
e955d825 | 357 | dma_addr_t |
83821d3f | 358 | pcibr_dma_map(struct pci_dev * hwdev, unsigned long phys_addr, size_t size, int dma_flags) |
1da177e4 LT |
359 | { |
360 | dma_addr_t dma_handle; | |
e955d825 | 361 | struct pcidev_info *pcidev_info = SN_PCIDEV_INFO(hwdev); |
1da177e4 | 362 | |
e955d825 MM |
363 | /* SN cannot support DMA addresses smaller than 32 bits. */ |
364 | if (hwdev->dma_mask < 0x7fffffff) { | |
365 | return 0; | |
366 | } | |
1da177e4 | 367 | |
e955d825 MM |
368 | if (hwdev->dma_mask == ~0UL) { |
369 | /* | |
370 | * Handle the most common case: 64 bit cards. This | |
371 | * call should always succeed. | |
372 | */ | |
373 | ||
374 | dma_handle = pcibr_dmatrans_direct64(pcidev_info, phys_addr, | |
83821d3f | 375 | PCI64_ATTR_PREF, dma_flags); |
e955d825 MM |
376 | } else { |
377 | /* Handle 32-63 bit cards via direct mapping */ | |
378 | dma_handle = pcibr_dmatrans_direct32(pcidev_info, phys_addr, | |
83821d3f | 379 | size, 0, dma_flags); |
e955d825 | 380 | if (!dma_handle) { |
1da177e4 | 381 | /* |
e955d825 MM |
382 | * It is a 32 bit card and we cannot do direct mapping, |
383 | * so we use an ATE. | |
1da177e4 LT |
384 | */ |
385 | ||
e955d825 | 386 | dma_handle = pcibr_dmamap_ate32(pcidev_info, phys_addr, |
83821d3f MM |
387 | size, PCI32_ATE_PREF, |
388 | dma_flags); | |
1da177e4 LT |
389 | } |
390 | } | |
391 | ||
392 | return dma_handle; | |
393 | } | |
394 | ||
e955d825 MM |
395 | dma_addr_t |
396 | pcibr_dma_map_consistent(struct pci_dev * hwdev, unsigned long phys_addr, | |
83821d3f | 397 | size_t size, int dma_flags) |
e955d825 MM |
398 | { |
399 | dma_addr_t dma_handle; | |
400 | struct pcidev_info *pcidev_info = SN_PCIDEV_INFO(hwdev); | |
401 | ||
402 | if (hwdev->dev.coherent_dma_mask == ~0UL) { | |
403 | dma_handle = pcibr_dmatrans_direct64(pcidev_info, phys_addr, | |
83821d3f | 404 | PCI64_ATTR_BAR, dma_flags); |
e955d825 MM |
405 | } else { |
406 | dma_handle = (dma_addr_t) pcibr_dmamap_ate32(pcidev_info, | |
407 | phys_addr, size, | |
83821d3f | 408 | PCI32_ATE_BAR, dma_flags); |
e955d825 MM |
409 | } |
410 | ||
411 | return dma_handle; | |
412 | } | |
413 | ||
1da177e4 | 414 | EXPORT_SYMBOL(sn_dma_flush); |