Commit | Line | Data |
---|---|---|
1da177e4 LT |
1 | /* |
2 | * Nvidia AGPGART routines. | |
3 | * Based upon a 2.4 agpgart diff by the folks from NVIDIA, and hacked up | |
f4432c5c | 4 | * to work in 2.5 by Dave Jones <davej@redhat.com> |
1da177e4 LT |
5 | */ |
6 | ||
7 | #include <linux/module.h> | |
8 | #include <linux/pci.h> | |
9 | #include <linux/init.h> | |
10 | #include <linux/agp_backend.h> | |
1da177e4 LT |
11 | #include <linux/page-flags.h> |
12 | #include <linux/mm.h> | |
05613bdd | 13 | #include <linux/jiffies.h> |
1da177e4 LT |
14 | #include "agp.h" |
15 | ||
16 | /* NVIDIA registers */ | |
17 | #define NVIDIA_0_APSIZE 0x80 | |
18 | #define NVIDIA_1_WBC 0xf0 | |
19 | #define NVIDIA_2_GARTCTRL 0xd0 | |
20 | #define NVIDIA_2_APBASE 0xd8 | |
21 | #define NVIDIA_2_APLIMIT 0xdc | |
22 | #define NVIDIA_2_ATTBASE(i) (0xe0 + (i) * 4) | |
23 | #define NVIDIA_3_APBASE 0x50 | |
24 | #define NVIDIA_3_APLIMIT 0x54 | |
25 | ||
26 | ||
27 | static struct _nvidia_private { | |
28 | struct pci_dev *dev_1; | |
29 | struct pci_dev *dev_2; | |
30 | struct pci_dev *dev_3; | |
31 | volatile u32 __iomem *aperture; | |
32 | int num_active_entries; | |
33 | off_t pg_offset; | |
34 | u32 wbc_mask; | |
35 | } nvidia_private; | |
36 | ||
37 | ||
38 | static int nvidia_fetch_size(void) | |
39 | { | |
40 | int i; | |
41 | u8 size_value; | |
42 | struct aper_size_info_8 *values; | |
43 | ||
44 | pci_read_config_byte(agp_bridge->dev, NVIDIA_0_APSIZE, &size_value); | |
45 | size_value &= 0x0f; | |
46 | values = A_SIZE_8(agp_bridge->driver->aperture_sizes); | |
47 | ||
48 | for (i = 0; i < agp_bridge->driver->num_aperture_sizes; i++) { | |
49 | if (size_value == values[i].size_value) { | |
50 | agp_bridge->previous_size = | |
51 | agp_bridge->current_size = (void *) (values + i); | |
52 | agp_bridge->aperture_size_idx = i; | |
53 | return values[i].size; | |
54 | } | |
55 | } | |
56 | ||
57 | return 0; | |
58 | } | |
59 | ||
60 | #define SYSCFG 0xC0010010 | |
61 | #define IORR_BASE0 0xC0010016 | |
62 | #define IORR_MASK0 0xC0010017 | |
63 | #define AMD_K7_NUM_IORR 2 | |
64 | ||
65 | static int nvidia_init_iorr(u32 base, u32 size) | |
66 | { | |
67 | u32 base_hi, base_lo; | |
68 | u32 mask_hi, mask_lo; | |
69 | u32 sys_hi, sys_lo; | |
70 | u32 iorr_addr, free_iorr_addr; | |
71 | ||
72 | /* Find the iorr that is already used for the base */ | |
73 | /* If not found, determine the uppermost available iorr */ | |
74 | free_iorr_addr = AMD_K7_NUM_IORR; | |
6a92a4e0 | 75 | for (iorr_addr = 0; iorr_addr < AMD_K7_NUM_IORR; iorr_addr++) { |
1da177e4 LT |
76 | rdmsr(IORR_BASE0 + 2 * iorr_addr, base_lo, base_hi); |
77 | rdmsr(IORR_MASK0 + 2 * iorr_addr, mask_lo, mask_hi); | |
78 | ||
79 | if ((base_lo & 0xfffff000) == (base & 0xfffff000)) | |
80 | break; | |
81 | ||
82 | if ((mask_lo & 0x00000800) == 0) | |
83 | free_iorr_addr = iorr_addr; | |
84 | } | |
6a92a4e0 | 85 | |
1da177e4 LT |
86 | if (iorr_addr >= AMD_K7_NUM_IORR) { |
87 | iorr_addr = free_iorr_addr; | |
88 | if (iorr_addr >= AMD_K7_NUM_IORR) | |
89 | return -EINVAL; | |
90 | } | |
91 | base_hi = 0x0; | |
92 | base_lo = (base & ~0xfff) | 0x18; | |
93 | mask_hi = 0xf; | |
94 | mask_lo = ((~(size - 1)) & 0xfffff000) | 0x800; | |
95 | wrmsr(IORR_BASE0 + 2 * iorr_addr, base_lo, base_hi); | |
96 | wrmsr(IORR_MASK0 + 2 * iorr_addr, mask_lo, mask_hi); | |
97 | ||
98 | rdmsr(SYSCFG, sys_lo, sys_hi); | |
99 | sys_lo |= 0x00100000; | |
100 | wrmsr(SYSCFG, sys_lo, sys_hi); | |
101 | ||
102 | return 0; | |
103 | } | |
104 | ||
105 | static int nvidia_configure(void) | |
106 | { | |
107 | int i, rc, num_dirs; | |
108 | u32 apbase, aplimit; | |
d68c5a27 | 109 | phys_addr_t apbase_phys; |
1da177e4 LT |
110 | struct aper_size_info_8 *current_size; |
111 | u32 temp; | |
112 | ||
113 | current_size = A_SIZE_8(agp_bridge->current_size); | |
114 | ||
115 | /* aperture size */ | |
116 | pci_write_config_byte(agp_bridge->dev, NVIDIA_0_APSIZE, | |
117 | current_size->size_value); | |
118 | ||
e501b3d8 BH |
119 | /* address to map to */ |
120 | apbase = pci_bus_address(agp_bridge->dev, AGP_APERTURE_BAR); | |
1da177e4 LT |
121 | agp_bridge->gart_bus_addr = apbase; |
122 | aplimit = apbase + (current_size->size * 1024 * 1024) - 1; | |
123 | pci_write_config_dword(nvidia_private.dev_2, NVIDIA_2_APBASE, apbase); | |
124 | pci_write_config_dword(nvidia_private.dev_2, NVIDIA_2_APLIMIT, aplimit); | |
125 | pci_write_config_dword(nvidia_private.dev_3, NVIDIA_3_APBASE, apbase); | |
126 | pci_write_config_dword(nvidia_private.dev_3, NVIDIA_3_APLIMIT, aplimit); | |
127 | if (0 != (rc = nvidia_init_iorr(apbase, current_size->size * 1024 * 1024))) | |
128 | return rc; | |
129 | ||
130 | /* directory size is 64k */ | |
131 | num_dirs = current_size->size / 64; | |
132 | nvidia_private.num_active_entries = current_size->num_entries; | |
133 | nvidia_private.pg_offset = 0; | |
134 | if (num_dirs == 0) { | |
135 | num_dirs = 1; | |
136 | nvidia_private.num_active_entries /= (64 / current_size->size); | |
137 | nvidia_private.pg_offset = (apbase & (64 * 1024 * 1024 - 1) & | |
138 | ~(current_size->size * 1024 * 1024 - 1)) / PAGE_SIZE; | |
139 | } | |
140 | ||
141 | /* attbase */ | |
6a92a4e0 | 142 | for (i = 0; i < 8; i++) { |
1da177e4 LT |
143 | pci_write_config_dword(nvidia_private.dev_2, NVIDIA_2_ATTBASE(i), |
144 | (agp_bridge->gatt_bus_addr + (i % num_dirs) * 64 * 1024) | 1); | |
145 | } | |
146 | ||
147 | /* gtlb control */ | |
148 | pci_read_config_dword(nvidia_private.dev_2, NVIDIA_2_GARTCTRL, &temp); | |
149 | pci_write_config_dword(nvidia_private.dev_2, NVIDIA_2_GARTCTRL, temp | 0x11); | |
150 | ||
151 | /* gart control */ | |
152 | pci_read_config_dword(agp_bridge->dev, NVIDIA_0_APSIZE, &temp); | |
153 | pci_write_config_dword(agp_bridge->dev, NVIDIA_0_APSIZE, temp | 0x100); | |
154 | ||
155 | /* map aperture */ | |
d68c5a27 | 156 | apbase_phys = pci_resource_start(agp_bridge->dev, AGP_APERTURE_BAR); |
1da177e4 | 157 | nvidia_private.aperture = |
d68c5a27 | 158 | (volatile u32 __iomem *) ioremap(apbase_phys, 33 * PAGE_SIZE); |
1da177e4 | 159 | |
5bdbc7dc ST |
160 | if (!nvidia_private.aperture) |
161 | return -ENOMEM; | |
162 | ||
1da177e4 LT |
163 | return 0; |
164 | } | |
165 | ||
166 | static void nvidia_cleanup(void) | |
167 | { | |
168 | struct aper_size_info_8 *previous_size; | |
169 | u32 temp; | |
170 | ||
171 | /* gart control */ | |
172 | pci_read_config_dword(agp_bridge->dev, NVIDIA_0_APSIZE, &temp); | |
173 | pci_write_config_dword(agp_bridge->dev, NVIDIA_0_APSIZE, temp & ~(0x100)); | |
174 | ||
175 | /* gtlb control */ | |
176 | pci_read_config_dword(nvidia_private.dev_2, NVIDIA_2_GARTCTRL, &temp); | |
177 | pci_write_config_dword(nvidia_private.dev_2, NVIDIA_2_GARTCTRL, temp & ~(0x11)); | |
178 | ||
179 | /* unmap aperture */ | |
180 | iounmap((void __iomem *) nvidia_private.aperture); | |
181 | ||
182 | /* restore previous aperture size */ | |
183 | previous_size = A_SIZE_8(agp_bridge->previous_size); | |
184 | pci_write_config_byte(agp_bridge->dev, NVIDIA_0_APSIZE, | |
185 | previous_size->size_value); | |
186 | ||
187 | /* restore iorr for previous aperture size */ | |
188 | nvidia_init_iorr(agp_bridge->gart_bus_addr, | |
189 | previous_size->size * 1024 * 1024); | |
190 | } | |
191 | ||
192 | ||
193 | /* | |
194 | * Note we can't use the generic routines, even though they are 99% the same. | |
195 | * Aperture sizes <64M still requires a full 64k GART directory, but | |
196 | * only use the portion of the TLB entries that correspond to the apertures | |
197 | * alignment inside the surrounding 64M block. | |
198 | */ | |
199 | extern int agp_memory_reserved; | |
200 | ||
201 | static int nvidia_insert_memory(struct agp_memory *mem, off_t pg_start, int type) | |
202 | { | |
203 | int i, j; | |
a64d2b37 | 204 | int mask_type; |
6a92a4e0 | 205 | |
a64d2b37 TH |
206 | mask_type = agp_generic_type_to_mask_type(mem->bridge, type); |
207 | if (mask_type != 0 || type != mem->type) | |
1da177e4 | 208 | return -EINVAL; |
6a92a4e0 | 209 | |
a64d2b37 TH |
210 | if (mem->page_count == 0) |
211 | return 0; | |
212 | ||
1da177e4 LT |
213 | if ((pg_start + mem->page_count) > |
214 | (nvidia_private.num_active_entries - agp_memory_reserved/PAGE_SIZE)) | |
215 | return -EINVAL; | |
6a92a4e0 DJ |
216 | |
217 | for (j = pg_start; j < (pg_start + mem->page_count); j++) { | |
1da177e4 LT |
218 | if (!PGE_EMPTY(agp_bridge, readl(agp_bridge->gatt_table+nvidia_private.pg_offset+j))) |
219 | return -EBUSY; | |
220 | } | |
221 | ||
c7258012 | 222 | if (!mem->is_flushed) { |
1da177e4 | 223 | global_cache_flush(); |
c7258012 | 224 | mem->is_flushed = true; |
1da177e4 LT |
225 | } |
226 | for (i = 0, j = pg_start; i < mem->page_count; i++, j++) { | |
227 | writel(agp_bridge->driver->mask_memory(agp_bridge, | |
6a12235c | 228 | page_to_phys(mem->pages[i]), mask_type), |
1da177e4 | 229 | agp_bridge->gatt_table+nvidia_private.pg_offset+j); |
1da177e4 | 230 | } |
a64d2b37 TH |
231 | |
232 | /* PCI Posting. */ | |
233 | readl(agp_bridge->gatt_table+nvidia_private.pg_offset+j - 1); | |
234 | ||
1da177e4 LT |
235 | agp_bridge->driver->tlb_flush(mem); |
236 | return 0; | |
237 | } | |
238 | ||
239 | ||
240 | static int nvidia_remove_memory(struct agp_memory *mem, off_t pg_start, int type) | |
241 | { | |
242 | int i; | |
243 | ||
a64d2b37 TH |
244 | int mask_type; |
245 | ||
246 | mask_type = agp_generic_type_to_mask_type(mem->bridge, type); | |
247 | if (mask_type != 0 || type != mem->type) | |
1da177e4 LT |
248 | return -EINVAL; |
249 | ||
a64d2b37 TH |
250 | if (mem->page_count == 0) |
251 | return 0; | |
252 | ||
1da177e4 LT |
253 | for (i = pg_start; i < (mem->page_count + pg_start); i++) |
254 | writel(agp_bridge->scratch_page, agp_bridge->gatt_table+nvidia_private.pg_offset+i); | |
255 | ||
256 | agp_bridge->driver->tlb_flush(mem); | |
257 | return 0; | |
258 | } | |
259 | ||
260 | ||
261 | static void nvidia_tlbflush(struct agp_memory *mem) | |
262 | { | |
263 | unsigned long end; | |
264 | u32 wbc_reg, temp; | |
265 | int i; | |
266 | ||
267 | /* flush chipset */ | |
268 | if (nvidia_private.wbc_mask) { | |
269 | pci_read_config_dword(nvidia_private.dev_1, NVIDIA_1_WBC, &wbc_reg); | |
270 | wbc_reg |= nvidia_private.wbc_mask; | |
271 | pci_write_config_dword(nvidia_private.dev_1, NVIDIA_1_WBC, wbc_reg); | |
272 | ||
273 | end = jiffies + 3*HZ; | |
274 | do { | |
275 | pci_read_config_dword(nvidia_private.dev_1, | |
276 | NVIDIA_1_WBC, &wbc_reg); | |
05613bdd | 277 | if (time_before_eq(end, jiffies)) { |
1da177e4 LT |
278 | printk(KERN_ERR PFX |
279 | "TLB flush took more than 3 seconds.\n"); | |
280 | } | |
281 | } while (wbc_reg & nvidia_private.wbc_mask); | |
282 | } | |
283 | ||
284 | /* flush TLB entries */ | |
6a92a4e0 | 285 | for (i = 0; i < 32 + 1; i++) |
1da177e4 | 286 | temp = readl(nvidia_private.aperture+(i * PAGE_SIZE / sizeof(u32))); |
6a92a4e0 | 287 | for (i = 0; i < 32 + 1; i++) |
1da177e4 LT |
288 | temp = readl(nvidia_private.aperture+(i * PAGE_SIZE / sizeof(u32))); |
289 | } | |
290 | ||
291 | ||
e5524f35 | 292 | static const struct aper_size_info_8 nvidia_generic_sizes[5] = |
1da177e4 LT |
293 | { |
294 | {512, 131072, 7, 0}, | |
295 | {256, 65536, 6, 8}, | |
296 | {128, 32768, 5, 12}, | |
297 | {64, 16384, 4, 14}, | |
298 | /* The 32M mode still requires a 64k gatt */ | |
299 | {32, 16384, 4, 15} | |
300 | }; | |
301 | ||
302 | ||
e5524f35 | 303 | static const struct gatt_mask nvidia_generic_masks[] = |
1da177e4 LT |
304 | { |
305 | { .mask = 1, .type = 0} | |
306 | }; | |
307 | ||
308 | ||
e5524f35 | 309 | static const struct agp_bridge_driver nvidia_driver = { |
1da177e4 LT |
310 | .owner = THIS_MODULE, |
311 | .aperture_sizes = nvidia_generic_sizes, | |
312 | .size_type = U8_APER_SIZE, | |
313 | .num_aperture_sizes = 5, | |
61cf0593 | 314 | .needs_scratch_page = true, |
1da177e4 LT |
315 | .configure = nvidia_configure, |
316 | .fetch_size = nvidia_fetch_size, | |
317 | .cleanup = nvidia_cleanup, | |
318 | .tlb_flush = nvidia_tlbflush, | |
319 | .mask_memory = agp_generic_mask_memory, | |
320 | .masks = nvidia_generic_masks, | |
321 | .agp_enable = agp_generic_enable, | |
322 | .cache_flush = global_cache_flush, | |
323 | .create_gatt_table = agp_generic_create_gatt_table, | |
324 | .free_gatt_table = agp_generic_free_gatt_table, | |
325 | .insert_memory = nvidia_insert_memory, | |
326 | .remove_memory = nvidia_remove_memory, | |
327 | .alloc_by_type = agp_generic_alloc_by_type, | |
328 | .free_by_type = agp_generic_free_by_type, | |
329 | .agp_alloc_page = agp_generic_alloc_page, | |
5f310b63 | 330 | .agp_alloc_pages = agp_generic_alloc_pages, |
1da177e4 | 331 | .agp_destroy_page = agp_generic_destroy_page, |
5f310b63 | 332 | .agp_destroy_pages = agp_generic_destroy_pages, |
a030ce44 | 333 | .agp_type_to_mask_type = agp_generic_type_to_mask_type, |
1da177e4 LT |
334 | }; |
335 | ||
bcd2982a GKH |
336 | static int agp_nvidia_probe(struct pci_dev *pdev, |
337 | const struct pci_device_id *ent) | |
1da177e4 LT |
338 | { |
339 | struct agp_bridge_data *bridge; | |
340 | u8 cap_ptr; | |
341 | ||
342 | nvidia_private.dev_1 = | |
1e415732 | 343 | pci_get_bus_and_slot((unsigned int)pdev->bus->number, PCI_DEVFN(0, 1)); |
1da177e4 | 344 | nvidia_private.dev_2 = |
1e415732 | 345 | pci_get_bus_and_slot((unsigned int)pdev->bus->number, PCI_DEVFN(0, 2)); |
1da177e4 | 346 | nvidia_private.dev_3 = |
1e415732 | 347 | pci_get_bus_and_slot((unsigned int)pdev->bus->number, PCI_DEVFN(30, 0)); |
6a92a4e0 | 348 | |
1da177e4 LT |
349 | if (!nvidia_private.dev_1 || !nvidia_private.dev_2 || !nvidia_private.dev_3) { |
350 | printk(KERN_INFO PFX "Detected an NVIDIA nForce/nForce2 " | |
351 | "chipset, but could not find the secondary devices.\n"); | |
352 | return -ENODEV; | |
353 | } | |
354 | ||
355 | cap_ptr = pci_find_capability(pdev, PCI_CAP_ID_AGP); | |
356 | if (!cap_ptr) | |
357 | return -ENODEV; | |
358 | ||
359 | switch (pdev->device) { | |
360 | case PCI_DEVICE_ID_NVIDIA_NFORCE: | |
361 | printk(KERN_INFO PFX "Detected NVIDIA nForce chipset\n"); | |
362 | nvidia_private.wbc_mask = 0x00010000; | |
363 | break; | |
364 | case PCI_DEVICE_ID_NVIDIA_NFORCE2: | |
365 | printk(KERN_INFO PFX "Detected NVIDIA nForce2 chipset\n"); | |
366 | nvidia_private.wbc_mask = 0x80000000; | |
367 | break; | |
368 | default: | |
369 | printk(KERN_ERR PFX "Unsupported NVIDIA chipset (device id: %04x)\n", | |
370 | pdev->device); | |
371 | return -ENODEV; | |
372 | } | |
373 | ||
374 | bridge = agp_alloc_bridge(); | |
375 | if (!bridge) | |
376 | return -ENOMEM; | |
377 | ||
378 | bridge->driver = &nvidia_driver; | |
379 | bridge->dev_private_data = &nvidia_private, | |
380 | bridge->dev = pdev; | |
381 | bridge->capndx = cap_ptr; | |
382 | ||
383 | /* Fill in the mode register */ | |
384 | pci_read_config_dword(pdev, | |
385 | bridge->capndx+PCI_AGP_STATUS, | |
386 | &bridge->mode); | |
387 | ||
388 | pci_set_drvdata(pdev, bridge); | |
389 | return agp_add_bridge(bridge); | |
390 | } | |
391 | ||
39af33fc | 392 | static void agp_nvidia_remove(struct pci_dev *pdev) |
1da177e4 LT |
393 | { |
394 | struct agp_bridge_data *bridge = pci_get_drvdata(pdev); | |
395 | ||
396 | agp_remove_bridge(bridge); | |
397 | agp_put_bridge(bridge); | |
398 | } | |
399 | ||
c5f2f261 DJ |
400 | #ifdef CONFIG_PM |
401 | static int agp_nvidia_suspend(struct pci_dev *pdev, pm_message_t state) | |
402 | { | |
43355503 YW |
403 | pci_save_state(pdev); |
404 | pci_set_power_state(pdev, PCI_D3hot); | |
c5f2f261 DJ |
405 | |
406 | return 0; | |
407 | } | |
408 | ||
409 | static int agp_nvidia_resume(struct pci_dev *pdev) | |
410 | { | |
c5f2f261 | 411 | /* set power state 0 and restore PCI space */ |
43355503 | 412 | pci_set_power_state(pdev, PCI_D0); |
c5f2f261 DJ |
413 | pci_restore_state(pdev); |
414 | ||
415 | /* reconfigure AGP hardware again */ | |
416 | nvidia_configure(); | |
417 | ||
418 | return 0; | |
419 | } | |
420 | #endif | |
421 | ||
422 | ||
1da177e4 LT |
423 | static struct pci_device_id agp_nvidia_pci_table[] = { |
424 | { | |
425 | .class = (PCI_CLASS_BRIDGE_HOST << 8), | |
426 | .class_mask = ~0, | |
427 | .vendor = PCI_VENDOR_ID_NVIDIA, | |
428 | .device = PCI_DEVICE_ID_NVIDIA_NFORCE, | |
429 | .subvendor = PCI_ANY_ID, | |
430 | .subdevice = PCI_ANY_ID, | |
431 | }, | |
432 | { | |
433 | .class = (PCI_CLASS_BRIDGE_HOST << 8), | |
434 | .class_mask = ~0, | |
435 | .vendor = PCI_VENDOR_ID_NVIDIA, | |
436 | .device = PCI_DEVICE_ID_NVIDIA_NFORCE2, | |
437 | .subvendor = PCI_ANY_ID, | |
438 | .subdevice = PCI_ANY_ID, | |
439 | }, | |
440 | { } | |
441 | }; | |
442 | ||
443 | MODULE_DEVICE_TABLE(pci, agp_nvidia_pci_table); | |
444 | ||
445 | static struct pci_driver agp_nvidia_pci_driver = { | |
446 | .name = "agpgart-nvidia", | |
447 | .id_table = agp_nvidia_pci_table, | |
448 | .probe = agp_nvidia_probe, | |
449 | .remove = agp_nvidia_remove, | |
c5f2f261 DJ |
450 | #ifdef CONFIG_PM |
451 | .suspend = agp_nvidia_suspend, | |
452 | .resume = agp_nvidia_resume, | |
453 | #endif | |
1da177e4 LT |
454 | }; |
455 | ||
456 | static int __init agp_nvidia_init(void) | |
457 | { | |
458 | if (agp_off) | |
459 | return -EINVAL; | |
460 | return pci_register_driver(&agp_nvidia_pci_driver); | |
461 | } | |
462 | ||
463 | static void __exit agp_nvidia_cleanup(void) | |
464 | { | |
465 | pci_unregister_driver(&agp_nvidia_pci_driver); | |
1e415732 AC |
466 | pci_dev_put(nvidia_private.dev_1); |
467 | pci_dev_put(nvidia_private.dev_2); | |
468 | pci_dev_put(nvidia_private.dev_3); | |
1da177e4 LT |
469 | } |
470 | ||
471 | module_init(agp_nvidia_init); | |
472 | module_exit(agp_nvidia_cleanup); | |
473 | ||
474 | MODULE_LICENSE("GPL and additional rights"); | |
475 | MODULE_AUTHOR("NVIDIA Corporation"); | |
476 |