Commit | Line | Data |
---|---|---|
1da177e4 LT |
1 | /* |
2 | * Copyright (c) 2004 Topspin Communications. All rights reserved. | |
2a1d9b7f | 3 | * Copyright (c) 2005 Mellanox Technologies. All rights reserved. |
1da177e4 LT |
4 | * |
5 | * This software is available to you under a choice of one of two | |
6 | * licenses. You may choose to be licensed under the terms of the GNU | |
7 | * General Public License (GPL) Version 2, available from the file | |
8 | * COPYING in the main directory of this source tree, or the | |
9 | * OpenIB.org BSD license below: | |
10 | * | |
11 | * Redistribution and use in source and binary forms, with or | |
12 | * without modification, are permitted provided that the following | |
13 | * conditions are met: | |
14 | * | |
15 | * - Redistributions of source code must retain the above | |
16 | * copyright notice, this list of conditions and the following | |
17 | * disclaimer. | |
18 | * | |
19 | * - Redistributions in binary form must reproduce the above | |
20 | * copyright notice, this list of conditions and the following | |
21 | * disclaimer in the documentation and/or other materials | |
22 | * provided with the distribution. | |
23 | * | |
24 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, | |
25 | * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF | |
26 | * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND | |
27 | * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS | |
28 | * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN | |
29 | * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN | |
30 | * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE | |
31 | * SOFTWARE. | |
32 | * | |
33 | * $Id: mthca_mr.c 1349 2004-12-16 21:09:43Z roland $ | |
34 | */ | |
35 | ||
36 | #include <linux/slab.h> | |
37 | #include <linux/init.h> | |
38 | #include <linux/errno.h> | |
39 | ||
40 | #include "mthca_dev.h" | |
41 | #include "mthca_cmd.h" | |
86562a13 | 42 | #include "mthca_memfree.h" |
1da177e4 | 43 | |
d56d6f95 RD |
44 | struct mthca_mtt { |
45 | struct mthca_buddy *buddy; | |
46 | int order; | |
47 | u32 first_seg; | |
48 | }; | |
49 | ||
1da177e4 LT |
50 | /* |
51 | * Must be packed because mtt_seg is 64 bits but only aligned to 32 bits. | |
52 | */ | |
53 | struct mthca_mpt_entry { | |
97f52eb4 SH |
54 | __be32 flags; |
55 | __be32 page_size; | |
56 | __be32 key; | |
57 | __be32 pd; | |
58 | __be64 start; | |
59 | __be64 length; | |
60 | __be32 lkey; | |
61 | __be32 window_count; | |
62 | __be32 window_count_limit; | |
63 | __be64 mtt_seg; | |
64 | __be32 mtt_sz; /* Arbel only */ | |
65 | u32 reserved[2]; | |
1da177e4 LT |
66 | } __attribute__((packed)); |
67 | ||
68 | #define MTHCA_MPT_FLAG_SW_OWNS (0xfUL << 28) | |
69 | #define MTHCA_MPT_FLAG_MIO (1 << 17) | |
70 | #define MTHCA_MPT_FLAG_BIND_ENABLE (1 << 15) | |
71 | #define MTHCA_MPT_FLAG_PHYSICAL (1 << 9) | |
72 | #define MTHCA_MPT_FLAG_REGION (1 << 8) | |
73 | ||
74 | #define MTHCA_MTT_FLAG_PRESENT 1 | |
75 | ||
e0f5fdca MT |
76 | #define MTHCA_MPT_STATUS_SW 0xF0 |
77 | #define MTHCA_MPT_STATUS_HW 0x00 | |
78 | ||
1da177e4 LT |
79 | /* |
80 | * Buddy allocator for MTT segments (currently not very efficient | |
81 | * since it doesn't keep a free list and just searches linearly | |
82 | * through the bitmaps) | |
83 | */ | |
84 | ||
9095e208 | 85 | static u32 mthca_buddy_alloc(struct mthca_buddy *buddy, int order) |
1da177e4 LT |
86 | { |
87 | int o; | |
88 | int m; | |
89 | u32 seg; | |
90 | ||
9095e208 | 91 | spin_lock(&buddy->lock); |
1da177e4 | 92 | |
9095e208 MT |
93 | for (o = order; o <= buddy->max_order; ++o) { |
94 | m = 1 << (buddy->max_order - o); | |
95 | seg = find_first_bit(buddy->bits[o], m); | |
1da177e4 LT |
96 | if (seg < m) |
97 | goto found; | |
98 | } | |
99 | ||
9095e208 | 100 | spin_unlock(&buddy->lock); |
1da177e4 LT |
101 | return -1; |
102 | ||
103 | found: | |
9095e208 | 104 | clear_bit(seg, buddy->bits[o]); |
1da177e4 LT |
105 | |
106 | while (o > order) { | |
107 | --o; | |
108 | seg <<= 1; | |
9095e208 | 109 | set_bit(seg ^ 1, buddy->bits[o]); |
1da177e4 LT |
110 | } |
111 | ||
9095e208 | 112 | spin_unlock(&buddy->lock); |
1da177e4 LT |
113 | |
114 | seg <<= order; | |
115 | ||
116 | return seg; | |
117 | } | |
118 | ||
9095e208 | 119 | static void mthca_buddy_free(struct mthca_buddy *buddy, u32 seg, int order) |
1da177e4 LT |
120 | { |
121 | seg >>= order; | |
122 | ||
9095e208 | 123 | spin_lock(&buddy->lock); |
1da177e4 | 124 | |
9095e208 MT |
125 | while (test_bit(seg ^ 1, buddy->bits[order])) { |
126 | clear_bit(seg ^ 1, buddy->bits[order]); | |
1da177e4 LT |
127 | seg >>= 1; |
128 | ++order; | |
129 | } | |
130 | ||
9095e208 | 131 | set_bit(seg, buddy->bits[order]); |
1da177e4 | 132 | |
9095e208 | 133 | spin_unlock(&buddy->lock); |
1da177e4 LT |
134 | } |
135 | ||
9095e208 | 136 | static int __devinit mthca_buddy_init(struct mthca_buddy *buddy, int max_order) |
86562a13 | 137 | { |
9095e208 MT |
138 | int i, s; |
139 | ||
140 | buddy->max_order = max_order; | |
141 | spin_lock_init(&buddy->lock); | |
142 | ||
143 | buddy->bits = kmalloc((buddy->max_order + 1) * sizeof (long *), | |
144 | GFP_KERNEL); | |
145 | if (!buddy->bits) | |
146 | goto err_out; | |
147 | ||
148 | memset(buddy->bits, 0, (buddy->max_order + 1) * sizeof (long *)); | |
149 | ||
150 | for (i = 0; i <= buddy->max_order; ++i) { | |
151 | s = BITS_TO_LONGS(1 << (buddy->max_order - i)); | |
152 | buddy->bits[i] = kmalloc(s * sizeof (long), GFP_KERNEL); | |
153 | if (!buddy->bits[i]) | |
154 | goto err_out_free; | |
155 | bitmap_zero(buddy->bits[i], | |
156 | 1 << (buddy->max_order - i)); | |
157 | } | |
158 | ||
159 | set_bit(0, buddy->bits[buddy->max_order]); | |
160 | ||
161 | return 0; | |
162 | ||
163 | err_out_free: | |
164 | for (i = 0; i <= buddy->max_order; ++i) | |
165 | kfree(buddy->bits[i]); | |
166 | ||
167 | kfree(buddy->bits); | |
168 | ||
169 | err_out: | |
170 | return -ENOMEM; | |
171 | } | |
172 | ||
173 | static void __devexit mthca_buddy_cleanup(struct mthca_buddy *buddy) | |
174 | { | |
175 | int i; | |
176 | ||
177 | for (i = 0; i <= buddy->max_order; ++i) | |
178 | kfree(buddy->bits[i]); | |
179 | ||
180 | kfree(buddy->bits); | |
181 | } | |
182 | ||
d56d6f95 RD |
183 | static u32 mthca_alloc_mtt_range(struct mthca_dev *dev, int order, |
184 | struct mthca_buddy *buddy) | |
9095e208 MT |
185 | { |
186 | u32 seg = mthca_buddy_alloc(buddy, order); | |
86562a13 RD |
187 | |
188 | if (seg == -1) | |
189 | return -1; | |
190 | ||
d10ddbf6 | 191 | if (mthca_is_memfree(dev)) |
86562a13 RD |
192 | if (mthca_table_get_range(dev, dev->mr_table.mtt_table, seg, |
193 | seg + (1 << order) - 1)) { | |
9095e208 | 194 | mthca_buddy_free(buddy, seg, order); |
86562a13 RD |
195 | seg = -1; |
196 | } | |
197 | ||
198 | return seg; | |
199 | } | |
200 | ||
d56d6f95 RD |
201 | static struct mthca_mtt *__mthca_alloc_mtt(struct mthca_dev *dev, int size, |
202 | struct mthca_buddy *buddy) | |
203 | { | |
204 | struct mthca_mtt *mtt; | |
205 | int i; | |
206 | ||
207 | if (size <= 0) | |
208 | return ERR_PTR(-EINVAL); | |
209 | ||
210 | mtt = kmalloc(sizeof *mtt, GFP_KERNEL); | |
211 | if (!mtt) | |
212 | return ERR_PTR(-ENOMEM); | |
213 | ||
214 | mtt->buddy = buddy; | |
215 | mtt->order = 0; | |
216 | for (i = MTHCA_MTT_SEG_SIZE / 8; i < size; i <<= 1) | |
217 | ++mtt->order; | |
218 | ||
219 | mtt->first_seg = mthca_alloc_mtt_range(dev, mtt->order, buddy); | |
220 | if (mtt->first_seg == -1) { | |
221 | kfree(mtt); | |
222 | return ERR_PTR(-ENOMEM); | |
223 | } | |
224 | ||
225 | return mtt; | |
226 | } | |
227 | ||
228 | struct mthca_mtt *mthca_alloc_mtt(struct mthca_dev *dev, int size) | |
229 | { | |
230 | return __mthca_alloc_mtt(dev, size, &dev->mr_table.mtt_buddy); | |
231 | } | |
232 | ||
233 | void mthca_free_mtt(struct mthca_dev *dev, struct mthca_mtt *mtt) | |
234 | { | |
235 | if (!mtt) | |
236 | return; | |
237 | ||
238 | mthca_buddy_free(mtt->buddy, mtt->first_seg, mtt->order); | |
239 | ||
240 | mthca_table_put_range(dev, dev->mr_table.mtt_table, | |
241 | mtt->first_seg, | |
242 | mtt->first_seg + (1 << mtt->order) - 1); | |
243 | ||
244 | kfree(mtt); | |
245 | } | |
246 | ||
247 | int mthca_write_mtt(struct mthca_dev *dev, struct mthca_mtt *mtt, | |
248 | int start_index, u64 *buffer_list, int list_len) | |
86562a13 | 249 | { |
ed878458 | 250 | struct mthca_mailbox *mailbox; |
97f52eb4 | 251 | __be64 *mtt_entry; |
d56d6f95 RD |
252 | int err = 0; |
253 | u8 status; | |
254 | int i; | |
255 | ||
ed878458 RD |
256 | mailbox = mthca_alloc_mailbox(dev, GFP_KERNEL); |
257 | if (IS_ERR(mailbox)) | |
258 | return PTR_ERR(mailbox); | |
259 | mtt_entry = mailbox->buf; | |
d56d6f95 RD |
260 | |
261 | while (list_len > 0) { | |
262 | mtt_entry[0] = cpu_to_be64(dev->mr_table.mtt_base + | |
263 | mtt->first_seg * MTHCA_MTT_SEG_SIZE + | |
264 | start_index * 8); | |
265 | mtt_entry[1] = 0; | |
ed878458 | 266 | for (i = 0; i < list_len && i < MTHCA_MAILBOX_SIZE / 8 - 2; ++i) |
d56d6f95 RD |
267 | mtt_entry[i + 2] = cpu_to_be64(buffer_list[i] | |
268 | MTHCA_MTT_FLAG_PRESENT); | |
269 | ||
270 | /* | |
271 | * If we have an odd number of entries to write, add | |
272 | * one more dummy entry for firmware efficiency. | |
273 | */ | |
274 | if (i & 1) | |
275 | mtt_entry[i + 2] = 0; | |
276 | ||
ed878458 | 277 | err = mthca_WRITE_MTT(dev, mailbox, (i + 1) & ~1, &status); |
d56d6f95 RD |
278 | if (err) { |
279 | mthca_warn(dev, "WRITE_MTT failed (%d)\n", err); | |
280 | goto out; | |
281 | } | |
282 | if (status) { | |
283 | mthca_warn(dev, "WRITE_MTT returned status 0x%02x\n", | |
284 | status); | |
285 | err = -EINVAL; | |
286 | goto out; | |
287 | } | |
288 | ||
289 | list_len -= i; | |
290 | start_index += i; | |
291 | buffer_list += i; | |
292 | } | |
293 | ||
294 | out: | |
ed878458 | 295 | mthca_free_mailbox(dev, mailbox); |
d56d6f95 | 296 | return err; |
86562a13 RD |
297 | } |
298 | ||
d0a9d25c MT |
299 | static inline u32 tavor_hw_index_to_key(u32 ind) |
300 | { | |
301 | return ind; | |
302 | } | |
303 | ||
304 | static inline u32 tavor_key_to_hw_index(u32 key) | |
305 | { | |
306 | return key; | |
307 | } | |
308 | ||
309 | static inline u32 arbel_hw_index_to_key(u32 ind) | |
310 | { | |
311 | return (ind >> 24) | (ind << 8); | |
312 | } | |
313 | ||
314 | static inline u32 arbel_key_to_hw_index(u32 key) | |
315 | { | |
316 | return (key << 24) | (key >> 8); | |
317 | } | |
318 | ||
1da177e4 LT |
319 | static inline u32 hw_index_to_key(struct mthca_dev *dev, u32 ind) |
320 | { | |
d10ddbf6 | 321 | if (mthca_is_memfree(dev)) |
d0a9d25c | 322 | return arbel_hw_index_to_key(ind); |
1da177e4 | 323 | else |
d0a9d25c | 324 | return tavor_hw_index_to_key(ind); |
1da177e4 LT |
325 | } |
326 | ||
327 | static inline u32 key_to_hw_index(struct mthca_dev *dev, u32 key) | |
328 | { | |
d10ddbf6 | 329 | if (mthca_is_memfree(dev)) |
d0a9d25c | 330 | return arbel_key_to_hw_index(key); |
1da177e4 | 331 | else |
d0a9d25c | 332 | return tavor_key_to_hw_index(key); |
1da177e4 LT |
333 | } |
334 | ||
d56d6f95 RD |
335 | int mthca_mr_alloc(struct mthca_dev *dev, u32 pd, int buffer_size_shift, |
336 | u64 iova, u64 total_size, u32 access, struct mthca_mr *mr) | |
1da177e4 | 337 | { |
ed878458 | 338 | struct mthca_mailbox *mailbox; |
1da177e4 LT |
339 | struct mthca_mpt_entry *mpt_entry; |
340 | u32 key; | |
d56d6f95 | 341 | int i; |
1da177e4 LT |
342 | int err; |
343 | u8 status; | |
344 | ||
345 | might_sleep(); | |
346 | ||
d56d6f95 RD |
347 | WARN_ON(buffer_size_shift >= 32); |
348 | ||
1da177e4 LT |
349 | key = mthca_alloc(&dev->mr_table.mpt_alloc); |
350 | if (key == -1) | |
351 | return -ENOMEM; | |
352 | mr->ibmr.rkey = mr->ibmr.lkey = hw_index_to_key(dev, key); | |
353 | ||
d10ddbf6 | 354 | if (mthca_is_memfree(dev)) { |
86562a13 RD |
355 | err = mthca_table_get(dev, dev->mr_table.mpt_table, key); |
356 | if (err) | |
357 | goto err_out_mpt_free; | |
358 | } | |
359 | ||
ed878458 RD |
360 | mailbox = mthca_alloc_mailbox(dev, GFP_KERNEL); |
361 | if (IS_ERR(mailbox)) { | |
362 | err = PTR_ERR(mailbox); | |
86562a13 | 363 | goto err_out_table; |
1da177e4 | 364 | } |
ed878458 | 365 | mpt_entry = mailbox->buf; |
1da177e4 LT |
366 | |
367 | mpt_entry->flags = cpu_to_be32(MTHCA_MPT_FLAG_SW_OWNS | | |
368 | MTHCA_MPT_FLAG_MIO | | |
1da177e4 LT |
369 | MTHCA_MPT_FLAG_REGION | |
370 | access); | |
d56d6f95 RD |
371 | if (!mr->mtt) |
372 | mpt_entry->flags |= cpu_to_be32(MTHCA_MPT_FLAG_PHYSICAL); | |
373 | ||
374 | mpt_entry->page_size = cpu_to_be32(buffer_size_shift - 12); | |
1da177e4 LT |
375 | mpt_entry->key = cpu_to_be32(key); |
376 | mpt_entry->pd = cpu_to_be32(pd); | |
d56d6f95 RD |
377 | mpt_entry->start = cpu_to_be64(iova); |
378 | mpt_entry->length = cpu_to_be64(total_size); | |
1da177e4 LT |
379 | |
380 | memset(&mpt_entry->lkey, 0, | |
381 | sizeof *mpt_entry - offsetof(struct mthca_mpt_entry, lkey)); | |
382 | ||
d56d6f95 RD |
383 | if (mr->mtt) |
384 | mpt_entry->mtt_seg = | |
385 | cpu_to_be64(dev->mr_table.mtt_base + | |
386 | mr->mtt->first_seg * MTHCA_MTT_SEG_SIZE); | |
387 | ||
388 | if (0) { | |
389 | mthca_dbg(dev, "Dumping MPT entry %08x:\n", mr->ibmr.lkey); | |
390 | for (i = 0; i < sizeof (struct mthca_mpt_entry) / 4; ++i) { | |
391 | if (i % 4 == 0) | |
392 | printk("[%02x] ", i * 4); | |
97f52eb4 | 393 | printk(" %08x", be32_to_cpu(((__be32 *) mpt_entry)[i])); |
d56d6f95 RD |
394 | if ((i + 1) % 4 == 0) |
395 | printk("\n"); | |
396 | } | |
397 | } | |
398 | ||
ed878458 | 399 | err = mthca_SW2HW_MPT(dev, mailbox, |
1da177e4 LT |
400 | key & (dev->limits.num_mpts - 1), |
401 | &status); | |
86562a13 | 402 | if (err) { |
1da177e4 | 403 | mthca_warn(dev, "SW2HW_MPT failed (%d)\n", err); |
d56d6f95 | 404 | goto err_out_mailbox; |
86562a13 | 405 | } else if (status) { |
1da177e4 LT |
406 | mthca_warn(dev, "SW2HW_MPT returned status 0x%02x\n", |
407 | status); | |
408 | err = -EINVAL; | |
d56d6f95 | 409 | goto err_out_mailbox; |
1da177e4 LT |
410 | } |
411 | ||
ed878458 | 412 | mthca_free_mailbox(dev, mailbox); |
1da177e4 | 413 | return err; |
86562a13 | 414 | |
d56d6f95 | 415 | err_out_mailbox: |
ed878458 | 416 | mthca_free_mailbox(dev, mailbox); |
d56d6f95 | 417 | |
86562a13 | 418 | err_out_table: |
a03a5a67 | 419 | mthca_table_put(dev, dev->mr_table.mpt_table, key); |
86562a13 RD |
420 | |
421 | err_out_mpt_free: | |
55645e9b | 422 | mthca_free(&dev->mr_table.mpt_alloc, key); |
86562a13 | 423 | return err; |
1da177e4 LT |
424 | } |
425 | ||
d56d6f95 RD |
426 | int mthca_mr_alloc_notrans(struct mthca_dev *dev, u32 pd, |
427 | u32 access, struct mthca_mr *mr) | |
428 | { | |
429 | mr->mtt = NULL; | |
430 | return mthca_mr_alloc(dev, pd, 12, 0, ~0ULL, access, mr); | |
431 | } | |
432 | ||
1da177e4 LT |
433 | int mthca_mr_alloc_phys(struct mthca_dev *dev, u32 pd, |
434 | u64 *buffer_list, int buffer_size_shift, | |
435 | int list_len, u64 iova, u64 total_size, | |
436 | u32 access, struct mthca_mr *mr) | |
437 | { | |
d56d6f95 | 438 | int err; |
1da177e4 | 439 | |
d56d6f95 RD |
440 | mr->mtt = mthca_alloc_mtt(dev, list_len); |
441 | if (IS_ERR(mr->mtt)) | |
442 | return PTR_ERR(mr->mtt); | |
1da177e4 | 443 | |
d56d6f95 | 444 | err = mthca_write_mtt(dev, mr->mtt, 0, buffer_list, list_len); |
1da177e4 | 445 | if (err) { |
d56d6f95 RD |
446 | mthca_free_mtt(dev, mr->mtt); |
447 | return err; | |
1da177e4 LT |
448 | } |
449 | ||
d56d6f95 RD |
450 | err = mthca_mr_alloc(dev, pd, buffer_size_shift, iova, |
451 | total_size, access, mr); | |
1da177e4 | 452 | if (err) |
d56d6f95 | 453 | mthca_free_mtt(dev, mr->mtt); |
1da177e4 | 454 | |
1da177e4 LT |
455 | return err; |
456 | } | |
457 | ||
e0f5fdca | 458 | /* Free mr or fmr */ |
d56d6f95 | 459 | static void mthca_free_region(struct mthca_dev *dev, u32 lkey) |
e0f5fdca | 460 | { |
a03a5a67 | 461 | mthca_table_put(dev, dev->mr_table.mpt_table, |
7f9f2dba | 462 | key_to_hw_index(dev, lkey)); |
e0f5fdca MT |
463 | |
464 | mthca_free(&dev->mr_table.mpt_alloc, key_to_hw_index(dev, lkey)); | |
465 | } | |
466 | ||
1da177e4 LT |
467 | void mthca_free_mr(struct mthca_dev *dev, struct mthca_mr *mr) |
468 | { | |
469 | int err; | |
470 | u8 status; | |
471 | ||
472 | might_sleep(); | |
473 | ||
474 | err = mthca_HW2SW_MPT(dev, NULL, | |
475 | key_to_hw_index(dev, mr->ibmr.lkey) & | |
476 | (dev->limits.num_mpts - 1), | |
477 | &status); | |
478 | if (err) | |
479 | mthca_warn(dev, "HW2SW_MPT failed (%d)\n", err); | |
480 | else if (status) | |
481 | mthca_warn(dev, "HW2SW_MPT returned status 0x%02x\n", | |
482 | status); | |
483 | ||
d56d6f95 RD |
484 | mthca_free_region(dev, mr->ibmr.lkey); |
485 | mthca_free_mtt(dev, mr->mtt); | |
e0f5fdca MT |
486 | } |
487 | ||
488 | int mthca_fmr_alloc(struct mthca_dev *dev, u32 pd, | |
489 | u32 access, struct mthca_fmr *mr) | |
490 | { | |
491 | struct mthca_mpt_entry *mpt_entry; | |
ed878458 | 492 | struct mthca_mailbox *mailbox; |
e0f5fdca MT |
493 | u64 mtt_seg; |
494 | u32 key, idx; | |
495 | u8 status; | |
496 | int list_len = mr->attr.max_pages; | |
497 | int err = -ENOMEM; | |
498 | int i; | |
499 | ||
500 | might_sleep(); | |
501 | ||
502 | if (mr->attr.page_size < 12 || mr->attr.page_size >= 32) | |
503 | return -EINVAL; | |
504 | ||
505 | /* For Arbel, all MTTs must fit in the same page. */ | |
d10ddbf6 | 506 | if (mthca_is_memfree(dev) && |
e0f5fdca MT |
507 | mr->attr.max_pages * sizeof *mr->mem.arbel.mtts > PAGE_SIZE) |
508 | return -EINVAL; | |
509 | ||
510 | mr->maps = 0; | |
511 | ||
512 | key = mthca_alloc(&dev->mr_table.mpt_alloc); | |
513 | if (key == -1) | |
514 | return -ENOMEM; | |
515 | ||
516 | idx = key & (dev->limits.num_mpts - 1); | |
517 | mr->ibmr.rkey = mr->ibmr.lkey = hw_index_to_key(dev, key); | |
518 | ||
d10ddbf6 | 519 | if (mthca_is_memfree(dev)) { |
e0f5fdca MT |
520 | err = mthca_table_get(dev, dev->mr_table.mpt_table, key); |
521 | if (err) | |
522 | goto err_out_mpt_free; | |
523 | ||
524 | mr->mem.arbel.mpt = mthca_table_find(dev->mr_table.mpt_table, key); | |
525 | BUG_ON(!mr->mem.arbel.mpt); | |
526 | } else | |
527 | mr->mem.tavor.mpt = dev->mr_table.tavor_fmr.mpt_base + | |
528 | sizeof *(mr->mem.tavor.mpt) * idx; | |
529 | ||
d56d6f95 RD |
530 | mr->mtt = __mthca_alloc_mtt(dev, list_len, dev->mr_table.fmr_mtt_buddy); |
531 | if (IS_ERR(mr->mtt)) | |
e0f5fdca MT |
532 | goto err_out_table; |
533 | ||
d56d6f95 | 534 | mtt_seg = mr->mtt->first_seg * MTHCA_MTT_SEG_SIZE; |
e0f5fdca | 535 | |
d10ddbf6 | 536 | if (mthca_is_memfree(dev)) { |
e0f5fdca | 537 | mr->mem.arbel.mtts = mthca_table_find(dev->mr_table.mtt_table, |
d56d6f95 | 538 | mr->mtt->first_seg); |
e0f5fdca MT |
539 | BUG_ON(!mr->mem.arbel.mtts); |
540 | } else | |
541 | mr->mem.tavor.mtts = dev->mr_table.tavor_fmr.mtt_base + mtt_seg; | |
542 | ||
ed878458 RD |
543 | mailbox = mthca_alloc_mailbox(dev, GFP_KERNEL); |
544 | if (IS_ERR(mailbox)) | |
e0f5fdca MT |
545 | goto err_out_free_mtt; |
546 | ||
ed878458 | 547 | mpt_entry = mailbox->buf; |
e0f5fdca MT |
548 | |
549 | mpt_entry->flags = cpu_to_be32(MTHCA_MPT_FLAG_SW_OWNS | | |
550 | MTHCA_MPT_FLAG_MIO | | |
551 | MTHCA_MPT_FLAG_REGION | | |
552 | access); | |
553 | ||
554 | mpt_entry->page_size = cpu_to_be32(mr->attr.page_size - 12); | |
555 | mpt_entry->key = cpu_to_be32(key); | |
556 | mpt_entry->pd = cpu_to_be32(pd); | |
557 | memset(&mpt_entry->start, 0, | |
558 | sizeof *mpt_entry - offsetof(struct mthca_mpt_entry, start)); | |
559 | mpt_entry->mtt_seg = cpu_to_be64(dev->mr_table.mtt_base + mtt_seg); | |
560 | ||
561 | if (0) { | |
562 | mthca_dbg(dev, "Dumping MPT entry %08x:\n", mr->ibmr.lkey); | |
563 | for (i = 0; i < sizeof (struct mthca_mpt_entry) / 4; ++i) { | |
564 | if (i % 4 == 0) | |
565 | printk("[%02x] ", i * 4); | |
97f52eb4 | 566 | printk(" %08x", be32_to_cpu(((__be32 *) mpt_entry)[i])); |
e0f5fdca MT |
567 | if ((i + 1) % 4 == 0) |
568 | printk("\n"); | |
569 | } | |
570 | } | |
571 | ||
ed878458 | 572 | err = mthca_SW2HW_MPT(dev, mailbox, |
e0f5fdca MT |
573 | key & (dev->limits.num_mpts - 1), |
574 | &status); | |
575 | if (err) { | |
576 | mthca_warn(dev, "SW2HW_MPT failed (%d)\n", err); | |
577 | goto err_out_mailbox_free; | |
578 | } | |
579 | if (status) { | |
580 | mthca_warn(dev, "SW2HW_MPT returned status 0x%02x\n", | |
581 | status); | |
582 | err = -EINVAL; | |
583 | goto err_out_mailbox_free; | |
584 | } | |
585 | ||
ed878458 | 586 | mthca_free_mailbox(dev, mailbox); |
e0f5fdca MT |
587 | return 0; |
588 | ||
589 | err_out_mailbox_free: | |
ed878458 | 590 | mthca_free_mailbox(dev, mailbox); |
e0f5fdca MT |
591 | |
592 | err_out_free_mtt: | |
d56d6f95 | 593 | mthca_free_mtt(dev, mr->mtt); |
1da177e4 | 594 | |
e0f5fdca | 595 | err_out_table: |
a03a5a67 | 596 | mthca_table_put(dev, dev->mr_table.mpt_table, key); |
e0f5fdca MT |
597 | |
598 | err_out_mpt_free: | |
599 | mthca_free(&dev->mr_table.mpt_alloc, mr->ibmr.lkey); | |
600 | return err; | |
601 | } | |
602 | ||
603 | int mthca_free_fmr(struct mthca_dev *dev, struct mthca_fmr *fmr) | |
604 | { | |
605 | if (fmr->maps) | |
606 | return -EBUSY; | |
607 | ||
d56d6f95 RD |
608 | mthca_free_region(dev, fmr->ibmr.lkey); |
609 | mthca_free_mtt(dev, fmr->mtt); | |
610 | ||
e0f5fdca MT |
611 | return 0; |
612 | } | |
613 | ||
614 | static inline int mthca_check_fmr(struct mthca_fmr *fmr, u64 *page_list, | |
615 | int list_len, u64 iova) | |
616 | { | |
617 | int i, page_mask; | |
618 | ||
619 | if (list_len > fmr->attr.max_pages) | |
620 | return -EINVAL; | |
621 | ||
622 | page_mask = (1 << fmr->attr.page_size) - 1; | |
623 | ||
624 | /* We are getting page lists, so va must be page aligned. */ | |
625 | if (iova & page_mask) | |
626 | return -EINVAL; | |
627 | ||
628 | /* Trust the user not to pass misaligned data in page_list */ | |
629 | if (0) | |
630 | for (i = 0; i < list_len; ++i) { | |
631 | if (page_list[i] & ~page_mask) | |
632 | return -EINVAL; | |
633 | } | |
634 | ||
635 | if (fmr->maps >= fmr->attr.max_maps) | |
636 | return -EINVAL; | |
637 | ||
638 | return 0; | |
639 | } | |
640 | ||
641 | ||
642 | int mthca_tavor_map_phys_fmr(struct ib_fmr *ibfmr, u64 *page_list, | |
643 | int list_len, u64 iova) | |
644 | { | |
645 | struct mthca_fmr *fmr = to_mfmr(ibfmr); | |
646 | struct mthca_dev *dev = to_mdev(ibfmr->device); | |
647 | struct mthca_mpt_entry mpt_entry; | |
648 | u32 key; | |
649 | int i, err; | |
650 | ||
651 | err = mthca_check_fmr(fmr, page_list, list_len, iova); | |
652 | if (err) | |
653 | return err; | |
654 | ||
655 | ++fmr->maps; | |
656 | ||
657 | key = tavor_key_to_hw_index(fmr->ibmr.lkey); | |
658 | key += dev->limits.num_mpts; | |
659 | fmr->ibmr.lkey = fmr->ibmr.rkey = tavor_hw_index_to_key(key); | |
660 | ||
661 | writeb(MTHCA_MPT_STATUS_SW, fmr->mem.tavor.mpt); | |
662 | ||
663 | for (i = 0; i < list_len; ++i) { | |
664 | __be64 mtt_entry = cpu_to_be64(page_list[i] | | |
665 | MTHCA_MTT_FLAG_PRESENT); | |
666 | mthca_write64_raw(mtt_entry, fmr->mem.tavor.mtts + i); | |
667 | } | |
668 | ||
669 | mpt_entry.lkey = cpu_to_be32(key); | |
670 | mpt_entry.length = cpu_to_be64(list_len * (1ull << fmr->attr.page_size)); | |
671 | mpt_entry.start = cpu_to_be64(iova); | |
672 | ||
97f52eb4 | 673 | __raw_writel((__force u32) mpt_entry.lkey, &fmr->mem.tavor.mpt->key); |
e0f5fdca MT |
674 | memcpy_toio(&fmr->mem.tavor.mpt->start, &mpt_entry.start, |
675 | offsetof(struct mthca_mpt_entry, window_count) - | |
676 | offsetof(struct mthca_mpt_entry, start)); | |
677 | ||
678 | writeb(MTHCA_MPT_STATUS_HW, fmr->mem.tavor.mpt); | |
679 | ||
680 | return 0; | |
681 | } | |
682 | ||
683 | int mthca_arbel_map_phys_fmr(struct ib_fmr *ibfmr, u64 *page_list, | |
684 | int list_len, u64 iova) | |
685 | { | |
686 | struct mthca_fmr *fmr = to_mfmr(ibfmr); | |
687 | struct mthca_dev *dev = to_mdev(ibfmr->device); | |
688 | u32 key; | |
689 | int i, err; | |
690 | ||
691 | err = mthca_check_fmr(fmr, page_list, list_len, iova); | |
692 | if (err) | |
693 | return err; | |
694 | ||
695 | ++fmr->maps; | |
696 | ||
697 | key = arbel_key_to_hw_index(fmr->ibmr.lkey); | |
698 | key += dev->limits.num_mpts; | |
699 | fmr->ibmr.lkey = fmr->ibmr.rkey = arbel_hw_index_to_key(key); | |
700 | ||
701 | *(u8 *) fmr->mem.arbel.mpt = MTHCA_MPT_STATUS_SW; | |
702 | ||
703 | wmb(); | |
704 | ||
705 | for (i = 0; i < list_len; ++i) | |
706 | fmr->mem.arbel.mtts[i] = cpu_to_be64(page_list[i] | | |
707 | MTHCA_MTT_FLAG_PRESENT); | |
708 | ||
709 | fmr->mem.arbel.mpt->key = cpu_to_be32(key); | |
710 | fmr->mem.arbel.mpt->lkey = cpu_to_be32(key); | |
711 | fmr->mem.arbel.mpt->length = cpu_to_be64(list_len * (1ull << fmr->attr.page_size)); | |
712 | fmr->mem.arbel.mpt->start = cpu_to_be64(iova); | |
713 | ||
714 | wmb(); | |
715 | ||
716 | *(u8 *) fmr->mem.arbel.mpt = MTHCA_MPT_STATUS_HW; | |
717 | ||
718 | wmb(); | |
719 | ||
720 | return 0; | |
721 | } | |
722 | ||
723 | void mthca_tavor_fmr_unmap(struct mthca_dev *dev, struct mthca_fmr *fmr) | |
724 | { | |
725 | u32 key; | |
726 | ||
727 | if (!fmr->maps) | |
728 | return; | |
729 | ||
730 | key = tavor_key_to_hw_index(fmr->ibmr.lkey); | |
731 | key &= dev->limits.num_mpts - 1; | |
732 | fmr->ibmr.lkey = fmr->ibmr.rkey = tavor_hw_index_to_key(key); | |
733 | ||
734 | fmr->maps = 0; | |
735 | ||
736 | writeb(MTHCA_MPT_STATUS_SW, fmr->mem.tavor.mpt); | |
737 | } | |
738 | ||
739 | void mthca_arbel_fmr_unmap(struct mthca_dev *dev, struct mthca_fmr *fmr) | |
740 | { | |
741 | u32 key; | |
742 | ||
743 | if (!fmr->maps) | |
744 | return; | |
745 | ||
746 | key = arbel_key_to_hw_index(fmr->ibmr.lkey); | |
747 | key &= dev->limits.num_mpts - 1; | |
748 | fmr->ibmr.lkey = fmr->ibmr.rkey = arbel_hw_index_to_key(key); | |
749 | ||
750 | fmr->maps = 0; | |
751 | ||
752 | *(u8 *) fmr->mem.arbel.mpt = MTHCA_MPT_STATUS_SW; | |
1da177e4 LT |
753 | } |
754 | ||
755 | int __devinit mthca_init_mr_table(struct mthca_dev *dev) | |
756 | { | |
e0f5fdca | 757 | int err, i; |
1da177e4 LT |
758 | |
759 | err = mthca_alloc_init(&dev->mr_table.mpt_alloc, | |
760 | dev->limits.num_mpts, | |
761 | ~0, dev->limits.reserved_mrws); | |
762 | if (err) | |
763 | return err; | |
764 | ||
d10ddbf6 | 765 | if (!mthca_is_memfree(dev) && |
e0f5fdca MT |
766 | (dev->mthca_flags & MTHCA_FLAG_DDR_HIDDEN)) |
767 | dev->limits.fmr_reserved_mtts = 0; | |
768 | else | |
769 | dev->mthca_flags |= MTHCA_FLAG_FMR; | |
770 | ||
9095e208 MT |
771 | err = mthca_buddy_init(&dev->mr_table.mtt_buddy, |
772 | fls(dev->limits.num_mtt_segs - 1)); | |
e0f5fdca | 773 | |
9095e208 MT |
774 | if (err) |
775 | goto err_mtt_buddy; | |
776 | ||
e0f5fdca MT |
777 | dev->mr_table.tavor_fmr.mpt_base = NULL; |
778 | dev->mr_table.tavor_fmr.mtt_base = NULL; | |
779 | ||
780 | if (dev->limits.fmr_reserved_mtts) { | |
781 | i = fls(dev->limits.fmr_reserved_mtts - 1); | |
782 | ||
783 | if (i >= 31) { | |
784 | mthca_warn(dev, "Unable to reserve 2^31 FMR MTTs.\n"); | |
785 | err = -EINVAL; | |
786 | goto err_fmr_mpt; | |
787 | } | |
788 | ||
789 | dev->mr_table.tavor_fmr.mpt_base = | |
790 | ioremap(dev->mr_table.mpt_base, | |
791 | (1 << i) * sizeof (struct mthca_mpt_entry)); | |
792 | ||
793 | if (!dev->mr_table.tavor_fmr.mpt_base) { | |
794 | mthca_warn(dev, "MPT ioremap for FMR failed.\n"); | |
795 | err = -ENOMEM; | |
796 | goto err_fmr_mpt; | |
797 | } | |
798 | ||
799 | dev->mr_table.tavor_fmr.mtt_base = | |
800 | ioremap(dev->mr_table.mtt_base, | |
801 | (1 << i) * MTHCA_MTT_SEG_SIZE); | |
802 | if (!dev->mr_table.tavor_fmr.mtt_base) { | |
803 | mthca_warn(dev, "MTT ioremap for FMR failed.\n"); | |
804 | err = -ENOMEM; | |
805 | goto err_fmr_mtt; | |
806 | } | |
807 | ||
808 | err = mthca_buddy_init(&dev->mr_table.tavor_fmr.mtt_buddy, i); | |
809 | if (err) | |
810 | goto err_fmr_mtt_buddy; | |
811 | ||
812 | /* Prevent regular MRs from using FMR keys */ | |
813 | err = mthca_buddy_alloc(&dev->mr_table.mtt_buddy, i); | |
814 | if (err) | |
815 | goto err_reserve_fmr; | |
816 | ||
817 | dev->mr_table.fmr_mtt_buddy = | |
818 | &dev->mr_table.tavor_fmr.mtt_buddy; | |
819 | } else | |
820 | dev->mr_table.fmr_mtt_buddy = &dev->mr_table.mtt_buddy; | |
821 | ||
822 | /* FMR table is always the first, take reserved MTTs out of there */ | |
9095e208 | 823 | if (dev->limits.reserved_mtts) { |
e0f5fdca MT |
824 | i = fls(dev->limits.reserved_mtts - 1); |
825 | ||
d56d6f95 RD |
826 | if (mthca_alloc_mtt_range(dev, i, |
827 | dev->mr_table.fmr_mtt_buddy) == -1) { | |
9095e208 | 828 | mthca_warn(dev, "MTT table of order %d is too small.\n", |
e0f5fdca | 829 | dev->mr_table.fmr_mtt_buddy->max_order); |
9095e208 | 830 | err = -ENOMEM; |
e0f5fdca | 831 | goto err_reserve_mtts; |
9095e208 | 832 | } |
1da177e4 LT |
833 | } |
834 | ||
1da177e4 LT |
835 | return 0; |
836 | ||
e0f5fdca MT |
837 | err_reserve_mtts: |
838 | err_reserve_fmr: | |
839 | if (dev->limits.fmr_reserved_mtts) | |
840 | mthca_buddy_cleanup(&dev->mr_table.tavor_fmr.mtt_buddy); | |
841 | ||
842 | err_fmr_mtt_buddy: | |
843 | if (dev->mr_table.tavor_fmr.mtt_base) | |
844 | iounmap(dev->mr_table.tavor_fmr.mtt_base); | |
845 | ||
846 | err_fmr_mtt: | |
847 | if (dev->mr_table.tavor_fmr.mpt_base) | |
848 | iounmap(dev->mr_table.tavor_fmr.mpt_base); | |
849 | ||
850 | err_fmr_mpt: | |
851 | mthca_buddy_cleanup(&dev->mr_table.mtt_buddy); | |
852 | ||
9095e208 | 853 | err_mtt_buddy: |
1da177e4 LT |
854 | mthca_alloc_cleanup(&dev->mr_table.mpt_alloc); |
855 | ||
856 | return err; | |
857 | } | |
858 | ||
859 | void __devexit mthca_cleanup_mr_table(struct mthca_dev *dev) | |
860 | { | |
1da177e4 | 861 | /* XXX check if any MRs are still allocated? */ |
e0f5fdca MT |
862 | if (dev->limits.fmr_reserved_mtts) |
863 | mthca_buddy_cleanup(&dev->mr_table.tavor_fmr.mtt_buddy); | |
864 | ||
9095e208 | 865 | mthca_buddy_cleanup(&dev->mr_table.mtt_buddy); |
e0f5fdca MT |
866 | |
867 | if (dev->mr_table.tavor_fmr.mtt_base) | |
868 | iounmap(dev->mr_table.tavor_fmr.mtt_base); | |
869 | if (dev->mr_table.tavor_fmr.mpt_base) | |
870 | iounmap(dev->mr_table.tavor_fmr.mpt_base); | |
871 | ||
1da177e4 LT |
872 | mthca_alloc_cleanup(&dev->mr_table.mpt_alloc); |
873 | } |