2 * Copyright (c) 2004 Topspin Communications. All rights reserved.
3 * Copyright (c) 2005 Sun Microsystems, Inc. All rights reserved.
5 * This software is available to you under a choice of one of two
6 * licenses. You may choose to be licensed under the terms of the GNU
7 * General Public License (GPL) Version 2, available from the file
8 * COPYING in the main directory of this source tree, or the
9 * OpenIB.org BSD license below:
11 * Redistribution and use in source and binary forms, with or
12 * without modification, are permitted provided that the following
15 * - Redistributions of source code must retain the above
16 * copyright notice, this list of conditions and the following
19 * - Redistributions in binary form must reproduce the above
20 * copyright notice, this list of conditions and the following
21 * disclaimer in the documentation and/or other materials
22 * provided with the distribution.
24 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
25 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
26 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
27 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
28 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
29 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
30 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
33 * $Id: mthca_av.c 1349 2004-12-16 21:09:43Z roland $
36 #include <linux/init.h>
37 #include <linux/string.h>
38 #include <linux/slab.h>
40 #include <rdma/ib_verbs.h>
41 #include <rdma/ib_cache.h>
43 #include "mthca_dev.h"
54 __be32 sl_tclass_flowlabel
;
58 int mthca_create_ah(struct mthca_dev
*dev
,
60 struct ib_ah_attr
*ah_attr
,
64 struct mthca_av
*av
= NULL
;
66 ah
->type
= MTHCA_AH_PCI_POOL
;
68 if (mthca_is_memfree(dev
)) {
69 ah
->av
= kmalloc(sizeof *ah
->av
, GFP_ATOMIC
);
73 ah
->type
= MTHCA_AH_KMALLOC
;
75 } else if (!atomic_read(&pd
->sqp_count
) &&
76 !(dev
->mthca_flags
& MTHCA_FLAG_DDR_HIDDEN
)) {
77 index
= mthca_alloc(&dev
->av_table
.alloc
);
79 /* fall back to allocate in host memory */
83 av
= kmalloc(sizeof *av
, GFP_ATOMIC
);
87 ah
->type
= MTHCA_AH_ON_HCA
;
88 ah
->avdma
= dev
->av_table
.ddr_av_base
+
89 index
* MTHCA_AV_SIZE
;
93 if (ah
->type
== MTHCA_AH_PCI_POOL
) {
94 ah
->av
= pci_pool_alloc(dev
->av_table
.pool
,
95 SLAB_ATOMIC
, &ah
->avdma
);
102 ah
->key
= pd
->ntmr
.ibmr
.lkey
;
104 memset(av
, 0, MTHCA_AV_SIZE
);
106 av
->port_pd
= cpu_to_be32(pd
->pd_num
| (ah_attr
->port_num
<< 24));
107 av
->g_slid
= ah_attr
->src_path_bits
;
108 av
->dlid
= cpu_to_be16(ah_attr
->dlid
);
109 av
->msg_sr
= (3 << 4) | /* 2K message */
110 ah_attr
->static_rate
;
111 av
->sl_tclass_flowlabel
= cpu_to_be32(ah_attr
->sl
<< 28);
112 if (ah_attr
->ah_flags
& IB_AH_GRH
) {
114 av
->gid_index
= (ah_attr
->port_num
- 1) * dev
->limits
.gid_table_len
+
115 ah_attr
->grh
.sgid_index
;
116 av
->hop_limit
= ah_attr
->grh
.hop_limit
;
117 av
->sl_tclass_flowlabel
|=
118 cpu_to_be32((ah_attr
->grh
.traffic_class
<< 20) |
119 ah_attr
->grh
.flow_label
);
120 memcpy(av
->dgid
, ah_attr
->grh
.dgid
.raw
, 16);
122 /* Arbel workaround -- low byte of GID must be 2 */
123 av
->dgid
[3] = cpu_to_be32(2);
129 mthca_dbg(dev
, "Created UDAV at %p/%08lx:\n",
130 av
, (unsigned long) ah
->avdma
);
131 for (j
= 0; j
< 8; ++j
)
132 printk(KERN_DEBUG
" [%2x] %08x\n",
133 j
* 4, be32_to_cpu(((__be32
*) av
)[j
]));
136 if (ah
->type
== MTHCA_AH_ON_HCA
) {
137 memcpy_toio(dev
->av_table
.av_map
+ index
* MTHCA_AV_SIZE
,
145 int mthca_destroy_ah(struct mthca_dev
*dev
, struct mthca_ah
*ah
)
148 case MTHCA_AH_ON_HCA
:
149 mthca_free(&dev
->av_table
.alloc
,
150 (ah
->avdma
- dev
->av_table
.ddr_av_base
) /
154 case MTHCA_AH_PCI_POOL
:
155 pci_pool_free(dev
->av_table
.pool
, ah
->av
, ah
->avdma
);
158 case MTHCA_AH_KMALLOC
:
166 int mthca_read_ah(struct mthca_dev
*dev
, struct mthca_ah
*ah
,
167 struct ib_ud_header
*header
)
169 if (ah
->type
== MTHCA_AH_ON_HCA
)
172 header
->lrh
.service_level
= be32_to_cpu(ah
->av
->sl_tclass_flowlabel
) >> 28;
173 header
->lrh
.destination_lid
= ah
->av
->dlid
;
174 header
->lrh
.source_lid
= cpu_to_be16(ah
->av
->g_slid
& 0x7f);
175 if (ah
->av
->g_slid
& 0x80) {
176 header
->grh_present
= 1;
177 header
->grh
.traffic_class
=
178 (be32_to_cpu(ah
->av
->sl_tclass_flowlabel
) >> 20) & 0xff;
179 header
->grh
.flow_label
=
180 ah
->av
->sl_tclass_flowlabel
& cpu_to_be32(0xfffff);
181 ib_get_cached_gid(&dev
->ib_dev
,
182 be32_to_cpu(ah
->av
->port_pd
) >> 24,
184 &header
->grh
.source_gid
);
185 memcpy(header
->grh
.destination_gid
.raw
,
188 header
->grh_present
= 0;
194 int __devinit
mthca_init_av_table(struct mthca_dev
*dev
)
198 if (mthca_is_memfree(dev
))
201 err
= mthca_alloc_init(&dev
->av_table
.alloc
,
202 dev
->av_table
.num_ddr_avs
,
203 dev
->av_table
.num_ddr_avs
- 1,
208 dev
->av_table
.pool
= pci_pool_create("mthca_av", dev
->pdev
,
211 if (!dev
->av_table
.pool
)
214 if (!(dev
->mthca_flags
& MTHCA_FLAG_DDR_HIDDEN
)) {
215 dev
->av_table
.av_map
= ioremap(pci_resource_start(dev
->pdev
, 4) +
216 dev
->av_table
.ddr_av_base
-
218 dev
->av_table
.num_ddr_avs
*
220 if (!dev
->av_table
.av_map
)
223 dev
->av_table
.av_map
= NULL
;
228 pci_pool_destroy(dev
->av_table
.pool
);
231 mthca_alloc_cleanup(&dev
->av_table
.alloc
);
235 void __devexit
mthca_cleanup_av_table(struct mthca_dev
*dev
)
237 if (mthca_is_memfree(dev
))
240 if (dev
->av_table
.av_map
)
241 iounmap(dev
->av_table
.av_map
);
242 pci_pool_destroy(dev
->av_table
.pool
);
243 mthca_alloc_cleanup(&dev
->av_table
.alloc
);