Commit | Line | Data |
---|---|---|
14d3a3b2 CH |
1 | /* |
2 | * Copyright (c) 2015 HGST, a Western Digital Company. | |
3 | * | |
4 | * This program is free software; you can redistribute it and/or modify it | |
5 | * under the terms and conditions of the GNU General Public License, | |
6 | * version 2, as published by the Free Software Foundation. | |
7 | * | |
8 | * This program is distributed in the hope it will be useful, but WITHOUT | |
9 | * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or | |
10 | * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for | |
11 | * more details. | |
12 | */ | |
13 | #include <linux/module.h> | |
14 | #include <linux/err.h> | |
15 | #include <linux/slab.h> | |
16 | #include <rdma/ib_verbs.h> | |
17 | ||
18 | /* # of WCs to poll for with a single call to ib_poll_cq */ | |
19 | #define IB_POLL_BATCH 16 | |
20 | ||
21 | /* # of WCs to iterate over before yielding */ | |
22 | #define IB_POLL_BUDGET_IRQ 256 | |
23 | #define IB_POLL_BUDGET_WORKQUEUE 65536 | |
24 | ||
25 | #define IB_POLL_FLAGS \ | |
26 | (IB_CQ_NEXT_COMP | IB_CQ_REPORT_MISSED_EVENTS) | |
27 | ||
28 | static int __ib_process_cq(struct ib_cq *cq, int budget) | |
29 | { | |
30 | int i, n, completed = 0; | |
31 | ||
32 | while ((n = ib_poll_cq(cq, IB_POLL_BATCH, cq->wc)) > 0) { | |
33 | for (i = 0; i < n; i++) { | |
34 | struct ib_wc *wc = &cq->wc[i]; | |
35 | ||
36 | if (wc->wr_cqe) | |
37 | wc->wr_cqe->done(cq, wc); | |
38 | else | |
39 | WARN_ON_ONCE(wc->status == IB_WC_SUCCESS); | |
40 | } | |
41 | ||
42 | completed += n; | |
43 | ||
44 | if (n != IB_POLL_BATCH || | |
45 | (budget != -1 && completed >= budget)) | |
46 | break; | |
47 | } | |
48 | ||
49 | return completed; | |
50 | } | |
51 | ||
52 | /** | |
53 | * ib_process_direct_cq - process a CQ in caller context | |
54 | * @cq: CQ to process | |
55 | * @budget: number of CQEs to poll for | |
56 | * | |
57 | * This function is used to process all outstanding CQ entries on a | |
58 | * %IB_POLL_DIRECT CQ. It does not offload CQ processing to a different | |
59 | * context and does not ask for completion interrupts from the HCA. | |
60 | * | |
61 | * Note: for compatibility reasons -1 can be passed in %budget for unlimited | |
62 | * polling. Do not use this feature in new code, it will be removed soon. | |
63 | */ | |
64 | int ib_process_cq_direct(struct ib_cq *cq, int budget) | |
65 | { | |
66 | WARN_ON_ONCE(cq->poll_ctx != IB_POLL_DIRECT); | |
67 | ||
68 | return __ib_process_cq(cq, budget); | |
69 | } | |
70 | EXPORT_SYMBOL(ib_process_cq_direct); | |
71 | ||
72 | static void ib_cq_completion_direct(struct ib_cq *cq, void *private) | |
73 | { | |
74 | WARN_ONCE(1, "got unsolicited completion for CQ 0x%p\n", cq); | |
75 | } | |
76 | ||
77 | static int ib_poll_handler(struct irq_poll *iop, int budget) | |
78 | { | |
79 | struct ib_cq *cq = container_of(iop, struct ib_cq, iop); | |
80 | int completed; | |
81 | ||
82 | completed = __ib_process_cq(cq, budget); | |
83 | if (completed < budget) { | |
84 | irq_poll_complete(&cq->iop); | |
85 | if (ib_req_notify_cq(cq, IB_POLL_FLAGS) > 0) | |
86 | irq_poll_sched(&cq->iop); | |
87 | } | |
88 | ||
89 | return completed; | |
90 | } | |
91 | ||
92 | static void ib_cq_completion_softirq(struct ib_cq *cq, void *private) | |
93 | { | |
94 | irq_poll_sched(&cq->iop); | |
95 | } | |
96 | ||
97 | static void ib_cq_poll_work(struct work_struct *work) | |
98 | { | |
99 | struct ib_cq *cq = container_of(work, struct ib_cq, work); | |
100 | int completed; | |
101 | ||
102 | completed = __ib_process_cq(cq, IB_POLL_BUDGET_WORKQUEUE); | |
103 | if (completed >= IB_POLL_BUDGET_WORKQUEUE || | |
104 | ib_req_notify_cq(cq, IB_POLL_FLAGS) > 0) | |
105 | queue_work(ib_comp_wq, &cq->work); | |
106 | } | |
107 | ||
108 | static void ib_cq_completion_workqueue(struct ib_cq *cq, void *private) | |
109 | { | |
110 | queue_work(ib_comp_wq, &cq->work); | |
111 | } | |
112 | ||
113 | /** | |
114 | * ib_alloc_cq - allocate a completion queue | |
115 | * @dev: device to allocate the CQ for | |
116 | * @private: driver private data, accessible from cq->cq_context | |
117 | * @nr_cqe: number of CQEs to allocate | |
118 | * @comp_vector: HCA completion vectors for this CQ | |
119 | * @poll_ctx: context to poll the CQ from. | |
120 | * | |
121 | * This is the proper interface to allocate a CQ for in-kernel users. A | |
122 | * CQ allocated with this interface will automatically be polled from the | |
123 | * specified context. The ULP needs must use wr->wr_cqe instead of wr->wr_id | |
124 | * to use this CQ abstraction. | |
125 | */ | |
126 | struct ib_cq *ib_alloc_cq(struct ib_device *dev, void *private, | |
127 | int nr_cqe, int comp_vector, enum ib_poll_context poll_ctx) | |
128 | { | |
129 | struct ib_cq_init_attr cq_attr = { | |
130 | .cqe = nr_cqe, | |
131 | .comp_vector = comp_vector, | |
132 | }; | |
133 | struct ib_cq *cq; | |
134 | int ret = -ENOMEM; | |
135 | ||
136 | cq = dev->create_cq(dev, &cq_attr, NULL, NULL); | |
137 | if (IS_ERR(cq)) | |
138 | return cq; | |
139 | ||
140 | cq->device = dev; | |
141 | cq->uobject = NULL; | |
142 | cq->event_handler = NULL; | |
143 | cq->cq_context = private; | |
144 | cq->poll_ctx = poll_ctx; | |
145 | atomic_set(&cq->usecnt, 0); | |
146 | ||
147 | cq->wc = kmalloc_array(IB_POLL_BATCH, sizeof(*cq->wc), GFP_KERNEL); | |
148 | if (!cq->wc) | |
149 | goto out_destroy_cq; | |
150 | ||
151 | switch (cq->poll_ctx) { | |
152 | case IB_POLL_DIRECT: | |
153 | cq->comp_handler = ib_cq_completion_direct; | |
154 | break; | |
155 | case IB_POLL_SOFTIRQ: | |
156 | cq->comp_handler = ib_cq_completion_softirq; | |
157 | ||
158 | irq_poll_init(&cq->iop, IB_POLL_BUDGET_IRQ, ib_poll_handler); | |
159 | ib_req_notify_cq(cq, IB_CQ_NEXT_COMP); | |
160 | break; | |
161 | case IB_POLL_WORKQUEUE: | |
162 | cq->comp_handler = ib_cq_completion_workqueue; | |
163 | INIT_WORK(&cq->work, ib_cq_poll_work); | |
164 | ib_req_notify_cq(cq, IB_CQ_NEXT_COMP); | |
165 | break; | |
166 | default: | |
167 | ret = -EINVAL; | |
168 | goto out_free_wc; | |
169 | } | |
170 | ||
171 | return cq; | |
172 | ||
173 | out_free_wc: | |
174 | kfree(cq->wc); | |
175 | out_destroy_cq: | |
176 | cq->device->destroy_cq(cq); | |
177 | return ERR_PTR(ret); | |
178 | } | |
179 | EXPORT_SYMBOL(ib_alloc_cq); | |
180 | ||
181 | /** | |
182 | * ib_free_cq - free a completion queue | |
183 | * @cq: completion queue to free. | |
184 | */ | |
185 | void ib_free_cq(struct ib_cq *cq) | |
186 | { | |
187 | int ret; | |
188 | ||
189 | if (WARN_ON_ONCE(atomic_read(&cq->usecnt))) | |
190 | return; | |
191 | ||
192 | switch (cq->poll_ctx) { | |
193 | case IB_POLL_DIRECT: | |
194 | break; | |
195 | case IB_POLL_SOFTIRQ: | |
196 | irq_poll_disable(&cq->iop); | |
197 | break; | |
198 | case IB_POLL_WORKQUEUE: | |
199 | flush_work(&cq->work); | |
200 | break; | |
201 | default: | |
202 | WARN_ON_ONCE(1); | |
203 | } | |
204 | ||
205 | kfree(cq->wc); | |
206 | ret = cq->device->destroy_cq(cq); | |
207 | WARN_ON_ONCE(ret); | |
208 | } | |
209 | EXPORT_SYMBOL(ib_free_cq); |