Commit | Line | Data |
---|---|---|
d7e09d03 PT |
1 | /* |
2 | * GPL HEADER START | |
3 | * | |
4 | * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. | |
5 | * | |
6 | * This program is free software; you can redistribute it and/or modify | |
7 | * it under the terms of the GNU General Public License version 2 only, | |
8 | * as published by the Free Software Foundation. | |
9 | * | |
10 | * This program is distributed in the hope that it will be useful, but | |
11 | * WITHOUT ANY WARRANTY; without even the implied warranty of | |
12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU | |
13 | * General Public License version 2 for more details (a copy is included | |
14 | * in the LICENSE file that accompanied this code). | |
15 | * | |
16 | * You should have received a copy of the GNU General Public License | |
17 | * version 2 along with this program; If not, see | |
18 | * http://www.sun.com/software/products/lustre/docs/GPLv2.pdf | |
19 | * | |
20 | * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa Clara, | |
21 | * CA 95054 USA or visit www.sun.com if you need additional information or | |
22 | * have any questions. | |
23 | * | |
24 | * GPL HEADER END | |
25 | */ | |
26 | /* | |
27 | * Copyright (c) 2007, 2010, Oracle and/or its affiliates. All rights reserved. | |
28 | * Use is subject to license terms. | |
29 | * | |
1dc563a6 | 30 | * Copyright (c) 2012, 2015, Intel Corporation. |
d7e09d03 PT |
31 | */ |
32 | /* | |
33 | * This file is part of Lustre, http://www.lustre.org/ | |
34 | * Lustre is a trademark of Sun Microsystems, Inc. | |
35 | * | |
36 | * lnet/selftest/rpc.c | |
37 | * | |
38 | * Author: Isaac Huang <isaac@clusterfs.com> | |
39 | * | |
40 | * 2012-05-13: Liang Zhen <liang@whamcloud.com> | |
41 | * - percpt data for service to improve smp performance | |
42 | * - code cleanup | |
43 | */ | |
44 | ||
45 | #define DEBUG_SUBSYSTEM S_LNET | |
46 | ||
47 | #include "selftest.h" | |
48 | ||
49 | typedef enum { | |
50 | SRPC_STATE_NONE, | |
51 | SRPC_STATE_NI_INIT, | |
52 | SRPC_STATE_EQ_INIT, | |
53 | SRPC_STATE_RUNNING, | |
54 | SRPC_STATE_STOPPING, | |
55 | } srpc_state_t; | |
56 | ||
62366bf1 | 57 | static struct smoketest_rpc { |
d7e09d03 PT |
58 | spinlock_t rpc_glock; /* global lock */ |
59 | srpc_service_t *rpc_services[SRPC_SERVICE_MAX_ID + 1]; | |
60 | lnet_handle_eq_t rpc_lnet_eq; /* _the_ LNet event queue */ | |
61 | srpc_state_t rpc_state; | |
62 | srpc_counters_t rpc_counters; | |
63 | __u64 rpc_matchbits; /* matchbits counter */ | |
64 | } srpc_data; | |
65 | ||
66 | static inline int | |
67 | srpc_serv_portal(int svc_id) | |
68 | { | |
69 | return svc_id < SRPC_FRAMEWORK_SERVICE_MAX_ID ? | |
70 | SRPC_FRAMEWORK_REQUEST_PORTAL : SRPC_REQUEST_PORTAL; | |
71 | } | |
72 | ||
73 | /* forward ref's */ | |
a659df62 | 74 | int srpc_handle_rpc(swi_workitem_t *wi); |
d7e09d03 | 75 | |
a659df62 | 76 | void srpc_get_counters(srpc_counters_t *cnt) |
d7e09d03 PT |
77 | { |
78 | spin_lock(&srpc_data.rpc_glock); | |
79 | *cnt = srpc_data.rpc_counters; | |
80 | spin_unlock(&srpc_data.rpc_glock); | |
81 | } | |
82 | ||
a659df62 | 83 | void srpc_set_counters(const srpc_counters_t *cnt) |
d7e09d03 PT |
84 | { |
85 | spin_lock(&srpc_data.rpc_glock); | |
86 | srpc_data.rpc_counters = *cnt; | |
87 | spin_unlock(&srpc_data.rpc_glock); | |
88 | } | |
89 | ||
8d94b6d2 | 90 | static int |
d7e09d03 PT |
91 | srpc_add_bulk_page(srpc_bulk_t *bk, struct page *pg, int i, int nob) |
92 | { | |
7c192dca | 93 | nob = min_t(int, nob, PAGE_CACHE_SIZE); |
d7e09d03 PT |
94 | |
95 | LASSERT(nob > 0); | |
96 | LASSERT(i >= 0 && i < bk->bk_niov); | |
97 | ||
98 | bk->bk_iovs[i].kiov_offset = 0; | |
99 | bk->bk_iovs[i].kiov_page = pg; | |
100 | bk->bk_iovs[i].kiov_len = nob; | |
101 | return nob; | |
102 | } | |
103 | ||
104 | void | |
a659df62 | 105 | srpc_free_bulk(srpc_bulk_t *bk) |
d7e09d03 | 106 | { |
74d68011 | 107 | int i; |
d7e09d03 PT |
108 | struct page *pg; |
109 | ||
a659df62 | 110 | LASSERT(bk != NULL); |
d7e09d03 PT |
111 | |
112 | for (i = 0; i < bk->bk_niov; i++) { | |
113 | pg = bk->bk_iovs[i].kiov_page; | |
f64a6f3c HE |
114 | if (pg == NULL) |
115 | break; | |
d7e09d03 PT |
116 | |
117 | __free_page(pg); | |
118 | } | |
119 | ||
120 | LIBCFS_FREE(bk, offsetof(srpc_bulk_t, bk_iovs[bk->bk_niov])); | |
121 | return; | |
122 | } | |
123 | ||
124 | srpc_bulk_t * | |
125 | srpc_alloc_bulk(int cpt, unsigned bulk_npg, unsigned bulk_len, int sink) | |
126 | { | |
74d68011 MS |
127 | srpc_bulk_t *bk; |
128 | int i; | |
d7e09d03 PT |
129 | |
130 | LASSERT(bulk_npg > 0 && bulk_npg <= LNET_MAX_IOV); | |
131 | ||
132 | LIBCFS_CPT_ALLOC(bk, lnet_cpt_table(), cpt, | |
133 | offsetof(srpc_bulk_t, bk_iovs[bulk_npg])); | |
134 | if (bk == NULL) { | |
135 | CERROR("Can't allocate descriptor for %d pages\n", bulk_npg); | |
136 | return NULL; | |
137 | } | |
138 | ||
139 | memset(bk, 0, offsetof(srpc_bulk_t, bk_iovs[bulk_npg])); | |
140 | bk->bk_sink = sink; | |
141 | bk->bk_len = bulk_len; | |
142 | bk->bk_niov = bulk_npg; | |
d7e09d03 PT |
143 | |
144 | for (i = 0; i < bulk_npg; i++) { | |
145 | struct page *pg; | |
74d68011 | 146 | int nob; |
d7e09d03 | 147 | |
49c02a75 | 148 | pg = alloc_pages_node(cfs_cpt_spread_node(lnet_cpt_table(), cpt), |
40113370 | 149 | GFP_KERNEL, 0); |
d7e09d03 PT |
150 | if (pg == NULL) { |
151 | CERROR("Can't allocate page %d of %d\n", i, bulk_npg); | |
152 | srpc_free_bulk(bk); | |
153 | return NULL; | |
154 | } | |
155 | ||
156 | nob = srpc_add_bulk_page(bk, pg, i, bulk_len); | |
157 | bulk_len -= nob; | |
158 | } | |
159 | ||
160 | return bk; | |
161 | } | |
162 | ||
163 | static inline __u64 | |
a659df62 | 164 | srpc_next_id(void) |
d7e09d03 PT |
165 | { |
166 | __u64 id; | |
167 | ||
168 | spin_lock(&srpc_data.rpc_glock); | |
169 | id = srpc_data.rpc_matchbits++; | |
170 | spin_unlock(&srpc_data.rpc_glock); | |
171 | return id; | |
172 | } | |
173 | ||
8d94b6d2 | 174 | static void |
d7e09d03 PT |
175 | srpc_init_server_rpc(struct srpc_server_rpc *rpc, |
176 | struct srpc_service_cd *scd, | |
177 | struct srpc_buffer *buffer) | |
178 | { | |
179 | memset(rpc, 0, sizeof(*rpc)); | |
180 | swi_init_workitem(&rpc->srpc_wi, rpc, srpc_handle_rpc, | |
181 | srpc_serv_is_framework(scd->scd_svc) ? | |
182 | lst_sched_serial : lst_sched_test[scd->scd_cpt]); | |
183 | ||
184 | rpc->srpc_ev.ev_fired = 1; /* no event expected now */ | |
185 | ||
186 | rpc->srpc_scd = scd; | |
187 | rpc->srpc_reqstbuf = buffer; | |
188 | rpc->srpc_peer = buffer->buf_peer; | |
189 | rpc->srpc_self = buffer->buf_self; | |
190 | LNetInvalidateHandle(&rpc->srpc_replymdh); | |
191 | } | |
192 | ||
193 | static void | |
194 | srpc_service_fini(struct srpc_service *svc) | |
195 | { | |
74d68011 MS |
196 | struct srpc_service_cd *scd; |
197 | struct srpc_server_rpc *rpc; | |
198 | struct srpc_buffer *buf; | |
199 | struct list_head *q; | |
200 | int i; | |
d7e09d03 PT |
201 | |
202 | if (svc->sv_cpt_data == NULL) | |
203 | return; | |
204 | ||
205 | cfs_percpt_for_each(scd, i, svc->sv_cpt_data) { | |
206 | while (1) { | |
207 | if (!list_empty(&scd->scd_buf_posted)) | |
208 | q = &scd->scd_buf_posted; | |
209 | else if (!list_empty(&scd->scd_buf_blocked)) | |
210 | q = &scd->scd_buf_blocked; | |
211 | else | |
212 | break; | |
213 | ||
214 | while (!list_empty(q)) { | |
c314c319 JS |
215 | buf = list_entry(q->next, struct srpc_buffer, |
216 | buf_list); | |
d7e09d03 PT |
217 | list_del(&buf->buf_list); |
218 | LIBCFS_FREE(buf, sizeof(*buf)); | |
219 | } | |
220 | } | |
221 | ||
222 | LASSERT(list_empty(&scd->scd_rpc_active)); | |
223 | ||
224 | while (!list_empty(&scd->scd_rpc_free)) { | |
225 | rpc = list_entry(scd->scd_rpc_free.next, | |
c314c319 JS |
226 | struct srpc_server_rpc, |
227 | srpc_list); | |
d7e09d03 PT |
228 | list_del(&rpc->srpc_list); |
229 | LIBCFS_FREE(rpc, sizeof(*rpc)); | |
230 | } | |
231 | } | |
232 | ||
233 | cfs_percpt_free(svc->sv_cpt_data); | |
234 | svc->sv_cpt_data = NULL; | |
235 | } | |
236 | ||
237 | static int | |
238 | srpc_service_nrpcs(struct srpc_service *svc) | |
239 | { | |
240 | int nrpcs = svc->sv_wi_total / svc->sv_ncpts; | |
241 | ||
242 | return srpc_serv_is_framework(svc) ? | |
243 | max(nrpcs, SFW_FRWK_WI_MIN) : max(nrpcs, SFW_TEST_WI_MIN); | |
244 | } | |
245 | ||
246 | int srpc_add_buffer(struct swi_workitem *wi); | |
247 | ||
248 | static int | |
249 | srpc_service_init(struct srpc_service *svc) | |
250 | { | |
74d68011 MS |
251 | struct srpc_service_cd *scd; |
252 | struct srpc_server_rpc *rpc; | |
253 | int nrpcs; | |
254 | int i; | |
255 | int j; | |
d7e09d03 PT |
256 | |
257 | svc->sv_shuttingdown = 0; | |
258 | ||
259 | svc->sv_cpt_data = cfs_percpt_alloc(lnet_cpt_table(), | |
260 | sizeof(struct srpc_service_cd)); | |
261 | if (svc->sv_cpt_data == NULL) | |
262 | return -ENOMEM; | |
263 | ||
264 | svc->sv_ncpts = srpc_serv_is_framework(svc) ? | |
265 | 1 : cfs_cpt_number(lnet_cpt_table()); | |
266 | nrpcs = srpc_service_nrpcs(svc); | |
267 | ||
268 | cfs_percpt_for_each(scd, i, svc->sv_cpt_data) { | |
269 | scd->scd_cpt = i; | |
270 | scd->scd_svc = svc; | |
271 | spin_lock_init(&scd->scd_lock); | |
272 | INIT_LIST_HEAD(&scd->scd_rpc_free); | |
273 | INIT_LIST_HEAD(&scd->scd_rpc_active); | |
274 | INIT_LIST_HEAD(&scd->scd_buf_posted); | |
275 | INIT_LIST_HEAD(&scd->scd_buf_blocked); | |
276 | ||
277 | scd->scd_ev.ev_data = scd; | |
278 | scd->scd_ev.ev_type = SRPC_REQUEST_RCVD; | |
279 | ||
4420cfd3 JS |
280 | /* |
281 | * NB: don't use lst_sched_serial for adding buffer, | |
282 | * see details in srpc_service_add_buffers() | |
283 | */ | |
d7e09d03 PT |
284 | swi_init_workitem(&scd->scd_buf_wi, scd, |
285 | srpc_add_buffer, lst_sched_test[i]); | |
286 | ||
287 | if (i != 0 && srpc_serv_is_framework(svc)) { | |
4420cfd3 JS |
288 | /* |
289 | * NB: framework service only needs srpc_service_cd for | |
d7e09d03 PT |
290 | * one partition, but we allocate for all to make |
291 | * it easier to implement, it will waste a little | |
4420cfd3 JS |
292 | * memory but nobody should care about this |
293 | */ | |
d7e09d03 PT |
294 | continue; |
295 | } | |
296 | ||
297 | for (j = 0; j < nrpcs; j++) { | |
298 | LIBCFS_CPT_ALLOC(rpc, lnet_cpt_table(), | |
299 | i, sizeof(*rpc)); | |
300 | if (rpc == NULL) { | |
301 | srpc_service_fini(svc); | |
302 | return -ENOMEM; | |
303 | } | |
304 | list_add(&rpc->srpc_list, &scd->scd_rpc_free); | |
305 | } | |
306 | } | |
307 | ||
308 | return 0; | |
309 | } | |
310 | ||
311 | int | |
312 | srpc_add_service(struct srpc_service *sv) | |
313 | { | |
314 | int id = sv->sv_id; | |
315 | ||
316 | LASSERT(0 <= id && id <= SRPC_SERVICE_MAX_ID); | |
317 | ||
318 | if (srpc_service_init(sv) != 0) | |
319 | return -ENOMEM; | |
320 | ||
321 | spin_lock(&srpc_data.rpc_glock); | |
322 | ||
323 | LASSERT(srpc_data.rpc_state == SRPC_STATE_RUNNING); | |
324 | ||
325 | if (srpc_data.rpc_services[id] != NULL) { | |
326 | spin_unlock(&srpc_data.rpc_glock); | |
327 | goto failed; | |
328 | } | |
329 | ||
330 | srpc_data.rpc_services[id] = sv; | |
331 | spin_unlock(&srpc_data.rpc_glock); | |
332 | ||
333 | CDEBUG(D_NET, "Adding service: id %d, name %s\n", id, sv->sv_name); | |
334 | return 0; | |
335 | ||
336 | failed: | |
337 | srpc_service_fini(sv); | |
338 | return -EBUSY; | |
339 | } | |
340 | ||
341 | int | |
a659df62 | 342 | srpc_remove_service(srpc_service_t *sv) |
d7e09d03 PT |
343 | { |
344 | int id = sv->sv_id; | |
345 | ||
346 | spin_lock(&srpc_data.rpc_glock); | |
347 | ||
348 | if (srpc_data.rpc_services[id] != sv) { | |
349 | spin_unlock(&srpc_data.rpc_glock); | |
350 | return -ENOENT; | |
351 | } | |
352 | ||
353 | srpc_data.rpc_services[id] = NULL; | |
354 | spin_unlock(&srpc_data.rpc_glock); | |
355 | return 0; | |
356 | } | |
357 | ||
8d94b6d2 | 358 | static int |
d7e09d03 PT |
359 | srpc_post_passive_rdma(int portal, int local, __u64 matchbits, void *buf, |
360 | int len, int options, lnet_process_id_t peer, | |
361 | lnet_handle_md_t *mdh, srpc_event_t *ev) | |
362 | { | |
74d68011 MS |
363 | int rc; |
364 | lnet_md_t md; | |
d7e09d03 PT |
365 | lnet_handle_me_t meh; |
366 | ||
367 | rc = LNetMEAttach(portal, peer, matchbits, 0, LNET_UNLINK, | |
368 | local ? LNET_INS_LOCAL : LNET_INS_AFTER, &meh); | |
369 | if (rc != 0) { | |
a659df62 HE |
370 | CERROR("LNetMEAttach failed: %d\n", rc); |
371 | LASSERT(rc == -ENOMEM); | |
d7e09d03 PT |
372 | return -ENOMEM; |
373 | } | |
374 | ||
375 | md.threshold = 1; | |
376 | md.user_ptr = ev; | |
377 | md.start = buf; | |
378 | md.length = len; | |
379 | md.options = options; | |
380 | md.eq_handle = srpc_data.rpc_lnet_eq; | |
381 | ||
382 | rc = LNetMDAttach(meh, md, LNET_UNLINK, mdh); | |
383 | if (rc != 0) { | |
a659df62 HE |
384 | CERROR("LNetMDAttach failed: %d\n", rc); |
385 | LASSERT(rc == -ENOMEM); | |
d7e09d03 PT |
386 | |
387 | rc = LNetMEUnlink(meh); | |
a659df62 | 388 | LASSERT(rc == 0); |
d7e09d03 PT |
389 | return -ENOMEM; |
390 | } | |
391 | ||
c314c319 JS |
392 | CDEBUG(D_NET, "Posted passive RDMA: peer %s, portal %d, matchbits %#llx\n", |
393 | libcfs_id2str(peer), portal, matchbits); | |
d7e09d03 PT |
394 | return 0; |
395 | } | |
396 | ||
8d94b6d2 | 397 | static int |
d7e09d03 PT |
398 | srpc_post_active_rdma(int portal, __u64 matchbits, void *buf, int len, |
399 | int options, lnet_process_id_t peer, lnet_nid_t self, | |
400 | lnet_handle_md_t *mdh, srpc_event_t *ev) | |
401 | { | |
74d68011 | 402 | int rc; |
d7e09d03 PT |
403 | lnet_md_t md; |
404 | ||
405 | md.user_ptr = ev; | |
406 | md.start = buf; | |
407 | md.length = len; | |
408 | md.eq_handle = srpc_data.rpc_lnet_eq; | |
409 | md.threshold = ((options & LNET_MD_OP_GET) != 0) ? 2 : 1; | |
410 | md.options = options & ~(LNET_MD_OP_PUT | LNET_MD_OP_GET); | |
411 | ||
412 | rc = LNetMDBind(md, LNET_UNLINK, mdh); | |
413 | if (rc != 0) { | |
a659df62 HE |
414 | CERROR("LNetMDBind failed: %d\n", rc); |
415 | LASSERT(rc == -ENOMEM); | |
d7e09d03 PT |
416 | return -ENOMEM; |
417 | } | |
418 | ||
4420cfd3 JS |
419 | /* |
420 | * this is kind of an abuse of the LNET_MD_OP_{PUT,GET} options. | |
d7e09d03 | 421 | * they're only meaningful for MDs attached to an ME (i.e. passive |
4420cfd3 JS |
422 | * buffers... |
423 | */ | |
d7e09d03 PT |
424 | if ((options & LNET_MD_OP_PUT) != 0) { |
425 | rc = LNetPut(self, *mdh, LNET_NOACK_REQ, peer, | |
426 | portal, matchbits, 0, 0); | |
427 | } else { | |
a659df62 | 428 | LASSERT((options & LNET_MD_OP_GET) != 0); |
d7e09d03 PT |
429 | |
430 | rc = LNetGet(self, *mdh, peer, portal, matchbits, 0); | |
431 | } | |
432 | ||
433 | if (rc != 0) { | |
a659df62 | 434 | CERROR("LNet%s(%s, %d, %lld) failed: %d\n", |
c314c319 JS |
435 | ((options & LNET_MD_OP_PUT) != 0) ? "Put" : "Get", |
436 | libcfs_id2str(peer), portal, matchbits, rc); | |
d7e09d03 | 437 | |
4420cfd3 JS |
438 | /* |
439 | * The forthcoming unlink event will complete this operation | |
d7e09d03 PT |
440 | * with failure, so fall through and return success here. |
441 | */ | |
442 | rc = LNetMDUnlink(*mdh); | |
a659df62 | 443 | LASSERT(rc == 0); |
d7e09d03 | 444 | } else { |
c314c319 JS |
445 | CDEBUG(D_NET, "Posted active RDMA: peer %s, portal %u, matchbits %#llx\n", |
446 | libcfs_id2str(peer), portal, matchbits); | |
d7e09d03 PT |
447 | } |
448 | return 0; | |
449 | } | |
450 | ||
8d94b6d2 | 451 | static int |
d7e09d03 PT |
452 | srpc_post_passive_rqtbuf(int service, int local, void *buf, int len, |
453 | lnet_handle_md_t *mdh, srpc_event_t *ev) | |
454 | { | |
455 | lnet_process_id_t any = {0}; | |
456 | ||
457 | any.nid = LNET_NID_ANY; | |
458 | any.pid = LNET_PID_ANY; | |
459 | ||
460 | return srpc_post_passive_rdma(srpc_serv_portal(service), | |
461 | local, service, buf, len, | |
462 | LNET_MD_OP_PUT, any, mdh, ev); | |
463 | } | |
464 | ||
8d94b6d2 | 465 | static int |
d7e09d03 | 466 | srpc_service_post_buffer(struct srpc_service_cd *scd, struct srpc_buffer *buf) |
dbfeb730 | 467 | __must_hold(&scd->scd_lock) |
d7e09d03 | 468 | { |
74d68011 MS |
469 | struct srpc_service *sv = scd->scd_svc; |
470 | struct srpc_msg *msg = &buf->buf_msg; | |
471 | int rc; | |
d7e09d03 PT |
472 | |
473 | LNetInvalidateHandle(&buf->buf_mdh); | |
474 | list_add(&buf->buf_list, &scd->scd_buf_posted); | |
475 | scd->scd_buf_nposted++; | |
476 | spin_unlock(&scd->scd_lock); | |
477 | ||
478 | rc = srpc_post_passive_rqtbuf(sv->sv_id, | |
479 | !srpc_serv_is_framework(sv), | |
480 | msg, sizeof(*msg), &buf->buf_mdh, | |
481 | &scd->scd_ev); | |
482 | ||
4420cfd3 JS |
483 | /* |
484 | * At this point, a RPC (new or delayed) may have arrived in | |
d7e09d03 | 485 | * msg and its event handler has been called. So we must add |
4420cfd3 JS |
486 | * buf to scd_buf_posted _before_ dropping scd_lock |
487 | */ | |
d7e09d03 PT |
488 | spin_lock(&scd->scd_lock); |
489 | ||
490 | if (rc == 0) { | |
491 | if (!sv->sv_shuttingdown) | |
492 | return 0; | |
493 | ||
494 | spin_unlock(&scd->scd_lock); | |
4420cfd3 JS |
495 | /* |
496 | * srpc_shutdown_service might have tried to unlink me | |
497 | * when my buf_mdh was still invalid | |
498 | */ | |
d7e09d03 PT |
499 | LNetMDUnlink(buf->buf_mdh); |
500 | spin_lock(&scd->scd_lock); | |
501 | return 0; | |
502 | } | |
503 | ||
504 | scd->scd_buf_nposted--; | |
505 | if (sv->sv_shuttingdown) | |
506 | return rc; /* don't allow to change scd_buf_posted */ | |
507 | ||
508 | list_del(&buf->buf_list); | |
509 | spin_unlock(&scd->scd_lock); | |
510 | ||
511 | LIBCFS_FREE(buf, sizeof(*buf)); | |
512 | ||
513 | spin_lock(&scd->scd_lock); | |
514 | return rc; | |
515 | } | |
516 | ||
517 | int | |
518 | srpc_add_buffer(struct swi_workitem *wi) | |
519 | { | |
74d68011 MS |
520 | struct srpc_service_cd *scd = wi->swi_workitem.wi_data; |
521 | struct srpc_buffer *buf; | |
522 | int rc = 0; | |
d7e09d03 | 523 | |
4420cfd3 JS |
524 | /* |
525 | * it's called by workitem scheduler threads, these threads | |
d7e09d03 | 526 | * should have been set CPT affinity, so buffers will be posted |
4420cfd3 JS |
527 | * on CPT local list of Portal |
528 | */ | |
d7e09d03 PT |
529 | spin_lock(&scd->scd_lock); |
530 | ||
531 | while (scd->scd_buf_adjust > 0 && | |
532 | !scd->scd_svc->sv_shuttingdown) { | |
533 | scd->scd_buf_adjust--; /* consume it */ | |
534 | scd->scd_buf_posting++; | |
535 | ||
536 | spin_unlock(&scd->scd_lock); | |
537 | ||
538 | LIBCFS_ALLOC(buf, sizeof(*buf)); | |
539 | if (buf == NULL) { | |
540 | CERROR("Failed to add new buf to service: %s\n", | |
541 | scd->scd_svc->sv_name); | |
542 | spin_lock(&scd->scd_lock); | |
543 | rc = -ENOMEM; | |
544 | break; | |
545 | } | |
546 | ||
547 | spin_lock(&scd->scd_lock); | |
548 | if (scd->scd_svc->sv_shuttingdown) { | |
549 | spin_unlock(&scd->scd_lock); | |
550 | LIBCFS_FREE(buf, sizeof(*buf)); | |
551 | ||
552 | spin_lock(&scd->scd_lock); | |
553 | rc = -ESHUTDOWN; | |
554 | break; | |
555 | } | |
556 | ||
557 | rc = srpc_service_post_buffer(scd, buf); | |
558 | if (rc != 0) | |
559 | break; /* buf has been freed inside */ | |
560 | ||
561 | LASSERT(scd->scd_buf_posting > 0); | |
562 | scd->scd_buf_posting--; | |
563 | scd->scd_buf_total++; | |
0c575417 | 564 | scd->scd_buf_low = max(2, scd->scd_buf_total / 4); |
d7e09d03 PT |
565 | } |
566 | ||
567 | if (rc != 0) { | |
d9f79e6b | 568 | scd->scd_buf_err_stamp = ktime_get_real_seconds(); |
d7e09d03 PT |
569 | scd->scd_buf_err = rc; |
570 | ||
571 | LASSERT(scd->scd_buf_posting > 0); | |
572 | scd->scd_buf_posting--; | |
573 | } | |
574 | ||
575 | spin_unlock(&scd->scd_lock); | |
576 | return 0; | |
577 | } | |
578 | ||
579 | int | |
580 | srpc_service_add_buffers(struct srpc_service *sv, int nbuffer) | |
581 | { | |
74d68011 MS |
582 | struct srpc_service_cd *scd; |
583 | int rc = 0; | |
584 | int i; | |
d7e09d03 PT |
585 | |
586 | LASSERTF(nbuffer > 0, "nbuffer must be positive: %d\n", nbuffer); | |
587 | ||
588 | cfs_percpt_for_each(scd, i, sv->sv_cpt_data) { | |
589 | spin_lock(&scd->scd_lock); | |
590 | ||
591 | scd->scd_buf_err = 0; | |
592 | scd->scd_buf_err_stamp = 0; | |
593 | scd->scd_buf_posting = 0; | |
594 | scd->scd_buf_adjust = nbuffer; | |
595 | /* start to post buffers */ | |
596 | swi_schedule_workitem(&scd->scd_buf_wi); | |
597 | spin_unlock(&scd->scd_lock); | |
598 | ||
599 | /* framework service only post buffer for one partition */ | |
600 | if (srpc_serv_is_framework(sv)) | |
601 | break; | |
602 | } | |
603 | ||
604 | cfs_percpt_for_each(scd, i, sv->sv_cpt_data) { | |
605 | spin_lock(&scd->scd_lock); | |
606 | /* | |
607 | * NB: srpc_service_add_buffers() can be called inside | |
608 | * thread context of lst_sched_serial, and we don't normally | |
609 | * allow to sleep inside thread context of WI scheduler | |
610 | * because it will block current scheduler thread from doing | |
611 | * anything else, even worse, it could deadlock if it's | |
612 | * waiting on result from another WI of the same scheduler. | |
613 | * However, it's safe at here because scd_buf_wi is scheduled | |
614 | * by thread in a different WI scheduler (lst_sched_test), | |
615 | * so we don't have any risk of deadlock, though this could | |
616 | * block all WIs pending on lst_sched_serial for a moment | |
617 | * which is not good but not fatal. | |
618 | */ | |
619 | lst_wait_until(scd->scd_buf_err != 0 || | |
620 | (scd->scd_buf_adjust == 0 && | |
621 | scd->scd_buf_posting == 0), | |
622 | scd->scd_lock, "waiting for adding buffer\n"); | |
623 | ||
624 | if (scd->scd_buf_err != 0 && rc == 0) | |
625 | rc = scd->scd_buf_err; | |
626 | ||
627 | spin_unlock(&scd->scd_lock); | |
628 | } | |
629 | ||
630 | return rc; | |
631 | } | |
632 | ||
633 | void | |
634 | srpc_service_remove_buffers(struct srpc_service *sv, int nbuffer) | |
635 | { | |
74d68011 MS |
636 | struct srpc_service_cd *scd; |
637 | int num; | |
638 | int i; | |
d7e09d03 PT |
639 | |
640 | LASSERT(!sv->sv_shuttingdown); | |
641 | ||
642 | cfs_percpt_for_each(scd, i, sv->sv_cpt_data) { | |
643 | spin_lock(&scd->scd_lock); | |
644 | ||
645 | num = scd->scd_buf_total + scd->scd_buf_posting; | |
646 | scd->scd_buf_adjust -= min(nbuffer, num); | |
647 | ||
648 | spin_unlock(&scd->scd_lock); | |
649 | } | |
650 | } | |
651 | ||
652 | /* returns 1 if sv has finished, otherwise 0 */ | |
653 | int | |
654 | srpc_finish_service(struct srpc_service *sv) | |
655 | { | |
74d68011 MS |
656 | struct srpc_service_cd *scd; |
657 | struct srpc_server_rpc *rpc; | |
658 | int i; | |
d7e09d03 PT |
659 | |
660 | LASSERT(sv->sv_shuttingdown); /* srpc_shutdown_service called */ | |
661 | ||
662 | cfs_percpt_for_each(scd, i, sv->sv_cpt_data) { | |
663 | spin_lock(&scd->scd_lock); | |
f8c47be4 SB |
664 | if (!swi_deschedule_workitem(&scd->scd_buf_wi)) { |
665 | spin_unlock(&scd->scd_lock); | |
d7e09d03 | 666 | return 0; |
f8c47be4 | 667 | } |
d7e09d03 PT |
668 | |
669 | if (scd->scd_buf_nposted > 0) { | |
670 | CDEBUG(D_NET, "waiting for %d posted buffers to unlink", | |
671 | scd->scd_buf_nposted); | |
672 | spin_unlock(&scd->scd_lock); | |
673 | return 0; | |
674 | } | |
675 | ||
676 | if (list_empty(&scd->scd_rpc_active)) { | |
677 | spin_unlock(&scd->scd_lock); | |
678 | continue; | |
679 | } | |
680 | ||
681 | rpc = list_entry(scd->scd_rpc_active.next, | |
c314c319 | 682 | struct srpc_server_rpc, srpc_list); |
2d00bd17 | 683 | CNETERR("Active RPC %p on shutdown: sv %s, peer %s, wi %s scheduled %d running %d, ev fired %d type %d status %d lnet %d\n", |
d7e09d03 PT |
684 | rpc, sv->sv_name, libcfs_id2str(rpc->srpc_peer), |
685 | swi_state2str(rpc->srpc_wi.swi_state), | |
686 | rpc->srpc_wi.swi_workitem.wi_scheduled, | |
687 | rpc->srpc_wi.swi_workitem.wi_running, | |
688 | rpc->srpc_ev.ev_fired, rpc->srpc_ev.ev_type, | |
689 | rpc->srpc_ev.ev_status, rpc->srpc_ev.ev_lnet); | |
690 | spin_unlock(&scd->scd_lock); | |
691 | return 0; | |
692 | } | |
693 | ||
694 | /* no lock needed from now on */ | |
695 | srpc_service_fini(sv); | |
696 | return 1; | |
697 | } | |
698 | ||
699 | /* called with sv->sv_lock held */ | |
8d94b6d2 | 700 | static void |
d7e09d03 | 701 | srpc_service_recycle_buffer(struct srpc_service_cd *scd, srpc_buffer_t *buf) |
48cbf265 | 702 | __must_hold(&scd->scd_lock) |
d7e09d03 PT |
703 | { |
704 | if (!scd->scd_svc->sv_shuttingdown && scd->scd_buf_adjust >= 0) { | |
705 | if (srpc_service_post_buffer(scd, buf) != 0) { | |
706 | CWARN("Failed to post %s buffer\n", | |
707 | scd->scd_svc->sv_name); | |
708 | } | |
709 | return; | |
710 | } | |
711 | ||
712 | /* service is shutting down, or we want to recycle some buffers */ | |
713 | scd->scd_buf_total--; | |
714 | ||
715 | if (scd->scd_buf_adjust < 0) { | |
716 | scd->scd_buf_adjust++; | |
717 | if (scd->scd_buf_adjust < 0 && | |
718 | scd->scd_buf_total == 0 && scd->scd_buf_posting == 0) { | |
719 | CDEBUG(D_INFO, | |
23ebb3fd | 720 | "Try to recycle %d buffers but nothing left\n", |
d7e09d03 PT |
721 | scd->scd_buf_adjust); |
722 | scd->scd_buf_adjust = 0; | |
723 | } | |
724 | } | |
725 | ||
726 | spin_unlock(&scd->scd_lock); | |
727 | LIBCFS_FREE(buf, sizeof(*buf)); | |
728 | spin_lock(&scd->scd_lock); | |
729 | } | |
730 | ||
731 | void | |
732 | srpc_abort_service(struct srpc_service *sv) | |
733 | { | |
74d68011 MS |
734 | struct srpc_service_cd *scd; |
735 | struct srpc_server_rpc *rpc; | |
736 | int i; | |
d7e09d03 PT |
737 | |
738 | CDEBUG(D_NET, "Aborting service: id %d, name %s\n", | |
739 | sv->sv_id, sv->sv_name); | |
740 | ||
741 | cfs_percpt_for_each(scd, i, sv->sv_cpt_data) { | |
742 | spin_lock(&scd->scd_lock); | |
743 | ||
4420cfd3 JS |
744 | /* |
745 | * schedule in-flight RPCs to notice the abort, NB: | |
d7e09d03 | 746 | * racing with incoming RPCs; complete fix should make test |
4420cfd3 JS |
747 | * RPCs carry session ID in its headers |
748 | */ | |
d7e09d03 PT |
749 | list_for_each_entry(rpc, &scd->scd_rpc_active, srpc_list) { |
750 | rpc->srpc_aborted = 1; | |
751 | swi_schedule_workitem(&rpc->srpc_wi); | |
752 | } | |
753 | ||
754 | spin_unlock(&scd->scd_lock); | |
755 | } | |
756 | } | |
757 | ||
758 | void | |
759 | srpc_shutdown_service(srpc_service_t *sv) | |
760 | { | |
74d68011 MS |
761 | struct srpc_service_cd *scd; |
762 | struct srpc_server_rpc *rpc; | |
763 | srpc_buffer_t *buf; | |
764 | int i; | |
d7e09d03 PT |
765 | |
766 | CDEBUG(D_NET, "Shutting down service: id %d, name %s\n", | |
767 | sv->sv_id, sv->sv_name); | |
768 | ||
769 | cfs_percpt_for_each(scd, i, sv->sv_cpt_data) | |
770 | spin_lock(&scd->scd_lock); | |
771 | ||
772 | sv->sv_shuttingdown = 1; /* i.e. no new active RPC */ | |
773 | ||
774 | cfs_percpt_for_each(scd, i, sv->sv_cpt_data) | |
775 | spin_unlock(&scd->scd_lock); | |
776 | ||
777 | cfs_percpt_for_each(scd, i, sv->sv_cpt_data) { | |
778 | spin_lock(&scd->scd_lock); | |
779 | ||
780 | /* schedule in-flight RPCs to notice the shutdown */ | |
781 | list_for_each_entry(rpc, &scd->scd_rpc_active, srpc_list) | |
782 | swi_schedule_workitem(&rpc->srpc_wi); | |
783 | ||
784 | spin_unlock(&scd->scd_lock); | |
785 | ||
4420cfd3 JS |
786 | /* |
787 | * OK to traverse scd_buf_posted without lock, since no one | |
788 | * touches scd_buf_posted now | |
789 | */ | |
d7e09d03 PT |
790 | list_for_each_entry(buf, &scd->scd_buf_posted, buf_list) |
791 | LNetMDUnlink(buf->buf_mdh); | |
792 | } | |
793 | } | |
794 | ||
8d94b6d2 | 795 | static int |
a659df62 | 796 | srpc_send_request(srpc_client_rpc_t *rpc) |
d7e09d03 PT |
797 | { |
798 | srpc_event_t *ev = &rpc->crpc_reqstev; | |
74d68011 | 799 | int rc; |
d7e09d03 PT |
800 | |
801 | ev->ev_fired = 0; | |
802 | ev->ev_data = rpc; | |
803 | ev->ev_type = SRPC_REQUEST_SENT; | |
804 | ||
ab419903 SB |
805 | rc = srpc_post_active_rdma(srpc_serv_portal(rpc->crpc_service), |
806 | rpc->crpc_service, &rpc->crpc_reqstmsg, | |
807 | sizeof(srpc_msg_t), LNET_MD_OP_PUT, | |
808 | rpc->crpc_dest, LNET_NID_ANY, | |
809 | &rpc->crpc_reqstmdh, ev); | |
d7e09d03 | 810 | if (rc != 0) { |
a659df62 | 811 | LASSERT(rc == -ENOMEM); |
d7e09d03 PT |
812 | ev->ev_fired = 1; /* no more event expected */ |
813 | } | |
814 | return rc; | |
815 | } | |
816 | ||
8d94b6d2 | 817 | static int |
a659df62 | 818 | srpc_prepare_reply(srpc_client_rpc_t *rpc) |
d7e09d03 PT |
819 | { |
820 | srpc_event_t *ev = &rpc->crpc_replyev; | |
74d68011 MS |
821 | __u64 *id = &rpc->crpc_reqstmsg.msg_body.reqst.rpyid; |
822 | int rc; | |
d7e09d03 PT |
823 | |
824 | ev->ev_fired = 0; | |
825 | ev->ev_data = rpc; | |
826 | ev->ev_type = SRPC_REPLY_RCVD; | |
827 | ||
828 | *id = srpc_next_id(); | |
829 | ||
830 | rc = srpc_post_passive_rdma(SRPC_RDMA_PORTAL, 0, *id, | |
831 | &rpc->crpc_replymsg, sizeof(srpc_msg_t), | |
832 | LNET_MD_OP_PUT, rpc->crpc_dest, | |
833 | &rpc->crpc_replymdh, ev); | |
834 | if (rc != 0) { | |
a659df62 | 835 | LASSERT(rc == -ENOMEM); |
d7e09d03 PT |
836 | ev->ev_fired = 1; /* no more event expected */ |
837 | } | |
838 | return rc; | |
839 | } | |
840 | ||
8d94b6d2 | 841 | static int |
a659df62 | 842 | srpc_prepare_bulk(srpc_client_rpc_t *rpc) |
d7e09d03 | 843 | { |
74d68011 | 844 | srpc_bulk_t *bk = &rpc->crpc_bulk; |
d7e09d03 PT |
845 | srpc_event_t *ev = &rpc->crpc_bulkev; |
846 | __u64 *id = &rpc->crpc_reqstmsg.msg_body.reqst.bulkid; | |
74d68011 MS |
847 | int rc; |
848 | int opt; | |
d7e09d03 | 849 | |
a659df62 | 850 | LASSERT(bk->bk_niov <= LNET_MAX_IOV); |
d7e09d03 | 851 | |
f64a6f3c HE |
852 | if (bk->bk_niov == 0) |
853 | return 0; /* nothing to do */ | |
d7e09d03 PT |
854 | |
855 | opt = bk->bk_sink ? LNET_MD_OP_PUT : LNET_MD_OP_GET; | |
856 | opt |= LNET_MD_KIOV; | |
857 | ||
858 | ev->ev_fired = 0; | |
859 | ev->ev_data = rpc; | |
860 | ev->ev_type = SRPC_BULK_REQ_RCVD; | |
861 | ||
862 | *id = srpc_next_id(); | |
863 | ||
864 | rc = srpc_post_passive_rdma(SRPC_RDMA_PORTAL, 0, *id, | |
865 | &bk->bk_iovs[0], bk->bk_niov, opt, | |
866 | rpc->crpc_dest, &bk->bk_mdh, ev); | |
867 | if (rc != 0) { | |
a659df62 | 868 | LASSERT(rc == -ENOMEM); |
d7e09d03 PT |
869 | ev->ev_fired = 1; /* no more event expected */ |
870 | } | |
871 | return rc; | |
872 | } | |
873 | ||
8d94b6d2 | 874 | static int |
168c7a13 | 875 | srpc_do_bulk(struct srpc_server_rpc *rpc) |
d7e09d03 | 876 | { |
74d68011 MS |
877 | srpc_event_t *ev = &rpc->srpc_ev; |
878 | srpc_bulk_t *bk = rpc->srpc_bulk; | |
879 | __u64 id = rpc->srpc_reqstbuf->buf_msg.msg_body.reqst.bulkid; | |
880 | int rc; | |
881 | int opt; | |
d7e09d03 | 882 | |
a659df62 | 883 | LASSERT(bk != NULL); |
d7e09d03 PT |
884 | |
885 | opt = bk->bk_sink ? LNET_MD_OP_GET : LNET_MD_OP_PUT; | |
886 | opt |= LNET_MD_KIOV; | |
887 | ||
888 | ev->ev_fired = 0; | |
889 | ev->ev_data = rpc; | |
890 | ev->ev_type = bk->bk_sink ? SRPC_BULK_GET_RPLD : SRPC_BULK_PUT_SENT; | |
891 | ||
892 | rc = srpc_post_active_rdma(SRPC_RDMA_PORTAL, id, | |
893 | &bk->bk_iovs[0], bk->bk_niov, opt, | |
894 | rpc->srpc_peer, rpc->srpc_self, | |
895 | &bk->bk_mdh, ev); | |
896 | if (rc != 0) | |
897 | ev->ev_fired = 1; /* no more event expected */ | |
898 | return rc; | |
899 | } | |
900 | ||
901 | /* only called from srpc_handle_rpc */ | |
8d94b6d2 | 902 | static void |
168c7a13 | 903 | srpc_server_rpc_done(struct srpc_server_rpc *rpc, int status) |
d7e09d03 | 904 | { |
74d68011 MS |
905 | struct srpc_service_cd *scd = rpc->srpc_scd; |
906 | struct srpc_service *sv = scd->scd_svc; | |
907 | srpc_buffer_t *buffer; | |
d7e09d03 | 908 | |
a659df62 | 909 | LASSERT(status != 0 || rpc->srpc_wi.swi_state == SWI_STATE_DONE); |
d7e09d03 PT |
910 | |
911 | rpc->srpc_status = status; | |
912 | ||
a659df62 | 913 | CDEBUG_LIMIT(status == 0 ? D_NET : D_NETERROR, |
c314c319 JS |
914 | "Server RPC %p done: service %s, peer %s, status %s:%d\n", |
915 | rpc, sv->sv_name, libcfs_id2str(rpc->srpc_peer), | |
916 | swi_state2str(rpc->srpc_wi.swi_state), status); | |
d7e09d03 PT |
917 | |
918 | if (status != 0) { | |
919 | spin_lock(&srpc_data.rpc_glock); | |
920 | srpc_data.rpc_counters.rpcs_dropped++; | |
921 | spin_unlock(&srpc_data.rpc_glock); | |
922 | } | |
923 | ||
924 | if (rpc->srpc_done != NULL) | |
925 | (*rpc->srpc_done) (rpc); | |
926 | LASSERT(rpc->srpc_bulk == NULL); | |
927 | ||
928 | spin_lock(&scd->scd_lock); | |
929 | ||
930 | if (rpc->srpc_reqstbuf != NULL) { | |
4420cfd3 JS |
931 | /* |
932 | * NB might drop sv_lock in srpc_service_recycle_buffer, but | |
933 | * sv won't go away for scd_rpc_active must not be empty | |
934 | */ | |
d7e09d03 PT |
935 | srpc_service_recycle_buffer(scd, rpc->srpc_reqstbuf); |
936 | rpc->srpc_reqstbuf = NULL; | |
937 | } | |
938 | ||
939 | list_del(&rpc->srpc_list); /* from scd->scd_rpc_active */ | |
940 | ||
941 | /* | |
942 | * No one can schedule me now since: | |
943 | * - I'm not on scd_rpc_active. | |
944 | * - all LNet events have been fired. | |
945 | * Cancel pending schedules and prevent future schedule attempts: | |
946 | */ | |
947 | LASSERT(rpc->srpc_ev.ev_fired); | |
948 | swi_exit_workitem(&rpc->srpc_wi); | |
949 | ||
950 | if (!sv->sv_shuttingdown && !list_empty(&scd->scd_buf_blocked)) { | |
951 | buffer = list_entry(scd->scd_buf_blocked.next, | |
c314c319 | 952 | srpc_buffer_t, buf_list); |
d7e09d03 PT |
953 | list_del(&buffer->buf_list); |
954 | ||
955 | srpc_init_server_rpc(rpc, scd, buffer); | |
956 | list_add_tail(&rpc->srpc_list, &scd->scd_rpc_active); | |
957 | swi_schedule_workitem(&rpc->srpc_wi); | |
958 | } else { | |
959 | list_add(&rpc->srpc_list, &scd->scd_rpc_free); | |
960 | } | |
961 | ||
962 | spin_unlock(&scd->scd_lock); | |
963 | return; | |
964 | } | |
965 | ||
966 | /* handles an incoming RPC */ | |
967 | int | |
968 | srpc_handle_rpc(swi_workitem_t *wi) | |
969 | { | |
74d68011 MS |
970 | struct srpc_server_rpc *rpc = wi->swi_workitem.wi_data; |
971 | struct srpc_service_cd *scd = rpc->srpc_scd; | |
972 | struct srpc_service *sv = scd->scd_svc; | |
973 | srpc_event_t *ev = &rpc->srpc_ev; | |
974 | int rc = 0; | |
d7e09d03 PT |
975 | |
976 | LASSERT(wi == &rpc->srpc_wi); | |
977 | ||
978 | spin_lock(&scd->scd_lock); | |
979 | ||
980 | if (sv->sv_shuttingdown || rpc->srpc_aborted) { | |
981 | spin_unlock(&scd->scd_lock); | |
982 | ||
983 | if (rpc->srpc_bulk != NULL) | |
984 | LNetMDUnlink(rpc->srpc_bulk->bk_mdh); | |
985 | LNetMDUnlink(rpc->srpc_replymdh); | |
986 | ||
987 | if (ev->ev_fired) { /* no more event, OK to finish */ | |
988 | srpc_server_rpc_done(rpc, -ESHUTDOWN); | |
989 | return 1; | |
990 | } | |
991 | return 0; | |
992 | } | |
993 | ||
994 | spin_unlock(&scd->scd_lock); | |
995 | ||
996 | switch (wi->swi_state) { | |
997 | default: | |
a659df62 | 998 | LBUG(); |
d7e09d03 | 999 | case SWI_STATE_NEWBORN: { |
74d68011 | 1000 | srpc_msg_t *msg; |
d7e09d03 PT |
1001 | srpc_generic_reply_t *reply; |
1002 | ||
1003 | msg = &rpc->srpc_reqstbuf->buf_msg; | |
1004 | reply = &rpc->srpc_replymsg.msg_body.reply; | |
1005 | ||
1006 | if (msg->msg_magic == 0) { | |
1007 | /* moaned already in srpc_lnet_ev_handler */ | |
1008 | srpc_server_rpc_done(rpc, EBADMSG); | |
1009 | return 1; | |
1010 | } | |
1011 | ||
1012 | srpc_unpack_msg_hdr(msg); | |
1013 | if (msg->msg_version != SRPC_MSG_VERSION) { | |
1014 | CWARN("Version mismatch: %u, %u expected, from %s\n", | |
1015 | msg->msg_version, SRPC_MSG_VERSION, | |
1016 | libcfs_id2str(rpc->srpc_peer)); | |
1017 | reply->status = EPROTO; | |
1018 | /* drop through and send reply */ | |
1019 | } else { | |
1020 | reply->status = 0; | |
1021 | rc = (*sv->sv_handler)(rpc); | |
1022 | LASSERT(reply->status == 0 || !rpc->srpc_bulk); | |
1023 | if (rc != 0) { | |
1024 | srpc_server_rpc_done(rpc, rc); | |
1025 | return 1; | |
1026 | } | |
1027 | } | |
1028 | ||
1029 | wi->swi_state = SWI_STATE_BULK_STARTED; | |
1030 | ||
1031 | if (rpc->srpc_bulk != NULL) { | |
1032 | rc = srpc_do_bulk(rpc); | |
1033 | if (rc == 0) | |
1034 | return 0; /* wait for bulk */ | |
1035 | ||
a659df62 | 1036 | LASSERT(ev->ev_fired); |
d7e09d03 PT |
1037 | ev->ev_status = rc; |
1038 | } | |
1039 | } | |
1040 | case SWI_STATE_BULK_STARTED: | |
a659df62 | 1041 | LASSERT(rpc->srpc_bulk == NULL || ev->ev_fired); |
d7e09d03 PT |
1042 | |
1043 | if (rpc->srpc_bulk != NULL) { | |
1044 | rc = ev->ev_status; | |
1045 | ||
1046 | if (sv->sv_bulk_ready != NULL) | |
1047 | rc = (*sv->sv_bulk_ready) (rpc, rc); | |
1048 | ||
1049 | if (rc != 0) { | |
1050 | srpc_server_rpc_done(rpc, rc); | |
1051 | return 1; | |
1052 | } | |
1053 | } | |
1054 | ||
1055 | wi->swi_state = SWI_STATE_REPLY_SUBMITTED; | |
1056 | rc = srpc_send_reply(rpc); | |
1057 | if (rc == 0) | |
1058 | return 0; /* wait for reply */ | |
1059 | srpc_server_rpc_done(rpc, rc); | |
1060 | return 1; | |
1061 | ||
1062 | case SWI_STATE_REPLY_SUBMITTED: | |
1063 | if (!ev->ev_fired) { | |
1064 | CERROR("RPC %p: bulk %p, service %d\n", | |
1065 | rpc, rpc->srpc_bulk, sv->sv_id); | |
1066 | CERROR("Event: status %d, type %d, lnet %d\n", | |
1067 | ev->ev_status, ev->ev_type, ev->ev_lnet); | |
a659df62 | 1068 | LASSERT(ev->ev_fired); |
d7e09d03 PT |
1069 | } |
1070 | ||
1071 | wi->swi_state = SWI_STATE_DONE; | |
1072 | srpc_server_rpc_done(rpc, ev->ev_status); | |
1073 | return 1; | |
1074 | } | |
1075 | ||
1076 | return 0; | |
1077 | } | |
1078 | ||
8d94b6d2 | 1079 | static void |
a659df62 | 1080 | srpc_client_rpc_expired(void *data) |
d7e09d03 PT |
1081 | { |
1082 | srpc_client_rpc_t *rpc = data; | |
1083 | ||
a659df62 | 1084 | CWARN("Client RPC expired: service %d, peer %s, timeout %d.\n", |
c314c319 JS |
1085 | rpc->crpc_service, libcfs_id2str(rpc->crpc_dest), |
1086 | rpc->crpc_timeout); | |
d7e09d03 PT |
1087 | |
1088 | spin_lock(&rpc->crpc_lock); | |
1089 | ||
1090 | rpc->crpc_timeout = 0; | |
1091 | srpc_abort_rpc(rpc, -ETIMEDOUT); | |
1092 | ||
1093 | spin_unlock(&rpc->crpc_lock); | |
1094 | ||
1095 | spin_lock(&srpc_data.rpc_glock); | |
1096 | srpc_data.rpc_counters.rpcs_expired++; | |
1097 | spin_unlock(&srpc_data.rpc_glock); | |
1098 | } | |
1099 | ||
1100 | inline void | |
a659df62 | 1101 | srpc_add_client_rpc_timer(srpc_client_rpc_t *rpc) |
d7e09d03 PT |
1102 | { |
1103 | stt_timer_t *timer = &rpc->crpc_timer; | |
1104 | ||
f64a6f3c HE |
1105 | if (rpc->crpc_timeout == 0) |
1106 | return; | |
d7e09d03 PT |
1107 | |
1108 | INIT_LIST_HEAD(&timer->stt_list); | |
1109 | timer->stt_data = rpc; | |
1110 | timer->stt_func = srpc_client_rpc_expired; | |
d9f79e6b | 1111 | timer->stt_expires = ktime_get_real_seconds() + rpc->crpc_timeout; |
d7e09d03 PT |
1112 | stt_add_timer(timer); |
1113 | return; | |
1114 | } | |
1115 | ||
1116 | /* | |
1117 | * Called with rpc->crpc_lock held. | |
1118 | * | |
1119 | * Upon exit the RPC expiry timer is not queued and the handler is not | |
4420cfd3 JS |
1120 | * running on any CPU. |
1121 | */ | |
8d94b6d2 | 1122 | static void |
a659df62 | 1123 | srpc_del_client_rpc_timer(srpc_client_rpc_t *rpc) |
d7e09d03 PT |
1124 | { |
1125 | /* timer not planted or already exploded */ | |
1126 | if (rpc->crpc_timeout == 0) | |
1127 | return; | |
1128 | ||
2b284326 | 1129 | /* timer successfully defused */ |
d7e09d03 PT |
1130 | if (stt_del_timer(&rpc->crpc_timer)) |
1131 | return; | |
1132 | ||
1133 | /* timer detonated, wait for it to explode */ | |
1134 | while (rpc->crpc_timeout != 0) { | |
1135 | spin_unlock(&rpc->crpc_lock); | |
1136 | ||
1137 | schedule(); | |
1138 | ||
1139 | spin_lock(&rpc->crpc_lock); | |
1140 | } | |
1141 | } | |
1142 | ||
8d94b6d2 | 1143 | static void |
a659df62 | 1144 | srpc_client_rpc_done(srpc_client_rpc_t *rpc, int status) |
d7e09d03 PT |
1145 | { |
1146 | swi_workitem_t *wi = &rpc->crpc_wi; | |
1147 | ||
1148 | LASSERT(status != 0 || wi->swi_state == SWI_STATE_DONE); | |
1149 | ||
1150 | spin_lock(&rpc->crpc_lock); | |
1151 | ||
1152 | rpc->crpc_closed = 1; | |
1153 | if (rpc->crpc_status == 0) | |
1154 | rpc->crpc_status = status; | |
1155 | ||
1156 | srpc_del_client_rpc_timer(rpc); | |
1157 | ||
a659df62 | 1158 | CDEBUG_LIMIT((status == 0) ? D_NET : D_NETERROR, |
c314c319 JS |
1159 | "Client RPC done: service %d, peer %s, status %s:%d:%d\n", |
1160 | rpc->crpc_service, libcfs_id2str(rpc->crpc_dest), | |
1161 | swi_state2str(wi->swi_state), rpc->crpc_aborted, status); | |
d7e09d03 PT |
1162 | |
1163 | /* | |
1164 | * No one can schedule me now since: | |
1165 | * - RPC timer has been defused. | |
1166 | * - all LNet events have been fired. | |
1167 | * - crpc_closed has been set, preventing srpc_abort_rpc from | |
1168 | * scheduling me. | |
1169 | * Cancel pending schedules and prevent future schedule attempts: | |
1170 | */ | |
a659df62 | 1171 | LASSERT(!srpc_event_pending(rpc)); |
d7e09d03 PT |
1172 | swi_exit_workitem(wi); |
1173 | ||
1174 | spin_unlock(&rpc->crpc_lock); | |
1175 | ||
1176 | (*rpc->crpc_done)(rpc); | |
1177 | return; | |
1178 | } | |
1179 | ||
1180 | /* sends an outgoing RPC */ | |
1181 | int | |
a659df62 | 1182 | srpc_send_rpc(swi_workitem_t *wi) |
d7e09d03 | 1183 | { |
74d68011 | 1184 | int rc = 0; |
d7e09d03 | 1185 | srpc_client_rpc_t *rpc; |
74d68011 MS |
1186 | srpc_msg_t *reply; |
1187 | int do_bulk; | |
d7e09d03 PT |
1188 | |
1189 | LASSERT(wi != NULL); | |
1190 | ||
1191 | rpc = wi->swi_workitem.wi_data; | |
1192 | ||
a659df62 HE |
1193 | LASSERT(rpc != NULL); |
1194 | LASSERT(wi == &rpc->crpc_wi); | |
d7e09d03 PT |
1195 | |
1196 | reply = &rpc->crpc_replymsg; | |
1197 | do_bulk = rpc->crpc_bulk.bk_niov > 0; | |
1198 | ||
1199 | spin_lock(&rpc->crpc_lock); | |
1200 | ||
1201 | if (rpc->crpc_aborted) { | |
1202 | spin_unlock(&rpc->crpc_lock); | |
1203 | goto abort; | |
1204 | } | |
1205 | ||
1206 | spin_unlock(&rpc->crpc_lock); | |
1207 | ||
1208 | switch (wi->swi_state) { | |
1209 | default: | |
a659df62 | 1210 | LBUG(); |
d7e09d03 | 1211 | case SWI_STATE_NEWBORN: |
a659df62 | 1212 | LASSERT(!srpc_event_pending(rpc)); |
d7e09d03 PT |
1213 | |
1214 | rc = srpc_prepare_reply(rpc); | |
1215 | if (rc != 0) { | |
1216 | srpc_client_rpc_done(rpc, rc); | |
1217 | return 1; | |
1218 | } | |
1219 | ||
1220 | rc = srpc_prepare_bulk(rpc); | |
f64a6f3c HE |
1221 | if (rc != 0) |
1222 | break; | |
d7e09d03 PT |
1223 | |
1224 | wi->swi_state = SWI_STATE_REQUEST_SUBMITTED; | |
1225 | rc = srpc_send_request(rpc); | |
1226 | break; | |
1227 | ||
1228 | case SWI_STATE_REQUEST_SUBMITTED: | |
4420cfd3 JS |
1229 | /* |
1230 | * CAVEAT EMPTOR: rqtev, rpyev, and bulkev may come in any | |
d7e09d03 | 1231 | * order; however, they're processed in a strict order: |
4420cfd3 JS |
1232 | * rqt, rpy, and bulk. |
1233 | */ | |
f64a6f3c HE |
1234 | if (!rpc->crpc_reqstev.ev_fired) |
1235 | break; | |
d7e09d03 PT |
1236 | |
1237 | rc = rpc->crpc_reqstev.ev_status; | |
f64a6f3c HE |
1238 | if (rc != 0) |
1239 | break; | |
d7e09d03 PT |
1240 | |
1241 | wi->swi_state = SWI_STATE_REQUEST_SENT; | |
1242 | /* perhaps more events, fall thru */ | |
1243 | case SWI_STATE_REQUEST_SENT: { | |
1244 | srpc_msg_type_t type = srpc_service2reply(rpc->crpc_service); | |
1245 | ||
f64a6f3c HE |
1246 | if (!rpc->crpc_replyev.ev_fired) |
1247 | break; | |
d7e09d03 PT |
1248 | |
1249 | rc = rpc->crpc_replyev.ev_status; | |
f64a6f3c HE |
1250 | if (rc != 0) |
1251 | break; | |
d7e09d03 PT |
1252 | |
1253 | srpc_unpack_msg_hdr(reply); | |
1254 | if (reply->msg_type != type || | |
1255 | (reply->msg_magic != SRPC_MSG_MAGIC && | |
1256 | reply->msg_magic != __swab32(SRPC_MSG_MAGIC))) { | |
2d00bd17 JP |
1257 | CWARN("Bad message from %s: type %u (%d expected), magic %u (%d expected).\n", |
1258 | libcfs_id2str(rpc->crpc_dest), | |
1259 | reply->msg_type, type, | |
1260 | reply->msg_magic, SRPC_MSG_MAGIC); | |
d7e09d03 PT |
1261 | rc = -EBADMSG; |
1262 | break; | |
1263 | } | |
1264 | ||
1265 | if (do_bulk && reply->msg_body.reply.status != 0) { | |
2d00bd17 JP |
1266 | CWARN("Remote error %d at %s, unlink bulk buffer in case peer didn't initiate bulk transfer\n", |
1267 | reply->msg_body.reply.status, | |
1268 | libcfs_id2str(rpc->crpc_dest)); | |
d7e09d03 PT |
1269 | LNetMDUnlink(rpc->crpc_bulk.bk_mdh); |
1270 | } | |
1271 | ||
1272 | wi->swi_state = SWI_STATE_REPLY_RECEIVED; | |
1273 | } | |
1274 | case SWI_STATE_REPLY_RECEIVED: | |
f64a6f3c HE |
1275 | if (do_bulk && !rpc->crpc_bulkev.ev_fired) |
1276 | break; | |
d7e09d03 PT |
1277 | |
1278 | rc = do_bulk ? rpc->crpc_bulkev.ev_status : 0; | |
1279 | ||
4420cfd3 JS |
1280 | /* |
1281 | * Bulk buffer was unlinked due to remote error. Clear error | |
d7e09d03 PT |
1282 | * since reply buffer still contains valid data. |
1283 | * NB rpc->crpc_done shouldn't look into bulk data in case of | |
4420cfd3 JS |
1284 | * remote error. |
1285 | */ | |
d7e09d03 PT |
1286 | if (do_bulk && rpc->crpc_bulkev.ev_lnet == LNET_EVENT_UNLINK && |
1287 | rpc->crpc_status == 0 && reply->msg_body.reply.status != 0) | |
1288 | rc = 0; | |
1289 | ||
1290 | wi->swi_state = SWI_STATE_DONE; | |
1291 | srpc_client_rpc_done(rpc, rc); | |
1292 | return 1; | |
1293 | } | |
1294 | ||
1295 | if (rc != 0) { | |
1296 | spin_lock(&rpc->crpc_lock); | |
1297 | srpc_abort_rpc(rpc, rc); | |
1298 | spin_unlock(&rpc->crpc_lock); | |
1299 | } | |
1300 | ||
1301 | abort: | |
1302 | if (rpc->crpc_aborted) { | |
1303 | LNetMDUnlink(rpc->crpc_reqstmdh); | |
1304 | LNetMDUnlink(rpc->crpc_replymdh); | |
1305 | LNetMDUnlink(rpc->crpc_bulk.bk_mdh); | |
1306 | ||
1307 | if (!srpc_event_pending(rpc)) { | |
1308 | srpc_client_rpc_done(rpc, -EINTR); | |
1309 | return 1; | |
1310 | } | |
1311 | } | |
1312 | return 0; | |
1313 | } | |
1314 | ||
1315 | srpc_client_rpc_t * | |
a659df62 | 1316 | srpc_create_client_rpc(lnet_process_id_t peer, int service, |
c314c319 JS |
1317 | int nbulkiov, int bulklen, |
1318 | void (*rpc_done)(srpc_client_rpc_t *), | |
1319 | void (*rpc_fini)(srpc_client_rpc_t *), void *priv) | |
d7e09d03 PT |
1320 | { |
1321 | srpc_client_rpc_t *rpc; | |
1322 | ||
1323 | LIBCFS_ALLOC(rpc, offsetof(srpc_client_rpc_t, | |
1324 | crpc_bulk.bk_iovs[nbulkiov])); | |
1325 | if (rpc == NULL) | |
1326 | return NULL; | |
1327 | ||
1328 | srpc_init_client_rpc(rpc, peer, service, nbulkiov, | |
1329 | bulklen, rpc_done, rpc_fini, priv); | |
1330 | return rpc; | |
1331 | } | |
1332 | ||
1333 | /* called with rpc->crpc_lock held */ | |
1334 | void | |
a659df62 | 1335 | srpc_abort_rpc(srpc_client_rpc_t *rpc, int why) |
d7e09d03 | 1336 | { |
a659df62 | 1337 | LASSERT(why != 0); |
d7e09d03 PT |
1338 | |
1339 | if (rpc->crpc_aborted || /* already aborted */ | |
1340 | rpc->crpc_closed) /* callback imminent */ | |
1341 | return; | |
1342 | ||
c314c319 JS |
1343 | CDEBUG(D_NET, "Aborting RPC: service %d, peer %s, state %s, why %d\n", |
1344 | rpc->crpc_service, libcfs_id2str(rpc->crpc_dest), | |
1345 | swi_state2str(rpc->crpc_wi.swi_state), why); | |
d7e09d03 PT |
1346 | |
1347 | rpc->crpc_aborted = 1; | |
1348 | rpc->crpc_status = why; | |
1349 | swi_schedule_workitem(&rpc->crpc_wi); | |
1350 | return; | |
1351 | } | |
1352 | ||
1353 | /* called with rpc->crpc_lock held */ | |
1354 | void | |
a659df62 | 1355 | srpc_post_rpc(srpc_client_rpc_t *rpc) |
d7e09d03 | 1356 | { |
a659df62 HE |
1357 | LASSERT(!rpc->crpc_aborted); |
1358 | LASSERT(srpc_data.rpc_state == SRPC_STATE_RUNNING); | |
d7e09d03 | 1359 | |
a659df62 | 1360 | CDEBUG(D_NET, "Posting RPC: peer %s, service %d, timeout %d\n", |
c314c319 JS |
1361 | libcfs_id2str(rpc->crpc_dest), rpc->crpc_service, |
1362 | rpc->crpc_timeout); | |
d7e09d03 PT |
1363 | |
1364 | srpc_add_client_rpc_timer(rpc); | |
1365 | swi_schedule_workitem(&rpc->crpc_wi); | |
1366 | return; | |
1367 | } | |
1368 | ||
d7e09d03 PT |
1369 | int |
1370 | srpc_send_reply(struct srpc_server_rpc *rpc) | |
1371 | { | |
74d68011 MS |
1372 | srpc_event_t *ev = &rpc->srpc_ev; |
1373 | struct srpc_msg *msg = &rpc->srpc_replymsg; | |
1374 | struct srpc_buffer *buffer = rpc->srpc_reqstbuf; | |
1375 | struct srpc_service_cd *scd = rpc->srpc_scd; | |
1376 | struct srpc_service *sv = scd->scd_svc; | |
1377 | __u64 rpyid; | |
1378 | int rc; | |
d7e09d03 PT |
1379 | |
1380 | LASSERT(buffer != NULL); | |
1381 | rpyid = buffer->buf_msg.msg_body.reqst.rpyid; | |
1382 | ||
1383 | spin_lock(&scd->scd_lock); | |
1384 | ||
1385 | if (!sv->sv_shuttingdown && !srpc_serv_is_framework(sv)) { | |
4420cfd3 JS |
1386 | /* |
1387 | * Repost buffer before replying since test client | |
1388 | * might send me another RPC once it gets the reply | |
1389 | */ | |
d7e09d03 PT |
1390 | if (srpc_service_post_buffer(scd, buffer) != 0) |
1391 | CWARN("Failed to repost %s buffer\n", sv->sv_name); | |
1392 | rpc->srpc_reqstbuf = NULL; | |
1393 | } | |
1394 | ||
1395 | spin_unlock(&scd->scd_lock); | |
1396 | ||
1397 | ev->ev_fired = 0; | |
1398 | ev->ev_data = rpc; | |
1399 | ev->ev_type = SRPC_REPLY_SENT; | |
1400 | ||
1401 | msg->msg_magic = SRPC_MSG_MAGIC; | |
1402 | msg->msg_version = SRPC_MSG_VERSION; | |
1403 | msg->msg_type = srpc_service2reply(sv->sv_id); | |
1404 | ||
1405 | rc = srpc_post_active_rdma(SRPC_RDMA_PORTAL, rpyid, msg, | |
1406 | sizeof(*msg), LNET_MD_OP_PUT, | |
1407 | rpc->srpc_peer, rpc->srpc_self, | |
1408 | &rpc->srpc_replymdh, ev); | |
1409 | if (rc != 0) | |
1410 | ev->ev_fired = 1; /* no more event expected */ | |
1411 | return rc; | |
1412 | } | |
1413 | ||
1414 | /* when in kernel always called with LNET_LOCK() held, and in thread context */ | |
8d94b6d2 | 1415 | static void |
d7e09d03 PT |
1416 | srpc_lnet_ev_handler(lnet_event_t *ev) |
1417 | { | |
74d68011 MS |
1418 | struct srpc_service_cd *scd; |
1419 | srpc_event_t *rpcev = ev->md.user_ptr; | |
d7e09d03 | 1420 | srpc_client_rpc_t *crpc; |
168c7a13 | 1421 | struct srpc_server_rpc *srpc; |
74d68011 MS |
1422 | srpc_buffer_t *buffer; |
1423 | srpc_service_t *sv; | |
1424 | srpc_msg_t *msg; | |
1425 | srpc_msg_type_t type; | |
d7e09d03 | 1426 | |
a659df62 | 1427 | LASSERT(!in_interrupt()); |
d7e09d03 PT |
1428 | |
1429 | if (ev->status != 0) { | |
1430 | spin_lock(&srpc_data.rpc_glock); | |
1431 | srpc_data.rpc_counters.errors++; | |
1432 | spin_unlock(&srpc_data.rpc_glock); | |
1433 | } | |
1434 | ||
1435 | rpcev->ev_lnet = ev->type; | |
1436 | ||
1437 | switch (rpcev->ev_type) { | |
1438 | default: | |
1439 | CERROR("Unknown event: status %d, type %d, lnet %d\n", | |
1440 | rpcev->ev_status, rpcev->ev_type, rpcev->ev_lnet); | |
a659df62 | 1441 | LBUG(); |
d7e09d03 PT |
1442 | case SRPC_REQUEST_SENT: |
1443 | if (ev->status == 0 && ev->type != LNET_EVENT_UNLINK) { | |
1444 | spin_lock(&srpc_data.rpc_glock); | |
1445 | srpc_data.rpc_counters.rpcs_sent++; | |
1446 | spin_unlock(&srpc_data.rpc_glock); | |
1447 | } | |
1448 | case SRPC_REPLY_RCVD: | |
1449 | case SRPC_BULK_REQ_RCVD: | |
1450 | crpc = rpcev->ev_data; | |
1451 | ||
1452 | if (rpcev != &crpc->crpc_reqstev && | |
1453 | rpcev != &crpc->crpc_replyev && | |
1454 | rpcev != &crpc->crpc_bulkev) { | |
1455 | CERROR("rpcev %p, crpc %p, reqstev %p, replyev %p, bulkev %p\n", | |
1456 | rpcev, crpc, &crpc->crpc_reqstev, | |
1457 | &crpc->crpc_replyev, &crpc->crpc_bulkev); | |
1458 | CERROR("Bad event: status %d, type %d, lnet %d\n", | |
1459 | rpcev->ev_status, rpcev->ev_type, rpcev->ev_lnet); | |
a659df62 | 1460 | LBUG(); |
d7e09d03 PT |
1461 | } |
1462 | ||
1463 | spin_lock(&crpc->crpc_lock); | |
1464 | ||
1465 | LASSERT(rpcev->ev_fired == 0); | |
1466 | rpcev->ev_fired = 1; | |
1467 | rpcev->ev_status = (ev->type == LNET_EVENT_UNLINK) ? | |
1468 | -EINTR : ev->status; | |
1469 | swi_schedule_workitem(&crpc->crpc_wi); | |
1470 | ||
1471 | spin_unlock(&crpc->crpc_lock); | |
1472 | break; | |
1473 | ||
1474 | case SRPC_REQUEST_RCVD: | |
1475 | scd = rpcev->ev_data; | |
1476 | sv = scd->scd_svc; | |
1477 | ||
1478 | LASSERT(rpcev == &scd->scd_ev); | |
1479 | ||
1480 | spin_lock(&scd->scd_lock); | |
1481 | ||
a659df62 HE |
1482 | LASSERT(ev->unlinked); |
1483 | LASSERT(ev->type == LNET_EVENT_PUT || | |
c314c319 | 1484 | ev->type == LNET_EVENT_UNLINK); |
a659df62 | 1485 | LASSERT(ev->type != LNET_EVENT_UNLINK || |
c314c319 | 1486 | sv->sv_shuttingdown); |
d7e09d03 PT |
1487 | |
1488 | buffer = container_of(ev->md.start, srpc_buffer_t, buf_msg); | |
1489 | buffer->buf_peer = ev->initiator; | |
1490 | buffer->buf_self = ev->target.nid; | |
1491 | ||
1492 | LASSERT(scd->scd_buf_nposted > 0); | |
1493 | scd->scd_buf_nposted--; | |
1494 | ||
1495 | if (sv->sv_shuttingdown) { | |
4420cfd3 JS |
1496 | /* |
1497 | * Leave buffer on scd->scd_buf_nposted since | |
1498 | * srpc_finish_service needs to traverse it. | |
1499 | */ | |
d7e09d03 PT |
1500 | spin_unlock(&scd->scd_lock); |
1501 | break; | |
1502 | } | |
1503 | ||
1504 | if (scd->scd_buf_err_stamp != 0 && | |
d9f79e6b | 1505 | scd->scd_buf_err_stamp < ktime_get_real_seconds()) { |
d7e09d03 PT |
1506 | /* re-enable adding buffer */ |
1507 | scd->scd_buf_err_stamp = 0; | |
1508 | scd->scd_buf_err = 0; | |
1509 | } | |
1510 | ||
1511 | if (scd->scd_buf_err == 0 && /* adding buffer is enabled */ | |
1512 | scd->scd_buf_adjust == 0 && | |
1513 | scd->scd_buf_nposted < scd->scd_buf_low) { | |
0c575417 | 1514 | scd->scd_buf_adjust = max(scd->scd_buf_total / 2, |
d7e09d03 PT |
1515 | SFW_TEST_WI_MIN); |
1516 | swi_schedule_workitem(&scd->scd_buf_wi); | |
1517 | } | |
1518 | ||
1519 | list_del(&buffer->buf_list); /* from scd->scd_buf_posted */ | |
1520 | msg = &buffer->buf_msg; | |
1521 | type = srpc_service2request(sv->sv_id); | |
1522 | ||
1523 | if (ev->status != 0 || ev->mlength != sizeof(*msg) || | |
1524 | (msg->msg_type != type && | |
1525 | msg->msg_type != __swab32(type)) || | |
1526 | (msg->msg_magic != SRPC_MSG_MAGIC && | |
1527 | msg->msg_magic != __swab32(SRPC_MSG_MAGIC))) { | |
2d00bd17 JP |
1528 | CERROR("Dropping RPC (%s) from %s: status %d mlength %d type %u magic %u.\n", |
1529 | sv->sv_name, libcfs_id2str(ev->initiator), | |
1530 | ev->status, ev->mlength, | |
1531 | msg->msg_type, msg->msg_magic); | |
d7e09d03 | 1532 | |
4420cfd3 JS |
1533 | /* |
1534 | * NB can't call srpc_service_recycle_buffer here since | |
d7e09d03 | 1535 | * it may call LNetM[DE]Attach. The invalid magic tells |
4420cfd3 JS |
1536 | * srpc_handle_rpc to drop this RPC |
1537 | */ | |
d7e09d03 PT |
1538 | msg->msg_magic = 0; |
1539 | } | |
1540 | ||
1541 | if (!list_empty(&scd->scd_rpc_free)) { | |
1542 | srpc = list_entry(scd->scd_rpc_free.next, | |
c314c319 JS |
1543 | struct srpc_server_rpc, |
1544 | srpc_list); | |
d7e09d03 PT |
1545 | list_del(&srpc->srpc_list); |
1546 | ||
1547 | srpc_init_server_rpc(srpc, scd, buffer); | |
1548 | list_add_tail(&srpc->srpc_list, | |
c314c319 | 1549 | &scd->scd_rpc_active); |
d7e09d03 PT |
1550 | swi_schedule_workitem(&srpc->srpc_wi); |
1551 | } else { | |
1552 | list_add_tail(&buffer->buf_list, | |
c314c319 | 1553 | &scd->scd_buf_blocked); |
d7e09d03 PT |
1554 | } |
1555 | ||
1556 | spin_unlock(&scd->scd_lock); | |
1557 | ||
1558 | spin_lock(&srpc_data.rpc_glock); | |
1559 | srpc_data.rpc_counters.rpcs_rcvd++; | |
1560 | spin_unlock(&srpc_data.rpc_glock); | |
1561 | break; | |
1562 | ||
1563 | case SRPC_BULK_GET_RPLD: | |
a659df62 | 1564 | LASSERT(ev->type == LNET_EVENT_SEND || |
c314c319 JS |
1565 | ev->type == LNET_EVENT_REPLY || |
1566 | ev->type == LNET_EVENT_UNLINK); | |
d7e09d03 PT |
1567 | |
1568 | if (!ev->unlinked) | |
1569 | break; /* wait for final event */ | |
1570 | ||
1571 | case SRPC_BULK_PUT_SENT: | |
1572 | if (ev->status == 0 && ev->type != LNET_EVENT_UNLINK) { | |
1573 | spin_lock(&srpc_data.rpc_glock); | |
1574 | ||
1575 | if (rpcev->ev_type == SRPC_BULK_GET_RPLD) | |
1576 | srpc_data.rpc_counters.bulk_get += ev->mlength; | |
1577 | else | |
1578 | srpc_data.rpc_counters.bulk_put += ev->mlength; | |
1579 | ||
1580 | spin_unlock(&srpc_data.rpc_glock); | |
1581 | } | |
1582 | case SRPC_REPLY_SENT: | |
1583 | srpc = rpcev->ev_data; | |
1584 | scd = srpc->srpc_scd; | |
1585 | ||
1586 | LASSERT(rpcev == &srpc->srpc_ev); | |
1587 | ||
1588 | spin_lock(&scd->scd_lock); | |
1589 | ||
1590 | rpcev->ev_fired = 1; | |
1591 | rpcev->ev_status = (ev->type == LNET_EVENT_UNLINK) ? | |
1592 | -EINTR : ev->status; | |
1593 | swi_schedule_workitem(&srpc->srpc_wi); | |
1594 | ||
1595 | spin_unlock(&scd->scd_lock); | |
1596 | break; | |
1597 | } | |
1598 | } | |
1599 | ||
d7e09d03 | 1600 | int |
a659df62 | 1601 | srpc_startup(void) |
d7e09d03 PT |
1602 | { |
1603 | int rc; | |
1604 | ||
1605 | memset(&srpc_data, 0, sizeof(struct smoketest_rpc)); | |
1606 | spin_lock_init(&srpc_data.rpc_glock); | |
1607 | ||
1608 | /* 1 second pause to avoid timestamp reuse */ | |
d3caf4d5 PT |
1609 | set_current_state(TASK_UNINTERRUPTIBLE); |
1610 | schedule_timeout(cfs_time_seconds(1)); | |
d9f79e6b | 1611 | srpc_data.rpc_matchbits = ((__u64)ktime_get_real_seconds()) << 48; |
d7e09d03 PT |
1612 | |
1613 | srpc_data.rpc_state = SRPC_STATE_NONE; | |
1614 | ||
1615 | rc = LNetNIInit(LUSTRE_SRV_LNET_PID); | |
1616 | if (rc < 0) { | |
a659df62 | 1617 | CERROR("LNetNIInit() has failed: %d\n", rc); |
d7e09d03 PT |
1618 | return rc; |
1619 | } | |
1620 | ||
1621 | srpc_data.rpc_state = SRPC_STATE_NI_INIT; | |
1622 | ||
1623 | LNetInvalidateHandle(&srpc_data.rpc_lnet_eq); | |
1624 | rc = LNetEQAlloc(0, srpc_lnet_ev_handler, &srpc_data.rpc_lnet_eq); | |
1625 | if (rc != 0) { | |
1626 | CERROR("LNetEQAlloc() has failed: %d\n", rc); | |
1627 | goto bail; | |
1628 | } | |
1629 | ||
1630 | rc = LNetSetLazyPortal(SRPC_FRAMEWORK_REQUEST_PORTAL); | |
1631 | LASSERT(rc == 0); | |
1632 | rc = LNetSetLazyPortal(SRPC_REQUEST_PORTAL); | |
1633 | LASSERT(rc == 0); | |
1634 | ||
1635 | srpc_data.rpc_state = SRPC_STATE_EQ_INIT; | |
1636 | ||
1637 | rc = stt_startup(); | |
1638 | ||
1639 | bail: | |
1640 | if (rc != 0) | |
1641 | srpc_shutdown(); | |
1642 | else | |
1643 | srpc_data.rpc_state = SRPC_STATE_RUNNING; | |
1644 | ||
1645 | return rc; | |
1646 | } | |
1647 | ||
1648 | void | |
a659df62 | 1649 | srpc_shutdown(void) |
d7e09d03 PT |
1650 | { |
1651 | int i; | |
1652 | int rc; | |
1653 | int state; | |
1654 | ||
1655 | state = srpc_data.rpc_state; | |
1656 | srpc_data.rpc_state = SRPC_STATE_STOPPING; | |
1657 | ||
1658 | switch (state) { | |
1659 | default: | |
a659df62 | 1660 | LBUG(); |
d7e09d03 PT |
1661 | case SRPC_STATE_RUNNING: |
1662 | spin_lock(&srpc_data.rpc_glock); | |
1663 | ||
1664 | for (i = 0; i <= SRPC_SERVICE_MAX_ID; i++) { | |
1665 | srpc_service_t *sv = srpc_data.rpc_services[i]; | |
1666 | ||
a659df62 | 1667 | LASSERTF(sv == NULL, |
c314c319 JS |
1668 | "service not empty: id %d, name %s\n", |
1669 | i, sv->sv_name); | |
d7e09d03 PT |
1670 | } |
1671 | ||
1672 | spin_unlock(&srpc_data.rpc_glock); | |
1673 | ||
1674 | stt_shutdown(); | |
1675 | ||
1676 | case SRPC_STATE_EQ_INIT: | |
1677 | rc = LNetClearLazyPortal(SRPC_FRAMEWORK_REQUEST_PORTAL); | |
1678 | rc = LNetClearLazyPortal(SRPC_REQUEST_PORTAL); | |
a659df62 | 1679 | LASSERT(rc == 0); |
d7e09d03 | 1680 | rc = LNetEQFree(srpc_data.rpc_lnet_eq); |
a659df62 | 1681 | LASSERT(rc == 0); /* the EQ should have no user by now */ |
d7e09d03 PT |
1682 | |
1683 | case SRPC_STATE_NI_INIT: | |
1684 | LNetNIFini(); | |
1685 | } | |
1686 | ||
1687 | return; | |
1688 | } |