| 1 | /* |
| 2 | * GPL HEADER START |
| 3 | * |
| 4 | * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. |
| 5 | * |
| 6 | * This program is free software; you can redistribute it and/or modify |
| 7 | * it under the terms of the GNU General Public License version 2 only, |
| 8 | * as published by the Free Software Foundation. |
| 9 | * |
| 10 | * This program is distributed in the hope that it will be useful, but |
| 11 | * WITHOUT ANY WARRANTY; without even the implied warranty of |
| 12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU |
| 13 | * General Public License version 2 for more details (a copy is included |
| 14 | * in the LICENSE file that accompanied this code). |
| 15 | * |
| 16 | * You should have received a copy of the GNU General Public License |
| 17 | * version 2 along with this program; If not, see |
| 18 | * http://www.sun.com/software/products/lustre/docs/GPLv2.pdf |
| 19 | * |
| 20 | * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa Clara, |
| 21 | * CA 95054 USA or visit www.sun.com if you need additional information or |
| 22 | * have any questions. |
| 23 | * |
| 24 | * GPL HEADER END |
| 25 | */ |
| 26 | /* |
| 27 | * Copyright (c) 2007, 2010, Oracle and/or its affiliates. All rights reserved. |
| 28 | * Use is subject to license terms. |
| 29 | * |
| 30 | * Copyright (c) 2011, 2015, Intel Corporation. |
| 31 | */ |
| 32 | /* |
| 33 | * This file is part of Lustre, http://www.lustre.org/ |
| 34 | * Lustre is a trademark of Sun Microsystems, Inc. |
| 35 | * |
| 36 | * lnet/selftest/brw_test.c |
| 37 | * |
| 38 | * Author: Isaac Huang <isaac@clusterfs.com> |
| 39 | */ |
| 40 | |
| 41 | #include "selftest.h" |
| 42 | |
| 43 | static int brw_srv_workitems = SFW_TEST_WI_MAX; |
| 44 | module_param(brw_srv_workitems, int, 0644); |
| 45 | MODULE_PARM_DESC(brw_srv_workitems, "# BRW server workitems"); |
| 46 | |
| 47 | static int brw_inject_errors; |
| 48 | module_param(brw_inject_errors, int, 0644); |
| 49 | MODULE_PARM_DESC(brw_inject_errors, "# data errors to inject randomly, zero by default"); |
| 50 | |
| 51 | static void |
| 52 | brw_client_fini(sfw_test_instance_t *tsi) |
| 53 | { |
| 54 | srpc_bulk_t *bulk; |
| 55 | sfw_test_unit_t *tsu; |
| 56 | |
| 57 | LASSERT(tsi->tsi_is_client); |
| 58 | |
| 59 | list_for_each_entry(tsu, &tsi->tsi_units, tsu_list) { |
| 60 | bulk = tsu->tsu_private; |
| 61 | if (!bulk) |
| 62 | continue; |
| 63 | |
| 64 | srpc_free_bulk(bulk); |
| 65 | tsu->tsu_private = NULL; |
| 66 | } |
| 67 | } |
| 68 | |
| 69 | static int |
| 70 | brw_client_init(sfw_test_instance_t *tsi) |
| 71 | { |
| 72 | sfw_session_t *sn = tsi->tsi_batch->bat_session; |
| 73 | int flags; |
| 74 | int npg; |
| 75 | int len; |
| 76 | int opc; |
| 77 | srpc_bulk_t *bulk; |
| 78 | sfw_test_unit_t *tsu; |
| 79 | |
| 80 | LASSERT(sn); |
| 81 | LASSERT(tsi->tsi_is_client); |
| 82 | |
| 83 | if (!(sn->sn_features & LST_FEAT_BULK_LEN)) { |
| 84 | test_bulk_req_t *breq = &tsi->tsi_u.bulk_v0; |
| 85 | |
| 86 | opc = breq->blk_opc; |
| 87 | flags = breq->blk_flags; |
| 88 | npg = breq->blk_npg; |
| 89 | /* |
| 90 | * NB: this is not going to work for variable page size, |
| 91 | * but we have to keep it for compatibility |
| 92 | */ |
| 93 | len = npg * PAGE_CACHE_SIZE; |
| 94 | |
| 95 | } else { |
| 96 | test_bulk_req_v1_t *breq = &tsi->tsi_u.bulk_v1; |
| 97 | |
| 98 | /* |
| 99 | * I should never get this step if it's unknown feature |
| 100 | * because make_session will reject unknown feature |
| 101 | */ |
| 102 | LASSERT(!(sn->sn_features & ~LST_FEATS_MASK)); |
| 103 | |
| 104 | opc = breq->blk_opc; |
| 105 | flags = breq->blk_flags; |
| 106 | len = breq->blk_len; |
| 107 | npg = (len + PAGE_CACHE_SIZE - 1) >> PAGE_CACHE_SHIFT; |
| 108 | } |
| 109 | |
| 110 | if (npg > LNET_MAX_IOV || npg <= 0) |
| 111 | return -EINVAL; |
| 112 | |
| 113 | if (opc != LST_BRW_READ && opc != LST_BRW_WRITE) |
| 114 | return -EINVAL; |
| 115 | |
| 116 | if (flags != LST_BRW_CHECK_NONE && |
| 117 | flags != LST_BRW_CHECK_FULL && flags != LST_BRW_CHECK_SIMPLE) |
| 118 | return -EINVAL; |
| 119 | |
| 120 | list_for_each_entry(tsu, &tsi->tsi_units, tsu_list) { |
| 121 | bulk = srpc_alloc_bulk(lnet_cpt_of_nid(tsu->tsu_dest.nid), |
| 122 | npg, len, opc == LST_BRW_READ); |
| 123 | if (!bulk) { |
| 124 | brw_client_fini(tsi); |
| 125 | return -ENOMEM; |
| 126 | } |
| 127 | |
| 128 | tsu->tsu_private = bulk; |
| 129 | } |
| 130 | |
| 131 | return 0; |
| 132 | } |
| 133 | |
| 134 | #define BRW_POISON 0xbeefbeefbeefbeefULL |
| 135 | #define BRW_MAGIC 0xeeb0eeb1eeb2eeb3ULL |
| 136 | #define BRW_MSIZE sizeof(__u64) |
| 137 | |
| 138 | static int |
| 139 | brw_inject_one_error(void) |
| 140 | { |
| 141 | struct timespec64 ts; |
| 142 | |
| 143 | if (brw_inject_errors <= 0) |
| 144 | return 0; |
| 145 | |
| 146 | ktime_get_ts64(&ts); |
| 147 | |
| 148 | if (!((ts.tv_nsec / NSEC_PER_USEC) & 1)) |
| 149 | return 0; |
| 150 | |
| 151 | return brw_inject_errors--; |
| 152 | } |
| 153 | |
| 154 | static void |
| 155 | brw_fill_page(struct page *pg, int pattern, __u64 magic) |
| 156 | { |
| 157 | char *addr = page_address(pg); |
| 158 | int i; |
| 159 | |
| 160 | LASSERT(addr); |
| 161 | |
| 162 | if (pattern == LST_BRW_CHECK_NONE) |
| 163 | return; |
| 164 | |
| 165 | if (magic == BRW_MAGIC) |
| 166 | magic += brw_inject_one_error(); |
| 167 | |
| 168 | if (pattern == LST_BRW_CHECK_SIMPLE) { |
| 169 | memcpy(addr, &magic, BRW_MSIZE); |
| 170 | addr += PAGE_CACHE_SIZE - BRW_MSIZE; |
| 171 | memcpy(addr, &magic, BRW_MSIZE); |
| 172 | return; |
| 173 | } |
| 174 | |
| 175 | if (pattern == LST_BRW_CHECK_FULL) { |
| 176 | for (i = 0; i < PAGE_CACHE_SIZE / BRW_MSIZE; i++) |
| 177 | memcpy(addr + i * BRW_MSIZE, &magic, BRW_MSIZE); |
| 178 | return; |
| 179 | } |
| 180 | |
| 181 | LBUG(); |
| 182 | } |
| 183 | |
| 184 | static int |
| 185 | brw_check_page(struct page *pg, int pattern, __u64 magic) |
| 186 | { |
| 187 | char *addr = page_address(pg); |
| 188 | __u64 data = 0; /* make compiler happy */ |
| 189 | int i; |
| 190 | |
| 191 | LASSERT(addr); |
| 192 | |
| 193 | if (pattern == LST_BRW_CHECK_NONE) |
| 194 | return 0; |
| 195 | |
| 196 | if (pattern == LST_BRW_CHECK_SIMPLE) { |
| 197 | data = *((__u64 *)addr); |
| 198 | if (data != magic) |
| 199 | goto bad_data; |
| 200 | |
| 201 | addr += PAGE_CACHE_SIZE - BRW_MSIZE; |
| 202 | data = *((__u64 *)addr); |
| 203 | if (data != magic) |
| 204 | goto bad_data; |
| 205 | |
| 206 | return 0; |
| 207 | } |
| 208 | |
| 209 | if (pattern == LST_BRW_CHECK_FULL) { |
| 210 | for (i = 0; i < PAGE_CACHE_SIZE / BRW_MSIZE; i++) { |
| 211 | data = *(((__u64 *)addr) + i); |
| 212 | if (data != magic) |
| 213 | goto bad_data; |
| 214 | } |
| 215 | |
| 216 | return 0; |
| 217 | } |
| 218 | |
| 219 | LBUG(); |
| 220 | |
| 221 | bad_data: |
| 222 | CERROR("Bad data in page %p: %#llx, %#llx expected\n", |
| 223 | pg, data, magic); |
| 224 | return 1; |
| 225 | } |
| 226 | |
| 227 | static void |
| 228 | brw_fill_bulk(srpc_bulk_t *bk, int pattern, __u64 magic) |
| 229 | { |
| 230 | int i; |
| 231 | struct page *pg; |
| 232 | |
| 233 | for (i = 0; i < bk->bk_niov; i++) { |
| 234 | pg = bk->bk_iovs[i].kiov_page; |
| 235 | brw_fill_page(pg, pattern, magic); |
| 236 | } |
| 237 | } |
| 238 | |
| 239 | static int |
| 240 | brw_check_bulk(srpc_bulk_t *bk, int pattern, __u64 magic) |
| 241 | { |
| 242 | int i; |
| 243 | struct page *pg; |
| 244 | |
| 245 | for (i = 0; i < bk->bk_niov; i++) { |
| 246 | pg = bk->bk_iovs[i].kiov_page; |
| 247 | if (brw_check_page(pg, pattern, magic)) { |
| 248 | CERROR("Bulk page %p (%d/%d) is corrupted!\n", |
| 249 | pg, i, bk->bk_niov); |
| 250 | return 1; |
| 251 | } |
| 252 | } |
| 253 | |
| 254 | return 0; |
| 255 | } |
| 256 | |
| 257 | static int |
| 258 | brw_client_prep_rpc(sfw_test_unit_t *tsu, |
| 259 | lnet_process_id_t dest, srpc_client_rpc_t **rpcpp) |
| 260 | { |
| 261 | srpc_bulk_t *bulk = tsu->tsu_private; |
| 262 | sfw_test_instance_t *tsi = tsu->tsu_instance; |
| 263 | sfw_session_t *sn = tsi->tsi_batch->bat_session; |
| 264 | srpc_client_rpc_t *rpc; |
| 265 | srpc_brw_reqst_t *req; |
| 266 | int flags; |
| 267 | int npg; |
| 268 | int len; |
| 269 | int opc; |
| 270 | int rc; |
| 271 | |
| 272 | LASSERT(sn); |
| 273 | LASSERT(bulk); |
| 274 | |
| 275 | if (!(sn->sn_features & LST_FEAT_BULK_LEN)) { |
| 276 | test_bulk_req_t *breq = &tsi->tsi_u.bulk_v0; |
| 277 | |
| 278 | opc = breq->blk_opc; |
| 279 | flags = breq->blk_flags; |
| 280 | npg = breq->blk_npg; |
| 281 | len = npg * PAGE_CACHE_SIZE; |
| 282 | |
| 283 | } else { |
| 284 | test_bulk_req_v1_t *breq = &tsi->tsi_u.bulk_v1; |
| 285 | |
| 286 | /* |
| 287 | * I should never get this step if it's unknown feature |
| 288 | * because make_session will reject unknown feature |
| 289 | */ |
| 290 | LASSERT(!(sn->sn_features & ~LST_FEATS_MASK)); |
| 291 | |
| 292 | opc = breq->blk_opc; |
| 293 | flags = breq->blk_flags; |
| 294 | len = breq->blk_len; |
| 295 | npg = (len + PAGE_CACHE_SIZE - 1) >> PAGE_CACHE_SHIFT; |
| 296 | } |
| 297 | |
| 298 | rc = sfw_create_test_rpc(tsu, dest, sn->sn_features, npg, len, &rpc); |
| 299 | if (rc) |
| 300 | return rc; |
| 301 | |
| 302 | memcpy(&rpc->crpc_bulk, bulk, offsetof(srpc_bulk_t, bk_iovs[npg])); |
| 303 | if (opc == LST_BRW_WRITE) |
| 304 | brw_fill_bulk(&rpc->crpc_bulk, flags, BRW_MAGIC); |
| 305 | else |
| 306 | brw_fill_bulk(&rpc->crpc_bulk, flags, BRW_POISON); |
| 307 | |
| 308 | req = &rpc->crpc_reqstmsg.msg_body.brw_reqst; |
| 309 | req->brw_flags = flags; |
| 310 | req->brw_rw = opc; |
| 311 | req->brw_len = len; |
| 312 | |
| 313 | *rpcpp = rpc; |
| 314 | return 0; |
| 315 | } |
| 316 | |
| 317 | static void |
| 318 | brw_client_done_rpc(sfw_test_unit_t *tsu, srpc_client_rpc_t *rpc) |
| 319 | { |
| 320 | __u64 magic = BRW_MAGIC; |
| 321 | sfw_test_instance_t *tsi = tsu->tsu_instance; |
| 322 | sfw_session_t *sn = tsi->tsi_batch->bat_session; |
| 323 | srpc_msg_t *msg = &rpc->crpc_replymsg; |
| 324 | srpc_brw_reply_t *reply = &msg->msg_body.brw_reply; |
| 325 | srpc_brw_reqst_t *reqst = &rpc->crpc_reqstmsg.msg_body.brw_reqst; |
| 326 | |
| 327 | LASSERT(sn); |
| 328 | |
| 329 | if (rpc->crpc_status) { |
| 330 | CERROR("BRW RPC to %s failed with %d\n", |
| 331 | libcfs_id2str(rpc->crpc_dest), rpc->crpc_status); |
| 332 | if (!tsi->tsi_stopping) /* rpc could have been aborted */ |
| 333 | atomic_inc(&sn->sn_brw_errors); |
| 334 | return; |
| 335 | } |
| 336 | |
| 337 | if (msg->msg_magic != SRPC_MSG_MAGIC) { |
| 338 | __swab64s(&magic); |
| 339 | __swab32s(&reply->brw_status); |
| 340 | } |
| 341 | |
| 342 | CDEBUG(reply->brw_status ? D_WARNING : D_NET, |
| 343 | "BRW RPC to %s finished with brw_status: %d\n", |
| 344 | libcfs_id2str(rpc->crpc_dest), reply->brw_status); |
| 345 | |
| 346 | if (reply->brw_status) { |
| 347 | atomic_inc(&sn->sn_brw_errors); |
| 348 | rpc->crpc_status = -(int)reply->brw_status; |
| 349 | return; |
| 350 | } |
| 351 | |
| 352 | if (reqst->brw_rw == LST_BRW_WRITE) |
| 353 | return; |
| 354 | |
| 355 | if (brw_check_bulk(&rpc->crpc_bulk, reqst->brw_flags, magic)) { |
| 356 | CERROR("Bulk data from %s is corrupted!\n", |
| 357 | libcfs_id2str(rpc->crpc_dest)); |
| 358 | atomic_inc(&sn->sn_brw_errors); |
| 359 | rpc->crpc_status = -EBADMSG; |
| 360 | } |
| 361 | } |
| 362 | |
| 363 | static void |
| 364 | brw_server_rpc_done(struct srpc_server_rpc *rpc) |
| 365 | { |
| 366 | srpc_bulk_t *blk = rpc->srpc_bulk; |
| 367 | |
| 368 | if (!blk) |
| 369 | return; |
| 370 | |
| 371 | if (rpc->srpc_status) |
| 372 | CERROR("Bulk transfer %s %s has failed: %d\n", |
| 373 | blk->bk_sink ? "from" : "to", |
| 374 | libcfs_id2str(rpc->srpc_peer), rpc->srpc_status); |
| 375 | else |
| 376 | CDEBUG(D_NET, "Transferred %d pages bulk data %s %s\n", |
| 377 | blk->bk_niov, blk->bk_sink ? "from" : "to", |
| 378 | libcfs_id2str(rpc->srpc_peer)); |
| 379 | |
| 380 | sfw_free_pages(rpc); |
| 381 | } |
| 382 | |
| 383 | static int |
| 384 | brw_bulk_ready(struct srpc_server_rpc *rpc, int status) |
| 385 | { |
| 386 | __u64 magic = BRW_MAGIC; |
| 387 | srpc_brw_reply_t *reply = &rpc->srpc_replymsg.msg_body.brw_reply; |
| 388 | srpc_brw_reqst_t *reqst; |
| 389 | srpc_msg_t *reqstmsg; |
| 390 | |
| 391 | LASSERT(rpc->srpc_bulk); |
| 392 | LASSERT(rpc->srpc_reqstbuf); |
| 393 | |
| 394 | reqstmsg = &rpc->srpc_reqstbuf->buf_msg; |
| 395 | reqst = &reqstmsg->msg_body.brw_reqst; |
| 396 | |
| 397 | if (status) { |
| 398 | CERROR("BRW bulk %s failed for RPC from %s: %d\n", |
| 399 | reqst->brw_rw == LST_BRW_READ ? "READ" : "WRITE", |
| 400 | libcfs_id2str(rpc->srpc_peer), status); |
| 401 | return -EIO; |
| 402 | } |
| 403 | |
| 404 | if (reqst->brw_rw == LST_BRW_READ) |
| 405 | return 0; |
| 406 | |
| 407 | if (reqstmsg->msg_magic != SRPC_MSG_MAGIC) |
| 408 | __swab64s(&magic); |
| 409 | |
| 410 | if (brw_check_bulk(rpc->srpc_bulk, reqst->brw_flags, magic)) { |
| 411 | CERROR("Bulk data from %s is corrupted!\n", |
| 412 | libcfs_id2str(rpc->srpc_peer)); |
| 413 | reply->brw_status = EBADMSG; |
| 414 | } |
| 415 | |
| 416 | return 0; |
| 417 | } |
| 418 | |
| 419 | static int |
| 420 | brw_server_handle(struct srpc_server_rpc *rpc) |
| 421 | { |
| 422 | struct srpc_service *sv = rpc->srpc_scd->scd_svc; |
| 423 | srpc_msg_t *replymsg = &rpc->srpc_replymsg; |
| 424 | srpc_msg_t *reqstmsg = &rpc->srpc_reqstbuf->buf_msg; |
| 425 | srpc_brw_reply_t *reply = &replymsg->msg_body.brw_reply; |
| 426 | srpc_brw_reqst_t *reqst = &reqstmsg->msg_body.brw_reqst; |
| 427 | int npg; |
| 428 | int rc; |
| 429 | |
| 430 | LASSERT(sv->sv_id == SRPC_SERVICE_BRW); |
| 431 | |
| 432 | if (reqstmsg->msg_magic != SRPC_MSG_MAGIC) { |
| 433 | LASSERT(reqstmsg->msg_magic == __swab32(SRPC_MSG_MAGIC)); |
| 434 | |
| 435 | __swab32s(&reqst->brw_rw); |
| 436 | __swab32s(&reqst->brw_len); |
| 437 | __swab32s(&reqst->brw_flags); |
| 438 | __swab64s(&reqst->brw_rpyid); |
| 439 | __swab64s(&reqst->brw_bulkid); |
| 440 | } |
| 441 | LASSERT(reqstmsg->msg_type == (__u32)srpc_service2request(sv->sv_id)); |
| 442 | |
| 443 | reply->brw_status = 0; |
| 444 | rpc->srpc_done = brw_server_rpc_done; |
| 445 | |
| 446 | if ((reqst->brw_rw != LST_BRW_READ && reqst->brw_rw != LST_BRW_WRITE) || |
| 447 | (reqst->brw_flags != LST_BRW_CHECK_NONE && |
| 448 | reqst->brw_flags != LST_BRW_CHECK_FULL && |
| 449 | reqst->brw_flags != LST_BRW_CHECK_SIMPLE)) { |
| 450 | reply->brw_status = EINVAL; |
| 451 | return 0; |
| 452 | } |
| 453 | |
| 454 | if (reqstmsg->msg_ses_feats & ~LST_FEATS_MASK) { |
| 455 | replymsg->msg_ses_feats = LST_FEATS_MASK; |
| 456 | reply->brw_status = EPROTO; |
| 457 | return 0; |
| 458 | } |
| 459 | |
| 460 | if (!(reqstmsg->msg_ses_feats & LST_FEAT_BULK_LEN)) { |
| 461 | /* compat with old version */ |
| 462 | if (reqst->brw_len & ~CFS_PAGE_MASK) { |
| 463 | reply->brw_status = EINVAL; |
| 464 | return 0; |
| 465 | } |
| 466 | npg = reqst->brw_len >> PAGE_CACHE_SHIFT; |
| 467 | |
| 468 | } else { |
| 469 | npg = (reqst->brw_len + PAGE_CACHE_SIZE - 1) >> PAGE_CACHE_SHIFT; |
| 470 | } |
| 471 | |
| 472 | replymsg->msg_ses_feats = reqstmsg->msg_ses_feats; |
| 473 | |
| 474 | if (!reqst->brw_len || npg > LNET_MAX_IOV) { |
| 475 | reply->brw_status = EINVAL; |
| 476 | return 0; |
| 477 | } |
| 478 | |
| 479 | rc = sfw_alloc_pages(rpc, rpc->srpc_scd->scd_cpt, npg, |
| 480 | reqst->brw_len, |
| 481 | reqst->brw_rw == LST_BRW_WRITE); |
| 482 | if (rc) |
| 483 | return rc; |
| 484 | |
| 485 | if (reqst->brw_rw == LST_BRW_READ) |
| 486 | brw_fill_bulk(rpc->srpc_bulk, reqst->brw_flags, BRW_MAGIC); |
| 487 | else |
| 488 | brw_fill_bulk(rpc->srpc_bulk, reqst->brw_flags, BRW_POISON); |
| 489 | |
| 490 | return 0; |
| 491 | } |
| 492 | |
| 493 | sfw_test_client_ops_t brw_test_client; |
| 494 | void brw_init_test_client(void) |
| 495 | { |
| 496 | brw_test_client.tso_init = brw_client_init; |
| 497 | brw_test_client.tso_fini = brw_client_fini; |
| 498 | brw_test_client.tso_prep_rpc = brw_client_prep_rpc; |
| 499 | brw_test_client.tso_done_rpc = brw_client_done_rpc; |
| 500 | }; |
| 501 | |
| 502 | srpc_service_t brw_test_service; |
| 503 | void brw_init_test_service(void) |
| 504 | { |
| 505 | brw_test_service.sv_id = SRPC_SERVICE_BRW; |
| 506 | brw_test_service.sv_name = "brw_test"; |
| 507 | brw_test_service.sv_handler = brw_server_handle; |
| 508 | brw_test_service.sv_bulk_ready = brw_bulk_ready; |
| 509 | brw_test_service.sv_wi_total = brw_srv_workitems; |
| 510 | } |