Commit | Line | Data |
---|---|---|
d7e09d03 PT |
1 | /* |
2 | * GPL HEADER START | |
3 | * | |
4 | * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. | |
5 | * | |
6 | * This program is free software; you can redistribute it and/or modify | |
7 | * it under the terms of the GNU General Public License version 2 only, | |
8 | * as published by the Free Software Foundation. | |
9 | * | |
10 | * This program is distributed in the hope that it will be useful, but | |
11 | * WITHOUT ANY WARRANTY; without even the implied warranty of | |
12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU | |
13 | * General Public License version 2 for more details (a copy is included | |
14 | * in the LICENSE file that accompanied this code). | |
15 | * | |
16 | * You should have received a copy of the GNU General Public License | |
17 | * version 2 along with this program; If not, see | |
18 | * http://www.sun.com/software/products/lustre/docs/GPLv2.pdf | |
19 | * | |
20 | * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa Clara, | |
21 | * CA 95054 USA or visit www.sun.com if you need additional information or | |
22 | * have any questions. | |
23 | * | |
24 | * GPL HEADER END | |
25 | */ | |
26 | /* | |
27 | * Copyright (c) 2008, 2010, Oracle and/or its affiliates. All rights reserved. | |
28 | * Use is subject to license terms. | |
29 | * | |
30 | * Copyright (c) 2011, 2012, Intel Corporation. | |
31 | */ | |
32 | /* | |
33 | * This file is part of Lustre, http://www.lustre.org/ | |
34 | * Lustre is a trademark of Sun Microsystems, Inc. | |
35 | * | |
36 | * Implementation of cl_io for LOV layer. | |
37 | * | |
38 | * Author: Nikita Danilov <nikita.danilov@sun.com> | |
39 | * Author: Jinshan Xiong <jinshan.xiong@whamcloud.com> | |
40 | */ | |
41 | ||
42 | #define DEBUG_SUBSYSTEM S_LOV | |
43 | ||
44 | #include "lov_cl_internal.h" | |
45 | ||
46 | /** \addtogroup lov | |
47 | * @{ | |
48 | */ | |
49 | ||
50 | static inline void lov_sub_enter(struct lov_io_sub *sub) | |
51 | { | |
52 | sub->sub_reenter++; | |
53 | } | |
54 | static inline void lov_sub_exit(struct lov_io_sub *sub) | |
55 | { | |
56 | sub->sub_reenter--; | |
57 | } | |
58 | ||
59 | static void lov_io_sub_fini(const struct lu_env *env, struct lov_io *lio, | |
60 | struct lov_io_sub *sub) | |
61 | { | |
d7e09d03 PT |
62 | if (sub->sub_io != NULL) { |
63 | if (sub->sub_io_initialized) { | |
64 | lov_sub_enter(sub); | |
65 | cl_io_fini(sub->sub_env, sub->sub_io); | |
66 | lov_sub_exit(sub); | |
67 | sub->sub_io_initialized = 0; | |
68 | lio->lis_active_subios--; | |
69 | } | |
70 | if (sub->sub_stripe == lio->lis_single_subio_index) | |
71 | lio->lis_single_subio_index = -1; | |
72 | else if (!sub->sub_borrowed) | |
73 | OBD_FREE_PTR(sub->sub_io); | |
74 | sub->sub_io = NULL; | |
75 | } | |
76 | if (sub->sub_env != NULL && !IS_ERR(sub->sub_env)) { | |
77 | if (!sub->sub_borrowed) | |
78 | cl_env_put(sub->sub_env, &sub->sub_refcheck); | |
79 | sub->sub_env = NULL; | |
80 | } | |
d7e09d03 PT |
81 | } |
82 | ||
83 | static void lov_io_sub_inherit(struct cl_io *io, struct lov_io *lio, | |
84 | int stripe, loff_t start, loff_t end) | |
85 | { | |
86 | struct lov_stripe_md *lsm = lio->lis_object->lo_lsm; | |
87 | struct cl_io *parent = lio->lis_cl.cis_io; | |
88 | ||
5f3a68f9 | 89 | switch (io->ci_type) { |
d7e09d03 PT |
90 | case CIT_SETATTR: { |
91 | io->u.ci_setattr.sa_attr = parent->u.ci_setattr.sa_attr; | |
92 | io->u.ci_setattr.sa_valid = parent->u.ci_setattr.sa_valid; | |
93 | io->u.ci_setattr.sa_capa = parent->u.ci_setattr.sa_capa; | |
94 | if (cl_io_is_trunc(io)) { | |
95 | loff_t new_size = parent->u.ci_setattr.sa_attr.lvb_size; | |
96 | ||
97 | new_size = lov_size_to_stripe(lsm, new_size, stripe); | |
98 | io->u.ci_setattr.sa_attr.lvb_size = new_size; | |
99 | } | |
100 | break; | |
101 | } | |
102 | case CIT_FAULT: { | |
103 | struct cl_object *obj = parent->ci_obj; | |
104 | loff_t off = cl_offset(obj, parent->u.ci_fault.ft_index); | |
105 | ||
106 | io->u.ci_fault = parent->u.ci_fault; | |
107 | off = lov_size_to_stripe(lsm, off, stripe); | |
108 | io->u.ci_fault.ft_index = cl_index(obj, off); | |
109 | break; | |
110 | } | |
111 | case CIT_FSYNC: { | |
112 | io->u.ci_fsync.fi_start = start; | |
113 | io->u.ci_fsync.fi_end = end; | |
114 | io->u.ci_fsync.fi_capa = parent->u.ci_fsync.fi_capa; | |
115 | io->u.ci_fsync.fi_fid = parent->u.ci_fsync.fi_fid; | |
116 | io->u.ci_fsync.fi_mode = parent->u.ci_fsync.fi_mode; | |
117 | break; | |
118 | } | |
119 | case CIT_READ: | |
120 | case CIT_WRITE: { | |
121 | io->u.ci_wr.wr_sync = cl_io_is_sync_write(parent); | |
122 | if (cl_io_is_append(parent)) { | |
123 | io->u.ci_wr.wr_append = 1; | |
124 | } else { | |
125 | io->u.ci_rw.crw_pos = start; | |
126 | io->u.ci_rw.crw_count = end - start; | |
127 | } | |
128 | break; | |
129 | } | |
130 | default: | |
131 | break; | |
132 | } | |
133 | } | |
134 | ||
135 | static int lov_io_sub_init(const struct lu_env *env, struct lov_io *lio, | |
136 | struct lov_io_sub *sub) | |
137 | { | |
138 | struct lov_object *lov = lio->lis_object; | |
139 | struct lov_device *ld = lu2lov_dev(lov2cl(lov)->co_lu.lo_dev); | |
140 | struct cl_io *sub_io; | |
141 | struct cl_object *sub_obj; | |
142 | struct cl_io *io = lio->lis_cl.cis_io; | |
143 | ||
144 | int stripe = sub->sub_stripe; | |
145 | int result; | |
146 | ||
147 | LASSERT(sub->sub_io == NULL); | |
148 | LASSERT(sub->sub_env == NULL); | |
149 | LASSERT(sub->sub_stripe < lio->lis_stripe_count); | |
d7e09d03 PT |
150 | |
151 | result = 0; | |
152 | sub->sub_io_initialized = 0; | |
153 | sub->sub_borrowed = 0; | |
154 | ||
155 | if (lio->lis_mem_frozen) { | |
156 | LASSERT(mutex_is_locked(&ld->ld_mutex)); | |
157 | sub->sub_io = &ld->ld_emrg[stripe]->emrg_subio; | |
158 | sub->sub_env = ld->ld_emrg[stripe]->emrg_env; | |
159 | sub->sub_borrowed = 1; | |
160 | } else { | |
161 | void *cookie; | |
162 | ||
163 | /* obtain new environment */ | |
164 | cookie = cl_env_reenter(); | |
165 | sub->sub_env = cl_env_get(&sub->sub_refcheck); | |
166 | cl_env_reexit(cookie); | |
167 | if (IS_ERR(sub->sub_env)) | |
168 | result = PTR_ERR(sub->sub_env); | |
169 | ||
170 | if (result == 0) { | |
171 | /* | |
172 | * First sub-io. Use ->lis_single_subio to | |
173 | * avoid dynamic allocation. | |
174 | */ | |
175 | if (lio->lis_active_subios == 0) { | |
176 | sub->sub_io = &lio->lis_single_subio; | |
177 | lio->lis_single_subio_index = stripe; | |
178 | } else { | |
179 | OBD_ALLOC_PTR(sub->sub_io); | |
180 | if (sub->sub_io == NULL) | |
181 | result = -ENOMEM; | |
182 | } | |
183 | } | |
184 | } | |
185 | ||
186 | if (result == 0) { | |
187 | sub_obj = lovsub2cl(lov_r0(lov)->lo_sub[stripe]); | |
188 | sub_io = sub->sub_io; | |
189 | ||
190 | sub_io->ci_obj = sub_obj; | |
191 | sub_io->ci_result = 0; | |
192 | ||
193 | sub_io->ci_parent = io; | |
194 | sub_io->ci_lockreq = io->ci_lockreq; | |
195 | sub_io->ci_type = io->ci_type; | |
196 | sub_io->ci_no_srvlock = io->ci_no_srvlock; | |
ec9bca9c | 197 | sub_io->ci_noatime = io->ci_noatime; |
d7e09d03 PT |
198 | |
199 | lov_sub_enter(sub); | |
200 | result = cl_io_sub_init(sub->sub_env, sub_io, | |
201 | io->ci_type, sub_obj); | |
202 | lov_sub_exit(sub); | |
203 | if (result >= 0) { | |
204 | lio->lis_active_subios++; | |
205 | sub->sub_io_initialized = 1; | |
206 | result = 0; | |
207 | } | |
208 | } | |
209 | if (result != 0) | |
210 | lov_io_sub_fini(env, lio, sub); | |
0a3bdb00 | 211 | return result; |
d7e09d03 PT |
212 | } |
213 | ||
214 | struct lov_io_sub *lov_sub_get(const struct lu_env *env, | |
215 | struct lov_io *lio, int stripe) | |
216 | { | |
217 | int rc; | |
218 | struct lov_io_sub *sub = &lio->lis_subs[stripe]; | |
219 | ||
220 | LASSERT(stripe < lio->lis_stripe_count); | |
d7e09d03 PT |
221 | |
222 | if (!sub->sub_io_initialized) { | |
223 | sub->sub_stripe = stripe; | |
224 | rc = lov_io_sub_init(env, lio, sub); | |
225 | } else | |
226 | rc = 0; | |
227 | if (rc == 0) | |
228 | lov_sub_enter(sub); | |
229 | else | |
230 | sub = ERR_PTR(rc); | |
0a3bdb00 | 231 | return sub; |
d7e09d03 PT |
232 | } |
233 | ||
234 | void lov_sub_put(struct lov_io_sub *sub) | |
235 | { | |
236 | lov_sub_exit(sub); | |
237 | } | |
238 | ||
239 | /***************************************************************************** | |
240 | * | |
241 | * Lov io operations. | |
242 | * | |
243 | */ | |
244 | ||
245 | static int lov_page_stripe(const struct cl_page *page) | |
246 | { | |
247 | struct lovsub_object *subobj; | |
248 | ||
d7e09d03 PT |
249 | subobj = lu2lovsub( |
250 | lu_object_locate(page->cp_child->cp_obj->co_lu.lo_header, | |
251 | &lovsub_device_type)); | |
252 | LASSERT(subobj != NULL); | |
0a3bdb00 | 253 | return subobj->lso_index; |
d7e09d03 PT |
254 | } |
255 | ||
256 | struct lov_io_sub *lov_page_subio(const struct lu_env *env, struct lov_io *lio, | |
257 | const struct cl_page_slice *slice) | |
258 | { | |
259 | struct lov_stripe_md *lsm = lio->lis_object->lo_lsm; | |
260 | struct cl_page *page = slice->cpl_page; | |
261 | int stripe; | |
262 | ||
263 | LASSERT(lio->lis_cl.cis_io != NULL); | |
264 | LASSERT(cl2lov(slice->cpl_obj) == lio->lis_object); | |
265 | LASSERT(lsm != NULL); | |
266 | LASSERT(lio->lis_nr_subios > 0); | |
d7e09d03 PT |
267 | |
268 | stripe = lov_page_stripe(page); | |
0a3bdb00 | 269 | return lov_sub_get(env, lio, stripe); |
d7e09d03 PT |
270 | } |
271 | ||
272 | ||
273 | static int lov_io_subio_init(const struct lu_env *env, struct lov_io *lio, | |
274 | struct cl_io *io) | |
275 | { | |
276 | struct lov_stripe_md *lsm = lio->lis_object->lo_lsm; | |
277 | int result; | |
278 | ||
279 | LASSERT(lio->lis_object != NULL); | |
d7e09d03 PT |
280 | |
281 | /* | |
282 | * Need to be optimized, we can't afford to allocate a piece of memory | |
283 | * when writing a page. -jay | |
284 | */ | |
285 | OBD_ALLOC_LARGE(lio->lis_subs, | |
ec83e611 | 286 | lsm->lsm_stripe_count * sizeof(lio->lis_subs[0])); |
d7e09d03 PT |
287 | if (lio->lis_subs != NULL) { |
288 | lio->lis_nr_subios = lio->lis_stripe_count; | |
289 | lio->lis_single_subio_index = -1; | |
290 | lio->lis_active_subios = 0; | |
291 | result = 0; | |
292 | } else | |
293 | result = -ENOMEM; | |
0a3bdb00 | 294 | return result; |
d7e09d03 PT |
295 | } |
296 | ||
297 | static void lov_io_slice_init(struct lov_io *lio, | |
298 | struct lov_object *obj, struct cl_io *io) | |
299 | { | |
d7e09d03 PT |
300 | io->ci_result = 0; |
301 | lio->lis_object = obj; | |
302 | ||
303 | LASSERT(obj->lo_lsm != NULL); | |
304 | lio->lis_stripe_count = obj->lo_lsm->lsm_stripe_count; | |
305 | ||
306 | switch (io->ci_type) { | |
307 | case CIT_READ: | |
308 | case CIT_WRITE: | |
309 | lio->lis_pos = io->u.ci_rw.crw_pos; | |
310 | lio->lis_endpos = io->u.ci_rw.crw_pos + io->u.ci_rw.crw_count; | |
311 | lio->lis_io_endpos = lio->lis_endpos; | |
312 | if (cl_io_is_append(io)) { | |
313 | LASSERT(io->ci_type == CIT_WRITE); | |
314 | lio->lis_pos = 0; | |
315 | lio->lis_endpos = OBD_OBJECT_EOF; | |
316 | } | |
317 | break; | |
318 | ||
319 | case CIT_SETATTR: | |
320 | if (cl_io_is_trunc(io)) | |
321 | lio->lis_pos = io->u.ci_setattr.sa_attr.lvb_size; | |
322 | else | |
323 | lio->lis_pos = 0; | |
324 | lio->lis_endpos = OBD_OBJECT_EOF; | |
325 | break; | |
326 | ||
327 | case CIT_FAULT: { | |
328 | pgoff_t index = io->u.ci_fault.ft_index; | |
329 | lio->lis_pos = cl_offset(io->ci_obj, index); | |
330 | lio->lis_endpos = cl_offset(io->ci_obj, index + 1); | |
331 | break; | |
332 | } | |
333 | ||
334 | case CIT_FSYNC: { | |
335 | lio->lis_pos = io->u.ci_fsync.fi_start; | |
336 | lio->lis_endpos = io->u.ci_fsync.fi_end; | |
337 | break; | |
338 | } | |
339 | ||
340 | case CIT_MISC: | |
341 | lio->lis_pos = 0; | |
342 | lio->lis_endpos = OBD_OBJECT_EOF; | |
343 | break; | |
344 | ||
345 | default: | |
346 | LBUG(); | |
347 | } | |
d7e09d03 PT |
348 | } |
349 | ||
350 | static void lov_io_fini(const struct lu_env *env, const struct cl_io_slice *ios) | |
351 | { | |
352 | struct lov_io *lio = cl2lov_io(env, ios); | |
353 | struct lov_object *lov = cl2lov(ios->cis_obj); | |
354 | int i; | |
355 | ||
d7e09d03 PT |
356 | if (lio->lis_subs != NULL) { |
357 | for (i = 0; i < lio->lis_nr_subios; i++) | |
358 | lov_io_sub_fini(env, lio, &lio->lis_subs[i]); | |
359 | OBD_FREE_LARGE(lio->lis_subs, | |
ec83e611 | 360 | lio->lis_nr_subios * sizeof(lio->lis_subs[0])); |
d7e09d03 PT |
361 | lio->lis_nr_subios = 0; |
362 | } | |
363 | ||
364 | LASSERT(atomic_read(&lov->lo_active_ios) > 0); | |
365 | if (atomic_dec_and_test(&lov->lo_active_ios)) | |
366 | wake_up_all(&lov->lo_waitq); | |
d7e09d03 PT |
367 | } |
368 | ||
369 | static obd_off lov_offset_mod(obd_off val, int delta) | |
370 | { | |
371 | if (val != OBD_OBJECT_EOF) | |
372 | val += delta; | |
373 | return val; | |
374 | } | |
375 | ||
376 | static int lov_io_iter_init(const struct lu_env *env, | |
377 | const struct cl_io_slice *ios) | |
378 | { | |
379 | struct lov_io *lio = cl2lov_io(env, ios); | |
380 | struct lov_stripe_md *lsm = lio->lis_object->lo_lsm; | |
381 | struct lov_io_sub *sub; | |
382 | obd_off endpos; | |
383 | obd_off start; | |
384 | obd_off end; | |
385 | int stripe; | |
386 | int rc = 0; | |
387 | ||
d7e09d03 PT |
388 | endpos = lov_offset_mod(lio->lis_endpos, -1); |
389 | for (stripe = 0; stripe < lio->lis_stripe_count; stripe++) { | |
390 | if (!lov_stripe_intersects(lsm, stripe, lio->lis_pos, | |
391 | endpos, &start, &end)) | |
392 | continue; | |
393 | ||
394 | end = lov_offset_mod(end, +1); | |
395 | sub = lov_sub_get(env, lio, stripe); | |
396 | if (!IS_ERR(sub)) { | |
397 | lov_io_sub_inherit(sub->sub_io, lio, stripe, | |
398 | start, end); | |
399 | rc = cl_io_iter_init(sub->sub_env, sub->sub_io); | |
400 | lov_sub_put(sub); | |
401 | CDEBUG(D_VFSTRACE, "shrink: %d ["LPU64", "LPU64")\n", | |
402 | stripe, start, end); | |
403 | } else | |
404 | rc = PTR_ERR(sub); | |
405 | ||
406 | if (!rc) | |
407 | list_add_tail(&sub->sub_linkage, &lio->lis_active); | |
408 | else | |
409 | break; | |
410 | } | |
0a3bdb00 | 411 | return rc; |
d7e09d03 PT |
412 | } |
413 | ||
414 | static int lov_io_rw_iter_init(const struct lu_env *env, | |
415 | const struct cl_io_slice *ios) | |
416 | { | |
417 | struct lov_io *lio = cl2lov_io(env, ios); | |
418 | struct cl_io *io = ios->cis_io; | |
419 | struct lov_stripe_md *lsm = lio->lis_object->lo_lsm; | |
b69835a2 | 420 | __u64 start = io->u.ci_rw.crw_pos; |
d7e09d03 PT |
421 | loff_t next; |
422 | unsigned long ssize = lsm->lsm_stripe_size; | |
423 | ||
424 | LASSERT(io->ci_type == CIT_READ || io->ci_type == CIT_WRITE); | |
d7e09d03 PT |
425 | |
426 | /* fast path for common case. */ | |
427 | if (lio->lis_nr_subios != 1 && !cl_io_is_append(io)) { | |
428 | ||
429 | lov_do_div64(start, ssize); | |
430 | next = (start + 1) * ssize; | |
431 | if (next <= start * ssize) | |
432 | next = ~0ull; | |
433 | ||
434 | io->ci_continue = next < lio->lis_io_endpos; | |
435 | io->u.ci_rw.crw_count = min_t(loff_t, lio->lis_io_endpos, | |
436 | next) - io->u.ci_rw.crw_pos; | |
437 | lio->lis_pos = io->u.ci_rw.crw_pos; | |
438 | lio->lis_endpos = io->u.ci_rw.crw_pos + io->u.ci_rw.crw_count; | |
439 | CDEBUG(D_VFSTRACE, "stripe: "LPU64" chunk: ["LPU64", "LPU64") " | |
440 | LPU64"\n", (__u64)start, lio->lis_pos, lio->lis_endpos, | |
441 | (__u64)lio->lis_io_endpos); | |
442 | } | |
443 | /* | |
444 | * XXX The following call should be optimized: we know, that | |
445 | * [lio->lis_pos, lio->lis_endpos) intersects with exactly one stripe. | |
446 | */ | |
0a3bdb00 | 447 | return lov_io_iter_init(env, ios); |
d7e09d03 PT |
448 | } |
449 | ||
450 | static int lov_io_call(const struct lu_env *env, struct lov_io *lio, | |
451 | int (*iofunc)(const struct lu_env *, struct cl_io *)) | |
452 | { | |
453 | struct cl_io *parent = lio->lis_cl.cis_io; | |
454 | struct lov_io_sub *sub; | |
455 | int rc = 0; | |
456 | ||
d7e09d03 PT |
457 | list_for_each_entry(sub, &lio->lis_active, sub_linkage) { |
458 | lov_sub_enter(sub); | |
459 | rc = iofunc(sub->sub_env, sub->sub_io); | |
460 | lov_sub_exit(sub); | |
461 | if (rc) | |
462 | break; | |
463 | ||
464 | if (parent->ci_result == 0) | |
465 | parent->ci_result = sub->sub_io->ci_result; | |
466 | } | |
0a3bdb00 | 467 | return rc; |
d7e09d03 PT |
468 | } |
469 | ||
470 | static int lov_io_lock(const struct lu_env *env, const struct cl_io_slice *ios) | |
471 | { | |
0a3bdb00 | 472 | return lov_io_call(env, cl2lov_io(env, ios), cl_io_lock); |
d7e09d03 PT |
473 | } |
474 | ||
475 | static int lov_io_start(const struct lu_env *env, const struct cl_io_slice *ios) | |
476 | { | |
0a3bdb00 | 477 | return lov_io_call(env, cl2lov_io(env, ios), cl_io_start); |
d7e09d03 PT |
478 | } |
479 | ||
480 | static int lov_io_end_wrapper(const struct lu_env *env, struct cl_io *io) | |
481 | { | |
d7e09d03 PT |
482 | /* |
483 | * It's possible that lov_io_start() wasn't called against this | |
484 | * sub-io, either because previous sub-io failed, or upper layer | |
485 | * completed IO. | |
486 | */ | |
487 | if (io->ci_state == CIS_IO_GOING) | |
488 | cl_io_end(env, io); | |
489 | else | |
490 | io->ci_state = CIS_IO_FINISHED; | |
0a3bdb00 | 491 | return 0; |
d7e09d03 PT |
492 | } |
493 | ||
494 | static int lov_io_iter_fini_wrapper(const struct lu_env *env, struct cl_io *io) | |
495 | { | |
496 | cl_io_iter_fini(env, io); | |
0a3bdb00 | 497 | return 0; |
d7e09d03 PT |
498 | } |
499 | ||
500 | static int lov_io_unlock_wrapper(const struct lu_env *env, struct cl_io *io) | |
501 | { | |
502 | cl_io_unlock(env, io); | |
0a3bdb00 | 503 | return 0; |
d7e09d03 PT |
504 | } |
505 | ||
506 | static void lov_io_end(const struct lu_env *env, const struct cl_io_slice *ios) | |
507 | { | |
508 | int rc; | |
509 | ||
510 | rc = lov_io_call(env, cl2lov_io(env, ios), lov_io_end_wrapper); | |
511 | LASSERT(rc == 0); | |
512 | } | |
513 | ||
514 | static void lov_io_iter_fini(const struct lu_env *env, | |
515 | const struct cl_io_slice *ios) | |
516 | { | |
517 | struct lov_io *lio = cl2lov_io(env, ios); | |
518 | int rc; | |
519 | ||
d7e09d03 PT |
520 | rc = lov_io_call(env, lio, lov_io_iter_fini_wrapper); |
521 | LASSERT(rc == 0); | |
522 | while (!list_empty(&lio->lis_active)) | |
523 | list_del_init(lio->lis_active.next); | |
d7e09d03 PT |
524 | } |
525 | ||
526 | static void lov_io_unlock(const struct lu_env *env, | |
527 | const struct cl_io_slice *ios) | |
528 | { | |
529 | int rc; | |
530 | ||
d7e09d03 PT |
531 | rc = lov_io_call(env, cl2lov_io(env, ios), lov_io_unlock_wrapper); |
532 | LASSERT(rc == 0); | |
d7e09d03 PT |
533 | } |
534 | ||
535 | ||
536 | static struct cl_page_list *lov_io_submit_qin(struct lov_device *ld, | |
537 | struct cl_page_list *qin, | |
538 | int idx, int alloc) | |
539 | { | |
540 | return alloc ? &qin[idx] : &ld->ld_emrg[idx]->emrg_page_list; | |
541 | } | |
542 | ||
543 | /** | |
544 | * lov implementation of cl_operations::cio_submit() method. It takes a list | |
545 | * of pages in \a queue, splits it into per-stripe sub-lists, invokes | |
546 | * cl_io_submit() on underlying devices to submit sub-lists, and then splices | |
547 | * everything back. | |
548 | * | |
549 | * Major complication of this function is a need to handle memory cleansing: | |
550 | * cl_io_submit() is called to write out pages as a part of VM memory | |
551 | * reclamation, and hence it may not fail due to memory shortages (system | |
552 | * dead-locks otherwise). To deal with this, some resources (sub-lists, | |
553 | * sub-environment, etc.) are allocated per-device on "startup" (i.e., in a | |
554 | * not-memory cleansing context), and in case of memory shortage, these | |
555 | * pre-allocated resources are used by lov_io_submit() under | |
556 | * lov_device::ld_mutex mutex. | |
557 | */ | |
558 | static int lov_io_submit(const struct lu_env *env, | |
559 | const struct cl_io_slice *ios, | |
560 | enum cl_req_type crt, struct cl_2queue *queue) | |
561 | { | |
562 | struct lov_io *lio = cl2lov_io(env, ios); | |
563 | struct lov_object *obj = lio->lis_object; | |
564 | struct lov_device *ld = lu2lov_dev(lov2cl(obj)->co_lu.lo_dev); | |
565 | struct cl_page_list *qin = &queue->c2_qin; | |
566 | struct cl_2queue *cl2q = &lov_env_info(env)->lti_cl2q; | |
567 | struct cl_page_list *stripes_qin = NULL; | |
568 | struct cl_page *page; | |
569 | struct cl_page *tmp; | |
570 | int stripe; | |
571 | ||
572 | #define QIN(stripe) lov_io_submit_qin(ld, stripes_qin, stripe, alloc) | |
573 | ||
574 | int rc = 0; | |
575 | int alloc = | |
576 | !(current->flags & PF_MEMALLOC); | |
29aaf496 | 577 | |
d7e09d03 PT |
578 | if (lio->lis_active_subios == 1) { |
579 | int idx = lio->lis_single_subio_index; | |
580 | struct lov_io_sub *sub; | |
581 | ||
582 | LASSERT(idx < lio->lis_nr_subios); | |
583 | sub = lov_sub_get(env, lio, idx); | |
584 | LASSERT(!IS_ERR(sub)); | |
585 | LASSERT(sub->sub_io == &lio->lis_single_subio); | |
586 | rc = cl_io_submit_rw(sub->sub_env, sub->sub_io, | |
587 | crt, queue); | |
588 | lov_sub_put(sub); | |
0a3bdb00 | 589 | return rc; |
d7e09d03 PT |
590 | } |
591 | ||
592 | LASSERT(lio->lis_subs != NULL); | |
593 | if (alloc) { | |
594 | OBD_ALLOC_LARGE(stripes_qin, | |
595 | sizeof(*stripes_qin) * lio->lis_nr_subios); | |
596 | if (stripes_qin == NULL) | |
0a3bdb00 | 597 | return -ENOMEM; |
d7e09d03 PT |
598 | |
599 | for (stripe = 0; stripe < lio->lis_nr_subios; stripe++) | |
600 | cl_page_list_init(&stripes_qin[stripe]); | |
601 | } else { | |
602 | /* | |
603 | * If we get here, it means pageout & swap doesn't help. | |
604 | * In order to not make things worse, even don't try to | |
605 | * allocate the memory with __GFP_NOWARN. -jay | |
606 | */ | |
607 | mutex_lock(&ld->ld_mutex); | |
608 | lio->lis_mem_frozen = 1; | |
609 | } | |
610 | ||
611 | cl_2queue_init(cl2q); | |
612 | cl_page_list_for_each_safe(page, tmp, qin) { | |
613 | stripe = lov_page_stripe(page); | |
614 | cl_page_list_move(QIN(stripe), qin, page); | |
615 | } | |
616 | ||
617 | for (stripe = 0; stripe < lio->lis_nr_subios; stripe++) { | |
618 | struct lov_io_sub *sub; | |
619 | struct cl_page_list *sub_qin = QIN(stripe); | |
620 | ||
621 | if (list_empty(&sub_qin->pl_pages)) | |
622 | continue; | |
623 | ||
624 | cl_page_list_splice(sub_qin, &cl2q->c2_qin); | |
625 | sub = lov_sub_get(env, lio, stripe); | |
626 | if (!IS_ERR(sub)) { | |
627 | rc = cl_io_submit_rw(sub->sub_env, sub->sub_io, | |
628 | crt, cl2q); | |
629 | lov_sub_put(sub); | |
630 | } else | |
631 | rc = PTR_ERR(sub); | |
632 | cl_page_list_splice(&cl2q->c2_qin, &queue->c2_qin); | |
633 | cl_page_list_splice(&cl2q->c2_qout, &queue->c2_qout); | |
634 | if (rc != 0) | |
635 | break; | |
636 | } | |
637 | ||
638 | for (stripe = 0; stripe < lio->lis_nr_subios; stripe++) { | |
639 | struct cl_page_list *sub_qin = QIN(stripe); | |
640 | ||
641 | if (list_empty(&sub_qin->pl_pages)) | |
642 | continue; | |
643 | ||
644 | cl_page_list_splice(sub_qin, qin); | |
645 | } | |
646 | ||
647 | if (alloc) { | |
648 | OBD_FREE_LARGE(stripes_qin, | |
649 | sizeof(*stripes_qin) * lio->lis_nr_subios); | |
650 | } else { | |
651 | int i; | |
652 | ||
653 | for (i = 0; i < lio->lis_nr_subios; i++) { | |
654 | struct cl_io *cio = lio->lis_subs[i].sub_io; | |
655 | ||
656 | if (cio && cio == &ld->ld_emrg[i]->emrg_subio) | |
657 | lov_io_sub_fini(env, lio, &lio->lis_subs[i]); | |
658 | } | |
659 | lio->lis_mem_frozen = 0; | |
660 | mutex_unlock(&ld->ld_mutex); | |
661 | } | |
662 | ||
0a3bdb00 | 663 | return rc; |
d7e09d03 PT |
664 | #undef QIN |
665 | } | |
666 | ||
667 | static int lov_io_prepare_write(const struct lu_env *env, | |
668 | const struct cl_io_slice *ios, | |
669 | const struct cl_page_slice *slice, | |
670 | unsigned from, unsigned to) | |
671 | { | |
672 | struct lov_io *lio = cl2lov_io(env, ios); | |
673 | struct cl_page *sub_page = lov_sub_page(slice); | |
674 | struct lov_io_sub *sub; | |
675 | int result; | |
676 | ||
d7e09d03 PT |
677 | sub = lov_page_subio(env, lio, slice); |
678 | if (!IS_ERR(sub)) { | |
679 | result = cl_io_prepare_write(sub->sub_env, sub->sub_io, | |
680 | sub_page, from, to); | |
681 | lov_sub_put(sub); | |
682 | } else | |
683 | result = PTR_ERR(sub); | |
0a3bdb00 | 684 | return result; |
d7e09d03 PT |
685 | } |
686 | ||
687 | static int lov_io_commit_write(const struct lu_env *env, | |
688 | const struct cl_io_slice *ios, | |
689 | const struct cl_page_slice *slice, | |
690 | unsigned from, unsigned to) | |
691 | { | |
692 | struct lov_io *lio = cl2lov_io(env, ios); | |
693 | struct cl_page *sub_page = lov_sub_page(slice); | |
694 | struct lov_io_sub *sub; | |
695 | int result; | |
696 | ||
d7e09d03 PT |
697 | sub = lov_page_subio(env, lio, slice); |
698 | if (!IS_ERR(sub)) { | |
699 | result = cl_io_commit_write(sub->sub_env, sub->sub_io, | |
700 | sub_page, from, to); | |
701 | lov_sub_put(sub); | |
702 | } else | |
703 | result = PTR_ERR(sub); | |
0a3bdb00 | 704 | return result; |
d7e09d03 PT |
705 | } |
706 | ||
707 | static int lov_io_fault_start(const struct lu_env *env, | |
708 | const struct cl_io_slice *ios) | |
709 | { | |
710 | struct cl_fault_io *fio; | |
711 | struct lov_io *lio; | |
712 | struct lov_io_sub *sub; | |
713 | ||
d7e09d03 PT |
714 | fio = &ios->cis_io->u.ci_fault; |
715 | lio = cl2lov_io(env, ios); | |
716 | sub = lov_sub_get(env, lio, lov_page_stripe(fio->ft_page)); | |
717 | sub->sub_io->u.ci_fault.ft_nob = fio->ft_nob; | |
718 | lov_sub_put(sub); | |
0a3bdb00 | 719 | return lov_io_start(env, ios); |
d7e09d03 PT |
720 | } |
721 | ||
722 | static void lov_io_fsync_end(const struct lu_env *env, | |
723 | const struct cl_io_slice *ios) | |
724 | { | |
725 | struct lov_io *lio = cl2lov_io(env, ios); | |
726 | struct lov_io_sub *sub; | |
727 | unsigned int *written = &ios->cis_io->u.ci_fsync.fi_nr_written; | |
d7e09d03 PT |
728 | |
729 | *written = 0; | |
730 | list_for_each_entry(sub, &lio->lis_active, sub_linkage) { | |
731 | struct cl_io *subio = sub->sub_io; | |
732 | ||
733 | lov_sub_enter(sub); | |
734 | lov_io_end_wrapper(sub->sub_env, subio); | |
735 | lov_sub_exit(sub); | |
736 | ||
737 | if (subio->ci_result == 0) | |
738 | *written += subio->u.ci_fsync.fi_nr_written; | |
739 | } | |
d7e09d03 PT |
740 | } |
741 | ||
742 | static const struct cl_io_operations lov_io_ops = { | |
743 | .op = { | |
744 | [CIT_READ] = { | |
745 | .cio_fini = lov_io_fini, | |
746 | .cio_iter_init = lov_io_rw_iter_init, | |
747 | .cio_iter_fini = lov_io_iter_fini, | |
748 | .cio_lock = lov_io_lock, | |
749 | .cio_unlock = lov_io_unlock, | |
750 | .cio_start = lov_io_start, | |
751 | .cio_end = lov_io_end | |
752 | }, | |
753 | [CIT_WRITE] = { | |
754 | .cio_fini = lov_io_fini, | |
755 | .cio_iter_init = lov_io_rw_iter_init, | |
756 | .cio_iter_fini = lov_io_iter_fini, | |
757 | .cio_lock = lov_io_lock, | |
758 | .cio_unlock = lov_io_unlock, | |
759 | .cio_start = lov_io_start, | |
760 | .cio_end = lov_io_end | |
761 | }, | |
762 | [CIT_SETATTR] = { | |
763 | .cio_fini = lov_io_fini, | |
764 | .cio_iter_init = lov_io_iter_init, | |
765 | .cio_iter_fini = lov_io_iter_fini, | |
766 | .cio_lock = lov_io_lock, | |
767 | .cio_unlock = lov_io_unlock, | |
768 | .cio_start = lov_io_start, | |
769 | .cio_end = lov_io_end | |
770 | }, | |
771 | [CIT_FAULT] = { | |
772 | .cio_fini = lov_io_fini, | |
773 | .cio_iter_init = lov_io_iter_init, | |
774 | .cio_iter_fini = lov_io_iter_fini, | |
775 | .cio_lock = lov_io_lock, | |
776 | .cio_unlock = lov_io_unlock, | |
777 | .cio_start = lov_io_fault_start, | |
778 | .cio_end = lov_io_end | |
779 | }, | |
780 | [CIT_FSYNC] = { | |
781 | .cio_fini = lov_io_fini, | |
782 | .cio_iter_init = lov_io_iter_init, | |
783 | .cio_iter_fini = lov_io_iter_fini, | |
784 | .cio_lock = lov_io_lock, | |
785 | .cio_unlock = lov_io_unlock, | |
786 | .cio_start = lov_io_start, | |
787 | .cio_end = lov_io_fsync_end | |
788 | }, | |
789 | [CIT_MISC] = { | |
790 | .cio_fini = lov_io_fini | |
791 | } | |
792 | }, | |
793 | .req_op = { | |
794 | [CRT_READ] = { | |
795 | .cio_submit = lov_io_submit | |
796 | }, | |
797 | [CRT_WRITE] = { | |
798 | .cio_submit = lov_io_submit | |
799 | } | |
800 | }, | |
801 | .cio_prepare_write = lov_io_prepare_write, | |
802 | .cio_commit_write = lov_io_commit_write | |
803 | }; | |
804 | ||
805 | /***************************************************************************** | |
806 | * | |
807 | * Empty lov io operations. | |
808 | * | |
809 | */ | |
810 | ||
811 | static void lov_empty_io_fini(const struct lu_env *env, | |
812 | const struct cl_io_slice *ios) | |
813 | { | |
814 | struct lov_object *lov = cl2lov(ios->cis_obj); | |
d7e09d03 PT |
815 | |
816 | if (atomic_dec_and_test(&lov->lo_active_ios)) | |
817 | wake_up_all(&lov->lo_waitq); | |
d7e09d03 PT |
818 | } |
819 | ||
820 | static void lov_empty_impossible(const struct lu_env *env, | |
821 | struct cl_io_slice *ios) | |
822 | { | |
823 | LBUG(); | |
824 | } | |
825 | ||
826 | #define LOV_EMPTY_IMPOSSIBLE ((void *)lov_empty_impossible) | |
827 | ||
828 | /** | |
829 | * An io operation vector for files without stripes. | |
830 | */ | |
831 | static const struct cl_io_operations lov_empty_io_ops = { | |
832 | .op = { | |
833 | [CIT_READ] = { | |
834 | .cio_fini = lov_empty_io_fini, | |
835 | #if 0 | |
836 | .cio_iter_init = LOV_EMPTY_IMPOSSIBLE, | |
837 | .cio_lock = LOV_EMPTY_IMPOSSIBLE, | |
838 | .cio_start = LOV_EMPTY_IMPOSSIBLE, | |
839 | .cio_end = LOV_EMPTY_IMPOSSIBLE | |
840 | #endif | |
841 | }, | |
842 | [CIT_WRITE] = { | |
843 | .cio_fini = lov_empty_io_fini, | |
844 | .cio_iter_init = LOV_EMPTY_IMPOSSIBLE, | |
845 | .cio_lock = LOV_EMPTY_IMPOSSIBLE, | |
846 | .cio_start = LOV_EMPTY_IMPOSSIBLE, | |
847 | .cio_end = LOV_EMPTY_IMPOSSIBLE | |
848 | }, | |
849 | [CIT_SETATTR] = { | |
850 | .cio_fini = lov_empty_io_fini, | |
851 | .cio_iter_init = LOV_EMPTY_IMPOSSIBLE, | |
852 | .cio_lock = LOV_EMPTY_IMPOSSIBLE, | |
853 | .cio_start = LOV_EMPTY_IMPOSSIBLE, | |
854 | .cio_end = LOV_EMPTY_IMPOSSIBLE | |
855 | }, | |
856 | [CIT_FAULT] = { | |
857 | .cio_fini = lov_empty_io_fini, | |
858 | .cio_iter_init = LOV_EMPTY_IMPOSSIBLE, | |
859 | .cio_lock = LOV_EMPTY_IMPOSSIBLE, | |
860 | .cio_start = LOV_EMPTY_IMPOSSIBLE, | |
861 | .cio_end = LOV_EMPTY_IMPOSSIBLE | |
862 | }, | |
863 | [CIT_FSYNC] = { | |
864 | .cio_fini = lov_empty_io_fini | |
865 | }, | |
866 | [CIT_MISC] = { | |
867 | .cio_fini = lov_empty_io_fini | |
868 | } | |
869 | }, | |
870 | .req_op = { | |
871 | [CRT_READ] = { | |
872 | .cio_submit = LOV_EMPTY_IMPOSSIBLE | |
873 | }, | |
874 | [CRT_WRITE] = { | |
875 | .cio_submit = LOV_EMPTY_IMPOSSIBLE | |
876 | } | |
877 | }, | |
878 | .cio_commit_write = LOV_EMPTY_IMPOSSIBLE | |
879 | }; | |
880 | ||
881 | int lov_io_init_raid0(const struct lu_env *env, struct cl_object *obj, | |
882 | struct cl_io *io) | |
883 | { | |
884 | struct lov_io *lio = lov_env_io(env); | |
885 | struct lov_object *lov = cl2lov(obj); | |
886 | ||
d7e09d03 PT |
887 | INIT_LIST_HEAD(&lio->lis_active); |
888 | lov_io_slice_init(lio, lov, io); | |
889 | if (io->ci_result == 0) { | |
890 | io->ci_result = lov_io_subio_init(env, lio, io); | |
891 | if (io->ci_result == 0) { | |
892 | cl_io_slice_add(io, &lio->lis_cl, obj, &lov_io_ops); | |
893 | atomic_inc(&lov->lo_active_ios); | |
894 | } | |
895 | } | |
0a3bdb00 | 896 | return io->ci_result; |
d7e09d03 PT |
897 | } |
898 | ||
899 | int lov_io_init_empty(const struct lu_env *env, struct cl_object *obj, | |
900 | struct cl_io *io) | |
901 | { | |
902 | struct lov_object *lov = cl2lov(obj); | |
903 | struct lov_io *lio = lov_env_io(env); | |
904 | int result; | |
d7e09d03 PT |
905 | |
906 | lio->lis_object = lov; | |
907 | switch (io->ci_type) { | |
908 | default: | |
909 | LBUG(); | |
910 | case CIT_MISC: | |
911 | case CIT_READ: | |
912 | result = 0; | |
913 | break; | |
914 | case CIT_FSYNC: | |
915 | case CIT_SETATTR: | |
916 | result = +1; | |
917 | break; | |
918 | case CIT_WRITE: | |
919 | result = -EBADF; | |
920 | break; | |
921 | case CIT_FAULT: | |
922 | result = -EFAULT; | |
923 | CERROR("Page fault on a file without stripes: "DFID"\n", | |
924 | PFID(lu_object_fid(&obj->co_lu))); | |
925 | break; | |
926 | } | |
927 | if (result == 0) { | |
928 | cl_io_slice_add(io, &lio->lis_cl, obj, &lov_empty_io_ops); | |
929 | atomic_inc(&lov->lo_active_ios); | |
930 | } | |
931 | ||
932 | io->ci_result = result < 0 ? result : 0; | |
0a3bdb00 | 933 | return result != 0; |
d7e09d03 PT |
934 | } |
935 | ||
5dd16419 JX |
936 | int lov_io_init_released(const struct lu_env *env, struct cl_object *obj, |
937 | struct cl_io *io) | |
938 | { | |
939 | struct lov_object *lov = cl2lov(obj); | |
940 | struct lov_io *lio = lov_env_io(env); | |
941 | int result; | |
5dd16419 JX |
942 | |
943 | LASSERT(lov->lo_lsm != NULL); | |
944 | lio->lis_object = lov; | |
945 | ||
946 | switch (io->ci_type) { | |
947 | default: | |
948 | LASSERTF(0, "invalid type %d\n", io->ci_type); | |
949 | case CIT_MISC: | |
950 | case CIT_FSYNC: | |
5ea17d6c | 951 | result = 1; |
5dd16419 | 952 | break; |
251c4317 | 953 | case CIT_SETATTR: |
5ea17d6c JL |
954 | /* the truncate to 0 is managed by MDT: |
955 | * - in open, for open O_TRUNC | |
956 | * - in setattr, for truncate | |
957 | */ | |
958 | /* the truncate is for size > 0 so triggers a restore */ | |
959 | if (cl_io_is_trunc(io)) | |
960 | io->ci_restore_needed = 1; | |
961 | result = -ENODATA; | |
962 | break; | |
5dd16419 JX |
963 | case CIT_READ: |
964 | case CIT_WRITE: | |
965 | case CIT_FAULT: | |
5ea17d6c JL |
966 | io->ci_restore_needed = 1; |
967 | result = -ENODATA; | |
5dd16419 JX |
968 | break; |
969 | } | |
970 | if (result == 0) { | |
971 | cl_io_slice_add(io, &lio->lis_cl, obj, &lov_empty_io_ops); | |
972 | atomic_inc(&lov->lo_active_ios); | |
973 | } | |
974 | ||
975 | io->ci_result = result < 0 ? result : 0; | |
0a3bdb00 | 976 | return result != 0; |
5dd16419 | 977 | } |
d7e09d03 | 978 | /** @} lov */ |