2 * Device operations for the pnfs nfs4 file layout driver.
5 * The Regents of the University of Michigan
8 * Dean Hildebrand <dhildebz@umich.edu>
9 * Garth Goodson <Garth.Goodson@netapp.com>
11 * Permission is granted to use, copy, create derivative works, and
12 * redistribute this software and such derivative works for any purpose,
13 * so long as the name of the University of Michigan is not used in
14 * any advertising or publicity pertaining to the use or distribution
15 * of this software without specific, written prior authorization. If
16 * the above copyright notice or any other identification of the
17 * University of Michigan is included in any copy of any portion of
18 * this software, then the disclaimer below must also be included.
20 * This software is provided as is, without representation or warranty
21 * of any kind either express or implied, including without limitation
22 * the implied warranties of merchantability, fitness for a particular
23 * purpose, or noninfringement. The Regents of the University of
24 * Michigan shall not be liable for any damages, including special,
25 * indirect, incidental, or consequential damages, with respect to any
26 * claim arising out of or in connection with the use of the software,
27 * even if it has been or is hereafter advised of the possibility of
31 #include <linux/nfs_fs.h>
32 #include <linux/vmalloc.h>
35 #include "nfs4filelayout.h"
37 #define NFSDBG_FACILITY NFSDBG_PNFS_LD
42 * Data servers can be mapped to different device ids.
43 * nfs4_pnfs_ds reference counting
44 * - set to 1 on allocation
45 * - incremented when a device id maps a data server already in the cache.
46 * - decremented when deviceid is removed from the cache.
48 DEFINE_SPINLOCK(nfs4_ds_cache_lock
);
49 static LIST_HEAD(nfs4_data_server_cache
);
53 print_ds(struct nfs4_pnfs_ds
*ds
)
56 printk("%s NULL device\n", __func__
);
59 printk(" ip_addr %x port %hu\n"
62 " cl_exchange_flags %x\n",
63 ntohl(ds
->ds_ip_addr
), ntohs(ds
->ds_port
),
64 atomic_read(&ds
->ds_count
), ds
->ds_clp
,
65 ds
->ds_clp
? ds
->ds_clp
->cl_exchange_flags
: 0);
68 /* nfs4_ds_cache_lock is held */
69 static struct nfs4_pnfs_ds
*
70 _data_server_lookup_locked(u32 ip_addr
, u32 port
)
72 struct nfs4_pnfs_ds
*ds
;
74 dprintk("_data_server_lookup: ip_addr=%x port=%hu\n",
75 ntohl(ip_addr
), ntohs(port
));
77 list_for_each_entry(ds
, &nfs4_data_server_cache
, ds_node
) {
78 if (ds
->ds_ip_addr
== ip_addr
&&
79 ds
->ds_port
== port
) {
87 * Create an rpc connection to the nfs4_pnfs_ds data server
88 * Currently only support IPv4
91 nfs4_ds_connect(struct nfs_server
*mds_srv
, struct nfs4_pnfs_ds
*ds
)
93 struct nfs_client
*clp
;
94 struct sockaddr_in sin
;
97 dprintk("--> %s ip:port %x:%hu au_flavor %d\n", __func__
,
98 ntohl(ds
->ds_ip_addr
), ntohs(ds
->ds_port
),
99 mds_srv
->nfs_client
->cl_rpcclient
->cl_auth
->au_flavor
);
101 sin
.sin_family
= AF_INET
;
102 sin
.sin_addr
.s_addr
= ds
->ds_ip_addr
;
103 sin
.sin_port
= ds
->ds_port
;
105 clp
= nfs4_set_ds_client(mds_srv
->nfs_client
, (struct sockaddr
*)&sin
,
106 sizeof(sin
), IPPROTO_TCP
);
108 status
= PTR_ERR(clp
);
112 if ((clp
->cl_exchange_flags
& EXCHGID4_FLAG_MASK_PNFS
) != 0) {
113 if (!is_ds_client(clp
)) {
118 dprintk("%s [existing] ip=%x, port=%hu\n", __func__
,
119 ntohl(ds
->ds_ip_addr
), ntohs(ds
->ds_port
));
124 * Do not set NFS_CS_CHECK_LEASE_TIME instead set the DS lease to
125 * be equal to the MDS lease. Renewal is scheduled in create_session.
127 spin_lock(&mds_srv
->nfs_client
->cl_lock
);
128 clp
->cl_lease_time
= mds_srv
->nfs_client
->cl_lease_time
;
129 spin_unlock(&mds_srv
->nfs_client
->cl_lock
);
130 clp
->cl_last_renewal
= jiffies
;
133 status
= nfs4_init_ds_session(clp
);
138 dprintk("%s [new] ip=%x, port=%hu\n", __func__
, ntohl(ds
->ds_ip_addr
),
148 destroy_ds(struct nfs4_pnfs_ds
*ds
)
150 dprintk("--> %s\n", __func__
);
155 nfs_put_client(ds
->ds_clp
);
160 nfs4_fl_free_deviceid(struct nfs4_file_layout_dsaddr
*dsaddr
)
162 struct nfs4_pnfs_ds
*ds
;
165 nfs4_print_deviceid(&dsaddr
->id_node
.deviceid
);
167 for (i
= 0; i
< dsaddr
->ds_num
; i
++) {
168 ds
= dsaddr
->ds_list
[i
];
170 if (atomic_dec_and_lock(&ds
->ds_count
,
171 &nfs4_ds_cache_lock
)) {
172 list_del_init(&ds
->ds_node
);
173 spin_unlock(&nfs4_ds_cache_lock
);
178 kfree(dsaddr
->stripe_indices
);
182 static struct nfs4_pnfs_ds
*
183 nfs4_pnfs_ds_add(struct inode
*inode
, u32 ip_addr
, u32 port
, gfp_t gfp_flags
)
185 struct nfs4_pnfs_ds
*tmp_ds
, *ds
;
187 ds
= kzalloc(sizeof(*tmp_ds
), gfp_flags
);
191 spin_lock(&nfs4_ds_cache_lock
);
192 tmp_ds
= _data_server_lookup_locked(ip_addr
, port
);
193 if (tmp_ds
== NULL
) {
194 ds
->ds_ip_addr
= ip_addr
;
196 atomic_set(&ds
->ds_count
, 1);
197 INIT_LIST_HEAD(&ds
->ds_node
);
199 list_add(&ds
->ds_node
, &nfs4_data_server_cache
);
200 dprintk("%s add new data server ip 0x%x\n", __func__
,
204 atomic_inc(&tmp_ds
->ds_count
);
205 dprintk("%s data server found ip 0x%x, inc'ed ds_count to %d\n",
206 __func__
, tmp_ds
->ds_ip_addr
,
207 atomic_read(&tmp_ds
->ds_count
));
210 spin_unlock(&nfs4_ds_cache_lock
);
216 * Currently only support ipv4, and one multi-path address.
218 static struct nfs4_pnfs_ds
*
219 decode_and_add_ds(struct xdr_stream
*streamp
, struct inode
*inode
, gfp_t gfp_flags
)
221 struct nfs4_pnfs_ds
*ds
= NULL
;
223 const char *ipend
, *pstr
;
230 p
= xdr_inline_decode(streamp
, 4);
233 nlen
= be32_to_cpup(p
++);
235 p
= xdr_inline_decode(streamp
, nlen
);
239 /* Check that netid is "tcp" */
240 if (nlen
!= 3 || memcmp((char *)p
, "tcp", 3)) {
241 dprintk("%s: ERROR: non ipv4 TCP r_netid\n", __func__
);
246 p
= xdr_inline_decode(streamp
, 4);
249 rlen
= be32_to_cpup(p
);
251 p
= xdr_inline_decode(streamp
, rlen
);
255 /* ipv6 length plus port is legal */
256 if (rlen
> INET6_ADDRSTRLEN
+ 8) {
257 dprintk("%s: Invalid address, length %d\n", __func__
,
261 buf
= kmalloc(rlen
+ 1, gfp_flags
);
263 dprintk("%s: Not enough memory\n", __func__
);
267 memcpy(buf
, p
, rlen
);
269 /* replace the port dots with dashes for the in4_pton() delimiter*/
270 for (i
= 0; i
< 2; i
++) {
271 char *res
= strrchr(buf
, '.');
273 dprintk("%s: Failed finding expected dots in port\n",
280 /* Currently only support ipv4 address */
281 if (in4_pton(buf
, rlen
, (u8
*)&ip_addr
, '-', &ipend
) == 0) {
282 dprintk("%s: Only ipv4 addresses supported\n", __func__
);
288 sscanf(pstr
, "-%d-%d", &tmp
[0], &tmp
[1]);
289 port
= htons((tmp
[0] << 8) | (tmp
[1]));
291 ds
= nfs4_pnfs_ds_add(inode
, ip_addr
, port
, gfp_flags
);
292 dprintk("%s: Decoded address and port %s\n", __func__
, buf
);
299 /* Decode opaque device data and return the result */
300 static struct nfs4_file_layout_dsaddr
*
301 decode_device(struct inode
*ino
, struct pnfs_device
*pdev
, gfp_t gfp_flags
)
309 struct nfs4_file_layout_dsaddr
*dsaddr
= NULL
;
310 struct xdr_stream stream
;
311 struct xdr_buf buf
= {
312 .pages
= pdev
->pages
,
313 .page_len
= pdev
->pglen
,
314 .buflen
= pdev
->pglen
,
317 struct page
*scratch
;
319 /* set up xdr stream */
320 scratch
= alloc_page(gfp_flags
);
324 xdr_init_decode(&stream
, &buf
, NULL
);
325 xdr_set_scratch_buffer(&stream
, page_address(scratch
), PAGE_SIZE
);
327 /* Get the stripe count (number of stripe index) */
328 p
= xdr_inline_decode(&stream
, 4);
330 goto out_err_free_scratch
;
332 cnt
= be32_to_cpup(p
);
333 dprintk("%s stripe count %d\n", __func__
, cnt
);
334 if (cnt
> NFS4_PNFS_MAX_STRIPE_CNT
) {
335 printk(KERN_WARNING
"%s: stripe count %d greater than "
336 "supported maximum %d\n", __func__
,
337 cnt
, NFS4_PNFS_MAX_STRIPE_CNT
);
338 goto out_err_free_scratch
;
341 /* read stripe indices */
342 stripe_indices
= kcalloc(cnt
, sizeof(u8
), gfp_flags
);
344 goto out_err_free_scratch
;
346 p
= xdr_inline_decode(&stream
, cnt
<< 2);
348 goto out_err_free_stripe_indices
;
350 indexp
= &stripe_indices
[0];
351 max_stripe_index
= 0;
352 for (i
= 0; i
< cnt
; i
++) {
353 *indexp
= be32_to_cpup(p
++);
354 max_stripe_index
= max(max_stripe_index
, *indexp
);
358 /* Check the multipath list count */
359 p
= xdr_inline_decode(&stream
, 4);
361 goto out_err_free_stripe_indices
;
363 num
= be32_to_cpup(p
);
364 dprintk("%s ds_num %u\n", __func__
, num
);
365 if (num
> NFS4_PNFS_MAX_MULTI_CNT
) {
366 printk(KERN_WARNING
"%s: multipath count %d greater than "
367 "supported maximum %d\n", __func__
,
368 num
, NFS4_PNFS_MAX_MULTI_CNT
);
369 goto out_err_free_stripe_indices
;
372 /* validate stripe indices are all < num */
373 if (max_stripe_index
>= num
) {
374 printk(KERN_WARNING
"%s: stripe index %u >= num ds %u\n",
375 __func__
, max_stripe_index
, num
);
376 goto out_err_free_stripe_indices
;
379 dsaddr
= kzalloc(sizeof(*dsaddr
) +
380 (sizeof(struct nfs4_pnfs_ds
*) * (num
- 1)),
383 goto out_err_free_stripe_indices
;
385 dsaddr
->stripe_count
= cnt
;
386 dsaddr
->stripe_indices
= stripe_indices
;
387 stripe_indices
= NULL
;
388 dsaddr
->ds_num
= num
;
389 nfs4_init_deviceid_node(&dsaddr
->id_node
,
390 NFS_SERVER(ino
)->pnfs_curr_ld
,
391 NFS_SERVER(ino
)->nfs_client
,
394 for (i
= 0; i
< dsaddr
->ds_num
; i
++) {
398 p
= xdr_inline_decode(&stream
, 4);
400 goto out_err_free_deviceid
;
402 mp_count
= be32_to_cpup(p
); /* multipath count */
405 "%s: Multipath count %d not supported, "
406 "skipping all greater than 1\n", __func__
,
409 for (j
= 0; j
< mp_count
; j
++) {
411 dsaddr
->ds_list
[i
] = decode_and_add_ds(&stream
,
413 if (dsaddr
->ds_list
[i
] == NULL
)
414 goto out_err_free_deviceid
;
417 /* skip extra multipath */
420 p
= xdr_inline_decode(&stream
, 4);
422 goto out_err_free_deviceid
;
423 len
= be32_to_cpup(p
);
425 p
= xdr_inline_decode(&stream
, len
);
427 goto out_err_free_deviceid
;
430 p
= xdr_inline_decode(&stream
, 4);
432 goto out_err_free_deviceid
;
433 len
= be32_to_cpup(p
);
435 p
= xdr_inline_decode(&stream
, len
);
437 goto out_err_free_deviceid
;
442 __free_page(scratch
);
445 out_err_free_deviceid
:
446 nfs4_fl_free_deviceid(dsaddr
);
447 /* stripe_indicies was part of dsaddr */
448 goto out_err_free_scratch
;
449 out_err_free_stripe_indices
:
450 kfree(stripe_indices
);
451 out_err_free_scratch
:
452 __free_page(scratch
);
454 dprintk("%s ERROR: returning NULL\n", __func__
);
459 * Decode the opaque device specified in 'dev' and add it to the cache of
462 static struct nfs4_file_layout_dsaddr
*
463 decode_and_add_device(struct inode
*inode
, struct pnfs_device
*dev
, gfp_t gfp_flags
)
465 struct nfs4_deviceid_node
*d
;
466 struct nfs4_file_layout_dsaddr
*n
, *new;
468 new = decode_device(inode
, dev
, gfp_flags
);
470 printk(KERN_WARNING
"%s: Could not decode or add device\n",
475 d
= nfs4_insert_deviceid_node(&new->id_node
);
476 n
= container_of(d
, struct nfs4_file_layout_dsaddr
, id_node
);
478 nfs4_fl_free_deviceid(new);
486 * Retrieve the information for dev_id, add it to the list
487 * of available devices, and return it.
489 struct nfs4_file_layout_dsaddr
*
490 get_device_info(struct inode
*inode
, struct nfs4_deviceid
*dev_id
, gfp_t gfp_flags
)
492 struct pnfs_device
*pdev
= NULL
;
495 struct page
**pages
= NULL
;
496 struct nfs4_file_layout_dsaddr
*dsaddr
= NULL
;
498 struct nfs_server
*server
= NFS_SERVER(inode
);
501 * Use the session max response size as the basis for setting
502 * GETDEVICEINFO's maxcount
504 max_resp_sz
= server
->nfs_client
->cl_session
->fc_attrs
.max_resp_sz
;
505 max_pages
= max_resp_sz
>> PAGE_SHIFT
;
506 dprintk("%s inode %p max_resp_sz %u max_pages %d\n",
507 __func__
, inode
, max_resp_sz
, max_pages
);
509 pdev
= kzalloc(sizeof(struct pnfs_device
), gfp_flags
);
513 pages
= kzalloc(max_pages
* sizeof(struct page
*), gfp_flags
);
518 for (i
= 0; i
< max_pages
; i
++) {
519 pages
[i
] = alloc_page(gfp_flags
);
524 memcpy(&pdev
->dev_id
, dev_id
, sizeof(*dev_id
));
525 pdev
->layout_type
= LAYOUT_NFSV4_1_FILES
;
528 pdev
->pglen
= PAGE_SIZE
* max_pages
;
531 rc
= nfs4_proc_getdeviceinfo(server
, pdev
);
532 dprintk("%s getdevice info returns %d\n", __func__
, rc
);
537 * Found new device, need to decode it and then add it to the
538 * list of known devices for this mountpoint.
540 dsaddr
= decode_and_add_device(inode
, pdev
, gfp_flags
);
542 for (i
= 0; i
< max_pages
; i
++)
543 __free_page(pages
[i
]);
546 dprintk("<-- %s dsaddr %p\n", __func__
, dsaddr
);
551 nfs4_fl_put_deviceid(struct nfs4_file_layout_dsaddr
*dsaddr
)
553 nfs4_put_deviceid_node(&dsaddr
->id_node
);
557 * Want res = (offset - layout->pattern_offset)/ layout->stripe_unit
558 * Then: ((res + fsi) % dsaddr->stripe_count)
561 nfs4_fl_calc_j_index(struct pnfs_layout_segment
*lseg
, loff_t offset
)
563 struct nfs4_filelayout_segment
*flseg
= FILELAYOUT_LSEG(lseg
);
566 tmp
= offset
- flseg
->pattern_offset
;
567 do_div(tmp
, flseg
->stripe_unit
);
568 tmp
+= flseg
->first_stripe_index
;
569 return do_div(tmp
, flseg
->dsaddr
->stripe_count
);
573 nfs4_fl_calc_ds_index(struct pnfs_layout_segment
*lseg
, u32 j
)
575 return FILELAYOUT_LSEG(lseg
)->dsaddr
->stripe_indices
[j
];
579 nfs4_fl_select_ds_fh(struct pnfs_layout_segment
*lseg
, u32 j
)
581 struct nfs4_filelayout_segment
*flseg
= FILELAYOUT_LSEG(lseg
);
584 if (flseg
->stripe_type
== STRIPE_SPARSE
) {
585 if (flseg
->num_fh
== 1)
587 else if (flseg
->num_fh
== 0)
588 /* Use the MDS OPEN fh set in nfs_read_rpcsetup */
591 i
= nfs4_fl_calc_ds_index(lseg
, j
);
594 return flseg
->fh_array
[i
];
598 filelayout_mark_devid_negative(struct nfs4_file_layout_dsaddr
*dsaddr
,
599 int err
, u32 ds_addr
)
601 u32
*p
= (u32
*)&dsaddr
->id_node
.deviceid
;
603 printk(KERN_ERR
"NFS: data server %x connection error %d."
604 " Deviceid [%x%x%x%x] marked out of use.\n",
605 ds_addr
, err
, p
[0], p
[1], p
[2], p
[3]);
607 spin_lock(&nfs4_ds_cache_lock
);
608 dsaddr
->flags
|= NFS4_DEVICE_ID_NEG_ENTRY
;
609 spin_unlock(&nfs4_ds_cache_lock
);
612 struct nfs4_pnfs_ds
*
613 nfs4_fl_prepare_ds(struct pnfs_layout_segment
*lseg
, u32 ds_idx
)
615 struct nfs4_file_layout_dsaddr
*dsaddr
= FILELAYOUT_LSEG(lseg
)->dsaddr
;
616 struct nfs4_pnfs_ds
*ds
= dsaddr
->ds_list
[ds_idx
];
619 printk(KERN_ERR
"%s: No data server for offset index %d\n",
625 struct nfs_server
*s
= NFS_SERVER(lseg
->pls_layout
->plh_inode
);
628 if (dsaddr
->flags
& NFS4_DEVICE_ID_NEG_ENTRY
) {
629 /* Already tried to connect, don't try again */
630 dprintk("%s Deviceid marked out of use\n", __func__
);
633 err
= nfs4_ds_connect(s
, ds
);
635 filelayout_mark_devid_negative(dsaddr
, err
,
636 ntohl(ds
->ds_ip_addr
));