2 * Request reply cache. This is currently a global cache, but this may
3 * change in the future and be a per-client cache.
5 * This code is heavily inspired by the 44BSD implementation, although
6 * it does things a bit differently.
8 * Copyright (C) 1995, 1996 Olaf Kirch <okir@monad.swb.de>
11 #include <linux/slab.h>
12 #include <linux/sunrpc/addr.h>
13 #include <linux/highmem.h>
14 #include <net/checksum.h>
19 #define NFSDDBG_FACILITY NFSDDBG_REPCACHE
23 static struct hlist_head
* cache_hash
;
24 static struct list_head lru_head
;
25 static struct kmem_cache
*drc_slab
;
26 static unsigned int num_drc_entries
;
27 static unsigned int max_drc_entries
;
30 * Calculate the hash index from an XID.
32 static inline u32
request_hash(u32 xid
)
36 return h
& (HASHSIZE
-1);
39 static int nfsd_cache_append(struct svc_rqst
*rqstp
, struct kvec
*vec
);
40 static void cache_cleaner_func(struct work_struct
*unused
);
41 static int nfsd_reply_cache_shrink(struct shrinker
*shrink
,
42 struct shrink_control
*sc
);
44 struct shrinker nfsd_reply_cache_shrinker
= {
45 .shrink
= nfsd_reply_cache_shrink
,
50 * locking for the reply cache:
51 * A cache entry is "single use" if c_state == RC_INPROG
52 * Otherwise, it when accessing _prev or _next, the lock must be held.
54 static DEFINE_SPINLOCK(cache_lock
);
55 static DECLARE_DELAYED_WORK(cache_cleaner
, cache_cleaner_func
);
58 * Put a cap on the size of the DRC based on the amount of available
59 * low memory in the machine.
71 * ...with a hard cap of 256k entries. In the worst case, each entry will be
72 * ~1k, so the above numbers should give a rough max of the amount of memory
76 nfsd_cache_size_limit(void)
79 unsigned long low_pages
= totalram_pages
- totalhigh_pages
;
81 limit
= (16 * int_sqrt(low_pages
)) << (PAGE_SHIFT
-10);
82 return min_t(unsigned int, limit
, 256*1024);
85 static struct svc_cacherep
*
86 nfsd_reply_cache_alloc(void)
88 struct svc_cacherep
*rp
;
90 rp
= kmem_cache_alloc(drc_slab
, GFP_KERNEL
);
92 rp
->c_state
= RC_UNUSED
;
93 rp
->c_type
= RC_NOCACHE
;
94 INIT_LIST_HEAD(&rp
->c_lru
);
95 INIT_HLIST_NODE(&rp
->c_hash
);
101 nfsd_reply_cache_free_locked(struct svc_cacherep
*rp
)
103 if (rp
->c_type
== RC_REPLBUFF
)
104 kfree(rp
->c_replvec
.iov_base
);
105 hlist_del(&rp
->c_hash
);
106 list_del(&rp
->c_lru
);
108 kmem_cache_free(drc_slab
, rp
);
112 nfsd_reply_cache_free(struct svc_cacherep
*rp
)
114 spin_lock(&cache_lock
);
115 nfsd_reply_cache_free_locked(rp
);
116 spin_unlock(&cache_lock
);
119 int nfsd_reply_cache_init(void)
121 register_shrinker(&nfsd_reply_cache_shrinker
);
122 drc_slab
= kmem_cache_create("nfsd_drc", sizeof(struct svc_cacherep
),
127 cache_hash
= kcalloc(HASHSIZE
, sizeof(struct hlist_head
), GFP_KERNEL
);
131 INIT_LIST_HEAD(&lru_head
);
132 max_drc_entries
= nfsd_cache_size_limit();
137 printk(KERN_ERR
"nfsd: failed to allocate reply cache\n");
138 nfsd_reply_cache_shutdown();
142 void nfsd_reply_cache_shutdown(void)
144 struct svc_cacherep
*rp
;
146 unregister_shrinker(&nfsd_reply_cache_shrinker
);
147 cancel_delayed_work_sync(&cache_cleaner
);
149 while (!list_empty(&lru_head
)) {
150 rp
= list_entry(lru_head
.next
, struct svc_cacherep
, c_lru
);
151 nfsd_reply_cache_free_locked(rp
);
158 kmem_cache_destroy(drc_slab
);
164 * Move cache entry to end of LRU list, and queue the cleaner to run if it's
165 * not already scheduled.
168 lru_put_end(struct svc_cacherep
*rp
)
170 rp
->c_timestamp
= jiffies
;
171 list_move_tail(&rp
->c_lru
, &lru_head
);
172 schedule_delayed_work(&cache_cleaner
, RC_EXPIRE
);
176 * Move a cache entry from one hash list to another
179 hash_refile(struct svc_cacherep
*rp
)
181 hlist_del_init(&rp
->c_hash
);
182 hlist_add_head(&rp
->c_hash
, cache_hash
+ request_hash(rp
->c_xid
));
186 nfsd_cache_entry_expired(struct svc_cacherep
*rp
)
188 return rp
->c_state
!= RC_INPROG
&&
189 time_after(jiffies
, rp
->c_timestamp
+ RC_EXPIRE
);
193 * Walk the LRU list and prune off entries that are older than RC_EXPIRE.
194 * Also prune the oldest ones when the total exceeds the max number of entries.
197 prune_cache_entries(void)
199 struct svc_cacherep
*rp
, *tmp
;
201 list_for_each_entry_safe(rp
, tmp
, &lru_head
, c_lru
) {
202 if (!nfsd_cache_entry_expired(rp
) &&
203 num_drc_entries
<= max_drc_entries
)
205 nfsd_reply_cache_free_locked(rp
);
209 * Conditionally rearm the job. If we cleaned out the list, then
210 * cancel any pending run (since there won't be any work to do).
211 * Otherwise, we rearm the job or modify the existing one to run in
212 * RC_EXPIRE since we just ran the pruner.
214 if (list_empty(&lru_head
))
215 cancel_delayed_work(&cache_cleaner
);
217 mod_delayed_work(system_wq
, &cache_cleaner
, RC_EXPIRE
);
221 cache_cleaner_func(struct work_struct
*unused
)
223 spin_lock(&cache_lock
);
224 prune_cache_entries();
225 spin_unlock(&cache_lock
);
229 nfsd_reply_cache_shrink(struct shrinker
*shrink
, struct shrink_control
*sc
)
233 spin_lock(&cache_lock
);
235 prune_cache_entries();
236 num
= num_drc_entries
;
237 spin_unlock(&cache_lock
);
243 * Walk an xdr_buf and get a CRC for at most the first RC_CSUMLEN bytes
246 nfsd_cache_csum(struct svc_rqst
*rqstp
)
251 struct xdr_buf
*buf
= &rqstp
->rq_arg
;
252 const unsigned char *p
= buf
->head
[0].iov_base
;
253 size_t csum_len
= min_t(size_t, buf
->head
[0].iov_len
+ buf
->page_len
,
255 size_t len
= min(buf
->head
[0].iov_len
, csum_len
);
257 /* rq_arg.head first */
258 csum
= csum_partial(p
, len
, 0);
261 /* Continue into page array */
262 idx
= buf
->page_base
/ PAGE_SIZE
;
263 base
= buf
->page_base
& ~PAGE_MASK
;
265 p
= page_address(buf
->pages
[idx
]) + base
;
266 len
= min_t(size_t, PAGE_SIZE
- base
, csum_len
);
267 csum
= csum_partial(p
, len
, csum
);
276 * Search the request hash for an entry that matches the given rqstp.
277 * Must be called with cache_lock held. Returns the found entry or
280 static struct svc_cacherep
*
281 nfsd_cache_search(struct svc_rqst
*rqstp
, __wsum csum
)
283 struct svc_cacherep
*rp
;
284 struct hlist_head
*rh
;
285 __be32 xid
= rqstp
->rq_xid
;
286 u32 proto
= rqstp
->rq_prot
,
287 vers
= rqstp
->rq_vers
,
288 proc
= rqstp
->rq_proc
;
290 rh
= &cache_hash
[request_hash(xid
)];
291 hlist_for_each_entry(rp
, rh
, c_hash
) {
292 if (xid
== rp
->c_xid
&& proc
== rp
->c_proc
&&
293 proto
== rp
->c_prot
&& vers
== rp
->c_vers
&&
294 rqstp
->rq_arg
.len
== rp
->c_len
&& csum
== rp
->c_csum
&&
295 rpc_cmp_addr(svc_addr(rqstp
), (struct sockaddr
*)&rp
->c_addr
) &&
296 rpc_get_port(svc_addr(rqstp
)) == rpc_get_port((struct sockaddr
*)&rp
->c_addr
))
303 * Try to find an entry matching the current call in the cache. When none
304 * is found, we try to grab the oldest expired entry off the LRU list. If
305 * a suitable one isn't there, then drop the cache_lock and allocate a
306 * new one, then search again in case one got inserted while this thread
307 * didn't hold the lock.
310 nfsd_cache_lookup(struct svc_rqst
*rqstp
)
312 struct svc_cacherep
*rp
, *found
;
313 __be32 xid
= rqstp
->rq_xid
;
314 u32 proto
= rqstp
->rq_prot
,
315 vers
= rqstp
->rq_vers
,
316 proc
= rqstp
->rq_proc
;
319 int type
= rqstp
->rq_cachetype
;
322 rqstp
->rq_cacherep
= NULL
;
323 if (type
== RC_NOCACHE
) {
324 nfsdstats
.rcnocache
++;
328 csum
= nfsd_cache_csum(rqstp
);
330 spin_lock(&cache_lock
);
333 rp
= nfsd_cache_search(rqstp
, csum
);
337 /* Try to use the first entry on the LRU */
338 if (!list_empty(&lru_head
)) {
339 rp
= list_first_entry(&lru_head
, struct svc_cacherep
, c_lru
);
340 if (nfsd_cache_entry_expired(rp
) ||
341 num_drc_entries
>= max_drc_entries
) {
343 prune_cache_entries();
348 /* Drop the lock and allocate a new entry */
349 spin_unlock(&cache_lock
);
350 rp
= nfsd_reply_cache_alloc();
352 dprintk("nfsd: unable to allocate DRC entry!\n");
355 spin_lock(&cache_lock
);
359 * Must search again just in case someone inserted one
360 * after we dropped the lock above.
362 found
= nfsd_cache_search(rqstp
, csum
);
364 nfsd_reply_cache_free_locked(rp
);
370 * We're keeping the one we just allocated. Are we now over the
371 * limit? Prune one off the tip of the LRU in trade for the one we
372 * just allocated if so.
374 if (num_drc_entries
>= max_drc_entries
)
375 nfsd_reply_cache_free_locked(list_first_entry(&lru_head
,
376 struct svc_cacherep
, c_lru
));
379 nfsdstats
.rcmisses
++;
380 rqstp
->rq_cacherep
= rp
;
381 rp
->c_state
= RC_INPROG
;
384 rpc_copy_addr((struct sockaddr
*)&rp
->c_addr
, svc_addr(rqstp
));
385 rpc_set_port((struct sockaddr
*)&rp
->c_addr
, rpc_get_port(svc_addr(rqstp
)));
388 rp
->c_len
= rqstp
->rq_arg
.len
;
394 /* release any buffer */
395 if (rp
->c_type
== RC_REPLBUFF
) {
396 kfree(rp
->c_replvec
.iov_base
);
397 rp
->c_replvec
.iov_base
= NULL
;
399 rp
->c_type
= RC_NOCACHE
;
401 spin_unlock(&cache_lock
);
406 /* We found a matching entry which is either in progress or done. */
407 age
= jiffies
- rp
->c_timestamp
;
411 /* Request being processed or excessive rexmits */
412 if (rp
->c_state
== RC_INPROG
|| age
< RC_DELAY
)
415 /* From the hall of fame of impractical attacks:
416 * Is this a user who tries to snoop on the cache? */
418 if (!rqstp
->rq_secure
&& rp
->c_secure
)
421 /* Compose RPC reply header */
422 switch (rp
->c_type
) {
426 svc_putu32(&rqstp
->rq_res
.head
[0], rp
->c_replstat
);
430 if (!nfsd_cache_append(rqstp
, &rp
->c_replvec
))
431 goto out
; /* should not happen */
435 printk(KERN_WARNING
"nfsd: bad repcache type %d\n", rp
->c_type
);
436 nfsd_reply_cache_free_locked(rp
);
443 * Update a cache entry. This is called from nfsd_dispatch when
444 * the procedure has been executed and the complete reply is in
447 * We're copying around data here rather than swapping buffers because
448 * the toplevel loop requires max-sized buffers, which would be a waste
449 * of memory for a cache with a max reply size of 100 bytes (diropokres).
451 * If we should start to use different types of cache entries tailored
452 * specifically for attrstat and fh's, we may save even more space.
454 * Also note that a cachetype of RC_NOCACHE can legally be passed when
455 * nfsd failed to encode a reply that otherwise would have been cached.
456 * In this case, nfsd_cache_update is called with statp == NULL.
459 nfsd_cache_update(struct svc_rqst
*rqstp
, int cachetype
, __be32
*statp
)
461 struct svc_cacherep
*rp
= rqstp
->rq_cacherep
;
462 struct kvec
*resv
= &rqstp
->rq_res
.head
[0], *cachv
;
468 len
= resv
->iov_len
- ((char*)statp
- (char*)resv
->iov_base
);
471 /* Don't cache excessive amounts of data and XDR failures */
472 if (!statp
|| len
> (256 >> 2)) {
473 nfsd_reply_cache_free(rp
);
480 printk("nfsd: RC_REPLSTAT/reply len %d!\n",len
);
481 rp
->c_replstat
= *statp
;
484 cachv
= &rp
->c_replvec
;
485 cachv
->iov_base
= kmalloc(len
<< 2, GFP_KERNEL
);
486 if (!cachv
->iov_base
) {
487 nfsd_reply_cache_free(rp
);
490 cachv
->iov_len
= len
<< 2;
491 memcpy(cachv
->iov_base
, statp
, len
<< 2);
494 nfsd_reply_cache_free(rp
);
497 spin_lock(&cache_lock
);
499 rp
->c_secure
= rqstp
->rq_secure
;
500 rp
->c_type
= cachetype
;
501 rp
->c_state
= RC_DONE
;
502 spin_unlock(&cache_lock
);
507 * Copy cached reply to current reply buffer. Should always fit.
508 * FIXME as reply is in a page, we should just attach the page, and
509 * keep a refcount....
512 nfsd_cache_append(struct svc_rqst
*rqstp
, struct kvec
*data
)
514 struct kvec
*vec
= &rqstp
->rq_res
.head
[0];
516 if (vec
->iov_len
+ data
->iov_len
> PAGE_SIZE
) {
517 printk(KERN_WARNING
"nfsd: cached reply too large (%Zd).\n",
521 memcpy((char*)vec
->iov_base
+ vec
->iov_len
, data
->iov_base
, data
->iov_len
);
522 vec
->iov_len
+= data
->iov_len
;