2 * Asynchronous refcounty things
4 * Copyright 2010, 2011 Kent Overstreet <kent.overstreet@gmail.com>
5 * Copyright 2012 Google, Inc.
8 #include <linux/debugfs.h>
9 #include <linux/module.h>
10 #include <linux/seq_file.h>
14 void closure_queue(struct closure
*cl
)
16 struct workqueue_struct
*wq
= cl
->wq
;
18 INIT_WORK(&cl
->work
, cl
->work
.func
);
19 BUG_ON(!queue_work(wq
, &cl
->work
));
23 EXPORT_SYMBOL_GPL(closure_queue
);
25 #define CL_FIELD(type, field) \
27 return &container_of(cl, struct type, cl)->field
29 static struct closure_waitlist
*closure_waitlist(struct closure
*cl
)
32 CL_FIELD(closure_with_waitlist
, wait
);
33 CL_FIELD(closure_with_waitlist_and_timer
, wait
);
39 static struct timer_list
*closure_timer(struct closure
*cl
)
42 CL_FIELD(closure_with_timer
, timer
);
43 CL_FIELD(closure_with_waitlist_and_timer
, timer
);
49 static inline void closure_put_after_sub(struct closure
*cl
, int flags
)
51 int r
= flags
& CLOSURE_REMAINING_MASK
;
53 BUG_ON(flags
& CLOSURE_GUARD_MASK
);
54 BUG_ON(!r
&& (flags
& ~(CLOSURE_DESTRUCTOR
|CLOSURE_BLOCKING
)));
56 /* Must deliver precisely one wakeup */
57 if (r
== 1 && (flags
& CLOSURE_SLEEPING
))
58 wake_up_process(cl
->task
);
61 if (cl
->fn
&& !(flags
& CLOSURE_DESTRUCTOR
)) {
62 /* CLOSURE_BLOCKING might be set - clear it */
63 atomic_set(&cl
->remaining
,
64 CLOSURE_REMAINING_INITIALIZER
);
67 struct closure
*parent
= cl
->parent
;
68 struct closure_waitlist
*wait
= closure_waitlist(cl
);
70 closure_debug_destroy(cl
);
72 atomic_set(&cl
->remaining
, -1);
75 closure_wake_up(wait
);
86 /* For clearing flags with the same atomic op as a put */
87 void closure_sub(struct closure
*cl
, int v
)
89 closure_put_after_sub(cl
, atomic_sub_return(v
, &cl
->remaining
));
91 EXPORT_SYMBOL_GPL(closure_sub
);
93 void closure_put(struct closure
*cl
)
95 closure_put_after_sub(cl
, atomic_dec_return(&cl
->remaining
));
97 EXPORT_SYMBOL_GPL(closure_put
);
99 static void set_waiting(struct closure
*cl
, unsigned long f
)
101 #ifdef CONFIG_BCACHE_CLOSURES_DEBUG
106 void __closure_wake_up(struct closure_waitlist
*wait_list
)
108 struct llist_node
*list
;
110 struct llist_node
*reverse
= NULL
;
112 list
= llist_del_all(&wait_list
->list
);
114 /* We first reverse the list to preserve FIFO ordering and fairness */
117 struct llist_node
*t
= list
;
118 list
= llist_next(list
);
124 /* Then do the wakeups */
127 cl
= container_of(reverse
, struct closure
, list
);
128 reverse
= llist_next(reverse
);
131 closure_sub(cl
, CLOSURE_WAITING
+ 1);
134 EXPORT_SYMBOL_GPL(__closure_wake_up
);
136 bool closure_wait(struct closure_waitlist
*list
, struct closure
*cl
)
138 if (atomic_read(&cl
->remaining
) & CLOSURE_WAITING
)
141 set_waiting(cl
, _RET_IP_
);
142 atomic_add(CLOSURE_WAITING
+ 1, &cl
->remaining
);
143 llist_add(&cl
->list
, &list
->list
);
147 EXPORT_SYMBOL_GPL(closure_wait
);
150 * closure_sync() - sleep until a closure a closure has nothing left to wait on
152 * Sleeps until the refcount hits 1 - the thread that's running the closure owns
155 void closure_sync(struct closure
*cl
)
158 __closure_start_sleep(cl
);
159 closure_set_ret_ip(cl
);
161 if ((atomic_read(&cl
->remaining
) &
162 CLOSURE_REMAINING_MASK
) == 1)
168 __closure_end_sleep(cl
);
170 EXPORT_SYMBOL_GPL(closure_sync
);
173 * closure_trylock() - try to acquire the closure, without waiting
174 * @cl: closure to lock
176 * Returns true if the closure was succesfully locked.
178 bool closure_trylock(struct closure
*cl
, struct closure
*parent
)
180 if (atomic_cmpxchg(&cl
->remaining
, -1,
181 CLOSURE_REMAINING_INITIALIZER
) != -1)
184 closure_set_ret_ip(cl
);
191 closure_debug_create(cl
);
194 EXPORT_SYMBOL_GPL(closure_trylock
);
196 void __closure_lock(struct closure
*cl
, struct closure
*parent
,
197 struct closure_waitlist
*wait_list
)
200 closure_init_stack(&wait
);
203 if (closure_trylock(cl
, parent
))
206 closure_wait_event_sync(wait_list
, &wait
,
207 atomic_read(&cl
->remaining
) == -1);
210 EXPORT_SYMBOL_GPL(__closure_lock
);
212 static void closure_delay_timer_fn(unsigned long data
)
214 struct closure
*cl
= (struct closure
*) data
;
215 closure_sub(cl
, CLOSURE_TIMER
+ 1);
218 void do_closure_timer_init(struct closure
*cl
)
220 struct timer_list
*timer
= closure_timer(cl
);
223 timer
->data
= (unsigned long) cl
;
224 timer
->function
= closure_delay_timer_fn
;
226 EXPORT_SYMBOL_GPL(do_closure_timer_init
);
228 bool __closure_delay(struct closure
*cl
, unsigned long delay
,
229 struct timer_list
*timer
)
231 if (atomic_read(&cl
->remaining
) & CLOSURE_TIMER
)
234 BUG_ON(timer_pending(timer
));
236 timer
->expires
= jiffies
+ delay
;
238 atomic_add(CLOSURE_TIMER
+ 1, &cl
->remaining
);
242 EXPORT_SYMBOL_GPL(__closure_delay
);
244 void __closure_flush(struct closure
*cl
, struct timer_list
*timer
)
246 if (del_timer(timer
))
247 closure_sub(cl
, CLOSURE_TIMER
+ 1);
249 EXPORT_SYMBOL_GPL(__closure_flush
);
251 void __closure_flush_sync(struct closure
*cl
, struct timer_list
*timer
)
253 if (del_timer_sync(timer
))
254 closure_sub(cl
, CLOSURE_TIMER
+ 1);
256 EXPORT_SYMBOL_GPL(__closure_flush_sync
);
258 #ifdef CONFIG_BCACHE_CLOSURES_DEBUG
260 static LIST_HEAD(closure_list
);
261 static DEFINE_SPINLOCK(closure_list_lock
);
263 void closure_debug_create(struct closure
*cl
)
267 BUG_ON(cl
->magic
== CLOSURE_MAGIC_ALIVE
);
268 cl
->magic
= CLOSURE_MAGIC_ALIVE
;
270 spin_lock_irqsave(&closure_list_lock
, flags
);
271 list_add(&cl
->all
, &closure_list
);
272 spin_unlock_irqrestore(&closure_list_lock
, flags
);
274 EXPORT_SYMBOL_GPL(closure_debug_create
);
276 void closure_debug_destroy(struct closure
*cl
)
280 BUG_ON(cl
->magic
!= CLOSURE_MAGIC_ALIVE
);
281 cl
->magic
= CLOSURE_MAGIC_DEAD
;
283 spin_lock_irqsave(&closure_list_lock
, flags
);
285 spin_unlock_irqrestore(&closure_list_lock
, flags
);
287 EXPORT_SYMBOL_GPL(closure_debug_destroy
);
289 static struct dentry
*debug
;
291 #define work_data_bits(work) ((unsigned long *)(&(work)->data))
293 static int debug_seq_show(struct seq_file
*f
, void *data
)
296 spin_lock_irq(&closure_list_lock
);
298 list_for_each_entry(cl
, &closure_list
, all
) {
299 int r
= atomic_read(&cl
->remaining
);
301 seq_printf(f
, "%p: %pF -> %pf p %p r %i ",
302 cl
, (void *) cl
->ip
, cl
->fn
, cl
->parent
,
303 r
& CLOSURE_REMAINING_MASK
);
305 seq_printf(f
, "%s%s%s%s%s%s\n",
306 test_bit(WORK_STRUCT_PENDING
,
307 work_data_bits(&cl
->work
)) ? "Q" : "",
308 r
& CLOSURE_RUNNING
? "R" : "",
309 r
& CLOSURE_BLOCKING
? "B" : "",
310 r
& CLOSURE_STACK
? "S" : "",
311 r
& CLOSURE_SLEEPING
? "Sl" : "",
312 r
& CLOSURE_TIMER
? "T" : "");
314 if (r
& CLOSURE_WAITING
)
315 seq_printf(f
, " W %pF\n",
316 (void *) cl
->waiting_on
);
321 spin_unlock_irq(&closure_list_lock
);
325 static int debug_seq_open(struct inode
*inode
, struct file
*file
)
327 return single_open(file
, debug_seq_show
, NULL
);
330 static const struct file_operations debug_ops
= {
331 .owner
= THIS_MODULE
,
332 .open
= debug_seq_open
,
334 .release
= single_release
337 void __init
closure_debug_init(void)
339 debug
= debugfs_create_file("closures", 0400, NULL
, NULL
, &debug_ops
);
344 MODULE_AUTHOR("Kent Overstreet <koverstreet@google.com>");
345 MODULE_LICENSE("GPL");