struct worker_pool {
struct global_cwq *gcwq; /* I: the owning gcwq */
+ unsigned int cpu; /* I: the associated cpu */
+ int id; /* I: pool ID */
unsigned int flags; /* X: flags */
struct list_head worklist; /* L: list of pending works */
struct timer_list idle_timer; /* L: worker idle timeout */
struct timer_list mayday_timer; /* L: SOS timer for workers */
+ /* workers are chained either in busy_hash or idle_list */
+ DECLARE_HASHTABLE(busy_hash, BUSY_WORKER_HASH_ORDER);
+ /* L: hash of busy workers */
+
struct mutex assoc_mutex; /* protect POOL_DISASSOCIATED */
struct ida worker_ida; /* L: for worker IDs */
};
*/
struct global_cwq {
spinlock_t lock; /* the gcwq lock */
- unsigned int cpu; /* I: the associated cpu */
-
- /* workers are chained either in busy_hash or pool idle_list */
- DECLARE_HASHTABLE(busy_hash, BUSY_WORKER_HASH_ORDER);
- /* L: hash of busy workers */
struct worker_pool pools[NR_STD_WORKER_POOLS];
/* normal and highpri pools */
for ((pool) = &(gcwq)->pools[0]; \
(pool) < &(gcwq)->pools[NR_STD_WORKER_POOLS]; (pool)++)
-#define for_each_busy_worker(worker, i, pos, gcwq) \
- hash_for_each(gcwq->busy_hash, i, pos, worker, hentry)
+#define for_each_busy_worker(worker, i, pos, pool) \
+ hash_for_each(pool->busy_hash, i, pos, worker, hentry)
static inline int __next_gcwq_cpu(int cpu, const struct cpumask *mask,
unsigned int sw)
[0 ... NR_STD_WORKER_POOLS - 1] = ATOMIC_INIT(0), /* always 0 */
};
+/* idr of all pools */
+static DEFINE_MUTEX(worker_pool_idr_mutex);
+static DEFINE_IDR(worker_pool_idr);
+
static int worker_thread(void *__worker);
-static unsigned int work_cpu(struct work_struct *work);
static int std_worker_pool_pri(struct worker_pool *pool)
{
return &unbound_global_cwq;
}
+/* allocate ID and assign it to @pool */
+static int worker_pool_assign_id(struct worker_pool *pool)
+{
+ int ret;
+
+ mutex_lock(&worker_pool_idr_mutex);
+ idr_pre_get(&worker_pool_idr, GFP_KERNEL);
+ ret = idr_get_new(&worker_pool_idr, pool, &pool->id);
+ mutex_unlock(&worker_pool_idr_mutex);
+
+ return ret;
+}
+
+/*
+ * Lookup worker_pool by id. The idr currently is built during boot and
+ * never modified. Don't worry about locking for now.
+ */
+static struct worker_pool *worker_pool_by_id(int pool_id)
+{
+ return idr_find(&worker_pool_idr, pool_id);
+}
+
static atomic_t *get_pool_nr_running(struct worker_pool *pool)
{
- int cpu = pool->gcwq->cpu;
+ int cpu = pool->cpu;
int idx = std_worker_pool_pri(pool);
if (cpu != WORK_CPU_UNBOUND)
/*
* While queued, %WORK_STRUCT_CWQ is set and non flag bits of a work's data
* contain the pointer to the queued cwq. Once execution starts, the flag
- * is cleared and the high bits contain OFFQ flags and CPU number.
+ * is cleared and the high bits contain OFFQ flags and pool ID.
*
- * set_work_cwq(), set_work_cpu_and_clear_pending(), mark_work_canceling()
- * and clear_work_data() can be used to set the cwq, cpu or clear
+ * set_work_cwq(), set_work_pool_and_clear_pending(), mark_work_canceling()
+ * and clear_work_data() can be used to set the cwq, pool or clear
* work->data. These functions should only be called while the work is
* owned - ie. while the PENDING bit is set.
*
- * get_work_[g]cwq() can be used to obtain the gcwq or cwq corresponding to
- * a work. gcwq is available once the work has been queued anywhere after
- * initialization until it is sync canceled. cwq is available only while
- * the work item is queued.
+ * get_work_pool() and get_work_cwq() can be used to obtain the pool or cwq
+ * corresponding to a work. Pool is available once the work has been
+ * queued anywhere after initialization until it is sync canceled. cwq is
+ * available only while the work item is queued.
*
* %WORK_OFFQ_CANCELING is used to mark a work item which is being
* canceled. While being canceled, a work item may have its PENDING set
WORK_STRUCT_PENDING | WORK_STRUCT_CWQ | extra_flags);
}
-static void set_work_cpu_and_clear_pending(struct work_struct *work,
- unsigned int cpu)
+static void set_work_pool_and_clear_pending(struct work_struct *work,
+ int pool_id)
{
/*
* The following wmb is paired with the implied mb in
* owner.
*/
smp_wmb();
- set_work_data(work, (unsigned long)cpu << WORK_OFFQ_CPU_SHIFT, 0);
+ set_work_data(work, (unsigned long)pool_id << WORK_OFFQ_POOL_SHIFT, 0);
}
static void clear_work_data(struct work_struct *work)
{
- smp_wmb(); /* see set_work_cpu_and_clear_pending() */
- set_work_data(work, WORK_STRUCT_NO_CPU, 0);
+ smp_wmb(); /* see set_work_pool_and_clear_pending() */
+ set_work_data(work, WORK_STRUCT_NO_POOL, 0);
}
static struct cpu_workqueue_struct *get_work_cwq(struct work_struct *work)
return NULL;
}
-static struct global_cwq *get_work_gcwq(struct work_struct *work)
+/**
+ * get_work_pool - return the worker_pool a given work was associated with
+ * @work: the work item of interest
+ *
+ * Return the worker_pool @work was last associated with. %NULL if none.
+ */
+static struct worker_pool *get_work_pool(struct work_struct *work)
{
unsigned long data = atomic_long_read(&work->data);
- unsigned int cpu;
+ struct worker_pool *pool;
+ int pool_id;
if (data & WORK_STRUCT_CWQ)
return ((struct cpu_workqueue_struct *)
- (data & WORK_STRUCT_WQ_DATA_MASK))->pool->gcwq;
+ (data & WORK_STRUCT_WQ_DATA_MASK))->pool;
- cpu = data >> WORK_OFFQ_CPU_SHIFT;
- if (cpu == WORK_CPU_NONE)
+ pool_id = data >> WORK_OFFQ_POOL_SHIFT;
+ if (pool_id == WORK_OFFQ_POOL_NONE)
return NULL;
- BUG_ON(cpu >= nr_cpu_ids && cpu != WORK_CPU_UNBOUND);
- return get_gcwq(cpu);
+ pool = worker_pool_by_id(pool_id);
+ WARN_ON_ONCE(!pool);
+ return pool;
+}
+
+/**
+ * get_work_pool_id - return the worker pool ID a given work is associated with
+ * @work: the work item of interest
+ *
+ * Return the worker_pool ID @work was last associated with.
+ * %WORK_OFFQ_POOL_NONE if none.
+ */
+static int get_work_pool_id(struct work_struct *work)
+{
+ struct worker_pool *pool = get_work_pool(work);
+
+ return pool ? pool->id : WORK_OFFQ_POOL_NONE;
+}
+
+static struct global_cwq *get_work_gcwq(struct work_struct *work)
+{
+ struct worker_pool *pool = get_work_pool(work);
+
+ return pool ? pool->gcwq : NULL;
}
static void mark_work_canceling(struct work_struct *work)
{
- struct global_cwq *gcwq = get_work_gcwq(work);
- unsigned long cpu = gcwq ? gcwq->cpu : WORK_CPU_NONE;
+ unsigned long pool_id = get_work_pool_id(work);
- set_work_data(work, (cpu << WORK_OFFQ_CPU_SHIFT) | WORK_OFFQ_CANCELING,
- WORK_STRUCT_PENDING);
+ pool_id <<= WORK_OFFQ_POOL_SHIFT;
+ set_work_data(work, pool_id | WORK_OFFQ_CANCELING, WORK_STRUCT_PENDING);
}
static bool work_is_canceling(struct work_struct *work)
struct worker *worker = kthread_data(task);
if (!(worker->flags & WORKER_NOT_RUNNING)) {
- WARN_ON_ONCE(worker->pool->gcwq->cpu != cpu);
+ WARN_ON_ONCE(worker->pool->cpu != cpu);
atomic_inc(get_pool_nr_running(worker->pool));
}
}
/**
* find_worker_executing_work - find worker which is executing a work
- * @gcwq: gcwq of interest
+ * @pool: pool of interest
* @work: work to find worker for
*
- * Find a worker which is executing @work on @gcwq by searching
- * @gcwq->busy_hash which is keyed by the address of @work. For a worker
+ * Find a worker which is executing @work on @pool by searching
+ * @pool->busy_hash which is keyed by the address of @work. For a worker
* to match, its current execution should match the address of @work and
* its work function. This is to avoid unwanted dependency between
* unrelated work executions through a work item being recycled while still
* Pointer to worker which is executing @work if found, NULL
* otherwise.
*/
-static struct worker *find_worker_executing_work(struct global_cwq *gcwq,
+static struct worker *find_worker_executing_work(struct worker_pool *pool,
struct work_struct *work)
{
struct worker *worker;
struct hlist_node *tmp;
- hash_for_each_possible(gcwq->busy_hash, worker, tmp, hentry,
+ hash_for_each_possible(pool->busy_hash, worker, tmp, hentry,
(unsigned long)work)
if (worker->current_work == work &&
worker->current_func == work->func)
unsigned int cpu;
for_each_gcwq_cpu(cpu) {
- struct global_cwq *gcwq = get_gcwq(cpu);
+ struct cpu_workqueue_struct *cwq = get_cwq(cpu, wq);
+ struct worker_pool *pool = cwq->pool;
+ struct global_cwq *gcwq = pool->gcwq;
struct worker *worker;
struct hlist_node *pos;
int i;
spin_lock_irqsave(&gcwq->lock, flags);
- for_each_busy_worker(worker, i, pos, gcwq) {
+ for_each_busy_worker(worker, i, pos, pool) {
if (worker->task != current)
continue;
spin_unlock_irqrestore(&gcwq->lock, flags);
/* determine gcwq to use */
if (!(wq->flags & WQ_UNBOUND)) {
- struct global_cwq *last_gcwq;
+ struct worker_pool *last_pool;
if (cpu == WORK_CPU_UNBOUND)
cpu = raw_smp_processor_id();
* non-reentrancy.
*/
gcwq = get_gcwq(cpu);
- last_gcwq = get_work_gcwq(work);
+ last_pool = get_work_pool(work);
- if (last_gcwq && last_gcwq != gcwq) {
+ if (last_pool && last_pool->gcwq != gcwq) {
+ struct global_cwq *last_gcwq = last_pool->gcwq;
struct worker *worker;
spin_lock(&last_gcwq->lock);
- worker = find_worker_executing_work(last_gcwq, work);
+ worker = find_worker_executing_work(last_pool, work);
if (worker && worker->current_cwq->wq == wq)
gcwq = last_gcwq;
}
/* gcwq determined, get cwq and queue */
- cwq = get_cwq(gcwq->cpu, wq);
+ cwq = get_cwq(gcwq->pools[0].cpu, wq);
trace_workqueue_queue_work(req_cpu, cwq, work);
if (WARN_ON(!list_empty(&work->entry))) {
/*
* This stores cwq for the moment, for the timer_fn. Note that the
- * work's gcwq is preserved to allow reentrance detection for
+ * work's pool is preserved to allow reentrance detection for
* delayed works.
*/
if (!(wq->flags & WQ_UNBOUND)) {
- struct global_cwq *gcwq = get_work_gcwq(work);
+ struct worker_pool *pool = get_work_pool(work);
/*
- * If we cannot get the last gcwq from @work directly,
+ * If we cannot get the last pool from @work directly,
* select the last CPU such that it avoids unnecessarily
* triggering non-reentrancy check in __queue_work().
*/
lcpu = cpu;
- if (gcwq)
- lcpu = gcwq->cpu;
+ if (pool)
+ lcpu = pool->cpu;
if (lcpu == WORK_CPU_UNBOUND)
lcpu = raw_smp_processor_id();
} else {
* against POOL_DISASSOCIATED.
*/
if (!(pool->flags & POOL_DISASSOCIATED))
- set_cpus_allowed_ptr(task, get_cpu_mask(gcwq->cpu));
+ set_cpus_allowed_ptr(task, get_cpu_mask(pool->cpu));
spin_lock_irq(&gcwq->lock);
if (pool->flags & POOL_DISASSOCIATED)
return false;
- if (task_cpu(task) == gcwq->cpu &&
+ if (task_cpu(task) == pool->cpu &&
cpumask_equal(¤t->cpus_allowed,
- get_cpu_mask(gcwq->cpu)))
+ get_cpu_mask(pool->cpu)))
return true;
spin_unlock_irq(&gcwq->lock);
*/
wake_up_process(worker->task);
}
- }
- /* rebind busy workers */
- for_each_busy_worker(worker, i, pos, gcwq) {
- struct work_struct *rebind_work = &worker->rebind_work;
- struct workqueue_struct *wq;
-
- if (test_and_set_bit(WORK_STRUCT_PENDING_BIT,
- work_data_bits(rebind_work)))
- continue;
+ /* rebind busy workers */
+ for_each_busy_worker(worker, i, pos, pool) {
+ struct work_struct *rebind_work = &worker->rebind_work;
+ struct workqueue_struct *wq;
- debug_work_activate(rebind_work);
+ if (test_and_set_bit(WORK_STRUCT_PENDING_BIT,
+ work_data_bits(rebind_work)))
+ continue;
- /*
- * wq doesn't really matter but let's keep @worker->pool
- * and @cwq->pool consistent for sanity.
- */
- if (std_worker_pool_pri(worker->pool))
- wq = system_highpri_wq;
- else
- wq = system_wq;
+ debug_work_activate(rebind_work);
- insert_work(get_cwq(gcwq->cpu, wq), rebind_work,
- worker->scheduled.next,
- work_color_to_flags(WORK_NO_COLOR));
+ /*
+ * wq doesn't really matter but let's keep
+ * @worker->pool and @cwq->pool consistent for
+ * sanity.
+ */
+ if (std_worker_pool_pri(worker->pool))
+ wq = system_highpri_wq;
+ else
+ wq = system_wq;
+
+ insert_work(get_cwq(pool->cpu, wq), rebind_work,
+ worker->scheduled.next,
+ work_color_to_flags(WORK_NO_COLOR));
+ }
}
}
worker->pool = pool;
worker->id = id;
- if (gcwq->cpu != WORK_CPU_UNBOUND)
+ if (pool->cpu != WORK_CPU_UNBOUND)
worker->task = kthread_create_on_node(worker_thread,
- worker, cpu_to_node(gcwq->cpu),
- "kworker/%u:%d%s", gcwq->cpu, id, pri);
+ worker, cpu_to_node(pool->cpu),
+ "kworker/%u:%d%s", pool->cpu, id, pri);
else
worker->task = kthread_create(worker_thread, worker,
"kworker/u:%d%s", id, pri);
* online, make sure every worker has %PF_THREAD_BOUND set.
*/
if (!(pool->flags & POOL_DISASSOCIATED)) {
- kthread_bind(worker->task, gcwq->cpu);
+ kthread_bind(worker->task, pool->cpu);
} else {
worker->task->flags |= PF_THREAD_BOUND;
worker->flags |= WORKER_UNBOUND;
return false;
/* mayday mayday mayday */
- cpu = cwq->pool->gcwq->cpu;
+ cpu = cwq->pool->cpu;
/* WORK_CPU_UNBOUND can't be set in cpumask, use cpu 0 instead */
if (cpu == WORK_CPU_UNBOUND)
cpu = 0;
*/
WARN_ON_ONCE(!(worker->flags & WORKER_UNBOUND) &&
!(pool->flags & POOL_DISASSOCIATED) &&
- raw_smp_processor_id() != gcwq->cpu);
+ raw_smp_processor_id() != pool->cpu);
/*
* A single work shouldn't be executed concurrently by
* already processing the work. If so, defer the work to the
* currently executing one.
*/
- collision = find_worker_executing_work(gcwq, work);
+ collision = find_worker_executing_work(pool, work);
if (unlikely(collision)) {
move_linked_works(work, &collision->scheduled, NULL);
return;
/* claim and dequeue */
debug_work_deactivate(work);
- hash_add(gcwq->busy_hash, &worker->hentry, (unsigned long)work);
+ hash_add(pool->busy_hash, &worker->hentry, (unsigned long)work);
worker->current_work = work;
worker->current_func = work->func;
worker->current_cwq = cwq;
wake_up_worker(pool);
/*
- * Record the last CPU and clear PENDING which should be the last
+ * Record the last pool and clear PENDING which should be the last
* update to @work. Also, do this inside @gcwq->lock so that
* PENDING and queued state changes happen together while IRQ is
* disabled.
*/
- set_work_cpu_and_clear_pending(work, gcwq->cpu);
+ set_work_pool_and_clear_pending(work, pool->id);
spin_unlock_irq(&gcwq->lock);
static bool start_flush_work(struct work_struct *work, struct wq_barrier *barr)
{
struct worker *worker = NULL;
+ struct worker_pool *pool;
struct global_cwq *gcwq;
struct cpu_workqueue_struct *cwq;
might_sleep();
- gcwq = get_work_gcwq(work);
- if (!gcwq)
+ pool = get_work_pool(work);
+ if (!pool)
return false;
+ gcwq = pool->gcwq;
spin_lock_irq(&gcwq->lock);
if (!list_empty(&work->entry)) {
if (unlikely(!cwq || gcwq != cwq->pool->gcwq))
goto already_gone;
} else {
- worker = find_worker_executing_work(gcwq, work);
+ worker = find_worker_executing_work(pool, work);
if (!worker)
goto already_gone;
cwq = worker->current_cwq;
if (unlikely(ret < 0))
return false;
- set_work_cpu_and_clear_pending(&dwork->work, work_cpu(&dwork->work));
+ set_work_pool_and_clear_pending(&dwork->work,
+ get_work_pool_id(&dwork->work));
local_irq_restore(flags);
return ret;
}
}
EXPORT_SYMBOL_GPL(workqueue_congested);
-/**
- * work_cpu - return the last known associated cpu for @work
- * @work: the work of interest
- *
- * RETURNS:
- * CPU number if @work was ever queued. WORK_CPU_NONE otherwise.
- */
-static unsigned int work_cpu(struct work_struct *work)
-{
- struct global_cwq *gcwq = get_work_gcwq(work);
-
- return gcwq ? gcwq->cpu : WORK_CPU_NONE;
-}
-
/**
* work_busy - test whether a work is currently pending or running
* @work: the work to be tested
*/
unsigned int work_busy(struct work_struct *work)
{
- struct global_cwq *gcwq = get_work_gcwq(work);
+ struct worker_pool *pool = get_work_pool(work);
+ struct global_cwq *gcwq;
unsigned long flags;
unsigned int ret = 0;
- if (!gcwq)
+ if (!pool)
return 0;
+ gcwq = pool->gcwq;
spin_lock_irqsave(&gcwq->lock, flags);
if (work_pending(work))
ret |= WORK_BUSY_PENDING;
- if (find_worker_executing_work(gcwq, work))
+ if (find_worker_executing_work(pool, work))
ret |= WORK_BUSY_RUNNING;
spin_unlock_irqrestore(&gcwq->lock, flags);
struct hlist_node *pos;
int i;
- BUG_ON(gcwq->cpu != smp_processor_id());
+ BUG_ON(gcwq->pools[0].cpu != smp_processor_id());
gcwq_claim_assoc_and_lock(gcwq);
* ones which are still executing works from before the last CPU
* down must be on the cpu. After this, they may become diasporas.
*/
- for_each_worker_pool(pool, gcwq)
+ for_each_worker_pool(pool, gcwq) {
list_for_each_entry(worker, &pool->idle_list, entry)
worker->flags |= WORKER_UNBOUND;
- for_each_busy_worker(worker, i, pos, gcwq)
- worker->flags |= WORKER_UNBOUND;
+ for_each_busy_worker(worker, i, pos, pool)
+ worker->flags |= WORKER_UNBOUND;
- for_each_worker_pool(pool, gcwq)
pool->flags |= POOL_DISASSOCIATED;
+ }
gcwq_release_assoc_and_unlock(gcwq);
{
unsigned int cpu;
- /* make sure we have enough bits for OFFQ CPU number */
- BUILD_BUG_ON((1LU << (BITS_PER_LONG - WORK_OFFQ_CPU_SHIFT)) <
- WORK_CPU_LAST);
+ /* make sure we have enough bits for OFFQ pool ID */
+ BUILD_BUG_ON((1LU << (BITS_PER_LONG - WORK_OFFQ_POOL_SHIFT)) <
+ WORK_CPU_LAST * NR_STD_WORKER_POOLS);
cpu_notifier(workqueue_cpu_up_callback, CPU_PRI_WORKQUEUE_UP);
hotcpu_notifier(workqueue_cpu_down_callback, CPU_PRI_WORKQUEUE_DOWN);
struct worker_pool *pool;
spin_lock_init(&gcwq->lock);
- gcwq->cpu = cpu;
-
- hash_init(gcwq->busy_hash);
for_each_worker_pool(pool, gcwq) {
pool->gcwq = gcwq;
+ pool->cpu = cpu;
pool->flags |= POOL_DISASSOCIATED;
INIT_LIST_HEAD(&pool->worklist);
INIT_LIST_HEAD(&pool->idle_list);
+ hash_init(pool->busy_hash);
init_timer_deferrable(&pool->idle_timer);
pool->idle_timer.function = idle_worker_timeout;
mutex_init(&pool->assoc_mutex);
ida_init(&pool->worker_ida);
+
+ /* alloc pool ID */
+ BUG_ON(worker_pool_assign_id(pool));
}
}
projects / deliverable / linux.git / blobdiff