#include <linux/idr.h>
#include <linux/hashtable.h>
-#include "workqueue_sched.h"
+#include "workqueue_internal.h"
enum {
/*
- * global_cwq flags
+ * worker_pool flags
*
- * A bound gcwq is either associated or disassociated with its CPU.
+ * A bound pool is either associated or disassociated with its CPU.
* While associated (!DISASSOCIATED), all workers are bound to the
* CPU and none has %WORKER_UNBOUND set and concurrency management
* is in effect.
*
* While DISASSOCIATED, the cpu may be offline and all workers have
* %WORKER_UNBOUND set and concurrency management disabled, and may
- * be executing on any CPU. The gcwq behaves as an unbound one.
+ * be executing on any CPU. The pool behaves as an unbound one.
*
* Note that DISASSOCIATED can be flipped only while holding
- * assoc_mutex of all pools on the gcwq to avoid changing binding
- * state while create_worker() is in progress.
+ * assoc_mutex to avoid changing binding state while
+ * create_worker() is in progress.
*/
- GCWQ_DISASSOCIATED = 1 << 0, /* cpu can't serve workers */
- GCWQ_FREEZING = 1 << 1, /* freeze in progress */
-
- /* pool flags */
POOL_MANAGE_WORKERS = 1 << 0, /* need to manage workers */
POOL_MANAGING_WORKERS = 1 << 1, /* managing workers */
+ POOL_DISASSOCIATED = 1 << 2, /* cpu can't serve workers */
+ POOL_FREEZING = 1 << 3, /* freeze in progress */
/* worker flags */
WORKER_STARTED = 1 << 0, /* started */
WORKER_NOT_RUNNING = WORKER_PREP | WORKER_UNBOUND |
WORKER_CPU_INTENSIVE,
- NR_WORKER_POOLS = 2, /* # worker pools per gcwq */
+ NR_STD_WORKER_POOLS = 2, /* # standard pools per cpu */
BUSY_WORKER_HASH_ORDER = 6, /* 64 pointers */
* X: During normal operation, modification requires gcwq->lock and
* should be done only from local cpu. Either disabling preemption
* on local cpu or grabbing gcwq->lock is enough for read access.
- * If GCWQ_DISASSOCIATED is set, it's identical to L.
+ * If POOL_DISASSOCIATED is set, it's identical to L.
*
* F: wq->flush_mutex protected.
*
* W: workqueue_lock protected.
*/
-struct global_cwq;
-struct worker_pool;
-
-/*
- * The poor guys doing the actual heavy lifting. All on-duty workers
- * are either serving the manager role, on idle list or on busy hash.
- */
-struct worker {
- /* on idle list while idle, on busy hash table while busy */
- union {
- struct list_head entry; /* L: while idle */
- struct hlist_node hentry; /* L: while busy */
- };
-
- struct work_struct *current_work; /* L: work being processed */
- work_func_t current_func; /* L: current_work's fn */
- struct cpu_workqueue_struct *current_cwq; /* L: current_work's cwq */
- struct list_head scheduled; /* L: scheduled works */
- struct task_struct *task; /* I: worker task */
- struct worker_pool *pool; /* I: the associated pool */
- /* 64 bytes boundary on 64bit, 32 on 32bit */
- unsigned long last_active; /* L: last active timestamp */
- unsigned int flags; /* X: flags */
- int id; /* I: worker id */
-
- /* for rebinding worker to CPU */
- struct work_struct rebind_work; /* L: for busy worker */
-};
+/* struct worker is defined in workqueue_internal.h */
struct worker_pool {
struct global_cwq *gcwq; /* I: the owning gcwq */
+ unsigned int cpu; /* I: the associated cpu */
+ int id; /* I: pool ID */
unsigned int flags; /* X: flags */
struct list_head worklist; /* L: list of pending works */
struct timer_list idle_timer; /* L: worker idle timeout */
struct timer_list mayday_timer; /* L: SOS timer for workers */
- struct mutex assoc_mutex; /* protect GCWQ_DISASSOCIATED */
+ /* workers are chained either in busy_hash or idle_list */
+ DECLARE_HASHTABLE(busy_hash, BUSY_WORKER_HASH_ORDER);
+ /* L: hash of busy workers */
+
+ struct mutex assoc_mutex; /* protect POOL_DISASSOCIATED */
struct ida worker_ida; /* L: for worker IDs */
};
*/
struct global_cwq {
spinlock_t lock; /* the gcwq lock */
- unsigned int cpu; /* I: the associated cpu */
- unsigned int flags; /* L: GCWQ_* flags */
- /* workers are chained either in busy_hash or pool idle_list */
- DECLARE_HASHTABLE(busy_hash, BUSY_WORKER_HASH_ORDER);
- /* L: hash of busy workers */
-
- struct worker_pool pools[NR_WORKER_POOLS];
+ struct worker_pool pools[NR_STD_WORKER_POOLS];
/* normal and highpri pools */
} ____cacheline_aligned_in_smp;
#define for_each_worker_pool(pool, gcwq) \
for ((pool) = &(gcwq)->pools[0]; \
- (pool) < &(gcwq)->pools[NR_WORKER_POOLS]; (pool)++)
+ (pool) < &(gcwq)->pools[NR_STD_WORKER_POOLS]; (pool)++)
-#define for_each_busy_worker(worker, i, pos, gcwq) \
- hash_for_each(gcwq->busy_hash, i, pos, worker, hentry)
+#define for_each_busy_worker(worker, i, pos, pool) \
+ hash_for_each(pool->busy_hash, i, pos, worker, hentry)
static inline int __next_gcwq_cpu(int cpu, const struct cpumask *mask,
unsigned int sw)
* try_to_wake_up(). Put it in a separate cacheline.
*/
static DEFINE_PER_CPU(struct global_cwq, global_cwq);
-static DEFINE_PER_CPU_SHARED_ALIGNED(atomic_t, pool_nr_running[NR_WORKER_POOLS]);
+static DEFINE_PER_CPU_SHARED_ALIGNED(atomic_t, pool_nr_running[NR_STD_WORKER_POOLS]);
/*
- * Global cpu workqueue and nr_running counter for unbound gcwq. The
- * gcwq is always online, has GCWQ_DISASSOCIATED set, and all its
- * workers have WORKER_UNBOUND set.
+ * Global cpu workqueue and nr_running counter for unbound gcwq. The pools
+ * for online CPUs have POOL_DISASSOCIATED set, and all their workers have
+ * WORKER_UNBOUND set.
*/
static struct global_cwq unbound_global_cwq;
-static atomic_t unbound_pool_nr_running[NR_WORKER_POOLS] = {
- [0 ... NR_WORKER_POOLS - 1] = ATOMIC_INIT(0), /* always 0 */
+static atomic_t unbound_pool_nr_running[NR_STD_WORKER_POOLS] = {
+ [0 ... NR_STD_WORKER_POOLS - 1] = ATOMIC_INIT(0), /* always 0 */
};
+/* idr of all pools */
+static DEFINE_MUTEX(worker_pool_idr_mutex);
+static DEFINE_IDR(worker_pool_idr);
+
static int worker_thread(void *__worker);
-static int worker_pool_pri(struct worker_pool *pool)
+static int std_worker_pool_pri(struct worker_pool *pool)
{
return pool - pool->gcwq->pools;
}
return &unbound_global_cwq;
}
+/* allocate ID and assign it to @pool */
+static int worker_pool_assign_id(struct worker_pool *pool)
+{
+ int ret;
+
+ mutex_lock(&worker_pool_idr_mutex);
+ idr_pre_get(&worker_pool_idr, GFP_KERNEL);
+ ret = idr_get_new(&worker_pool_idr, pool, &pool->id);
+ mutex_unlock(&worker_pool_idr_mutex);
+
+ return ret;
+}
+
+/*
+ * Lookup worker_pool by id. The idr currently is built during boot and
+ * never modified. Don't worry about locking for now.
+ */
+static struct worker_pool *worker_pool_by_id(int pool_id)
+{
+ return idr_find(&worker_pool_idr, pool_id);
+}
+
static atomic_t *get_pool_nr_running(struct worker_pool *pool)
{
- int cpu = pool->gcwq->cpu;
- int idx = worker_pool_pri(pool);
+ int cpu = pool->cpu;
+ int idx = std_worker_pool_pri(pool);
if (cpu != WORK_CPU_UNBOUND)
return &per_cpu(pool_nr_running, cpu)[idx];
/*
* While queued, %WORK_STRUCT_CWQ is set and non flag bits of a work's data
* contain the pointer to the queued cwq. Once execution starts, the flag
- * is cleared and the high bits contain OFFQ flags and CPU number.
+ * is cleared and the high bits contain OFFQ flags and pool ID.
*
- * set_work_cwq(), set_work_cpu_and_clear_pending(), mark_work_canceling()
- * and clear_work_data() can be used to set the cwq, cpu or clear
+ * set_work_cwq(), set_work_pool_and_clear_pending(), mark_work_canceling()
+ * and clear_work_data() can be used to set the cwq, pool or clear
* work->data. These functions should only be called while the work is
* owned - ie. while the PENDING bit is set.
*
- * get_work_[g]cwq() can be used to obtain the gcwq or cwq corresponding to
- * a work. gcwq is available once the work has been queued anywhere after
- * initialization until it is sync canceled. cwq is available only while
- * the work item is queued.
+ * get_work_pool() and get_work_cwq() can be used to obtain the pool or cwq
+ * corresponding to a work. Pool is available once the work has been
+ * queued anywhere after initialization until it is sync canceled. cwq is
+ * available only while the work item is queued.
*
* %WORK_OFFQ_CANCELING is used to mark a work item which is being
* canceled. While being canceled, a work item may have its PENDING set
WORK_STRUCT_PENDING | WORK_STRUCT_CWQ | extra_flags);
}
-static void set_work_cpu_and_clear_pending(struct work_struct *work,
- unsigned int cpu)
+static void set_work_pool_and_clear_pending(struct work_struct *work,
+ int pool_id)
{
/*
* The following wmb is paired with the implied mb in
* owner.
*/
smp_wmb();
- set_work_data(work, (unsigned long)cpu << WORK_OFFQ_CPU_SHIFT, 0);
+ set_work_data(work, (unsigned long)pool_id << WORK_OFFQ_POOL_SHIFT, 0);
}
static void clear_work_data(struct work_struct *work)
{
- smp_wmb(); /* see set_work_cpu_and_clear_pending() */
- set_work_data(work, WORK_STRUCT_NO_CPU, 0);
+ smp_wmb(); /* see set_work_pool_and_clear_pending() */
+ set_work_data(work, WORK_STRUCT_NO_POOL, 0);
}
static struct cpu_workqueue_struct *get_work_cwq(struct work_struct *work)
return NULL;
}
-static struct global_cwq *get_work_gcwq(struct work_struct *work)
+/**
+ * get_work_pool - return the worker_pool a given work was associated with
+ * @work: the work item of interest
+ *
+ * Return the worker_pool @work was last associated with. %NULL if none.
+ */
+static struct worker_pool *get_work_pool(struct work_struct *work)
{
unsigned long data = atomic_long_read(&work->data);
- unsigned int cpu;
+ struct worker_pool *pool;
+ int pool_id;
if (data & WORK_STRUCT_CWQ)
return ((struct cpu_workqueue_struct *)
- (data & WORK_STRUCT_WQ_DATA_MASK))->pool->gcwq;
+ (data & WORK_STRUCT_WQ_DATA_MASK))->pool;
- cpu = data >> WORK_OFFQ_CPU_SHIFT;
- if (cpu == WORK_CPU_NONE)
+ pool_id = data >> WORK_OFFQ_POOL_SHIFT;
+ if (pool_id == WORK_OFFQ_POOL_NONE)
return NULL;
- BUG_ON(cpu >= nr_cpu_ids && cpu != WORK_CPU_UNBOUND);
- return get_gcwq(cpu);
+ pool = worker_pool_by_id(pool_id);
+ WARN_ON_ONCE(!pool);
+ return pool;
+}
+
+/**
+ * get_work_pool_id - return the worker pool ID a given work is associated with
+ * @work: the work item of interest
+ *
+ * Return the worker_pool ID @work was last associated with.
+ * %WORK_OFFQ_POOL_NONE if none.
+ */
+static int get_work_pool_id(struct work_struct *work)
+{
+ struct worker_pool *pool = get_work_pool(work);
+
+ return pool ? pool->id : WORK_OFFQ_POOL_NONE;
+}
+
+static struct global_cwq *get_work_gcwq(struct work_struct *work)
+{
+ struct worker_pool *pool = get_work_pool(work);
+
+ return pool ? pool->gcwq : NULL;
}
static void mark_work_canceling(struct work_struct *work)
{
- struct global_cwq *gcwq = get_work_gcwq(work);
- unsigned long cpu = gcwq ? gcwq->cpu : WORK_CPU_NONE;
+ unsigned long pool_id = get_work_pool_id(work);
- set_work_data(work, (cpu << WORK_OFFQ_CPU_SHIFT) | WORK_OFFQ_CANCELING,
- WORK_STRUCT_PENDING);
+ pool_id <<= WORK_OFFQ_POOL_SHIFT;
+ set_work_data(work, pool_id | WORK_OFFQ_CANCELING, WORK_STRUCT_PENDING);
}
static bool work_is_canceling(struct work_struct *work)
struct worker *worker = kthread_data(task);
if (!(worker->flags & WORKER_NOT_RUNNING)) {
- WARN_ON_ONCE(worker->pool->gcwq->cpu != cpu);
+ WARN_ON_ONCE(worker->pool->cpu != cpu);
atomic_inc(get_pool_nr_running(worker->pool));
}
}
unsigned int cpu)
{
struct worker *worker = kthread_data(task), *to_wakeup = NULL;
- struct worker_pool *pool = worker->pool;
- atomic_t *nr_running = get_pool_nr_running(pool);
+ struct worker_pool *pool;
+ atomic_t *nr_running;
+ /*
+ * Rescuers, which may not have all the fields set up like normal
+ * workers, also reach here, let's not access anything before
+ * checking NOT_RUNNING.
+ */
if (worker->flags & WORKER_NOT_RUNNING)
return NULL;
+ pool = worker->pool;
+ nr_running = get_pool_nr_running(pool);
+
/* this can only happen on the local cpu */
BUG_ON(cpu != raw_smp_processor_id());
/**
* find_worker_executing_work - find worker which is executing a work
- * @gcwq: gcwq of interest
+ * @pool: pool of interest
* @work: work to find worker for
*
- * Find a worker which is executing @work on @gcwq by searching
- * @gcwq->busy_hash which is keyed by the address of @work. For a worker
+ * Find a worker which is executing @work on @pool by searching
+ * @pool->busy_hash which is keyed by the address of @work. For a worker
* to match, its current execution should match the address of @work and
* its work function. This is to avoid unwanted dependency between
* unrelated work executions through a work item being recycled while still
* Pointer to worker which is executing @work if found, NULL
* otherwise.
*/
-static struct worker *find_worker_executing_work(struct global_cwq *gcwq,
+static struct worker *find_worker_executing_work(struct worker_pool *pool,
struct work_struct *work)
{
struct worker *worker;
struct hlist_node *tmp;
- hash_for_each_possible(gcwq->busy_hash, worker, tmp, hentry,
+ hash_for_each_possible(pool->busy_hash, worker, tmp, hentry,
(unsigned long)work)
if (worker->current_work == work &&
worker->current_func == work->func)
unsigned int cpu;
for_each_gcwq_cpu(cpu) {
- struct global_cwq *gcwq = get_gcwq(cpu);
+ struct cpu_workqueue_struct *cwq = get_cwq(cpu, wq);
+ struct worker_pool *pool = cwq->pool;
+ struct global_cwq *gcwq = pool->gcwq;
struct worker *worker;
struct hlist_node *pos;
int i;
spin_lock_irqsave(&gcwq->lock, flags);
- for_each_busy_worker(worker, i, pos, gcwq) {
+ for_each_busy_worker(worker, i, pos, pool) {
if (worker->task != current)
continue;
spin_unlock_irqrestore(&gcwq->lock, flags);
/* determine gcwq to use */
if (!(wq->flags & WQ_UNBOUND)) {
- struct global_cwq *last_gcwq;
+ struct worker_pool *last_pool;
if (cpu == WORK_CPU_UNBOUND)
cpu = raw_smp_processor_id();
* non-reentrancy.
*/
gcwq = get_gcwq(cpu);
- last_gcwq = get_work_gcwq(work);
+ last_pool = get_work_pool(work);
- if (last_gcwq && last_gcwq != gcwq) {
+ if (last_pool && last_pool->gcwq != gcwq) {
+ struct global_cwq *last_gcwq = last_pool->gcwq;
struct worker *worker;
spin_lock(&last_gcwq->lock);
- worker = find_worker_executing_work(last_gcwq, work);
+ worker = find_worker_executing_work(last_pool, work);
if (worker && worker->current_cwq->wq == wq)
gcwq = last_gcwq;
}
/* gcwq determined, get cwq and queue */
- cwq = get_cwq(gcwq->cpu, wq);
+ cwq = get_cwq(gcwq->pools[0].cpu, wq);
trace_workqueue_queue_work(req_cpu, cwq, work);
if (WARN_ON(!list_empty(&work->entry))) {
/*
* This stores cwq for the moment, for the timer_fn. Note that the
- * work's gcwq is preserved to allow reentrance detection for
+ * work's pool is preserved to allow reentrance detection for
* delayed works.
*/
if (!(wq->flags & WQ_UNBOUND)) {
- struct global_cwq *gcwq = get_work_gcwq(work);
+ struct worker_pool *pool = get_work_pool(work);
/*
- * If we cannot get the last gcwq from @work directly,
+ * If we cannot get the last pool from @work directly,
* select the last CPU such that it avoids unnecessarily
* triggering non-reentrancy check in __queue_work().
*/
lcpu = cpu;
- if (gcwq)
- lcpu = gcwq->cpu;
+ if (pool)
+ lcpu = pool->cpu;
if (lcpu == WORK_CPU_UNBOUND)
lcpu = raw_smp_processor_id();
} else {
static void worker_enter_idle(struct worker *worker)
{
struct worker_pool *pool = worker->pool;
- struct global_cwq *gcwq = pool->gcwq;
BUG_ON(worker->flags & WORKER_IDLE);
BUG_ON(!list_empty(&worker->entry) &&
* nr_running, the warning may trigger spuriously. Check iff
* unbind is not in progress.
*/
- WARN_ON_ONCE(!(gcwq->flags & GCWQ_DISASSOCIATED) &&
+ WARN_ON_ONCE(!(pool->flags & POOL_DISASSOCIATED) &&
pool->nr_workers == pool->nr_idle &&
atomic_read(get_pool_nr_running(pool)));
}
* [dis]associated in the meantime.
*
* This function tries set_cpus_allowed() and locks gcwq and verifies the
- * binding against %GCWQ_DISASSOCIATED which is set during
+ * binding against %POOL_DISASSOCIATED which is set during
* %CPU_DOWN_PREPARE and cleared during %CPU_ONLINE, so if the worker
* enters idle state or fetches works without dropping lock, it can
* guarantee the scheduling requirement described in the first paragraph.
static bool worker_maybe_bind_and_lock(struct worker *worker)
__acquires(&gcwq->lock)
{
- struct global_cwq *gcwq = worker->pool->gcwq;
+ struct worker_pool *pool = worker->pool;
+ struct global_cwq *gcwq = pool->gcwq;
struct task_struct *task = worker->task;
while (true) {
* The following call may fail, succeed or succeed
* without actually migrating the task to the cpu if
* it races with cpu hotunplug operation. Verify
- * against GCWQ_DISASSOCIATED.
+ * against POOL_DISASSOCIATED.
*/
- if (!(gcwq->flags & GCWQ_DISASSOCIATED))
- set_cpus_allowed_ptr(task, get_cpu_mask(gcwq->cpu));
+ if (!(pool->flags & POOL_DISASSOCIATED))
+ set_cpus_allowed_ptr(task, get_cpu_mask(pool->cpu));
spin_lock_irq(&gcwq->lock);
- if (gcwq->flags & GCWQ_DISASSOCIATED)
+ if (pool->flags & POOL_DISASSOCIATED)
return false;
- if (task_cpu(task) == gcwq->cpu &&
+ if (task_cpu(task) == pool->cpu &&
cpumask_equal(¤t->cpus_allowed,
- get_cpu_mask(gcwq->cpu)))
+ get_cpu_mask(pool->cpu)))
return true;
spin_unlock_irq(&gcwq->lock);
*/
wake_up_process(worker->task);
}
- }
- /* rebind busy workers */
- for_each_busy_worker(worker, i, pos, gcwq) {
- struct work_struct *rebind_work = &worker->rebind_work;
- struct workqueue_struct *wq;
+ /* rebind busy workers */
+ for_each_busy_worker(worker, i, pos, pool) {
+ struct work_struct *rebind_work = &worker->rebind_work;
+ struct workqueue_struct *wq;
- if (test_and_set_bit(WORK_STRUCT_PENDING_BIT,
- work_data_bits(rebind_work)))
- continue;
-
- debug_work_activate(rebind_work);
+ if (test_and_set_bit(WORK_STRUCT_PENDING_BIT,
+ work_data_bits(rebind_work)))
+ continue;
- /*
- * wq doesn't really matter but let's keep @worker->pool
- * and @cwq->pool consistent for sanity.
- */
- if (worker_pool_pri(worker->pool))
- wq = system_highpri_wq;
- else
- wq = system_wq;
+ debug_work_activate(rebind_work);
- insert_work(get_cwq(gcwq->cpu, wq), rebind_work,
- worker->scheduled.next,
- work_color_to_flags(WORK_NO_COLOR));
+ /*
+ * wq doesn't really matter but let's keep
+ * @worker->pool and @cwq->pool consistent for
+ * sanity.
+ */
+ if (std_worker_pool_pri(worker->pool))
+ wq = system_highpri_wq;
+ else
+ wq = system_wq;
+
+ insert_work(get_cwq(pool->cpu, wq), rebind_work,
+ worker->scheduled.next,
+ work_color_to_flags(WORK_NO_COLOR));
+ }
}
}
static struct worker *create_worker(struct worker_pool *pool)
{
struct global_cwq *gcwq = pool->gcwq;
- const char *pri = worker_pool_pri(pool) ? "H" : "";
+ const char *pri = std_worker_pool_pri(pool) ? "H" : "";
struct worker *worker = NULL;
int id = -1;
worker->pool = pool;
worker->id = id;
- if (gcwq->cpu != WORK_CPU_UNBOUND)
+ if (pool->cpu != WORK_CPU_UNBOUND)
worker->task = kthread_create_on_node(worker_thread,
- worker, cpu_to_node(gcwq->cpu),
- "kworker/%u:%d%s", gcwq->cpu, id, pri);
+ worker, cpu_to_node(pool->cpu),
+ "kworker/%u:%d%s", pool->cpu, id, pri);
else
worker->task = kthread_create(worker_thread, worker,
"kworker/u:%d%s", id, pri);
if (IS_ERR(worker->task))
goto fail;
- if (worker_pool_pri(pool))
+ if (std_worker_pool_pri(pool))
set_user_nice(worker->task, HIGHPRI_NICE_LEVEL);
/*
* Determine CPU binding of the new worker depending on
- * %GCWQ_DISASSOCIATED. The caller is responsible for ensuring the
+ * %POOL_DISASSOCIATED. The caller is responsible for ensuring the
* flag remains stable across this function. See the comments
* above the flag definition for details.
*
* As an unbound worker may later become a regular one if CPU comes
* online, make sure every worker has %PF_THREAD_BOUND set.
*/
- if (!(gcwq->flags & GCWQ_DISASSOCIATED)) {
- kthread_bind(worker->task, gcwq->cpu);
+ if (!(pool->flags & POOL_DISASSOCIATED)) {
+ kthread_bind(worker->task, pool->cpu);
} else {
worker->task->flags |= PF_THREAD_BOUND;
worker->flags |= WORKER_UNBOUND;
return false;
/* mayday mayday mayday */
- cpu = cwq->pool->gcwq->cpu;
+ cpu = cwq->pool->cpu;
/* WORK_CPU_UNBOUND can't be set in cpumask, use cpu 0 instead */
if (cpu == WORK_CPU_UNBOUND)
cpu = 0;
/*
* Ensure we're on the correct CPU. DISASSOCIATED test is
* necessary to avoid spurious warnings from rescuers servicing the
- * unbound or a disassociated gcwq.
+ * unbound or a disassociated pool.
*/
WARN_ON_ONCE(!(worker->flags & WORKER_UNBOUND) &&
- !(gcwq->flags & GCWQ_DISASSOCIATED) &&
- raw_smp_processor_id() != gcwq->cpu);
+ !(pool->flags & POOL_DISASSOCIATED) &&
+ raw_smp_processor_id() != pool->cpu);
/*
* A single work shouldn't be executed concurrently by
* already processing the work. If so, defer the work to the
* currently executing one.
*/
- collision = find_worker_executing_work(gcwq, work);
+ collision = find_worker_executing_work(pool, work);
if (unlikely(collision)) {
move_linked_works(work, &collision->scheduled, NULL);
return;
/* claim and dequeue */
debug_work_deactivate(work);
- hash_add(gcwq->busy_hash, &worker->hentry, (unsigned long)work);
+ hash_add(pool->busy_hash, &worker->hentry, (unsigned long)work);
worker->current_work = work;
worker->current_func = work->func;
worker->current_cwq = cwq;
wake_up_worker(pool);
/*
- * Record the last CPU and clear PENDING which should be the last
+ * Record the last pool and clear PENDING which should be the last
* update to @work. Also, do this inside @gcwq->lock so that
* PENDING and queued state changes happen together while IRQ is
* disabled.
*/
- set_work_cpu_and_clear_pending(work, gcwq->cpu);
+ set_work_pool_and_clear_pending(work, pool->id);
spin_unlock_irq(&gcwq->lock);
/**
* rescuer_thread - the rescuer thread function
- * @__wq: the associated workqueue
+ * @__rescuer: self
*
* Workqueue rescuer thread function. There's one rescuer for each
* workqueue which has WQ_RESCUER set.
*
* This should happen rarely.
*/
-static int rescuer_thread(void *__wq)
+static int rescuer_thread(void *__rescuer)
{
- struct workqueue_struct *wq = __wq;
- struct worker *rescuer = wq->rescuer;
+ struct worker *rescuer = __rescuer;
+ struct workqueue_struct *wq = rescuer->rescue_wq;
struct list_head *scheduled = &rescuer->scheduled;
bool is_unbound = wq->flags & WQ_UNBOUND;
unsigned int cpu;
set_user_nice(current, RESCUER_NICE_LEVEL);
+
+ /*
+ * Mark rescuer as worker too. As WORKER_PREP is never cleared, it
+ * doesn't participate in concurrency management.
+ */
+ rescuer->task->flags |= PF_WQ_WORKER;
repeat:
set_current_state(TASK_INTERRUPTIBLE);
if (kthread_should_stop()) {
__set_current_state(TASK_RUNNING);
+ rescuer->task->flags &= ~PF_WQ_WORKER;
return 0;
}
spin_unlock_irq(&gcwq->lock);
}
+ /* rescuers should never participate in concurrency management */
+ WARN_ON_ONCE(!(rescuer->flags & WORKER_NOT_RUNNING));
schedule();
goto repeat;
}
static bool start_flush_work(struct work_struct *work, struct wq_barrier *barr)
{
struct worker *worker = NULL;
+ struct worker_pool *pool;
struct global_cwq *gcwq;
struct cpu_workqueue_struct *cwq;
might_sleep();
- gcwq = get_work_gcwq(work);
- if (!gcwq)
+ pool = get_work_pool(work);
+ if (!pool)
return false;
+ gcwq = pool->gcwq;
spin_lock_irq(&gcwq->lock);
if (!list_empty(&work->entry)) {
if (unlikely(!cwq || gcwq != cwq->pool->gcwq))
goto already_gone;
} else {
- worker = find_worker_executing_work(gcwq, work);
+ worker = find_worker_executing_work(pool, work);
if (!worker)
goto already_gone;
cwq = worker->current_cwq;
if (unlikely(ret < 0))
return false;
- set_work_cpu_and_clear_pending(&dwork->work, work_cpu(&dwork->work));
+ set_work_pool_and_clear_pending(&dwork->work,
+ get_work_pool_id(&dwork->work));
local_irq_restore(flags);
return ret;
}
if (!rescuer)
goto err;
- rescuer->task = kthread_create(rescuer_thread, wq, "%s",
+ rescuer->rescue_wq = wq;
+ rescuer->task = kthread_create(rescuer_thread, rescuer, "%s",
wq->name);
if (IS_ERR(rescuer->task))
goto err;
wq->saved_max_active = max_active;
for_each_cwq_cpu(cpu, wq) {
- struct global_cwq *gcwq = get_gcwq(cpu);
+ struct cpu_workqueue_struct *cwq = get_cwq(cpu, wq);
+ struct worker_pool *pool = cwq->pool;
+ struct global_cwq *gcwq = pool->gcwq;
spin_lock_irq(&gcwq->lock);
if (!(wq->flags & WQ_FREEZABLE) ||
- !(gcwq->flags & GCWQ_FREEZING))
- cwq_set_max_active(get_cwq(gcwq->cpu, wq), max_active);
+ !(pool->flags & POOL_FREEZING))
+ cwq_set_max_active(cwq, max_active);
spin_unlock_irq(&gcwq->lock);
}
}
EXPORT_SYMBOL_GPL(workqueue_congested);
-/**
- * work_cpu - return the last known associated cpu for @work
- * @work: the work of interest
- *
- * RETURNS:
- * CPU number if @work was ever queued. WORK_CPU_NONE otherwise.
- */
-unsigned int work_cpu(struct work_struct *work)
-{
- struct global_cwq *gcwq = get_work_gcwq(work);
-
- return gcwq ? gcwq->cpu : WORK_CPU_NONE;
-}
-EXPORT_SYMBOL_GPL(work_cpu);
-
/**
* work_busy - test whether a work is currently pending or running
* @work: the work to be tested
*/
unsigned int work_busy(struct work_struct *work)
{
- struct global_cwq *gcwq = get_work_gcwq(work);
+ struct worker_pool *pool = get_work_pool(work);
+ struct global_cwq *gcwq;
unsigned long flags;
unsigned int ret = 0;
- if (!gcwq)
+ if (!pool)
return 0;
+ gcwq = pool->gcwq;
spin_lock_irqsave(&gcwq->lock, flags);
if (work_pending(work))
ret |= WORK_BUSY_PENDING;
- if (find_worker_executing_work(gcwq, work))
+ if (find_worker_executing_work(pool, work))
ret |= WORK_BUSY_RUNNING;
spin_unlock_irqrestore(&gcwq->lock, flags);
* gcwqs serve mix of short, long and very long running works making
* blocked draining impractical.
*
- * This is solved by allowing a gcwq to be disassociated from the CPU
+ * This is solved by allowing the pools to be disassociated from the CPU
* running as an unbound one and allowing it to be reattached later if the
* cpu comes back online.
*/
struct hlist_node *pos;
int i;
- BUG_ON(gcwq->cpu != smp_processor_id());
+ BUG_ON(gcwq->pools[0].cpu != smp_processor_id());
gcwq_claim_assoc_and_lock(gcwq);
* ones which are still executing works from before the last CPU
* down must be on the cpu. After this, they may become diasporas.
*/
- for_each_worker_pool(pool, gcwq)
+ for_each_worker_pool(pool, gcwq) {
list_for_each_entry(worker, &pool->idle_list, entry)
worker->flags |= WORKER_UNBOUND;
- for_each_busy_worker(worker, i, pos, gcwq)
- worker->flags |= WORKER_UNBOUND;
+ for_each_busy_worker(worker, i, pos, pool)
+ worker->flags |= WORKER_UNBOUND;
- gcwq->flags |= GCWQ_DISASSOCIATED;
+ pool->flags |= POOL_DISASSOCIATED;
+ }
gcwq_release_assoc_and_unlock(gcwq);
case CPU_DOWN_FAILED:
case CPU_ONLINE:
gcwq_claim_assoc_and_lock(gcwq);
- gcwq->flags &= ~GCWQ_DISASSOCIATED;
+ for_each_worker_pool(pool, gcwq)
+ pool->flags &= ~POOL_DISASSOCIATED;
rebind_workers(gcwq);
gcwq_release_assoc_and_unlock(gcwq);
break;
for_each_gcwq_cpu(cpu) {
struct global_cwq *gcwq = get_gcwq(cpu);
+ struct worker_pool *pool;
struct workqueue_struct *wq;
spin_lock_irq(&gcwq->lock);
- BUG_ON(gcwq->flags & GCWQ_FREEZING);
- gcwq->flags |= GCWQ_FREEZING;
+ for_each_worker_pool(pool, gcwq) {
+ WARN_ON_ONCE(pool->flags & POOL_FREEZING);
+ pool->flags |= POOL_FREEZING;
+ }
list_for_each_entry(wq, &workqueues, list) {
struct cpu_workqueue_struct *cwq = get_cwq(cpu, wq);
spin_lock_irq(&gcwq->lock);
- BUG_ON(!(gcwq->flags & GCWQ_FREEZING));
- gcwq->flags &= ~GCWQ_FREEZING;
+ for_each_worker_pool(pool, gcwq) {
+ WARN_ON_ONCE(!(pool->flags & POOL_FREEZING));
+ pool->flags &= ~POOL_FREEZING;
+ }
list_for_each_entry(wq, &workqueues, list) {
struct cpu_workqueue_struct *cwq = get_cwq(cpu, wq);
{
unsigned int cpu;
- /* make sure we have enough bits for OFFQ CPU number */
- BUILD_BUG_ON((1LU << (BITS_PER_LONG - WORK_OFFQ_CPU_SHIFT)) <
- WORK_CPU_LAST);
+ /* make sure we have enough bits for OFFQ pool ID */
+ BUILD_BUG_ON((1LU << (BITS_PER_LONG - WORK_OFFQ_POOL_SHIFT)) <
+ WORK_CPU_LAST * NR_STD_WORKER_POOLS);
cpu_notifier(workqueue_cpu_up_callback, CPU_PRI_WORKQUEUE_UP);
hotcpu_notifier(workqueue_cpu_down_callback, CPU_PRI_WORKQUEUE_DOWN);
struct worker_pool *pool;
spin_lock_init(&gcwq->lock);
- gcwq->cpu = cpu;
- gcwq->flags |= GCWQ_DISASSOCIATED;
-
- hash_init(gcwq->busy_hash);
for_each_worker_pool(pool, gcwq) {
pool->gcwq = gcwq;
+ pool->cpu = cpu;
+ pool->flags |= POOL_DISASSOCIATED;
INIT_LIST_HEAD(&pool->worklist);
INIT_LIST_HEAD(&pool->idle_list);
+ hash_init(pool->busy_hash);
init_timer_deferrable(&pool->idle_timer);
pool->idle_timer.function = idle_worker_timeout;
mutex_init(&pool->assoc_mutex);
ida_init(&pool->worker_ida);
+
+ /* alloc pool ID */
+ BUG_ON(worker_pool_assign_id(pool));
}
}
struct global_cwq *gcwq = get_gcwq(cpu);
struct worker_pool *pool;
- if (cpu != WORK_CPU_UNBOUND)
- gcwq->flags &= ~GCWQ_DISASSOCIATED;
-
for_each_worker_pool(pool, gcwq) {
struct worker *worker;
+ if (cpu != WORK_CPU_UNBOUND)
+ pool->flags &= ~POOL_DISASSOCIATED;
+
worker = create_worker(pool);
BUG_ON(!worker);
spin_lock_irq(&gcwq->lock);
projects / deliverable / linux.git / blobdiff