version 1.29, 2005/02/12 21:39:00 |
version 1.29.6.10, 2008/02/27 08:36:55 |
|
|
/* $NetBSD$ */ |
/* $NetBSD$ */ |
|
|
/*- |
/*- |
* Copyright (c) 2001 The NetBSD Foundation, Inc. |
* Copyright (c) 2001, 2006, 2007 The NetBSD Foundation, Inc. |
* All rights reserved. |
* All rights reserved. |
* |
* |
* This code is derived from software contributed to The NetBSD Foundation |
* This code is derived from software contributed to The NetBSD Foundation |
* by Nathan J. Williams. |
* by Nathan J. Williams, and Andrew Doran. |
* |
* |
* Redistribution and use in source and binary forms, with or without |
* Redistribution and use in source and binary forms, with or without |
* modification, are permitted provided that the following conditions |
* modification, are permitted provided that the following conditions |
|
|
* POSSIBILITY OF SUCH DAMAGE. |
* POSSIBILITY OF SUCH DAMAGE. |
*/ |
*/ |
|
|
|
/* |
|
* Overview |
|
* |
|
* Lightweight processes (LWPs) are the basic unit or thread of |
|
* execution within the kernel. The core state of an LWP is described |
|
* by "struct lwp", also known as lwp_t. |
|
* |
|
* Each LWP is contained within a process (described by "struct proc"), |
|
* Every process contains at least one LWP, but may contain more. The |
|
* process describes attributes shared among all of its LWPs such as a |
|
* private address space, global execution state (stopped, active, |
|
* zombie, ...), signal disposition and so on. On a multiprocessor |
|
* machine, multiple LWPs be executing concurrently in the kernel. |
|
* |
|
* Execution states |
|
* |
|
* At any given time, an LWP has overall state that is described by |
|
* lwp::l_stat. The states are broken into two sets below. The first |
|
* set is guaranteed to represent the absolute, current state of the |
|
* LWP: |
|
* |
|
* LSONPROC |
|
* |
|
* On processor: the LWP is executing on a CPU, either in the |
|
* kernel or in user space. |
|
* |
|
* LSRUN |
|
* |
|
* Runnable: the LWP is parked on a run queue, and may soon be |
|
* chosen to run by a idle processor, or by a processor that |
|
* has been asked to preempt a currently runnning but lower |
|
* priority LWP. If the LWP is not swapped in (L_INMEM == 0) |
|
* then the LWP is not on a run queue, but may be soon. |
|
* |
|
* LSIDL |
|
* |
|
* Idle: the LWP has been created but has not yet executed, |
|
* or it has ceased executing a unit of work and is waiting |
|
* to be started again. |
|
* |
|
* LSSUSPENDED: |
|
* |
|
* Suspended: the LWP has had its execution suspended by |
|
* another LWP in the same process using the _lwp_suspend() |
|
* system call. User-level LWPs also enter the suspended |
|
* state when the system is shutting down. |
|
* |
|
* The second set represent a "statement of intent" on behalf of the |
|
* LWP. The LWP may in fact be executing on a processor, may be |
|
* sleeping or idle. It is expected to take the necessary action to |
|
* stop executing or become "running" again within a short timeframe. |
|
* The LW_RUNNING flag in lwp::l_flag indicates that an LWP is running. |
|
* Importantly, in indicates that its state is tied to a CPU. |
|
* |
|
* LSZOMB: |
|
* |
|
* Dead or dying: the LWP has released most of its resources |
|
* and is a) about to switch away into oblivion b) has already |
|
* switched away. When it switches away, its few remaining |
|
* resources can be collected. |
|
* |
|
* LSSLEEP: |
|
* |
|
* Sleeping: the LWP has entered itself onto a sleep queue, and |
|
* has switched away or will switch away shortly to allow other |
|
* LWPs to run on the CPU. |
|
* |
|
* LSSTOP: |
|
* |
|
* Stopped: the LWP has been stopped as a result of a job |
|
* control signal, or as a result of the ptrace() interface. |
|
* |
|
* Stopped LWPs may run briefly within the kernel to handle |
|
* signals that they receive, but will not return to user space |
|
* until their process' state is changed away from stopped. |
|
* |
|
* Single LWPs within a process can not be set stopped |
|
* selectively: all actions that can stop or continue LWPs |
|
* occur at the process level. |
|
* |
|
* State transitions |
|
* |
|
* Note that the LSSTOP state may only be set when returning to |
|
* user space in userret(), or when sleeping interruptably. The |
|
* LSSUSPENDED state may only be set in userret(). Before setting |
|
* those states, we try to ensure that the LWPs will release all |
|
* locks that they hold, and at a minimum try to ensure that the |
|
* LWP can be set runnable again by a signal. |
|
* |
|
* LWPs may transition states in the following ways: |
|
* |
|
* RUN -------> ONPROC ONPROC -----> RUN |
|
* > STOPPED > SLEEP |
|
* > SUSPENDED > STOPPED |
|
* > SUSPENDED |
|
* > ZOMB |
|
* |
|
* STOPPED ---> RUN SUSPENDED --> RUN |
|
* > SLEEP > SLEEP |
|
* |
|
* SLEEP -----> ONPROC IDL --------> RUN |
|
* > RUN > SUSPENDED |
|
* > STOPPED > STOPPED |
|
* > SUSPENDED |
|
* |
|
* Other state transitions are possible with kernel threads (eg |
|
* ONPROC -> IDL), but only happen under tightly controlled |
|
* circumstances the side effects are understood. |
|
* |
|
* Locking |
|
* |
|
* The majority of fields in 'struct lwp' are covered by a single, |
|
* general spin lock pointed to by lwp::l_mutex. The locks covering |
|
* each field are documented in sys/lwp.h. |
|
* |
|
* State transitions must be made with the LWP's general lock held, |
|
* and may cause the LWP's lock pointer to change. Manipulation of |
|
* the general lock is not performed directly, but through calls to |
|
* lwp_lock(), lwp_relock() and similar. |
|
* |
|
* States and their associated locks: |
|
* |
|
* LSONPROC, LSZOMB: |
|
* |
|
* Always covered by spc_lwplock, which protects running LWPs. |
|
* This is a per-CPU lock. |
|
* |
|
* LSIDL, LSRUN: |
|
* |
|
* Always covered by spc_mutex, which protects the run queues. |
|
* This may be a per-CPU lock, depending on the scheduler. |
|
* |
|
* LSSLEEP: |
|
* |
|
* Covered by a lock associated with the sleep queue that the |
|
* LWP resides on, indirectly referenced by l_sleepq->sq_mutex. |
|
* |
|
* LSSTOP, LSSUSPENDED: |
|
* |
|
* If the LWP was previously sleeping (l_wchan != NULL), then |
|
* l_mutex references the sleep queue lock. If the LWP was |
|
* runnable or on the CPU when halted, or has been removed from |
|
* the sleep queue since halted, then the lock is spc_lwplock. |
|
* |
|
* The lock order is as follows: |
|
* |
|
* spc::spc_lwplock -> |
|
* sleepq_t::sq_mutex -> |
|
* tschain_t::tc_mutex -> |
|
* spc::spc_mutex |
|
* |
|
* Each process has an scheduler state lock (proc::p_smutex), and a |
|
* number of counters on LWPs and their states: p_nzlwps, p_nrlwps, and |
|
* so on. When an LWP is to be entered into or removed from one of the |
|
* following states, p_mutex must be held and the process wide counters |
|
* adjusted: |
|
* |
|
* LSIDL, LSZOMB, LSSTOP, LSSUSPENDED |
|
* |
|
* Note that an LWP is considered running or likely to run soon if in |
|
* one of the following states. This affects the value of p_nrlwps: |
|
* |
|
* LSRUN, LSONPROC, LSSLEEP |
|
* |
|
* p_smutex does not need to be held when transitioning among these |
|
* three states. |
|
*/ |
|
|
#include <sys/cdefs.h> |
#include <sys/cdefs.h> |
__KERNEL_RCSID(0, "$NetBSD$"); |
__KERNEL_RCSID(0, "$NetBSD$"); |
|
|
|
#include "opt_ddb.h" |
#include "opt_multiprocessor.h" |
#include "opt_multiprocessor.h" |
|
#include "opt_lockdebug.h" |
|
|
|
#define _LWP_API_PRIVATE |
|
|
#include <sys/param.h> |
#include <sys/param.h> |
#include <sys/systm.h> |
#include <sys/systm.h> |
|
#include <sys/cpu.h> |
#include <sys/pool.h> |
#include <sys/pool.h> |
#include <sys/lock.h> |
|
#include <sys/proc.h> |
#include <sys/proc.h> |
#include <sys/sa.h> |
|
#include <sys/savar.h> |
|
#include <sys/types.h> |
|
#include <sys/ucontext.h> |
|
#include <sys/resourcevar.h> |
|
#include <sys/mount.h> |
|
#include <sys/syscallargs.h> |
#include <sys/syscallargs.h> |
|
#include <sys/syscall_stats.h> |
|
#include <sys/kauth.h> |
|
#include <sys/sleepq.h> |
|
#include <sys/user.h> |
|
#include <sys/lockdebug.h> |
|
#include <sys/kmem.h> |
|
#include <sys/pset.h> |
|
#include <sys/intr.h> |
|
#include <sys/lwpctl.h> |
|
#include <sys/atomic.h> |
|
|
#include <uvm/uvm_extern.h> |
#include <uvm/uvm_extern.h> |
|
#include <uvm/uvm_object.h> |
|
|
struct lwplist alllwp; |
struct lwplist alllwp = LIST_HEAD_INITIALIZER(alllwp); |
|
|
#define LWP_DEBUG |
POOL_INIT(lwp_uc_pool, sizeof(ucontext_t), 0, 0, 0, "lwpucpl", |
|
&pool_allocator_nointr, IPL_NONE); |
|
|
#ifdef LWP_DEBUG |
static pool_cache_t lwp_cache; |
int lwp_debug = 0; |
static specificdata_domain_t lwp_specificdata_domain; |
#define DPRINTF(x) if (lwp_debug) printf x |
|
#else |
void |
#define DPRINTF(x) |
lwpinit(void) |
#endif |
|
/* ARGSUSED */ |
|
int |
|
sys__lwp_create(struct lwp *l, void *v, register_t *retval) |
|
{ |
{ |
struct sys__lwp_create_args /* { |
|
syscallarg(const ucontext_t *) ucp; |
|
syscallarg(u_long) flags; |
|
syscallarg(lwpid_t *) new_lwp; |
|
} */ *uap = v; |
|
struct proc *p = l->l_proc; |
|
struct lwp *l2; |
|
vaddr_t uaddr; |
|
boolean_t inmem; |
|
ucontext_t *newuc; |
|
int s, error; |
|
|
|
newuc = pool_get(&lwp_uc_pool, PR_WAITOK); |
lwp_specificdata_domain = specificdata_domain_create(); |
|
KASSERT(lwp_specificdata_domain != NULL); |
|
lwp_sys_init(); |
|
lwp_cache = pool_cache_init(sizeof(lwp_t), MIN_LWP_ALIGNMENT, 0, 0, |
|
"lwppl", NULL, IPL_NONE, NULL, NULL, NULL); |
|
} |
|
|
error = copyin(SCARG(uap, ucp), newuc, sizeof(*newuc)); |
/* |
if (error) |
* Set an suspended. |
return (error); |
* |
|
* Must be called with p_smutex held, and the LWP locked. Will unlock the |
|
* LWP before return. |
|
*/ |
|
int |
|
lwp_suspend(struct lwp *curl, struct lwp *t) |
|
{ |
|
int error; |
|
|
/* XXX check against resource limits */ |
KASSERT(mutex_owned(&t->l_proc->p_smutex)); |
|
KASSERT(lwp_locked(t, NULL)); |
|
|
inmem = uvm_uarea_alloc(&uaddr); |
KASSERT(curl != t || curl->l_stat == LSONPROC); |
if (__predict_false(uaddr == 0)) { |
|
return (ENOMEM); |
|
} |
|
|
|
/* XXX flags: |
/* |
* __LWP_ASLWP is probably needed for Solaris compat. |
* If the current LWP has been told to exit, we must not suspend anyone |
|
* else or deadlock could occur. We won't return to userspace. |
*/ |
*/ |
|
if ((curl->l_stat & (LW_WEXIT | LW_WCORE)) != 0) { |
newlwp(l, p, uaddr, inmem, |
lwp_unlock(t); |
SCARG(uap, flags) & LWP_DETACHED, |
return (EDEADLK); |
NULL, 0, startlwp, newuc, &l2); |
|
|
|
if ((SCARG(uap, flags) & LWP_SUSPENDED) == 0) { |
|
SCHED_LOCK(s); |
|
l2->l_stat = LSRUN; |
|
setrunqueue(l2); |
|
SCHED_UNLOCK(s); |
|
simple_lock(&p->p_lock); |
|
p->p_nrlwps++; |
|
simple_unlock(&p->p_lock); |
|
} else { |
|
l2->l_stat = LSSUSPENDED; |
|
} |
} |
|
|
error = copyout(&l2->l_lid, SCARG(uap, new_lwp), |
error = 0; |
sizeof(l2->l_lid)); |
|
if (error) |
|
return (error); |
|
|
|
return (0); |
|
} |
|
|
|
|
switch (t->l_stat) { |
|
case LSRUN: |
|
case LSONPROC: |
|
t->l_flag |= LW_WSUSPEND; |
|
lwp_need_userret(t); |
|
lwp_unlock(t); |
|
break; |
|
|
int |
case LSSLEEP: |
sys__lwp_exit(struct lwp *l, void *v, register_t *retval) |
t->l_flag |= LW_WSUSPEND; |
{ |
|
|
|
lwp_exit(l); |
/* |
/* NOTREACHED */ |
* Kick the LWP and try to get it to the kernel boundary |
return (0); |
* so that it will release any locks that it holds. |
} |
* setrunnable() will release the lock. |
|
*/ |
|
if ((t->l_flag & LW_SINTR) != 0) |
|
setrunnable(t); |
|
else |
|
lwp_unlock(t); |
|
break; |
|
|
|
case LSSUSPENDED: |
|
lwp_unlock(t); |
|
break; |
|
|
int |
case LSSTOP: |
sys__lwp_self(struct lwp *l, void *v, register_t *retval) |
t->l_flag |= LW_WSUSPEND; |
{ |
setrunnable(t); |
|
break; |
|
|
*retval = l->l_lid; |
case LSIDL: |
|
case LSZOMB: |
|
error = EINTR; /* It's what Solaris does..... */ |
|
lwp_unlock(t); |
|
break; |
|
} |
|
|
return (0); |
return (error); |
} |
} |
|
|
|
/* |
int |
* Restart a suspended LWP. |
sys__lwp_getprivate(struct lwp *l, void *v, register_t *retval) |
* |
|
* Must be called with p_smutex held, and the LWP locked. Will unlock the |
|
* LWP before return. |
|
*/ |
|
void |
|
lwp_continue(struct lwp *l) |
{ |
{ |
|
|
*retval = (uintptr_t) l->l_private; |
KASSERT(mutex_owned(&l->l_proc->p_smutex)); |
|
KASSERT(lwp_locked(l, NULL)); |
return (0); |
|
} |
|
|
|
|
/* If rebooting or not suspended, then just bail out. */ |
|
if ((l->l_flag & LW_WREBOOT) != 0) { |
|
lwp_unlock(l); |
|
return; |
|
} |
|
|
int |
l->l_flag &= ~LW_WSUSPEND; |
sys__lwp_setprivate(struct lwp *l, void *v, register_t *retval) |
|
{ |
|
struct sys__lwp_setprivate_args /* { |
|
syscallarg(void *) ptr; |
|
} */ *uap = v; |
|
|
|
l->l_private = SCARG(uap, ptr); |
if (l->l_stat != LSSUSPENDED) { |
|
lwp_unlock(l); |
|
return; |
|
} |
|
|
return (0); |
/* setrunnable() will release the lock. */ |
|
setrunnable(l); |
} |
} |
|
|
|
/* |
|
* Wait for an LWP within the current process to exit. If 'lid' is |
|
* non-zero, we are waiting for a specific LWP. |
|
* |
|
* Must be called with p->p_smutex held. |
|
*/ |
int |
int |
sys__lwp_suspend(struct lwp *l, void *v, register_t *retval) |
lwp_wait1(struct lwp *l, lwpid_t lid, lwpid_t *departed, int flags) |
{ |
{ |
struct sys__lwp_suspend_args /* { |
|
syscallarg(lwpid_t) target; |
|
} */ *uap = v; |
|
int target_lid; |
|
struct proc *p = l->l_proc; |
struct proc *p = l->l_proc; |
struct lwp *t; |
struct lwp *l2; |
struct lwp *t2; |
int nfound, error; |
|
lwpid_t curlid; |
target_lid = SCARG(uap, target); |
bool exiting; |
|
|
LIST_FOREACH(t, &p->p_lwps, l_sibling) |
KASSERT(mutex_owned(&p->p_smutex)); |
if (t->l_lid == target_lid) |
|
break; |
p->p_nlwpwait++; |
|
l->l_waitingfor = lid; |
|
curlid = l->l_lid; |
|
exiting = ((flags & LWPWAIT_EXITCONTROL) != 0); |
|
|
if (t == NULL) |
for (;;) { |
return (ESRCH); |
/* |
|
* Avoid a race between exit1() and sigexit(): if the |
|
* process is dumping core, then we need to bail out: call |
|
* into lwp_userret() where we will be suspended until the |
|
* deed is done. |
|
*/ |
|
if ((p->p_sflag & PS_WCORE) != 0) { |
|
mutex_exit(&p->p_smutex); |
|
lwp_userret(l); |
|
#ifdef DIAGNOSTIC |
|
panic("lwp_wait1"); |
|
#endif |
|
/* NOTREACHED */ |
|
} |
|
|
if (t == l) { |
|
/* |
/* |
* Check for deadlock, which is only possible |
* First off, drain any detached LWP that is waiting to be |
* when we're suspending ourself. |
* reaped. |
*/ |
*/ |
LIST_FOREACH(t2, &p->p_lwps, l_sibling) { |
while ((l2 = p->p_zomblwp) != NULL) { |
if ((t2 != l) && (t2->l_stat != LSSUSPENDED)) |
p->p_zomblwp = NULL; |
break; |
lwp_free(l2, false, false);/* releases proc mutex */ |
|
mutex_enter(&p->p_smutex); |
} |
} |
|
|
if (t2 == NULL) /* All other LWPs are suspended */ |
/* |
return (EDEADLK); |
* Now look for an LWP to collect. If the whole process is |
} |
* exiting, count detached LWPs as eligible to be collected, |
|
* but don't drain them here. |
|
*/ |
|
nfound = 0; |
|
error = 0; |
|
LIST_FOREACH(l2, &p->p_lwps, l_sibling) { |
|
/* |
|
* If a specific wait and the target is waiting on |
|
* us, then avoid deadlock. This also traps LWPs |
|
* that try to wait on themselves. |
|
* |
|
* Note that this does not handle more complicated |
|
* cycles, like: t1 -> t2 -> t3 -> t1. The process |
|
* can still be killed so it is not a major problem. |
|
*/ |
|
if (l2->l_lid == lid && l2->l_waitingfor == curlid) { |
|
error = EDEADLK; |
|
break; |
|
} |
|
if (l2 == l) |
|
continue; |
|
if ((l2->l_prflag & LPR_DETACHED) != 0) { |
|
nfound += exiting; |
|
continue; |
|
} |
|
if (lid != 0) { |
|
if (l2->l_lid != lid) |
|
continue; |
|
/* |
|
* Mark this LWP as the first waiter, if there |
|
* is no other. |
|
*/ |
|
if (l2->l_waiter == 0) |
|
l2->l_waiter = curlid; |
|
} else if (l2->l_waiter != 0) { |
|
/* |
|
* It already has a waiter - so don't |
|
* collect it. If the waiter doesn't |
|
* grab it we'll get another chance |
|
* later. |
|
*/ |
|
nfound++; |
|
continue; |
|
} |
|
nfound++; |
|
|
return lwp_suspend(l, t); |
/* No need to lock the LWP in order to see LSZOMB. */ |
} |
if (l2->l_stat != LSZOMB) |
|
continue; |
|
|
|
/* |
|
* We're no longer waiting. Reset the "first waiter" |
|
* pointer on the target, in case it was us. |
|
*/ |
|
l->l_waitingfor = 0; |
|
l2->l_waiter = 0; |
|
p->p_nlwpwait--; |
|
if (departed) |
|
*departed = l2->l_lid; |
|
sched_lwp_collect(l2); |
|
|
inline int |
/* lwp_free() releases the proc lock. */ |
lwp_suspend(struct lwp *l, struct lwp *t) |
lwp_free(l2, false, false); |
{ |
mutex_enter(&p->p_smutex); |
struct proc *p = t->l_proc; |
return 0; |
int s; |
} |
|
|
if (t == l) { |
if (error != 0) |
SCHED_LOCK(s); |
|
l->l_stat = LSSUSPENDED; |
|
/* XXX NJWLWP check if this makes sense here: */ |
|
p->p_stats->p_ru.ru_nvcsw++; |
|
mi_switch(l, NULL); |
|
SCHED_ASSERT_UNLOCKED(); |
|
splx(s); |
|
} else { |
|
switch (t->l_stat) { |
|
case LSSUSPENDED: |
|
return (0); /* _lwp_suspend() is idempotent */ |
|
case LSRUN: |
|
SCHED_LOCK(s); |
|
remrunqueue(t); |
|
t->l_stat = LSSUSPENDED; |
|
SCHED_UNLOCK(s); |
|
simple_lock(&p->p_lock); |
|
p->p_nrlwps--; |
|
simple_unlock(&p->p_lock); |
|
break; |
|
case LSSLEEP: |
|
t->l_stat = LSSUSPENDED; |
|
break; |
break; |
case LSIDL: |
if (nfound == 0) { |
case LSZOMB: |
error = ESRCH; |
return (EINTR); /* It's what Solaris does..... */ |
|
case LSSTOP: |
|
panic("_lwp_suspend: Stopped LWP in running process!"); |
|
break; |
break; |
case LSONPROC: |
|
/* XXX multiprocessor LWPs? Implement me! */ |
|
return (EINVAL); |
|
} |
} |
} |
|
|
|
return (0); |
|
} |
|
|
|
|
/* |
|
* The kernel is careful to ensure that it can not deadlock |
|
* when exiting - just keep waiting. |
|
*/ |
|
if (exiting) { |
|
KASSERT(p->p_nlwps > 1); |
|
cv_wait(&p->p_lwpcv, &p->p_smutex); |
|
continue; |
|
} |
|
|
int |
/* |
sys__lwp_continue(struct lwp *l, void *v, register_t *retval) |
* If all other LWPs are waiting for exits or suspends |
{ |
* and the supply of zombies and potential zombies is |
struct sys__lwp_continue_args /* { |
* exhausted, then we are about to deadlock. |
syscallarg(lwpid_t) target; |
* |
} */ *uap = v; |
* If the process is exiting (and this LWP is not the one |
int s, target_lid; |
* that is coordinating the exit) then bail out now. |
struct proc *p = l->l_proc; |
*/ |
struct lwp *t; |
if ((p->p_sflag & PS_WEXIT) != 0 || |
|
p->p_nrlwps + p->p_nzlwps - p->p_ndlwps <= p->p_nlwpwait) { |
target_lid = SCARG(uap, target); |
error = EDEADLK; |
|
|
LIST_FOREACH(t, &p->p_lwps, l_sibling) |
|
if (t->l_lid == target_lid) |
|
break; |
break; |
|
} |
|
|
if (t == NULL) |
/* |
return (ESRCH); |
* Sit around and wait for something to happen. We'll be |
|
* awoken if any of the conditions examined change: if an |
SCHED_LOCK(s); |
* LWP exits, is collected, or is detached. |
lwp_continue(t); |
*/ |
SCHED_UNLOCK(s); |
if ((error = cv_wait_sig(&p->p_lwpcv, &p->p_smutex)) != 0) |
|
|
return (0); |
|
} |
|
|
|
void |
|
lwp_continue(struct lwp *l) |
|
{ |
|
|
|
DPRINTF(("lwp_continue of %d.%d (%s), state %d, wchan %p\n", |
|
l->l_proc->p_pid, l->l_lid, l->l_proc->p_comm, l->l_stat, |
|
l->l_wchan)); |
|
|
|
if (l->l_stat != LSSUSPENDED) |
|
return; |
|
|
|
if (l->l_wchan == 0) { |
|
/* LWP was runnable before being suspended. */ |
|
setrunnable(l); |
|
} else { |
|
/* LWP was sleeping before being suspended. */ |
|
l->l_stat = LSSLEEP; |
|
} |
|
} |
|
|
|
int |
|
sys__lwp_wakeup(struct lwp *l, void *v, register_t *retval) |
|
{ |
|
struct sys__lwp_wakeup_args /* { |
|
syscallarg(lwpid_t) target; |
|
} */ *uap = v; |
|
lwpid_t target_lid; |
|
struct lwp *t; |
|
struct proc *p; |
|
int error; |
|
int s; |
|
|
|
p = l->l_proc; |
|
target_lid = SCARG(uap, target); |
|
|
|
SCHED_LOCK(s); |
|
|
|
LIST_FOREACH(t, &p->p_lwps, l_sibling) |
|
if (t->l_lid == target_lid) |
|
break; |
break; |
|
|
if (t == NULL) { |
|
error = ESRCH; |
|
goto exit; |
|
} |
|
|
|
if (t->l_stat != LSSLEEP) { |
|
error = ENODEV; |
|
goto exit; |
|
} |
} |
|
|
if ((t->l_flag & L_SINTR) == 0) { |
|
error = EBUSY; |
|
goto exit; |
|
} |
|
/* |
/* |
* Tell ltsleep to wakeup. |
* We didn't find any LWPs to collect, we may have received a |
|
* signal, or some other condition has caused us to bail out. |
|
* |
|
* If waiting on a specific LWP, clear the waiters marker: some |
|
* other LWP may want it. Then, kick all the remaining waiters |
|
* so that they can re-check for zombies and for deadlock. |
*/ |
*/ |
t->l_flag |= L_CANCELLED; |
if (lid != 0) { |
|
LIST_FOREACH(l2, &p->p_lwps, l_sibling) { |
setrunnable(t); |
if (l2->l_lid == lid) { |
error = 0; |
if (l2->l_waiter == curlid) |
exit: |
l2->l_waiter = 0; |
SCHED_UNLOCK(s); |
break; |
|
|
return error; |
|
} |
|
|
|
int |
|
sys__lwp_wait(struct lwp *l, void *v, register_t *retval) |
|
{ |
|
struct sys__lwp_wait_args /* { |
|
syscallarg(lwpid_t) wait_for; |
|
syscallarg(lwpid_t *) departed; |
|
} */ *uap = v; |
|
int error; |
|
lwpid_t dep; |
|
|
|
error = lwp_wait1(l, SCARG(uap, wait_for), &dep, 0); |
|
if (error) |
|
return (error); |
|
|
|
if (SCARG(uap, departed)) { |
|
error = copyout(&dep, SCARG(uap, departed), |
|
sizeof(dep)); |
|
if (error) |
|
return (error); |
|
} |
|
|
|
return (0); |
|
} |
|
|
|
|
|
int |
|
lwp_wait1(struct lwp *l, lwpid_t lid, lwpid_t *departed, int flags) |
|
{ |
|
struct proc *p = l->l_proc; |
|
struct lwp *l2, *l3; |
|
int nfound, error, wpri; |
|
static const char waitstr1[] = "lwpwait"; |
|
static const char waitstr2[] = "lwpwait2"; |
|
|
|
DPRINTF(("lwp_wait1: %d.%d waiting for %d.\n", |
|
p->p_pid, l->l_lid, lid)); |
|
|
|
if (lid == l->l_lid) |
|
return (EDEADLK); /* Waiting for ourselves makes no sense. */ |
|
|
|
wpri = PWAIT | |
|
((flags & LWPWAIT_EXITCONTROL) ? PNOEXITERR : PCATCH); |
|
loop: |
|
nfound = 0; |
|
LIST_FOREACH(l2, &p->p_lwps, l_sibling) { |
|
if ((l2 == l) || (l2->l_flag & L_DETACHED) || |
|
((lid != 0) && (lid != l2->l_lid))) |
|
continue; |
|
|
|
nfound++; |
|
if (l2->l_stat == LSZOMB) { |
|
if (departed) |
|
*departed = l2->l_lid; |
|
|
|
simple_lock(&p->p_lock); |
|
LIST_REMOVE(l2, l_sibling); |
|
p->p_nlwps--; |
|
p->p_nzlwps--; |
|
simple_unlock(&p->p_lock); |
|
/* XXX decrement limits */ |
|
|
|
pool_put(&lwp_pool, l2); |
|
|
|
return (0); |
|
} else if (l2->l_stat == LSSLEEP || |
|
l2->l_stat == LSSUSPENDED) { |
|
/* Deadlock checks. |
|
* 1. If all other LWPs are waiting for exits |
|
* or suspended, we would deadlock. |
|
*/ |
|
|
|
LIST_FOREACH(l3, &p->p_lwps, l_sibling) { |
|
if (l3 != l && (l3->l_stat != LSSUSPENDED) && |
|
!(l3->l_stat == LSSLEEP && |
|
l3->l_wchan == (caddr_t) &p->p_nlwps)) |
|
break; |
|
} |
} |
if (l3 == NULL) /* Everyone else is waiting. */ |
|
return (EDEADLK); |
|
|
|
/* XXX we'd like to check for a cycle of waiting |
|
* LWPs (specific LID waits, not any-LWP waits) |
|
* and detect that sort of deadlock, but we don't |
|
* have a good place to store the lwp that is |
|
* being waited for. wchan is already filled with |
|
* &p->p_nlwps, and putting the lwp address in |
|
* there for deadlock tracing would require |
|
* exiting LWPs to call wakeup on both their |
|
* own address and &p->p_nlwps, to get threads |
|
* sleeping on any LWP exiting. |
|
* |
|
* Revisit later. Maybe another auxillary |
|
* storage location associated with sleeping |
|
* is in order. |
|
*/ |
|
} |
} |
} |
} |
|
p->p_nlwpwait--; |
|
l->l_waitingfor = 0; |
|
cv_broadcast(&p->p_lwpcv); |
|
|
if (nfound == 0) |
return error; |
return (ESRCH); |
|
|
|
if ((error = tsleep((caddr_t) &p->p_nlwps, wpri, |
|
(lid != 0) ? waitstr1 : waitstr2, 0)) != 0) |
|
return (error); |
|
|
|
goto loop; |
|
} |
} |
|
|
|
/* |
|
* Create a new LWP within process 'p2', using LWP 'l1' as a template. |
|
* The new LWP is created in state LSIDL and must be set running, |
|
* suspended, or stopped by the caller. |
|
*/ |
int |
int |
newlwp(struct lwp *l1, struct proc *p2, vaddr_t uaddr, boolean_t inmem, |
lwp_create(lwp_t *l1, proc_t *p2, vaddr_t uaddr, bool inmem, int flags, |
int flags, void *stack, size_t stacksize, |
void *stack, size_t stacksize, void (*func)(void *), void *arg, |
void (*func)(void *), void *arg, struct lwp **rnewlwpp) |
lwp_t **rnewlwpp, int sclass) |
{ |
{ |
struct lwp *l2; |
struct lwp *l2, *isfree; |
int s; |
turnstile_t *ts; |
|
|
l2 = pool_get(&lwp_pool, PR_WAITOK); |
/* |
|
* First off, reap any detached LWP waiting to be collected. |
|
* We can re-use its LWP structure and turnstile. |
|
*/ |
|
isfree = NULL; |
|
if (p2->p_zomblwp != NULL) { |
|
mutex_enter(&p2->p_smutex); |
|
if ((isfree = p2->p_zomblwp) != NULL) { |
|
p2->p_zomblwp = NULL; |
|
lwp_free(isfree, true, false);/* releases proc mutex */ |
|
} else |
|
mutex_exit(&p2->p_smutex); |
|
} |
|
if (isfree == NULL) { |
|
l2 = pool_cache_get(lwp_cache, PR_WAITOK); |
|
memset(l2, 0, sizeof(*l2)); |
|
l2->l_ts = pool_cache_get(turnstile_cache, PR_WAITOK); |
|
SLIST_INIT(&l2->l_pi_lenders); |
|
} else { |
|
l2 = isfree; |
|
ts = l2->l_ts; |
|
KASSERT(l2->l_inheritedprio == -1); |
|
KASSERT(SLIST_EMPTY(&l2->l_pi_lenders)); |
|
memset(l2, 0, sizeof(*l2)); |
|
l2->l_ts = ts; |
|
} |
|
|
l2->l_stat = LSIDL; |
l2->l_stat = LSIDL; |
l2->l_forw = l2->l_back = NULL; |
|
l2->l_proc = p2; |
l2->l_proc = p2; |
|
l2->l_refcnt = 1; |
memset(&l2->l_startzero, 0, |
l2->l_class = sclass; |
(unsigned) ((caddr_t)&l2->l_endzero - |
l2->l_kpriority = l1->l_kpriority; |
(caddr_t)&l2->l_startzero)); |
l2->l_kpribase = PRI_KERNEL; |
memcpy(&l2->l_startcopy, &l1->l_startcopy, |
l2->l_priority = l1->l_priority; |
(unsigned) ((caddr_t)&l2->l_endcopy - |
l2->l_inheritedprio = -1; |
(caddr_t)&l2->l_startcopy)); |
l2->l_mutex = l1->l_cpu->ci_schedstate.spc_mutex; |
|
|
#if !defined(MULTIPROCESSOR) |
|
/* |
|
* In the single-processor case, all processes will always run |
|
* on the same CPU. So, initialize the child's CPU to the parent's |
|
* now. In the multiprocessor case, the child's CPU will be |
|
* initialized in the low-level context switch code when the |
|
* process runs. |
|
*/ |
|
KASSERT(l1->l_cpu != NULL); |
|
l2->l_cpu = l1->l_cpu; |
l2->l_cpu = l1->l_cpu; |
#else |
l2->l_flag = inmem ? LW_INMEM : 0; |
/* |
l2->l_pflag = LP_MPSAFE; |
* zero child's CPU pointer so we don't get trash. |
|
*/ |
|
l2->l_cpu = NULL; |
|
#endif /* ! MULTIPROCESSOR */ |
|
|
|
l2->l_flag = inmem ? L_INMEM : 0; |
if (p2->p_flag & PK_SYSTEM) { |
l2->l_flag |= (flags & LWP_DETACHED) ? L_DETACHED : 0; |
/* Mark it as a system LWP and not a candidate for swapping */ |
|
l2->l_flag |= LW_SYSTEM; |
|
} |
|
|
callout_init(&l2->l_tsleep_ch); |
lwp_initspecific(l2); |
|
sched_lwp_fork(l1, l2); |
|
lwp_update_creds(l2); |
|
callout_init(&l2->l_timeout_ch, CALLOUT_MPSAFE); |
|
callout_setfunc(&l2->l_timeout_ch, sleepq_timeout, l2); |
|
mutex_init(&l2->l_swaplock, MUTEX_DEFAULT, IPL_NONE); |
|
cv_init(&l2->l_sigcv, "sigwait"); |
|
l2->l_syncobj = &sched_syncobj; |
|
|
if (rnewlwpp != NULL) |
if (rnewlwpp != NULL) |
*rnewlwpp = l2; |
*rnewlwpp = l2; |
|
|
l2->l_addr = (struct user *)uaddr; |
l2->l_addr = UAREA_TO_USER(uaddr); |
uvm_lwp_fork(l1, l2, stack, stacksize, func, |
uvm_lwp_fork(l1, l2, stack, stacksize, func, |
(arg != NULL) ? arg : l2); |
(arg != NULL) ? arg : l2); |
|
|
simple_lock(&p2->p_lock); |
mutex_enter(&p2->p_smutex); |
l2->l_lid = ++p2->p_nlwpid; |
|
|
if ((flags & LWP_DETACHED) != 0) { |
|
l2->l_prflag = LPR_DETACHED; |
|
p2->p_ndlwps++; |
|
} else |
|
l2->l_prflag = 0; |
|
|
|
l2->l_sigmask = l1->l_sigmask; |
|
CIRCLEQ_INIT(&l2->l_sigpend.sp_info); |
|
sigemptyset(&l2->l_sigpend.sp_set); |
|
|
|
p2->p_nlwpid++; |
|
if (p2->p_nlwpid == 0) |
|
p2->p_nlwpid++; |
|
l2->l_lid = p2->p_nlwpid; |
LIST_INSERT_HEAD(&p2->p_lwps, l2, l_sibling); |
LIST_INSERT_HEAD(&p2->p_lwps, l2, l_sibling); |
p2->p_nlwps++; |
p2->p_nlwps++; |
simple_unlock(&p2->p_lock); |
|
|
|
/* XXX should be locked differently... */ |
mutex_exit(&p2->p_smutex); |
s = proclist_lock_write(); |
|
|
mutex_enter(&proclist_lock); |
LIST_INSERT_HEAD(&alllwp, l2, l_list); |
LIST_INSERT_HEAD(&alllwp, l2, l_list); |
proclist_unlock_write(s); |
mutex_exit(&proclist_lock); |
|
|
|
if ((p2->p_flag & PK_SYSTEM) == 0) { |
|
/* Locking is needed, since LWP is in the list of all LWPs */ |
|
lwp_lock(l2); |
|
/* Inherit a processor-set */ |
|
l2->l_psid = l1->l_psid; |
|
/* Inherit an affinity */ |
|
memcpy(&l2->l_affinity, &l1->l_affinity, sizeof(cpuset_t)); |
|
/* Look for a CPU to start */ |
|
l2->l_cpu = sched_takecpu(l2); |
|
lwp_unlock_to(l2, l2->l_cpu->ci_schedstate.spc_mutex); |
|
} |
|
|
|
SYSCALL_TIME_LWP_INIT(l2); |
|
|
if (p2->p_emul->e_lwp_fork) |
if (p2->p_emul->e_lwp_fork) |
(*p2->p_emul->e_lwp_fork)(l1, l2); |
(*p2->p_emul->e_lwp_fork)(l1, l2); |
Line 520 newlwp(struct lwp *l1, struct proc *p2, |
|
Line 650 newlwp(struct lwp *l1, struct proc *p2, |
|
return (0); |
return (0); |
} |
} |
|
|
|
/* |
|
* Called by MD code when a new LWP begins execution. Must be called |
|
* with the previous LWP locked (so at splsched), or if there is no |
|
* previous LWP, at splsched. |
|
*/ |
|
void |
|
lwp_startup(struct lwp *prev, struct lwp *new) |
|
{ |
|
|
|
if (prev != NULL) { |
|
/* |
|
* Normalize the count of the spin-mutexes, it was |
|
* increased in mi_switch(). Unmark the state of |
|
* context switch - it is finished for previous LWP. |
|
*/ |
|
curcpu()->ci_mtx_count++; |
|
membar_exit(); |
|
prev->l_ctxswtch = 0; |
|
} |
|
spl0(); |
|
pmap_activate(new); |
|
LOCKDEBUG_BARRIER(NULL, 0); |
|
if ((new->l_pflag & LP_MPSAFE) == 0) { |
|
KERNEL_LOCK(1, new); |
|
} |
|
} |
|
|
/* |
/* |
* Quit the process. This will call cpu_exit, which will call cpu_switch, |
* Exit an LWP. |
* so this can only be used meaningfully if you're willing to switch away. |
|
* Calling with l!=curlwp would be weird. |
|
*/ |
*/ |
void |
void |
lwp_exit(struct lwp *l) |
lwp_exit(struct lwp *l) |
{ |
{ |
struct proc *p = l->l_proc; |
struct proc *p = l->l_proc; |
int s; |
struct lwp *l2; |
|
bool current; |
|
|
DPRINTF(("lwp_exit: %d.%d exiting.\n", p->p_pid, l->l_lid)); |
current = (l == curlwp); |
DPRINTF((" nlwps: %d nrlwps %d nzlwps: %d\n", |
|
p->p_nlwps, p->p_nrlwps, p->p_nzlwps)); |
|
|
|
if (p->p_emul->e_lwp_exit) |
KASSERT(current || l->l_stat == LSIDL); |
(*p->p_emul->e_lwp_exit)(l); |
|
|
/* |
|
* Verify that we hold no locks other than the kernel lock. |
|
*/ |
|
#ifdef MULTIPROCESSOR |
|
LOCKDEBUG_BARRIER(&kernel_lock, 0); |
|
#else |
|
LOCKDEBUG_BARRIER(NULL, 0); |
|
#endif |
|
|
/* |
/* |
* If we are the last live LWP in a process, we need to exit |
* If we are the last live LWP in a process, we need to exit the |
* the entire process (if that's not already going on). We do |
* entire process. We do so with an exit status of zero, because |
* so with an exit status of zero, because it's a "controlled" |
* it's a "controlled" exit, and because that's what Solaris does. |
* exit, and because that's what Solaris does. |
* |
*/ |
* We are not quite a zombie yet, but for accounting purposes we |
if (((p->p_nlwps - p->p_nzlwps) == 1) && ((p->p_flag & P_WEXIT) == 0)) { |
* must increment the count of zombies here. |
DPRINTF(("lwp_exit: %d.%d calling exit1()\n", |
* |
p->p_pid, l->l_lid)); |
* Note: the last LWP's specificdata will be deleted here. |
|
*/ |
|
mutex_enter(&p->p_smutex); |
|
if (p->p_nlwps - p->p_nzlwps == 1) { |
|
KASSERT(current == true); |
|
/* XXXSMP kernel_lock not held */ |
exit1(l, 0); |
exit1(l, 0); |
/* NOTREACHED */ |
/* NOTREACHED */ |
} |
} |
|
p->p_nzlwps++; |
|
mutex_exit(&p->p_smutex); |
|
|
s = proclist_lock_write(); |
if (p->p_emul->e_lwp_exit) |
LIST_REMOVE(l, l_list); |
(*p->p_emul->e_lwp_exit)(l); |
proclist_unlock_write(s); |
|
|
|
/* Free MD LWP resources */ |
|
#ifndef __NO_CPU_LWP_FREE |
|
cpu_lwp_free(l, 0); |
|
#endif |
|
|
|
simple_lock(&p->p_lock); |
|
p->p_nrlwps--; |
|
simple_unlock(&p->p_lock); |
|
|
|
l->l_stat = LSDEAD; |
/* Delete the specificdata while it's still safe to sleep. */ |
|
specificdata_fini(lwp_specificdata_domain, &l->l_specdataref); |
|
|
/* This LWP no longer needs to hold the kernel lock. */ |
/* |
KERNEL_PROC_UNLOCK(l); |
* Release our cached credentials. |
|
*/ |
|
kauth_cred_free(l->l_cred); |
|
callout_destroy(&l->l_timeout_ch); |
|
|
pmap_deactivate(l); |
/* |
|
* While we can still block, mark the LWP as unswappable to |
|
* prevent conflicts with the with the swapper. |
|
*/ |
|
if (current) |
|
uvm_lwp_hold(l); |
|
|
/* cpu_exit() will not return */ |
/* |
cpu_exit(l); |
* Remove the LWP from the global list. |
|
*/ |
|
mutex_enter(&proclist_lock); |
|
mutex_enter(&proclist_mutex); |
|
LIST_REMOVE(l, l_list); |
|
mutex_exit(&proclist_mutex); |
|
mutex_exit(&proclist_lock); |
|
|
|
/* |
|
* Get rid of all references to the LWP that others (e.g. procfs) |
|
* may have, and mark the LWP as a zombie. If the LWP is detached, |
|
* mark it waiting for collection in the proc structure. Note that |
|
* before we can do that, we need to free any other dead, deatched |
|
* LWP waiting to meet its maker. |
|
* |
|
* XXXSMP disable preemption. |
|
*/ |
|
mutex_enter(&p->p_smutex); |
|
lwp_drainrefs(l); |
|
|
|
if ((l->l_prflag & LPR_DETACHED) != 0) { |
|
while ((l2 = p->p_zomblwp) != NULL) { |
|
p->p_zomblwp = NULL; |
|
lwp_free(l2, false, false);/* releases proc mutex */ |
|
mutex_enter(&p->p_smutex); |
|
l->l_refcnt++; |
|
lwp_drainrefs(l); |
|
} |
|
p->p_zomblwp = l; |
|
} |
|
|
|
/* |
|
* If we find a pending signal for the process and we have been |
|
* asked to check for signals, then we loose: arrange to have |
|
* all other LWPs in the process check for signals. |
|
*/ |
|
if ((l->l_flag & LW_PENDSIG) != 0 && |
|
firstsig(&p->p_sigpend.sp_set) != 0) { |
|
LIST_FOREACH(l2, &p->p_lwps, l_sibling) { |
|
lwp_lock(l2); |
|
l2->l_flag |= LW_PENDSIG; |
|
lwp_unlock(l2); |
|
} |
|
} |
|
|
|
lwp_lock(l); |
|
l->l_stat = LSZOMB; |
|
if (l->l_name != NULL) |
|
strcpy(l->l_name, "(zombie)"); |
|
lwp_unlock(l); |
|
p->p_nrlwps--; |
|
cv_broadcast(&p->p_lwpcv); |
|
if (l->l_lwpctl != NULL) |
|
l->l_lwpctl->lc_curcpu = LWPCTL_CPU_EXITED; |
|
mutex_exit(&p->p_smutex); |
|
|
|
/* |
|
* We can no longer block. At this point, lwp_free() may already |
|
* be gunning for us. On a multi-CPU system, we may be off p_lwps. |
|
* |
|
* Free MD LWP resources. |
|
*/ |
|
#ifndef __NO_CPU_LWP_FREE |
|
cpu_lwp_free(l, 0); |
|
#endif |
|
|
|
if (current) { |
|
pmap_deactivate(l); |
|
|
|
/* |
|
* Release the kernel lock, and switch away into |
|
* oblivion. |
|
*/ |
|
#ifdef notyet |
|
/* XXXSMP hold in lwp_userret() */ |
|
KERNEL_UNLOCK_LAST(l); |
|
#else |
|
KERNEL_UNLOCK_ALL(l, NULL); |
|
#endif |
|
lwp_exit_switchaway(l); |
|
} |
|
} |
|
|
|
void |
|
lwp_exit_switchaway(struct lwp *l) |
|
{ |
|
struct cpu_info *ci; |
|
struct lwp *idlelwp; |
|
|
|
/* Unlocked, but is for statistics only. */ |
|
uvmexp.swtch++; |
|
|
|
(void)splsched(); |
|
l->l_flag &= ~LW_RUNNING; |
|
ci = curcpu(); |
|
idlelwp = ci->ci_data.cpu_idlelwp; |
|
idlelwp->l_stat = LSONPROC; |
|
|
|
/* |
|
* cpu_onproc must be updated with the CPU locked, as |
|
* aston() may try to set a AST pending on the LWP (and |
|
* it does so with the CPU locked). Otherwise, the LWP |
|
* may be destroyed before the AST can be set, leading |
|
* to a user-after-free. |
|
*/ |
|
spc_lock(ci); |
|
ci->ci_data.cpu_onproc = idlelwp; |
|
spc_unlock(ci); |
|
cpu_switchto(NULL, idlelwp, false); |
} |
} |
|
|
/* |
/* |
* We are called from cpu_exit() once it is safe to schedule the |
* Free a dead LWP's remaining resources. |
* dead process's resources to be freed (i.e., once we've switched to |
|
* the idle PCB for the current CPU). |
|
* |
* |
* NOTE: One must be careful with locking in this routine. It's |
* XXXLWP limits. |
* called from a critical section in machine-dependent code, so |
|
* we should refrain from changing any interrupt state. |
|
*/ |
*/ |
void |
void |
lwp_exit2(struct lwp *l) |
lwp_free(struct lwp *l, bool recycle, bool last) |
{ |
{ |
struct proc *p; |
struct proc *p = l->l_proc; |
|
ksiginfoq_t kq; |
|
|
|
KASSERT(l != curlwp); |
|
|
KERNEL_LOCK(LK_EXCLUSIVE); |
|
/* |
/* |
* Free the VM resources we're still holding on to. |
* If this was not the last LWP in the process, then adjust |
|
* counters and unlock. |
*/ |
*/ |
uvm_lwp_exit(l); |
if (!last) { |
|
/* |
if (l->l_flag & L_DETACHED) { |
* Add the LWP's run time to the process' base value. |
/* Nobody waits for detached LWPs. */ |
* This needs to co-incide with coming off p_lwps. |
|
*/ |
|
bintime_add(&p->p_rtime, &l->l_rtime); |
|
p->p_pctcpu += l->l_pctcpu; |
|
LIST_REMOVE(l, l_sibling); |
|
p->p_nlwps--; |
|
p->p_nzlwps--; |
|
if ((l->l_prflag & LPR_DETACHED) != 0) |
|
p->p_ndlwps--; |
|
|
if ((l->l_flag & L_PROCEXIT) == 0) { |
/* |
LIST_REMOVE(l, l_sibling); |
* Have any LWPs sleeping in lwp_wait() recheck for |
p = l->l_proc; |
* deadlock. |
p->p_nlwps--; |
*/ |
} |
cv_broadcast(&p->p_lwpcv); |
|
mutex_exit(&p->p_smutex); |
|
} |
|
|
pool_put(&lwp_pool, l); |
#ifdef MULTIPROCESSOR |
KERNEL_UNLOCK(); |
/* |
} else { |
* In the unlikely event that the LWP is still on the CPU, |
l->l_stat = LSZOMB; |
* then spin until it has switched away. We need to release |
p = l->l_proc; |
* all locks to avoid deadlock against interrupt handlers on |
p->p_nzlwps++; |
* the target CPU. |
KERNEL_UNLOCK(); |
*/ |
wakeup(&p->p_nlwps); |
if ((l->l_flag & LW_RUNNING) != 0 || l->l_cpu->ci_curlwp == l) { |
|
int count; |
|
(void)count; /* XXXgcc */ |
|
KERNEL_UNLOCK_ALL(curlwp, &count); |
|
while ((l->l_flag & LW_RUNNING) != 0 || |
|
l->l_cpu->ci_curlwp == l) |
|
SPINLOCK_BACKOFF_HOOK; |
|
KERNEL_LOCK(count, curlwp); |
} |
} |
|
#endif |
|
|
|
/* |
|
* Destroy the LWP's remaining signal information. |
|
*/ |
|
ksiginfo_queue_init(&kq); |
|
sigclear(&l->l_sigpend, NULL, &kq); |
|
ksiginfo_queue_drain(&kq); |
|
cv_destroy(&l->l_sigcv); |
|
mutex_destroy(&l->l_swaplock); |
|
|
|
/* |
|
* Free the LWP's turnstile and the LWP structure itself unless the |
|
* caller wants to recycle them. Also, free the scheduler specific |
|
* data. |
|
* |
|
* We can't return turnstile0 to the pool (it didn't come from it), |
|
* so if it comes up just drop it quietly and move on. |
|
* |
|
* We don't recycle the VM resources at this time. |
|
*/ |
|
if (l->l_lwpctl != NULL) |
|
lwp_ctl_free(l); |
|
sched_lwp_exit(l); |
|
|
|
if (!recycle && l->l_ts != &turnstile0) |
|
pool_cache_put(turnstile_cache, l->l_ts); |
|
if (l->l_name != NULL) |
|
kmem_free(l->l_name, MAXCOMLEN); |
|
#ifndef __NO_CPU_LWP_FREE |
|
cpu_lwp_free2(l); |
|
#endif |
|
KASSERT((l->l_flag & LW_INMEM) != 0); |
|
uvm_lwp_exit(l); |
|
KASSERT(SLIST_EMPTY(&l->l_pi_lenders)); |
|
KASSERT(l->l_inheritedprio == -1); |
|
if (!recycle) |
|
pool_cache_put(lwp_cache, l); |
} |
} |
|
|
/* |
/* |
* Pick a LWP to represent the process for those operations which |
* Pick a LWP to represent the process for those operations which |
* want information about a "process" that is actually associated |
* want information about a "process" that is actually associated |
* with a LWP. |
* with a LWP. |
|
* |
|
* If 'locking' is false, no locking or lock checks are performed. |
|
* This is intended for use by DDB. |
|
* |
|
* We don't bother locking the LWP here, since code that uses this |
|
* interface is broken by design and an exact match is not required. |
*/ |
*/ |
struct lwp * |
struct lwp * |
proc_representative_lwp(struct proc *p) |
proc_representative_lwp(struct proc *p, int *nrlwps, int locking) |
{ |
{ |
struct lwp *l, *onproc, *running, *sleeping, *stopped, *suspended; |
struct lwp *l, *onproc, *running, *sleeping, *stopped, *suspended; |
struct lwp *signalled; |
struct lwp *signalled; |
|
int cnt; |
|
|
|
if (locking) { |
|
KASSERT(mutex_owned(&p->p_smutex)); |
|
} |
|
|
/* Trivial case: only one LWP */ |
/* Trivial case: only one LWP */ |
if (p->p_nlwps == 1) |
if (p->p_nlwps == 1) { |
return (LIST_FIRST(&p->p_lwps)); |
l = LIST_FIRST(&p->p_lwps); |
|
if (nrlwps) |
|
*nrlwps = (l->l_stat == LSONPROC || l->l_stat == LSRUN); |
|
return l; |
|
} |
|
|
|
cnt = 0; |
switch (p->p_stat) { |
switch (p->p_stat) { |
case SSTOP: |
case SSTOP: |
case SACTIVE: |
case SACTIVE: |
Line 638 proc_representative_lwp(struct proc *p) |
|
Line 983 proc_representative_lwp(struct proc *p) |
|
onproc = running = sleeping = stopped = suspended = NULL; |
onproc = running = sleeping = stopped = suspended = NULL; |
signalled = NULL; |
signalled = NULL; |
LIST_FOREACH(l, &p->p_lwps, l_sibling) { |
LIST_FOREACH(l, &p->p_lwps, l_sibling) { |
|
if ((l->l_flag & LW_IDLE) != 0) { |
|
continue; |
|
} |
if (l->l_lid == p->p_sigctx.ps_lwp) |
if (l->l_lid == p->p_sigctx.ps_lwp) |
signalled = l; |
signalled = l; |
switch (l->l_stat) { |
switch (l->l_stat) { |
case LSONPROC: |
case LSONPROC: |
onproc = l; |
onproc = l; |
|
cnt++; |
break; |
break; |
case LSRUN: |
case LSRUN: |
running = l; |
running = l; |
|
cnt++; |
break; |
break; |
case LSSLEEP: |
case LSSLEEP: |
sleeping = l; |
sleeping = l; |
Line 658 proc_representative_lwp(struct proc *p) |
|
Line 1008 proc_representative_lwp(struct proc *p) |
|
break; |
break; |
} |
} |
} |
} |
|
if (nrlwps) |
|
*nrlwps = cnt; |
if (signalled) |
if (signalled) |
return signalled; |
l = signalled; |
if (onproc) |
else if (onproc) |
return onproc; |
l = onproc; |
if (running) |
else if (running) |
return running; |
l = running; |
if (sleeping) |
else if (sleeping) |
return sleeping; |
l = sleeping; |
if (stopped) |
else if (stopped) |
return stopped; |
l = stopped; |
if (suspended) |
else if (suspended) |
return suspended; |
l = suspended; |
break; |
else |
case SZOMB: |
break; |
/* Doesn't really matter... */ |
return l; |
return (LIST_FIRST(&p->p_lwps)); |
|
#ifdef DIAGNOSTIC |
#ifdef DIAGNOSTIC |
case SIDL: |
case SIDL: |
|
case SZOMB: |
|
case SDYING: |
|
case SDEAD: |
|
if (locking) |
|
mutex_exit(&p->p_smutex); |
/* We have more than one LWP and we're in SIDL? |
/* We have more than one LWP and we're in SIDL? |
* How'd that happen? |
* How'd that happen? |
*/ |
*/ |
panic("Too many LWPs (%d) in SIDL process %d (%s)", |
panic("Too many LWPs in idle/dying process %d (%s) stat = %d", |
p->p_nrlwps, p->p_pid, p->p_comm); |
p->p_pid, p->p_comm, p->p_stat); |
|
break; |
default: |
default: |
|
if (locking) |
|
mutex_exit(&p->p_smutex); |
panic("Process %d (%s) in unknown state %d", |
panic("Process %d (%s) in unknown state %d", |
p->p_pid, p->p_comm, p->p_stat); |
p->p_pid, p->p_comm, p->p_stat); |
#endif |
#endif |
} |
} |
|
|
|
if (locking) |
|
mutex_exit(&p->p_smutex); |
panic("proc_representative_lwp: couldn't find a lwp for process" |
panic("proc_representative_lwp: couldn't find a lwp for process" |
" %d (%s)", p->p_pid, p->p_comm); |
" %d (%s)", p->p_pid, p->p_comm); |
/* NOTREACHED */ |
/* NOTREACHED */ |
return NULL; |
return NULL; |
} |
} |
|
|
|
/* |
|
* Migrate the LWP to the another CPU. Unlocks the LWP. |
|
*/ |
|
void |
|
lwp_migrate(lwp_t *l, struct cpu_info *ci) |
|
{ |
|
struct schedstate_percpu *spc; |
|
KASSERT(lwp_locked(l, NULL)); |
|
|
|
if (l->l_cpu == ci) { |
|
lwp_unlock(l); |
|
return; |
|
} |
|
|
|
spc = &ci->ci_schedstate; |
|
switch (l->l_stat) { |
|
case LSRUN: |
|
if (l->l_flag & LW_INMEM) { |
|
l->l_target_cpu = ci; |
|
break; |
|
} |
|
case LSIDL: |
|
l->l_cpu = ci; |
|
lwp_unlock_to(l, spc->spc_mutex); |
|
KASSERT(!mutex_owned(spc->spc_mutex)); |
|
return; |
|
case LSSLEEP: |
|
l->l_cpu = ci; |
|
break; |
|
case LSSTOP: |
|
case LSSUSPENDED: |
|
if (l->l_wchan != NULL) { |
|
l->l_cpu = ci; |
|
break; |
|
} |
|
case LSONPROC: |
|
l->l_target_cpu = ci; |
|
break; |
|
} |
|
lwp_unlock(l); |
|
} |
|
|
|
/* |
|
* Find the LWP in the process. Arguments may be zero, in such case, |
|
* the calling process and first LWP in the list will be used. |
|
* On success - returns LWP locked. |
|
*/ |
|
struct lwp * |
|
lwp_find2(pid_t pid, lwpid_t lid) |
|
{ |
|
proc_t *p; |
|
lwp_t *l; |
|
|
|
/* Find the process */ |
|
p = (pid == 0) ? curlwp->l_proc : p_find(pid, PFIND_UNLOCK_FAIL); |
|
if (p == NULL) |
|
return NULL; |
|
mutex_enter(&p->p_smutex); |
|
if (pid != 0) { |
|
/* Case of p_find */ |
|
mutex_exit(&proclist_lock); |
|
} |
|
|
|
/* Find the thread */ |
|
l = (lid == 0) ? LIST_FIRST(&p->p_lwps) : lwp_find(p, lid); |
|
if (l != NULL) |
|
lwp_lock(l); |
|
mutex_exit(&p->p_smutex); |
|
|
|
return l; |
|
} |
|
|
|
/* |
|
* Look up a live LWP within the speicifed process, and return it locked. |
|
* |
|
* Must be called with p->p_smutex held. |
|
*/ |
|
struct lwp * |
|
lwp_find(struct proc *p, int id) |
|
{ |
|
struct lwp *l; |
|
|
|
KASSERT(mutex_owned(&p->p_smutex)); |
|
|
|
LIST_FOREACH(l, &p->p_lwps, l_sibling) { |
|
if (l->l_lid == id) |
|
break; |
|
} |
|
|
|
/* |
|
* No need to lock - all of these conditions will |
|
* be visible with the process level mutex held. |
|
*/ |
|
if (l != NULL && (l->l_stat == LSIDL || l->l_stat == LSZOMB)) |
|
l = NULL; |
|
|
|
return l; |
|
} |
|
|
|
/* |
|
* Update an LWP's cached credentials to mirror the process' master copy. |
|
* |
|
* This happens early in the syscall path, on user trap, and on LWP |
|
* creation. A long-running LWP can also voluntarily choose to update |
|
* it's credentials by calling this routine. This may be called from |
|
* LWP_CACHE_CREDS(), which checks l->l_cred != p->p_cred beforehand. |
|
*/ |
|
void |
|
lwp_update_creds(struct lwp *l) |
|
{ |
|
kauth_cred_t oc; |
|
struct proc *p; |
|
|
|
p = l->l_proc; |
|
oc = l->l_cred; |
|
|
|
mutex_enter(&p->p_mutex); |
|
kauth_cred_hold(p->p_cred); |
|
l->l_cred = p->p_cred; |
|
mutex_exit(&p->p_mutex); |
|
if (oc != NULL) |
|
kauth_cred_free(oc); |
|
} |
|
|
|
/* |
|
* Verify that an LWP is locked, and optionally verify that the lock matches |
|
* one we specify. |
|
*/ |
|
int |
|
lwp_locked(struct lwp *l, kmutex_t *mtx) |
|
{ |
|
kmutex_t *cur = l->l_mutex; |
|
|
|
return mutex_owned(cur) && (mtx == cur || mtx == NULL); |
|
} |
|
|
|
/* |
|
* Lock an LWP. |
|
*/ |
|
void |
|
lwp_lock_retry(struct lwp *l, kmutex_t *old) |
|
{ |
|
|
|
/* |
|
* XXXgcc ignoring kmutex_t * volatile on i386 |
|
* |
|
* gcc version 4.1.2 20061021 prerelease (NetBSD nb1 20061021) |
|
*/ |
|
#if 1 |
|
while (l->l_mutex != old) { |
|
#else |
|
for (;;) { |
|
#endif |
|
mutex_spin_exit(old); |
|
old = l->l_mutex; |
|
mutex_spin_enter(old); |
|
|
|
/* |
|
* mutex_enter() will have posted a read barrier. Re-test |
|
* l->l_mutex. If it has changed, we need to try again. |
|
*/ |
|
#if 1 |
|
} |
|
#else |
|
} while (__predict_false(l->l_mutex != old)); |
|
#endif |
|
} |
|
|
|
/* |
|
* Lend a new mutex to an LWP. The old mutex must be held. |
|
*/ |
|
void |
|
lwp_setlock(struct lwp *l, kmutex_t *new) |
|
{ |
|
|
|
KASSERT(mutex_owned(l->l_mutex)); |
|
|
|
membar_producer(); |
|
l->l_mutex = new; |
|
} |
|
|
|
/* |
|
* Lend a new mutex to an LWP, and release the old mutex. The old mutex |
|
* must be held. |
|
*/ |
|
void |
|
lwp_unlock_to(struct lwp *l, kmutex_t *new) |
|
{ |
|
kmutex_t *old; |
|
|
|
KASSERT(mutex_owned(l->l_mutex)); |
|
|
|
old = l->l_mutex; |
|
membar_producer(); |
|
l->l_mutex = new; |
|
mutex_spin_exit(old); |
|
} |
|
|
|
/* |
|
* Acquire a new mutex, and donate it to an LWP. The LWP must already be |
|
* locked. |
|
*/ |
|
void |
|
lwp_relock(struct lwp *l, kmutex_t *new) |
|
{ |
|
kmutex_t *old; |
|
|
|
KASSERT(mutex_owned(l->l_mutex)); |
|
|
|
old = l->l_mutex; |
|
if (old != new) { |
|
mutex_spin_enter(new); |
|
l->l_mutex = new; |
|
mutex_spin_exit(old); |
|
} |
|
} |
|
|
|
int |
|
lwp_trylock(struct lwp *l) |
|
{ |
|
kmutex_t *old; |
|
|
|
for (;;) { |
|
if (!mutex_tryenter(old = l->l_mutex)) |
|
return 0; |
|
if (__predict_true(l->l_mutex == old)) |
|
return 1; |
|
mutex_spin_exit(old); |
|
} |
|
} |
|
|
|
/* |
|
* Handle exceptions for mi_userret(). Called if a member of LW_USERRET is |
|
* set. |
|
*/ |
|
void |
|
lwp_userret(struct lwp *l) |
|
{ |
|
struct proc *p; |
|
void (*hook)(void); |
|
int sig; |
|
|
|
p = l->l_proc; |
|
|
|
#ifndef __HAVE_FAST_SOFTINTS |
|
/* Run pending soft interrupts. */ |
|
if (l->l_cpu->ci_data.cpu_softints != 0) |
|
softint_overlay(); |
|
#endif |
|
|
|
/* |
|
* It should be safe to do this read unlocked on a multiprocessor |
|
* system.. |
|
*/ |
|
while ((l->l_flag & LW_USERRET) != 0) { |
|
/* |
|
* Process pending signals first, unless the process |
|
* is dumping core or exiting, where we will instead |
|
* enter the L_WSUSPEND case below. |
|
*/ |
|
if ((l->l_flag & (LW_PENDSIG | LW_WCORE | LW_WEXIT)) == |
|
LW_PENDSIG) { |
|
mutex_enter(&p->p_smutex); |
|
while ((sig = issignal(l)) != 0) |
|
postsig(sig); |
|
mutex_exit(&p->p_smutex); |
|
} |
|
|
|
/* |
|
* Core-dump or suspend pending. |
|
* |
|
* In case of core dump, suspend ourselves, so that the |
|
* kernel stack and therefore the userland registers saved |
|
* in the trapframe are around for coredump() to write them |
|
* out. We issue a wakeup on p->p_lwpcv so that sigexit() |
|
* will write the core file out once all other LWPs are |
|
* suspended. |
|
*/ |
|
if ((l->l_flag & LW_WSUSPEND) != 0) { |
|
mutex_enter(&p->p_smutex); |
|
p->p_nrlwps--; |
|
cv_broadcast(&p->p_lwpcv); |
|
lwp_lock(l); |
|
l->l_stat = LSSUSPENDED; |
|
mutex_exit(&p->p_smutex); |
|
mi_switch(l); |
|
} |
|
|
|
/* Process is exiting. */ |
|
if ((l->l_flag & LW_WEXIT) != 0) { |
|
lwp_exit(l); |
|
KASSERT(0); |
|
/* NOTREACHED */ |
|
} |
|
|
|
/* Call userret hook; used by Linux emulation. */ |
|
if ((l->l_flag & LW_WUSERRET) != 0) { |
|
lwp_lock(l); |
|
l->l_flag &= ~LW_WUSERRET; |
|
lwp_unlock(l); |
|
hook = p->p_userret; |
|
p->p_userret = NULL; |
|
(*hook)(); |
|
} |
|
} |
|
} |
|
|
|
/* |
|
* Force an LWP to enter the kernel, to take a trip through lwp_userret(). |
|
*/ |
|
void |
|
lwp_need_userret(struct lwp *l) |
|
{ |
|
KASSERT(lwp_locked(l, NULL)); |
|
|
|
/* |
|
* Since the tests in lwp_userret() are done unlocked, make sure |
|
* that the condition will be seen before forcing the LWP to enter |
|
* kernel mode. |
|
*/ |
|
membar_producer(); |
|
cpu_signotify(l); |
|
} |
|
|
|
/* |
|
* Add one reference to an LWP. This will prevent the LWP from |
|
* exiting, thus keep the lwp structure and PCB around to inspect. |
|
*/ |
|
void |
|
lwp_addref(struct lwp *l) |
|
{ |
|
|
|
KASSERT(mutex_owned(&l->l_proc->p_smutex)); |
|
KASSERT(l->l_stat != LSZOMB); |
|
KASSERT(l->l_refcnt != 0); |
|
|
|
l->l_refcnt++; |
|
} |
|
|
|
/* |
|
* Remove one reference to an LWP. If this is the last reference, |
|
* then we must finalize the LWP's death. |
|
*/ |
|
void |
|
lwp_delref(struct lwp *l) |
|
{ |
|
struct proc *p = l->l_proc; |
|
|
|
mutex_enter(&p->p_smutex); |
|
KASSERT(l->l_stat != LSZOMB); |
|
KASSERT(l->l_refcnt > 0); |
|
if (--l->l_refcnt == 0) |
|
cv_broadcast(&p->p_lwpcv); |
|
mutex_exit(&p->p_smutex); |
|
} |
|
|
|
/* |
|
* Drain all references to the current LWP. |
|
*/ |
|
void |
|
lwp_drainrefs(struct lwp *l) |
|
{ |
|
struct proc *p = l->l_proc; |
|
|
|
KASSERT(mutex_owned(&p->p_smutex)); |
|
KASSERT(l->l_refcnt != 0); |
|
|
|
l->l_refcnt--; |
|
while (l->l_refcnt != 0) |
|
cv_wait(&p->p_lwpcv, &p->p_smutex); |
|
} |
|
|
|
/* |
|
* lwp_specific_key_create -- |
|
* Create a key for subsystem lwp-specific data. |
|
*/ |
|
int |
|
lwp_specific_key_create(specificdata_key_t *keyp, specificdata_dtor_t dtor) |
|
{ |
|
|
|
return (specificdata_key_create(lwp_specificdata_domain, keyp, dtor)); |
|
} |
|
|
|
/* |
|
* lwp_specific_key_delete -- |
|
* Delete a key for subsystem lwp-specific data. |
|
*/ |
|
void |
|
lwp_specific_key_delete(specificdata_key_t key) |
|
{ |
|
|
|
specificdata_key_delete(lwp_specificdata_domain, key); |
|
} |
|
|
|
/* |
|
* lwp_initspecific -- |
|
* Initialize an LWP's specificdata container. |
|
*/ |
|
void |
|
lwp_initspecific(struct lwp *l) |
|
{ |
|
int error; |
|
|
|
error = specificdata_init(lwp_specificdata_domain, &l->l_specdataref); |
|
KASSERT(error == 0); |
|
} |
|
|
|
/* |
|
* lwp_finispecific -- |
|
* Finalize an LWP's specificdata container. |
|
*/ |
|
void |
|
lwp_finispecific(struct lwp *l) |
|
{ |
|
|
|
specificdata_fini(lwp_specificdata_domain, &l->l_specdataref); |
|
} |
|
|
|
/* |
|
* lwp_getspecific -- |
|
* Return lwp-specific data corresponding to the specified key. |
|
* |
|
* Note: LWP specific data is NOT INTERLOCKED. An LWP should access |
|
* only its OWN SPECIFIC DATA. If it is necessary to access another |
|
* LWP's specifc data, care must be taken to ensure that doing so |
|
* would not cause internal data structure inconsistency (i.e. caller |
|
* can guarantee that the target LWP is not inside an lwp_getspecific() |
|
* or lwp_setspecific() call). |
|
*/ |
|
void * |
|
lwp_getspecific(specificdata_key_t key) |
|
{ |
|
|
|
return (specificdata_getspecific_unlocked(lwp_specificdata_domain, |
|
&curlwp->l_specdataref, key)); |
|
} |
|
|
|
void * |
|
_lwp_getspecific_by_lwp(struct lwp *l, specificdata_key_t key) |
|
{ |
|
|
|
return (specificdata_getspecific_unlocked(lwp_specificdata_domain, |
|
&l->l_specdataref, key)); |
|
} |
|
|
|
/* |
|
* lwp_setspecific -- |
|
* Set lwp-specific data corresponding to the specified key. |
|
*/ |
|
void |
|
lwp_setspecific(specificdata_key_t key, void *data) |
|
{ |
|
|
|
specificdata_setspecific(lwp_specificdata_domain, |
|
&curlwp->l_specdataref, key, data); |
|
} |
|
|
|
/* |
|
* Allocate a new lwpctl structure for a user LWP. |
|
*/ |
|
int |
|
lwp_ctl_alloc(vaddr_t *uaddr) |
|
{ |
|
lcproc_t *lp; |
|
u_int bit, i, offset; |
|
struct uvm_object *uao; |
|
int error; |
|
lcpage_t *lcp; |
|
proc_t *p; |
|
lwp_t *l; |
|
|
|
l = curlwp; |
|
p = l->l_proc; |
|
|
|
if (l->l_lcpage != NULL) { |
|
lcp = l->l_lcpage; |
|
*uaddr = lcp->lcp_uaddr + (vaddr_t)l->l_lwpctl - lcp->lcp_kaddr; |
|
return (EINVAL); |
|
} |
|
|
|
/* First time around, allocate header structure for the process. */ |
|
if ((lp = p->p_lwpctl) == NULL) { |
|
lp = kmem_alloc(sizeof(*lp), KM_SLEEP); |
|
mutex_init(&lp->lp_lock, MUTEX_DEFAULT, IPL_NONE); |
|
lp->lp_uao = NULL; |
|
TAILQ_INIT(&lp->lp_pages); |
|
mutex_enter(&p->p_mutex); |
|
if (p->p_lwpctl == NULL) { |
|
p->p_lwpctl = lp; |
|
mutex_exit(&p->p_mutex); |
|
} else { |
|
mutex_exit(&p->p_mutex); |
|
mutex_destroy(&lp->lp_lock); |
|
kmem_free(lp, sizeof(*lp)); |
|
lp = p->p_lwpctl; |
|
} |
|
} |
|
|
|
/* |
|
* Set up an anonymous memory region to hold the shared pages. |
|
* Map them into the process' address space. The user vmspace |
|
* gets the first reference on the UAO. |
|
*/ |
|
mutex_enter(&lp->lp_lock); |
|
if (lp->lp_uao == NULL) { |
|
lp->lp_uao = uao_create(LWPCTL_UAREA_SZ, 0); |
|
lp->lp_cur = 0; |
|
lp->lp_max = LWPCTL_UAREA_SZ; |
|
lp->lp_uva = p->p_emul->e_vm_default_addr(p, |
|
(vaddr_t)p->p_vmspace->vm_daddr, LWPCTL_UAREA_SZ); |
|
error = uvm_map(&p->p_vmspace->vm_map, &lp->lp_uva, |
|
LWPCTL_UAREA_SZ, lp->lp_uao, 0, 0, UVM_MAPFLAG(UVM_PROT_RW, |
|
UVM_PROT_RW, UVM_INH_NONE, UVM_ADV_NORMAL, 0)); |
|
if (error != 0) { |
|
uao_detach(lp->lp_uao); |
|
lp->lp_uao = NULL; |
|
mutex_exit(&lp->lp_lock); |
|
return error; |
|
} |
|
} |
|
|
|
/* Get a free block and allocate for this LWP. */ |
|
TAILQ_FOREACH(lcp, &lp->lp_pages, lcp_chain) { |
|
if (lcp->lcp_nfree != 0) |
|
break; |
|
} |
|
if (lcp == NULL) { |
|
/* Nothing available - try to set up a free page. */ |
|
if (lp->lp_cur == lp->lp_max) { |
|
mutex_exit(&lp->lp_lock); |
|
return ENOMEM; |
|
} |
|
lcp = kmem_alloc(LWPCTL_LCPAGE_SZ, KM_SLEEP); |
|
if (lcp == NULL) { |
|
mutex_exit(&lp->lp_lock); |
|
return ENOMEM; |
|
} |
|
/* |
|
* Wire the next page down in kernel space. Since this |
|
* is a new mapping, we must add a reference. |
|
*/ |
|
uao = lp->lp_uao; |
|
(*uao->pgops->pgo_reference)(uao); |
|
error = uvm_map(kernel_map, &lcp->lcp_kaddr, PAGE_SIZE, |
|
uao, lp->lp_cur, PAGE_SIZE, |
|
UVM_MAPFLAG(UVM_PROT_RW, UVM_PROT_RW, |
|
UVM_INH_NONE, UVM_ADV_RANDOM, 0)); |
|
if (error != 0) { |
|
mutex_exit(&lp->lp_lock); |
|
kmem_free(lcp, LWPCTL_LCPAGE_SZ); |
|
(*uao->pgops->pgo_detach)(uao); |
|
return error; |
|
} |
|
error = uvm_map_pageable(kernel_map, lcp->lcp_kaddr, |
|
lcp->lcp_kaddr + PAGE_SIZE, FALSE, 0); |
|
if (error != 0) { |
|
mutex_exit(&lp->lp_lock); |
|
uvm_unmap(kernel_map, lcp->lcp_kaddr, |
|
lcp->lcp_kaddr + PAGE_SIZE); |
|
kmem_free(lcp, LWPCTL_LCPAGE_SZ); |
|
return error; |
|
} |
|
/* Prepare the page descriptor and link into the list. */ |
|
lcp->lcp_uaddr = lp->lp_uva + lp->lp_cur; |
|
lp->lp_cur += PAGE_SIZE; |
|
lcp->lcp_nfree = LWPCTL_PER_PAGE; |
|
lcp->lcp_rotor = 0; |
|
memset(lcp->lcp_bitmap, 0xff, LWPCTL_BITMAP_SZ); |
|
TAILQ_INSERT_HEAD(&lp->lp_pages, lcp, lcp_chain); |
|
} |
|
for (i = lcp->lcp_rotor; lcp->lcp_bitmap[i] == 0;) { |
|
if (++i >= LWPCTL_BITMAP_ENTRIES) |
|
i = 0; |
|
} |
|
bit = ffs(lcp->lcp_bitmap[i]) - 1; |
|
lcp->lcp_bitmap[i] ^= (1 << bit); |
|
lcp->lcp_rotor = i; |
|
lcp->lcp_nfree--; |
|
l->l_lcpage = lcp; |
|
offset = (i << 5) + bit; |
|
l->l_lwpctl = (lwpctl_t *)lcp->lcp_kaddr + offset; |
|
*uaddr = lcp->lcp_uaddr + offset * sizeof(lwpctl_t); |
|
mutex_exit(&lp->lp_lock); |
|
|
|
l->l_lwpctl->lc_curcpu = (short)curcpu()->ci_data.cpu_index; |
|
|
|
return 0; |
|
} |
|
|
|
/* |
|
* Free an lwpctl structure back to the per-process list. |
|
*/ |
|
void |
|
lwp_ctl_free(lwp_t *l) |
|
{ |
|
lcproc_t *lp; |
|
lcpage_t *lcp; |
|
u_int map, offset; |
|
|
|
lp = l->l_proc->p_lwpctl; |
|
KASSERT(lp != NULL); |
|
|
|
lcp = l->l_lcpage; |
|
offset = (u_int)((lwpctl_t *)l->l_lwpctl - (lwpctl_t *)lcp->lcp_kaddr); |
|
KASSERT(offset < LWPCTL_PER_PAGE); |
|
|
|
mutex_enter(&lp->lp_lock); |
|
lcp->lcp_nfree++; |
|
map = offset >> 5; |
|
lcp->lcp_bitmap[map] |= (1 << (offset & 31)); |
|
if (lcp->lcp_bitmap[lcp->lcp_rotor] == 0) |
|
lcp->lcp_rotor = map; |
|
if (TAILQ_FIRST(&lp->lp_pages)->lcp_nfree == 0) { |
|
TAILQ_REMOVE(&lp->lp_pages, lcp, lcp_chain); |
|
TAILQ_INSERT_HEAD(&lp->lp_pages, lcp, lcp_chain); |
|
} |
|
mutex_exit(&lp->lp_lock); |
|
} |
|
|
|
/* |
|
* Process is exiting; tear down lwpctl state. This can only be safely |
|
* called by the last LWP in the process. |
|
*/ |
|
void |
|
lwp_ctl_exit(void) |
|
{ |
|
lcpage_t *lcp, *next; |
|
lcproc_t *lp; |
|
proc_t *p; |
|
lwp_t *l; |
|
|
|
l = curlwp; |
|
l->l_lwpctl = NULL; |
|
p = l->l_proc; |
|
lp = p->p_lwpctl; |
|
|
|
KASSERT(lp != NULL); |
|
KASSERT(p->p_nlwps == 1); |
|
|
|
for (lcp = TAILQ_FIRST(&lp->lp_pages); lcp != NULL; lcp = next) { |
|
next = TAILQ_NEXT(lcp, lcp_chain); |
|
uvm_unmap(kernel_map, lcp->lcp_kaddr, |
|
lcp->lcp_kaddr + PAGE_SIZE); |
|
kmem_free(lcp, LWPCTL_LCPAGE_SZ); |
|
} |
|
|
|
if (lp->lp_uao != NULL) { |
|
uvm_unmap(&p->p_vmspace->vm_map, lp->lp_uva, |
|
lp->lp_uva + LWPCTL_UAREA_SZ); |
|
} |
|
|
|
mutex_destroy(&lp->lp_lock); |
|
kmem_free(lp, sizeof(*lp)); |
|
p->p_lwpctl = NULL; |
|
} |
|
|
|
#if defined(DDB) |
|
void |
|
lwp_whatis(uintptr_t addr, void (*pr)(const char *, ...)) |
|
{ |
|
lwp_t *l; |
|
|
|
LIST_FOREACH(l, &alllwp, l_list) { |
|
uintptr_t stack = (uintptr_t)KSTACK_LOWEST_ADDR(l); |
|
|
|
if (addr < stack || stack + KSTACK_SIZE <= addr) { |
|
continue; |
|
} |
|
(*pr)("%p is %p+%zu, LWP %p's stack\n", |
|
(void *)addr, (void *)stack, |
|
(size_t)(addr - stack), l); |
|
} |
|
} |
|
#endif /* defined(DDB) */ |