Please note that diffs are not public domain; they are subject to the copyright notices on the relevant files. =================================================================== RCS file: /ftp/cvs/cvsroot/src/sys/kern/kern_sleepq.c,v rcsdiff: /ftp/cvs/cvsroot/src/sys/kern/kern_sleepq.c,v: warning: Unknown phrases like `commitid ...;' are present. retrieving revision 1.21.6.5 retrieving revision 1.22 diff -u -p -r1.21.6.5 -r1.22 --- src/sys/kern/kern_sleepq.c 2009/01/17 13:29:19 1.21.6.5 +++ src/sys/kern/kern_sleepq.c 2008/03/17 16:54:51 1.22 @@ -1,4 +1,4 @@ -/* $NetBSD: kern_sleepq.c,v 1.21.6.5 2009/01/17 13:29:19 mjf Exp $ */ +/* $NetBSD: kern_sleepq.c,v 1.22 2008/03/17 16:54:51 ad Exp $ */ /*- * Copyright (c) 2006, 2007, 2008 The NetBSD Foundation, Inc. @@ -15,6 +15,13 @@ * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. + * 3. All advertising materials mentioning features or use of this software + * must display the following acknowledgement: + * This product includes software developed by the NetBSD + * Foundation, Inc. and its contributors. + * 4. Neither the name of The NetBSD Foundation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED @@ -35,7 +42,7 @@ */ #include -__KERNEL_RCSID(0, "$NetBSD: kern_sleepq.c,v 1.21.6.5 2009/01/17 13:29:19 mjf Exp $"); +__KERNEL_RCSID(0, "$NetBSD: kern_sleepq.c,v 1.22 2008/03/17 16:54:51 ad Exp $"); #include #include @@ -43,8 +50,6 @@ __KERNEL_RCSID(0, "$NetBSD: kern_sleepq. #include #include #include -#include -#include #include #include #include @@ -52,8 +57,6 @@ __KERNEL_RCSID(0, "$NetBSD: kern_sleepq. #include -#include "opt_sa.h" - int sleepq_sigtoerror(lwp_t *, int); /* General purpose sleep table, used by ltsleep() and condition variables. */ @@ -74,7 +77,7 @@ sleeptab_init(sleeptab_t *st) sq = &st->st_queues[i].st_queue; mutex_init(&st->st_queues[i].st_mutex, MUTEX_DEFAULT, IPL_SCHED); - sleepq_init(sq); + sleepq_init(sq, &st->st_queues[i].st_mutex); } } @@ -84,10 +87,12 @@ sleeptab_init(sleeptab_t *st) * Prepare a sleep queue for use. */ void -sleepq_init(sleepq_t *sq) +sleepq_init(sleepq_t *sq, kmutex_t *mtx) { - TAILQ_INIT(sq); + sq->sq_waiters = 0; + sq->sq_mutex = mtx; + TAILQ_INIT(&sq->sq_queue); } /* @@ -102,10 +107,21 @@ sleepq_remove(sleepq_t *sq, lwp_t *l) { struct schedstate_percpu *spc; struct cpu_info *ci; + pri_t pri; - KASSERT(lwp_locked(l, NULL)); + KASSERT(lwp_locked(l, sq->sq_mutex)); + KASSERT(sq->sq_waiters > 0); + + sq->sq_waiters--; + TAILQ_REMOVE(&sq->sq_queue, l, l_sleepchain); + +#ifdef DIAGNOSTIC + if (sq->sq_waiters == 0) + KASSERT(TAILQ_FIRST(&sq->sq_queue) == NULL); + else + KASSERT(TAILQ_FIRST(&sq->sq_queue) != NULL); +#endif - TAILQ_REMOVE(sq, l, l_sleepchain); l->l_syncobj = &sched_syncobj; l->l_wchan = NULL; l->l_sleepq = NULL; @@ -128,19 +144,18 @@ sleepq_remove(sleepq_t *sq, lwp_t *l) * If the LWP is still on the CPU, mark it as LSONPROC. It may be * about to call mi_switch(), in which case it will yield. */ - if ((l->l_pflag & LP_RUNNING) != 0) { + if ((l->l_flag & LW_RUNNING) != 0) { l->l_stat = LSONPROC; l->l_slptime = 0; lwp_setlock(l, spc->spc_lwplock); return 0; } - /* Update sleep time delta, call the wake-up handler of scheduler */ - l->l_slpticksum += (hardclock_ticks - l->l_slpticks); + /* + * Call the wake-up handler of scheduler. + * It might change the CPU for this thread. + */ sched_wakeup(l); - - /* Look for a CPU to wake up */ - l->l_cpu = sched_takecpu(l); ci = l->l_cpu; spc = &ci->ci_schedstate; @@ -149,15 +164,17 @@ sleepq_remove(sleepq_t *sq, lwp_t *l) */ spc_lock(ci); lwp_setlock(l, spc->spc_mutex); -#ifdef KERN_SA - if (l->l_proc->p_sa != NULL) - sa_awaken(l); -#endif /* KERN_SA */ sched_setrunnable(l); l->l_stat = LSRUN; l->l_slptime = 0; if ((l->l_flag & LW_INMEM) != 0) { sched_enqueue(l, false); + pri = lwp_eprio(l); + /* XXX This test is not good enough! */ + if (pri > spc->spc_curpriority) { + cpu_need_resched(ci, + (pri >= PRI_KERNEL ? RESCHED_IMMED : 0)); + } spc_unlock(ci); return 0; } @@ -177,7 +194,7 @@ sleepq_insert(sleepq_t *sq, lwp_t *l, sy const int pri = lwp_eprio(l); if ((sobj->sobj_flag & SOBJ_SLEEPQ_SORTED) != 0) { - TAILQ_FOREACH(l2, sq, l_sleepchain) { + TAILQ_FOREACH(l2, &sq->sq_queue, l_sleepchain) { if (lwp_eprio(l2) < pri) { TAILQ_INSERT_BEFORE(l2, l, l_sleepchain); return; @@ -186,9 +203,9 @@ sleepq_insert(sleepq_t *sq, lwp_t *l, sy } if ((sobj->sobj_flag & SOBJ_SLEEPQ_LIFO) != 0) - TAILQ_INSERT_HEAD(sq, l, l_sleepchain); + TAILQ_INSERT_HEAD(&sq->sq_queue, l, l_sleepchain); else - TAILQ_INSERT_TAIL(sq, l, l_sleepchain); + TAILQ_INSERT_TAIL(&sq->sq_queue, l, l_sleepchain); } /* @@ -203,7 +220,7 @@ sleepq_enqueue(sleepq_t *sq, wchan_t wch { lwp_t *l = curlwp; - KASSERT(lwp_locked(l, NULL)); + KASSERT(mutex_owned(sq->sq_mutex)); KASSERT(l->l_stat == LSONPROC); KASSERT(l->l_wchan == NULL && l->l_sleepq == NULL); @@ -215,10 +232,8 @@ sleepq_enqueue(sleepq_t *sq, wchan_t wch l->l_stat = LSSLEEP; l->l_sleeperr = 0; + sq->sq_waiters++; sleepq_insert(sq, l, sobj); - - /* Save the time when thread has slept */ - l->l_slpticks = hardclock_ticks; sched_slept(l); } @@ -236,7 +251,6 @@ sleepq_block(int timo, bool catch) struct proc *p; lwp_t *l = curlwp; bool early = false; - int biglocks = l->l_biglocks; ktrcsw(1, 0); @@ -260,13 +274,7 @@ sleepq_block(int timo, bool catch) } else { if (timo) callout_schedule(&l->l_timeout_ch, timo); - -#ifdef KERN_SA - if (((l->l_flag & LW_SA) != 0) && (~l->l_pflag & LP_SA_NOBLOCK)) - sa_switch(l); - else -#endif - mi_switch(l); + mi_switch(l); /* The LWP and sleep queue are now unlocked. */ if (timo) { @@ -274,7 +282,7 @@ sleepq_block(int timo, bool catch) * Even if the callout appears to have fired, we need to * stop it in order to synchronise with other CPUs. */ - if (callout_halt(&l->l_timeout_ch, NULL)) + if (callout_stop(&l->l_timeout_ch)) error = EWOULDBLOCK; } } @@ -284,24 +292,16 @@ sleepq_block(int timo, bool catch) if ((l->l_flag & (LW_CANCELLED | LW_WEXIT | LW_WCORE)) != 0) error = EINTR; else if ((l->l_flag & LW_PENDSIG) != 0) { - /* - * Acquiring p_lock may cause us to recurse - * through the sleep path and back into this - * routine, but is safe because LWPs sleeping - * on locks are non-interruptable. We will - * not recurse again. - */ - mutex_enter(p->p_lock); + mutex_enter(&p->p_smutex); if ((sig = issignal(l)) != 0) error = sleepq_sigtoerror(l, sig); - mutex_exit(p->p_lock); + mutex_exit(&p->p_smutex); } } ktrcsw(0, 0); - if (__predict_false(biglocks != 0)) { - KERNEL_LOCK(biglocks, NULL); - } + + KERNEL_LOCK(l->l_biglocks, l); return error; } @@ -311,16 +311,15 @@ sleepq_block(int timo, bool catch) * Wake zero or more LWPs blocked on a single wait channel. */ lwp_t * -sleepq_wake(sleepq_t *sq, wchan_t wchan, u_int expected, kmutex_t *mp) +sleepq_wake(sleepq_t *sq, wchan_t wchan, u_int expected) { lwp_t *l, *next; int swapin = 0; - KASSERT(mutex_owned(mp)); + KASSERT(mutex_owned(sq->sq_mutex)); - for (l = TAILQ_FIRST(sq); l != NULL; l = next) { + for (l = TAILQ_FIRST(&sq->sq_queue); l != NULL; l = next) { KASSERT(l->l_sleepq == sq); - KASSERT(l->l_mutex == mp); next = TAILQ_NEXT(l, l_sleepchain); if (l->l_wchan != wchan) continue; @@ -329,7 +328,7 @@ sleepq_wake(sleepq_t *sq, wchan_t wchan, break; } - mutex_spin_exit(mp); + sleepq_unlock(sq); /* * If there are newly awakend threads that need to be swapped in, @@ -352,16 +351,16 @@ u_int sleepq_unsleep(lwp_t *l, bool cleanup) { sleepq_t *sq = l->l_sleepq; - kmutex_t *mp = l->l_mutex; int swapin; - KASSERT(lwp_locked(l, mp)); + KASSERT(lwp_locked(l, NULL)); KASSERT(l->l_wchan != NULL); + KASSERT(l->l_mutex == sq->sq_mutex); swapin = sleepq_remove(sq, l); if (cleanup) { - mutex_spin_exit(mp); + sleepq_unlock(sq); if (swapin) uvm_kick_scheduler(); } @@ -406,7 +405,7 @@ sleepq_sigtoerror(lwp_t *l, int sig) struct proc *p = l->l_proc; int error; - KASSERT(mutex_owned(p->p_lock)); + KASSERT(mutex_owned(&p->p_smutex)); /* * If this sleep was canceled, don't let the syscall restart. @@ -455,29 +454,14 @@ sleepq_changepri(lwp_t *l, pri_t pri) sleepq_t *sq = l->l_sleepq; pri_t opri; - KASSERT(lwp_locked(l, NULL)); + KASSERT(lwp_locked(l, sq->sq_mutex)); opri = lwp_eprio(l); l->l_priority = pri; - - if (lwp_eprio(l) == opri) { - return; - } - if ((l->l_syncobj->sobj_flag & SOBJ_SLEEPQ_SORTED) == 0) { - return; - } - - /* - * Don't let the sleep queue become empty, even briefly. - * cv_signal() and cv_broadcast() inspect it without the - * sleep queue lock held and need to see a non-empty queue - * head if there are waiters. - */ - if (TAILQ_FIRST(sq) == l && TAILQ_NEXT(l, l_sleepchain) == NULL) { - return; + if (lwp_eprio(l) != opri) { + TAILQ_REMOVE(&sq->sq_queue, l, l_sleepchain); + sleepq_insert(sq, l, l->l_syncobj); } - TAILQ_REMOVE(sq, l, l_sleepchain); - sleepq_insert(sq, l, l->l_syncobj); } void @@ -486,27 +470,14 @@ sleepq_lendpri(lwp_t *l, pri_t pri) sleepq_t *sq = l->l_sleepq; pri_t opri; - KASSERT(lwp_locked(l, NULL)); + KASSERT(lwp_locked(l, sq->sq_mutex)); opri = lwp_eprio(l); l->l_inheritedprio = pri; - if (lwp_eprio(l) == opri) { - return; - } - if ((l->l_syncobj->sobj_flag & SOBJ_SLEEPQ_SORTED) == 0) { - return; - } - - /* - * Don't let the sleep queue become empty, even briefly. - * cv_signal() and cv_broadcast() inspect it without the - * sleep queue lock held and need to see a non-empty queue - * head if there are waiters. - */ - if (TAILQ_FIRST(sq) == l && TAILQ_NEXT(l, l_sleepchain) == NULL) { - return; + if (lwp_eprio(l) != opri && + (l->l_syncobj->sobj_flag & SOBJ_SLEEPQ_SORTED) != 0) { + TAILQ_REMOVE(&sq->sq_queue, l, l_sleepchain); + sleepq_insert(sq, l, l->l_syncobj); } - TAILQ_REMOVE(sq, l, l_sleepchain); - sleepq_insert(sq, l, l->l_syncobj); }