src/sys/uvm/uvm_loan.c - diff

Return to uvm_loan.c CVS log

Up to [cvs.NetBSD.org] / src / sys / uvm

Please note that diffs are not public domain; they are subject to the copyright notices on the relevant files.

Diff for /src/sys/uvm/uvm_loan.c between version 1.65 and 1.65.4.2

version 1.65, 2007/02/22 06:05:01

version 1.65.4.2, 2007/07/03 13:14:03

Line 183 uvm_loanentry(struct uvm_faultinfo *ufi,

rv = -1;

}

/* locked: if (rv > 0) => map, amap, uobj [o.w. unlocked] */

LOCK_ASSERT(rv > 0 || aref->ar_amap == NULL ||

KASSERT(rv > 0 || aref->ar_amap == NULL ||

!simple_lock_held(&aref->ar_amap->am_l));

!mutex_owned(&aref->ar_amap->am_l));

LOCK_ASSERT(rv > 0 || uobj == NULL ||

KASSERT(rv > 0 || uobj == NULL ||

!simple_lock_held(&uobj->vmobjlock));

!mutex_owned(&uobj->vmobjlock));

/* total failure */

if (rv < 0) {

Line 359 uvm_loananon(struct uvm_faultinfo *ufi,

if (flags & UVM_LOAN_TOANON) {

simple_lock(&anon->an_lock);

mutex_enter(&anon->an_lock);

pg = anon->an_page;

if (pg && (pg->pqflags & PQ_ANON) != 0 && anon->an_ref == 1) {

if (pg->wire_count > 0) {

Line 374 uvm_loananon(struct uvm_faultinfo *ufi,

anon->an_ref++;

**output = anon;

(*output)++;

simple_unlock(&anon->an_lock);

mutex_exit(&anon->an_lock);

UVMHIST_LOG(loanhist, "->A done", 0,0,0,0);

return (1);

}

Line 385 uvm_loananon(struct uvm_faultinfo *ufi,

* this for us.

simple_lock(&anon->an_lock);

mutex_enter(&anon->an_lock);

error = uvmfault_anonget(ufi, ufi->entry->aref.ar_amap, anon);

Line 416 uvm_loananon(struct uvm_faultinfo *ufi,

pg = anon->an_page;

uvm_lock_pageq();

mutex_enter(&uvm_pageqlock);

if (pg->wire_count > 0) {

uvm_unlock_pageq();

mutex_exit(&uvm_pageqlock);

UVMHIST_LOG(loanhist, "->K wired %p", pg,0,0,0);

KASSERT(pg->uobject == NULL);

uvmfault_unlockall(ufi, ufi->entry->aref.ar_amap,

Line 430 uvm_loananon(struct uvm_faultinfo *ufi,

}

pg->loan_count++;

uvm_pageactivate(pg);

uvm_unlock_pageq();

mutex_exit(&uvm_pageqlock);

**output = pg;

(*output)++;

/* unlock anon and return success */

if (pg->uobject)

simple_unlock(&pg->uobject->vmobjlock);

mutex_exit(&pg->uobject->vmobjlock);

simple_unlock(&anon->an_lock);

mutex_exit(&anon->an_lock);

UVMHIST_LOG(loanhist, "->K done", 0,0,0,0);

return (1);

}

Line 465 uvm_loanpage(struct vm_page **pgpp, int

KASSERT(pg->uobject != NULL);

KASSERT(pg->uobject == pgpp[0]->uobject);

KASSERT(!(pg->flags & (PG_RELEASED|PG_PAGEOUT)));

LOCK_ASSERT(simple_lock_held(&pg->uobject->vmobjlock));

KASSERT(mutex_owned(&pg->uobject->vmobjlock));

KASSERT(pg->flags & PG_BUSY);

uvm_lock_pageq();

mutex_enter(&uvm_pageqlock);

if (pg->wire_count > 0) {

uvm_unlock_pageq();

mutex_exit(&uvm_pageqlock);

UVMHIST_LOG(loanhist, "wired %p", pg,0,0,0);

error = EBUSY;

break;

Line 480 uvm_loanpage(struct vm_page **pgpp, int

}

pg->loan_count++;

uvm_pageactivate(pg);

uvm_unlock_pageq();

mutex_exit(&uvm_pageqlock);

}

uvm_page_unbusy(pgpp, npages);

Line 489 uvm_loanpage(struct vm_page **pgpp, int

* backout what we've done

struct simplelock *slock = &pgpp[0]->uobject->vmobjlock;

kmutex_t *slock = &pgpp[0]->uobject->vmobjlock;

simple_unlock(slock);

mutex_exit(slock);

uvm_unloan(pgpp, i, UVM_LOAN_TOPAGE);

simple_lock(slock);

mutex_enter(slock);

}

UVMHIST_LOG(loanhist, "done %d", error,0,0,0);

Line 521 uvm_loanuobjpages(struct uvm_object *uob

struct vm_page **pgpp;

int error;

int i;

struct simplelock *slock;

kmutex_t *slock;

pgpp = origpgpp;

for (ndone = 0; ndone < orignpages; ) {

Line 530 uvm_loanuobjpages(struct uvm_object *uob

int npendloan = 0xdead; /* XXX gcc */

reget:

npages = MIN(UVM_LOAN_GET_CHUNK, orignpages - ndone);

simple_lock(&uobj->vmobjlock);

mutex_enter(&uobj->vmobjlock);

error = (*uobj->pgops->pgo_get)(uobj,

pgoff + (ndone << PAGE_SHIFT), pgpp, &npages, 0,

VM_PROT_READ, 0, PGO_SYNCIO);

Line 546 reget:

/* loan and unbusy pages */

slock = NULL;

for (i = 0; i < npages; i++) {

struct simplelock *nextslock; /* slock for next page */

kmutex_t *nextslock; /* slock for next page */

struct vm_page *pg = *pgpp;

/* XXX assuming that the page is owned by uobj */

Line 558 reget:

KASSERT(npendloan > 0);

error = uvm_loanpage(pgpp - npendloan,

npendloan);

simple_unlock(slock);

mutex_exit(slock);

if (error)

goto fail;

ndone += npendloan;

Line 566 reget:

}

slock = nextslock;

npendloan = 0;

simple_lock(slock);

mutex_enter(slock);

}

if ((pg->flags & PG_RELEASED) != 0) {

* release pages and try again.

simple_unlock(slock);

mutex_exit(slock);

for (; i < npages; i++) {

pg = pgpp[i];

slock = &pg->uobject->vmobjlock;

simple_lock(slock);

mutex_enter(slock);

uvm_lock_pageq();

mutex_enter(&uvm_pageqlock);

uvm_page_unbusy(&pg, 1);

uvm_unlock_pageq();

mutex_exit(&uvm_pageqlock);

simple_unlock(slock);

mutex_exit(slock);

}

goto reget;

}

Line 594 reget:

KASSERT(slock != NULL);

KASSERT(npendloan > 0);

error = uvm_loanpage(pgpp - npendloan, npendloan);

simple_unlock(slock);

mutex_exit(slock);

if (error)

goto fail;

ndone += npendloan;

Line 638 uvm_loanuobj(struct uvm_faultinfo *ufi,

* XXXCDC: duplicate code with uvm_fault().

simple_lock(&uobj->vmobjlock);

mutex_enter(&uobj->vmobjlock);

if (uobj->pgops->pgo_get) { /* try locked pgo_get */

npages = 1;

pg = NULL;

Line 689 uvm_loanuobj(struct uvm_faultinfo *ufi,

if (locked && amap)

amap_lock(amap);

uobj = pg->uobject;

simple_lock(&uobj->vmobjlock);

mutex_enter(&uobj->vmobjlock);

* verify that the page has not be released and re-verify

Line 714 uvm_loanuobj(struct uvm_faultinfo *ufi,

wakeup(pg);

}

if (pg->flags & PG_RELEASED) {

uvm_lock_pageq();

mutex_enter(&uvm_pageqlock);

uvm_pagefree(pg);

uvm_unlock_pageq();

mutex_exit(&uvm_pageqlock);

simple_unlock(&uobj->vmobjlock);

mutex_exit(&uobj->vmobjlock);

return (0);

}

uvm_lock_pageq();

mutex_enter(&uvm_pageqlock);

uvm_pageactivate(pg);

uvm_unlock_pageq();

mutex_exit(&uvm_pageqlock);

pg->flags &= ~(PG_BUSY|PG_WANTED);

UVM_PAGE_OWN(pg, NULL);

simple_unlock(&uobj->vmobjlock);

mutex_exit(&uobj->vmobjlock);

return (0);

}

Line 743 uvm_loanuobj(struct uvm_faultinfo *ufi,

uvmfault_unlockall(ufi, amap, uobj, NULL);

return (-1);

}

simple_unlock(&uobj->vmobjlock);

mutex_exit(&uobj->vmobjlock);

**output = pg;

(*output)++;

return (1);

Line 758 uvm_loanuobj(struct uvm_faultinfo *ufi,

if (pg->uanon) {

anon = pg->uanon;

simple_lock(&anon->an_lock);

mutex_enter(&anon->an_lock);

anon->an_ref++;

simple_unlock(&anon->an_lock);

mutex_exit(&anon->an_lock);

if (pg->flags & PG_WANTED) {

wakeup(pg);

}

pg->flags &= ~(PG_WANTED|PG_BUSY);

UVM_PAGE_OWN(pg, NULL);

simple_unlock(&uobj->vmobjlock);

mutex_exit(&uobj->vmobjlock);

**output = anon;

(*output)++;

return (1);

Line 782 uvm_loanuobj(struct uvm_faultinfo *ufi,

}

anon->an_page = pg;

pg->uanon = anon;

uvm_lock_pageq();

mutex_enter(&uvm_pageqlock);

if (pg->wire_count > 0) {

uvm_unlock_pageq();

mutex_exit(&uvm_pageqlock);

UVMHIST_LOG(loanhist, "wired %p", pg,0,0,0);

pg->uanon = NULL;

anon->an_page = NULL;

anon->an_ref--;

simple_unlock(&anon->an_lock);

mutex_exit(&anon->an_lock);

uvm_anfree(anon);

goto fail;

}

Line 798 uvm_loanuobj(struct uvm_faultinfo *ufi,

}

pg->loan_count++;

uvm_pageactivate(pg);

uvm_unlock_pageq();

mutex_exit(&uvm_pageqlock);

if (pg->flags & PG_WANTED) {

wakeup(pg);

}

pg->flags &= ~(PG_WANTED|PG_BUSY);

UVM_PAGE_OWN(pg, NULL);

simple_unlock(&uobj->vmobjlock);

mutex_exit(&uobj->vmobjlock);

simple_unlock(&anon->an_lock);

mutex_exit(&anon->an_lock);

**output = anon;

(*output)++;

return (1);

Line 846 uvm_loanzero(struct uvm_faultinfo *ufi,

UVMHIST_FUNC(__func__); UVMHIST_CALLED(loanhist);

again:

simple_lock(&uvm_loanzero_object.vmobjlock);

mutex_enter(&uvm_loanzero_object.vmobjlock);

* first, get ahold of our single zero page.

Line 856 again:

TAILQ_FIRST(&uvm_loanzero_object.memq)) == NULL)) {

while ((pg = uvm_pagealloc(&uvm_loanzero_object, 0, NULL,

UVM_PGA_ZERO)) == NULL) {

simple_unlock(&uvm_loanzero_object.vmobjlock);

mutex_exit(&uvm_loanzero_object.vmobjlock);

uvmfault_unlockall(ufi, amap, NULL, NULL);

uvm_wait("loanzero");

if (!uvmfault_relock(ufi)) {

Line 871 again:

/* got a zero'd page. */

pg->flags &= ~(PG_WANTED|PG_BUSY|PG_FAKE);

pg->flags |= PG_RDONLY;

uvm_lock_pageq();

mutex_enter(&uvm_pageqlock);

uvm_pageactivate(pg);

uvm_unlock_pageq();

mutex_exit(&uvm_pageqlock);

UVM_PAGE_OWN(pg, NULL);

}

if ((flags & UVM_LOAN_TOANON) == 0) { /* loaning to kernel-page */

uvm_lock_pageq();

mutex_enter(&uvm_pageqlock);

pg->loan_count++;

uvm_unlock_pageq();

mutex_exit(&uvm_pageqlock);

simple_unlock(&uvm_loanzero_object.vmobjlock);

mutex_exit(&uvm_loanzero_object.vmobjlock);

**output = pg;

(*output)++;

return (1);

Line 895 again:

if (pg->uanon) {

anon = pg->uanon;

simple_lock(&anon->an_lock);

mutex_enter(&anon->an_lock);

anon->an_ref++;

simple_unlock(&anon->an_lock);

mutex_exit(&anon->an_lock);

simple_unlock(&uvm_loanzero_object.vmobjlock);

mutex_exit(&uvm_loanzero_object.vmobjlock);

**output = anon;

(*output)++;

return (1);

Line 911 again:

anon = uvm_analloc();

if (anon == NULL) {

/* out of swap causes us to fail */

simple_unlock(&uvm_loanzero_object.vmobjlock);

mutex_exit(&uvm_loanzero_object.vmobjlock);

uvmfault_unlockall(ufi, amap, NULL, NULL);

return (-1);

}

anon->an_page = pg;

pg->uanon = anon;

uvm_lock_pageq();

mutex_enter(&uvm_pageqlock);

pg->loan_count++;

uvm_pageactivate(pg);

uvm_unlock_pageq();

mutex_exit(&uvm_pageqlock);

simple_unlock(&anon->an_lock);

mutex_exit(&anon->an_lock);

simple_unlock(&uvm_loanzero_object.vmobjlock);

mutex_exit(&uvm_loanzero_object.vmobjlock);

**output = anon;

(*output)++;

return (1);

Line 944 uvm_unloananon(struct vm_anon **aloans,

int refs;

anon = *aloans++;

simple_lock(&anon->an_lock);

mutex_enter(&anon->an_lock);

refs = --anon->an_ref;

simple_unlock(&anon->an_lock);

mutex_exit(&anon->an_lock);

if (refs == 0) {

uvm_anfree(anon);

Line 964 static void

uvm_unloanpage(struct vm_page **ploans, int npages)

{

struct vm_page *pg;

struct simplelock *slock;

kmutex_t *slock;

uvm_lock_pageq();

mutex_enter(&uvm_pageqlock);

while (npages-- > 0) {

pg = *ploans++;

Line 983 uvm_unloanpage(struct vm_page **ploans,

} else {

slock = &pg->uanon->an_lock;

}

if (simple_lock_try(slock)) {

if (mutex_tryenter(slock)) {

break;

}

uvm_unlock_pageq();

mutex_exit(&uvm_pageqlock);

uvm_lock_pageq();

yield();

mutex_enter(&uvm_pageqlock);

slock = NULL;

}

Line 1016 uvm_unloanpage(struct vm_page **ploans,

Line 1017 uvm_unloanpage(struct vm_page **ploans,

uvm_pagefree(pg);

}

if (slock != NULL) {

simple_unlock(slock);

mutex_exit(slock);

}

uvm_unlock_pageq();

mutex_exit(&uvm_pageqlock);

}

Line 1055 ulz_put(struct uvm_object *uobj, voff_t

Line 1056 ulz_put(struct uvm_object *uobj, voff_t

if ((flags & PGO_FREE) == 0) {

simple_unlock(&uobj->vmobjlock);

mutex_exit(&uobj->vmobjlock);

return 0;

}

Line 1068 ulz_put(struct uvm_object *uobj, voff_t

Line 1069 ulz_put(struct uvm_object *uobj, voff_t

KASSERT(pg != NULL);

KASSERT(TAILQ_NEXT(pg, listq) == NULL);

uvm_lock_pageq();

mutex_enter(&uvm_pageqlock);

if (pg->uanon)

uvm_pageactivate(pg);

else

uvm_pagedequeue(pg);

uvm_unlock_pageq();

mutex_exit(&uvm_pageqlock);

simple_unlock(&uobj->vmobjlock);

mutex_exit(&uobj->vmobjlock);

return 0;

}

Line 1096 void

Line 1097 void

uvm_loan_init(void)

{

simple_lock_init(&uvm_loanzero_object.vmobjlock);

mutex_init(&uvm_loanzero_object.vmobjlock, MUTEX_DEFAULT, IPL_NONE);

TAILQ_INIT(&uvm_loanzero_object.memq);

uvm_loanzero_object.pgops = &ulz_pager;

Line 1120 uvm_loanbreak(struct vm_page *uobjpage)

Line 1121 uvm_loanbreak(struct vm_page *uobjpage)

#endif

KASSERT(uobj != NULL);

LOCK_ASSERT(simple_lock_held(&uobj->vmobjlock));

KASSERT(mutex_owned(&uobj->vmobjlock));

KASSERT(uobjpage->flags & PG_BUSY);

/* alloc new un-owned page */

Line 1155 uvm_loanbreak(struct vm_page *uobjpage)

Line 1156 uvm_loanbreak(struct vm_page *uobjpage)

uobjpage->flags &= ~(PG_WANTED|PG_BUSY);

UVM_PAGE_OWN(uobjpage, NULL);

uvm_lock_pageq();

mutex_enter(&uvm_pageqlock);

* replace uobjpage with new page.

Line 1178 uvm_loanbreak(struct vm_page *uobjpage)

Line 1179 uvm_loanbreak(struct vm_page *uobjpage)

/* install new page */

uvm_pageactivate(pg);

uvm_unlock_pageq();

mutex_exit(&uvm_pageqlock);

* done! loan is broken and "pg" is

CVSweb <webmaster@jp.NetBSD.org>