[BACK]Return to uvm_loan.c CVS log [TXT][DIR] Up to [cvs.NetBSD.org] / src / sys / uvm

Annotation of src/sys/uvm/uvm_loan.c, Revision 1.53

1.53    ! yamt        1: /*     $NetBSD: uvm_loan.c,v 1.52 2004/11/23 04:51:56 yamt Exp $       */
1.1       mrg         2:
                      3: /*
                      4:  *
                      5:  * Copyright (c) 1997 Charles D. Cranor and Washington University.
                      6:  * All rights reserved.
                      7:  *
                      8:  * Redistribution and use in source and binary forms, with or without
                      9:  * modification, are permitted provided that the following conditions
                     10:  * are met:
                     11:  * 1. Redistributions of source code must retain the above copyright
                     12:  *    notice, this list of conditions and the following disclaimer.
                     13:  * 2. Redistributions in binary form must reproduce the above copyright
                     14:  *    notice, this list of conditions and the following disclaimer in the
                     15:  *    documentation and/or other materials provided with the distribution.
                     16:  * 3. All advertising materials mentioning features or use of this software
                     17:  *    must display the following acknowledgement:
                     18:  *      This product includes software developed by Charles D. Cranor and
                     19:  *      Washington University.
                     20:  * 4. The name of the author may not be used to endorse or promote products
                     21:  *    derived from this software without specific prior written permission.
                     22:  *
                     23:  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
                     24:  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
                     25:  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
                     26:  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
                     27:  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
                     28:  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
                     29:  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
                     30:  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
                     31:  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
                     32:  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
1.4       mrg        33:  *
                     34:  * from: Id: uvm_loan.c,v 1.1.6.4 1998/02/06 05:08:43 chs Exp
1.1       mrg        35:  */
                     36:
                     37: /*
                     38:  * uvm_loan.c: page loanout handler
                     39:  */
1.35      lukem      40:
                     41: #include <sys/cdefs.h>
1.53    ! yamt       42: __KERNEL_RCSID(0, "$NetBSD: uvm_loan.c,v 1.52 2004/11/23 04:51:56 yamt Exp $");
1.1       mrg        43:
                     44: #include <sys/param.h>
                     45: #include <sys/systm.h>
                     46: #include <sys/kernel.h>
                     47: #include <sys/proc.h>
                     48: #include <sys/malloc.h>
                     49: #include <sys/mman.h>
                     50:
                     51: #include <uvm/uvm.h>
                     52:
                     53: /*
1.29      chs        54:  * "loaned" pages are pages which are (read-only, copy-on-write) loaned
1.1       mrg        55:  * from the VM system to other parts of the kernel.   this allows page
                     56:  * copying to be avoided (e.g. you can loan pages from objs/anons to
                     57:  * the mbuf system).
                     58:  *
                     59:  * there are 3 types of loans possible:
                     60:  *  O->K  uvm_object page to wired kernel page (e.g. mbuf data area)
1.16      thorpej    61:  *  A->K  anon page to wired kernel page (e.g. mbuf data area)
1.1       mrg        62:  *  O->A  uvm_object to anon loan (e.g. vnode page to an anon)
                     63:  * note that it possible to have an O page loaned to both an A and K
                     64:  * at the same time.
                     65:  *
                     66:  * loans are tracked by pg->loan_count.  an O->A page will have both
                     67:  * a uvm_object and a vm_anon, but PQ_ANON will not be set.   this sort
                     68:  * of page is considered "owned" by the uvm_object (not the anon).
                     69:  *
1.16      thorpej    70:  * each loan of a page to the kernel bumps the pg->wire_count.  the
                     71:  * kernel mappings for these pages will be read-only and wired.  since
                     72:  * the page will also be wired, it will not be a candidate for pageout,
                     73:  * and thus will never be pmap_page_protect()'d with VM_PROT_NONE.  a
                     74:  * write fault in the kernel to one of these pages will not cause
                     75:  * copy-on-write.  instead, the page fault is considered fatal.  this
                     76:  * is because the kernel mapping will have no way to look up the
                     77:  * object/anon which the page is owned by.  this is a good side-effect,
                     78:  * since a kernel write to a loaned page is an error.
1.1       mrg        79:  *
1.29      chs        80:  * owners that want to free their pages and discover that they are
1.1       mrg        81:  * loaned out simply "disown" them (the page becomes an orphan).  these
                     82:  * pages should be freed when the last loan is dropped.   in some cases
                     83:  * an anon may "adopt" an orphaned page.
                     84:  *
                     85:  * locking: to read pg->loan_count either the owner or the page queues
                     86:  * must be locked.   to modify pg->loan_count, both the owner of the page
                     87:  * and the PQs must be locked.   pg->flags is (as always) locked by
                     88:  * the owner of the page.
                     89:  *
                     90:  * note that locking from the "loaned" side is tricky since the object
                     91:  * getting the loaned page has no reference to the page's owner and thus
                     92:  * the owner could "die" at any time.   in order to prevent the owner
                     93:  * from dying the page queues should be locked.   this forces us to sometimes
                     94:  * use "try" locking.
                     95:  *
                     96:  * loans are typically broken by the following events:
1.29      chs        97:  *  1. user-level xwrite fault to a loaned page
1.1       mrg        98:  *  2. pageout of clean+inactive O->A loaned page
                     99:  *  3. owner frees page (e.g. pager flush)
                    100:  *
                    101:  * note that loaning a page causes all mappings of the page to become
                    102:  * read-only (via pmap_page_protect).   this could have an unexpected
1.16      thorpej   103:  * effect on normal "wired" pages if one is not careful (XXX).
1.1       mrg       104:  */
                    105:
                    106: /*
                    107:  * local prototypes
                    108:  */
                    109:
1.50      junyoung  110: static int     uvm_loananon(struct uvm_faultinfo *, void ***,
                    111:                             int, struct vm_anon *);
                    112: static int     uvm_loanentry(struct uvm_faultinfo *, void ***, int);
                    113: static int     uvm_loanuobj(struct uvm_faultinfo *, void ***,
                    114:                             int, vaddr_t);
                    115: static int     uvm_loanzero(struct uvm_faultinfo *, void ***, int);
                    116: static void    uvm_unloananon(struct vm_anon **, int);
                    117: static void    uvm_unloanpage(struct vm_page **, int);
1.51      yamt      118: static int     uvm_loanpage(struct vm_page **, int);
1.33      jdolecek  119:
1.1       mrg       120:
                    121: /*
                    122:  * inlines
                    123:  */
                    124:
                    125: /*
                    126:  * uvm_loanentry: loan out pages in a map entry (helper fn for uvm_loan())
                    127:  *
                    128:  * => "ufi" is the result of a successful map lookup (meaning that
1.31      chuck     129:  *     on entry the map is locked by the caller)
1.28      chuck     130:  * => we may unlock and then relock the map if needed (for I/O)
1.1       mrg       131:  * => we put our output result in "output"
1.31      chuck     132:  * => we always return with the map unlocked
1.28      chuck     133:  * => possible return values:
                    134:  *     -1 == error, map is unlocked
                    135:  *      0 == map relock error (try again!), map is unlocked
1.31      chuck     136:  *     >0 == number of pages we loaned, map is unlocked
1.1       mrg       137:  */
                    138:
1.6       mrg       139: static __inline int
                    140: uvm_loanentry(ufi, output, flags)
                    141:        struct uvm_faultinfo *ufi;
                    142:        void ***output;
                    143:        int flags;
1.1       mrg       144: {
1.10      chuck     145:        vaddr_t curaddr = ufi->orig_rvaddr;
1.9       eeh       146:        vsize_t togo = ufi->size;
1.6       mrg       147:        struct vm_aref *aref = &ufi->entry->aref;
                    148:        struct uvm_object *uobj = ufi->entry->object.uvm_obj;
                    149:        struct vm_anon *anon;
                    150:        int rv, result = 0;
                    151:
1.52      yamt      152:        UVMHIST_FUNC(__func__); UVMHIST_CALLED(loanhist);
                    153:
1.6       mrg       154:        /*
1.28      chuck     155:         * lock us the rest of the way down (we unlock before return)
1.6       mrg       156:         */
                    157:        if (aref->ar_amap)
1.13      chuck     158:                amap_lock(aref->ar_amap);
1.6       mrg       159:        if (uobj)
                    160:                simple_lock(&uobj->vmobjlock);
                    161:
                    162:        /*
                    163:         * loop until done
                    164:         */
                    165:        while (togo) {
                    166:
                    167:                /*
                    168:                 * find the page we want.   check the anon layer first.
                    169:                 */
                    170:
                    171:                if (aref->ar_amap) {
                    172:                        anon = amap_lookup(aref, curaddr - ufi->entry->start);
                    173:                } else {
                    174:                        anon = NULL;
                    175:                }
                    176:
1.28      chuck     177:                /* locked: map, amap, uobj */
1.6       mrg       178:                if (anon) {
                    179:                        rv = uvm_loananon(ufi, output, flags, anon);
                    180:                } else if (uobj) {
                    181:                        rv = uvm_loanuobj(ufi, output, flags, curaddr);
                    182:                } else if (UVM_ET_ISCOPYONWRITE(ufi->entry)) {
                    183:                        rv = uvm_loanzero(ufi, output, flags);
                    184:                } else {
1.47      yamt      185:                        uvmfault_unlockall(ufi, aref->ar_amap, uobj, NULL);
1.34      chs       186:                        rv = -1;
1.6       mrg       187:                }
1.31      chuck     188:                /* locked: if (rv > 0) => map, amap, uobj  [o.w. unlocked] */
1.51      yamt      189:                LOCK_ASSERT(rv > 0 || aref->ar_amap == NULL ||
                    190:                    !simple_lock_held(&aref->ar_amap->am_l));
                    191:                LOCK_ASSERT(rv > 0 || uobj == NULL ||
                    192:                    !simple_lock_held(&uobj->vmobjlock));
1.6       mrg       193:
                    194:                /* total failure */
1.52      yamt      195:                if (rv < 0) {
                    196:                        UVMHIST_LOG(loanhist, "failure %d", rv, 0,0,0);
1.34      chs       197:                        return (-1);
1.52      yamt      198:                }
1.6       mrg       199:
                    200:                /* relock failed, need to do another lookup */
1.52      yamt      201:                if (rv == 0) {
                    202:                        UVMHIST_LOG(loanhist, "relock failure %d", result
                    203:                            ,0,0,0);
1.34      chs       204:                        return (result);
1.52      yamt      205:                }
1.6       mrg       206:
                    207:                /*
                    208:                 * got it... advance to next page
                    209:                 */
1.34      chs       210:
1.6       mrg       211:                result++;
                    212:                togo -= PAGE_SIZE;
                    213:                curaddr += PAGE_SIZE;
                    214:        }
                    215:
                    216:        /*
1.31      chuck     217:         * unlock what we locked, unlock the maps and return
1.6       mrg       218:         */
1.34      chs       219:
1.28      chuck     220:        if (aref->ar_amap)
                    221:                amap_unlock(aref->ar_amap);
                    222:        if (uobj)
                    223:                simple_unlock(&uobj->vmobjlock);
1.31      chuck     224:        uvmfault_unlockmaps(ufi, FALSE);
1.52      yamt      225:        UVMHIST_LOG(loanhist, "done %d", result, 0,0,0);
1.34      chs       226:        return (result);
1.1       mrg       227: }
                    228:
                    229: /*
                    230:  * normal functions
                    231:  */
                    232:
                    233: /*
1.28      chuck     234:  * uvm_loan: loan pages in a map out to anons or to the kernel
1.29      chs       235:  *
1.1       mrg       236:  * => map should be unlocked
                    237:  * => start and len should be multiples of PAGE_SIZE
                    238:  * => result is either an array of anon's or vm_pages (depending on flags)
                    239:  * => flag values: UVM_LOAN_TOANON - loan to anons
                    240:  *                 UVM_LOAN_TOPAGE - loan to wired kernel page
                    241:  *    one and only one of these flags must be set!
1.28      chuck     242:  * => returns 0 (success), or an appropriate error number
1.1       mrg       243:  */
                    244:
1.6       mrg       245: int
1.34      chs       246: uvm_loan(map, start, len, v, flags)
1.6       mrg       247:        struct vm_map *map;
1.9       eeh       248:        vaddr_t start;
                    249:        vsize_t len;
1.34      chs       250:        void *v;
1.6       mrg       251:        int flags;
1.1       mrg       252: {
1.6       mrg       253:        struct uvm_faultinfo ufi;
1.34      chs       254:        void **result, **output;
1.25      chs       255:        int rv, error;
1.6       mrg       256:
1.52      yamt      257:        UVMHIST_FUNC(__func__); UVMHIST_CALLED(loanhist);
                    258:
1.6       mrg       259:        /*
                    260:         * ensure that one and only one of the flags is set
                    261:         */
                    262:
1.25      chs       263:        KASSERT(((flags & UVM_LOAN_TOANON) == 0) ^
                    264:                ((flags & UVM_LOAN_TOPAGE) == 0));
                    265:        KASSERT((map->flags & VM_MAP_INTRSAFE) == 0);
1.6       mrg       266:
                    267:        /*
1.34      chs       268:         * "output" is a pointer to the current place to put the loaned page.
1.6       mrg       269:         */
                    270:
1.34      chs       271:        result = v;
1.6       mrg       272:        output = &result[0];    /* start at the beginning ... */
                    273:
                    274:        /*
                    275:         * while we've got pages to do
                    276:         */
                    277:
                    278:        while (len > 0) {
                    279:
                    280:                /*
                    281:                 * fill in params for a call to uvmfault_lookup
                    282:                 */
                    283:
                    284:                ufi.orig_map = map;
                    285:                ufi.orig_rvaddr = start;
                    286:                ufi.orig_size = len;
1.29      chs       287:
1.6       mrg       288:                /*
                    289:                 * do the lookup, the only time this will fail is if we hit on
                    290:                 * an unmapped region (an error)
                    291:                 */
                    292:
1.25      chs       293:                if (!uvmfault_lookup(&ufi, FALSE)) {
                    294:                        error = ENOENT;
1.6       mrg       295:                        goto fail;
1.25      chs       296:                }
1.6       mrg       297:
                    298:                /*
1.28      chuck     299:                 * map now locked.  now do the loanout...
1.6       mrg       300:                 */
1.34      chs       301:
1.6       mrg       302:                rv = uvm_loanentry(&ufi, &output, flags);
1.25      chs       303:                if (rv < 0) {
1.28      chuck     304:                        /* all unlocked due to error */
1.25      chs       305:                        error = EINVAL;
1.6       mrg       306:                        goto fail;
1.25      chs       307:                }
1.6       mrg       308:
                    309:                /*
1.31      chuck     310:                 * done!  the map is unlocked.  advance, if possible.
1.28      chuck     311:                 *
1.50      junyoung  312:                 * XXXCDC: could be recoded to hold the map lock with
                    313:                 *         smarter code (but it only happens on map entry
1.31      chuck     314:                 *         boundaries, so it isn't that bad).
1.6       mrg       315:                 */
1.34      chs       316:
1.28      chuck     317:                if (rv) {
                    318:                        rv <<= PAGE_SHIFT;
                    319:                        len -= rv;
                    320:                        start += rv;
                    321:                }
1.6       mrg       322:        }
1.52      yamt      323:        UVMHIST_LOG(loanhist, "success", 0,0,0,0);
1.25      chs       324:        return 0;
1.1       mrg       325:
                    326: fail:
1.6       mrg       327:        /*
1.34      chs       328:         * failed to complete loans.  drop any loans and return failure code.
1.28      chuck     329:         * map is already unlocked.
1.6       mrg       330:         */
1.34      chs       331:
1.6       mrg       332:        if (output - result) {
1.34      chs       333:                if (flags & UVM_LOAN_TOANON) {
1.6       mrg       334:                        uvm_unloananon((struct vm_anon **)result,
1.34      chs       335:                            output - result);
                    336:                } else {
1.6       mrg       337:                        uvm_unloanpage((struct vm_page **)result,
1.34      chs       338:                            output - result);
                    339:                }
1.6       mrg       340:        }
1.52      yamt      341:        UVMHIST_LOG(loanhist, "error %d", error,0,0,0);
1.26      jdolecek  342:        return (error);
1.1       mrg       343: }
                    344:
                    345: /*
                    346:  * uvm_loananon: loan a page from an anon out
1.29      chs       347:  *
1.28      chuck     348:  * => called with map, amap, uobj locked
1.1       mrg       349:  * => return value:
                    350:  *     -1 = fatal error, everything is unlocked, abort.
                    351:  *      0 = lookup in ufi went stale, everything unlocked, relookup and
                    352:  *             try again
                    353:  *      1 = got it, everything still locked
                    354:  */
                    355:
1.6       mrg       356: int
                    357: uvm_loananon(ufi, output, flags, anon)
                    358:        struct uvm_faultinfo *ufi;
                    359:        void ***output;
                    360:        int flags;
                    361:        struct vm_anon *anon;
1.1       mrg       362: {
1.6       mrg       363:        struct vm_page *pg;
1.34      chs       364:        int error;
1.1       mrg       365:
1.52      yamt      366:        UVMHIST_FUNC(__func__); UVMHIST_CALLED(loanhist);
                    367:
1.6       mrg       368:        /*
1.28      chuck     369:         * if we are loaning to "another" anon then it is easy, we just
1.6       mrg       370:         * bump the reference count on the current anon and return a
1.28      chuck     371:         * pointer to it (it becomes copy-on-write shared).
1.6       mrg       372:         */
1.34      chs       373:
1.6       mrg       374:        if (flags & UVM_LOAN_TOANON) {
                    375:                simple_lock(&anon->an_lock);
1.53    ! yamt      376:                pg = anon->an_page;
1.34      chs       377:                if (pg && (pg->pqflags & PQ_ANON) != 0 && anon->an_ref == 1) {
1.51      yamt      378:                        if (pg->wire_count > 0) {
1.52      yamt      379:                                UVMHIST_LOG(loanhist, "->A wired %p", pg,0,0,0);
1.51      yamt      380:                                uvmfault_unlockall(ufi,
                    381:                                    ufi->entry->aref.ar_amap,
                    382:                                    ufi->entry->object.uvm_obj, anon);
                    383:                                return (-1);
                    384:                        }
1.19      chs       385:                        pmap_page_protect(pg, VM_PROT_READ);
1.34      chs       386:                }
1.6       mrg       387:                anon->an_ref++;
                    388:                **output = anon;
1.34      chs       389:                (*output)++;
1.6       mrg       390:                simple_unlock(&anon->an_lock);
1.52      yamt      391:                UVMHIST_LOG(loanhist, "->A done", 0,0,0,0);
1.34      chs       392:                return (1);
1.6       mrg       393:        }
                    394:
                    395:        /*
                    396:         * we are loaning to a kernel-page.   we need to get the page
                    397:         * resident so we can wire it.   uvmfault_anonget will handle
                    398:         * this for us.
                    399:         */
                    400:
                    401:        simple_lock(&anon->an_lock);
1.34      chs       402:        error = uvmfault_anonget(ufi, ufi->entry->aref.ar_amap, anon);
1.6       mrg       403:
                    404:        /*
                    405:         * if we were unable to get the anon, then uvmfault_anonget has
                    406:         * unlocked everything and returned an error code.
                    407:         */
1.34      chs       408:
                    409:        if (error) {
1.52      yamt      410:                UVMHIST_LOG(loanhist, "error %d", error,0,0,0);
1.6       mrg       411:
                    412:                /* need to refault (i.e. refresh our lookup) ? */
1.34      chs       413:                if (error == ERESTART) {
                    414:                        return (0);
                    415:                }
1.6       mrg       416:
                    417:                /* "try again"?   sleep a bit and retry ... */
1.34      chs       418:                if (error == EAGAIN) {
1.24      chs       419:                        tsleep(&lbolt, PVM, "loanagain", 0);
1.34      chs       420:                        return (0);
1.6       mrg       421:                }
                    422:
                    423:                /* otherwise flag it as an error */
1.34      chs       424:                return (-1);
1.6       mrg       425:        }
                    426:
                    427:        /*
                    428:         * we have the page and its owner locked: do the loan now.
                    429:         */
                    430:
1.53    ! yamt      431:        pg = anon->an_page;
1.6       mrg       432:        uvm_lock_pageq();
1.51      yamt      433:        if (pg->wire_count > 0) {
                    434:                uvm_unlock_pageq();
1.52      yamt      435:                UVMHIST_LOG(loanhist, "->K wired %p", pg,0,0,0);
1.51      yamt      436:                KASSERT(pg->uobject == NULL);
                    437:                uvmfault_unlockall(ufi, ufi->entry->aref.ar_amap,
                    438:                    ufi->entry->object.uvm_obj, anon);
                    439:                return (-1);
                    440:        }
1.34      chs       441:        if (pg->loan_count == 0) {
1.19      chs       442:                pmap_page_protect(pg, VM_PROT_READ);
1.34      chs       443:        }
1.6       mrg       444:        pg->loan_count++;
1.34      chs       445:        uvm_pagedequeue(pg);
1.6       mrg       446:        uvm_unlock_pageq();
                    447:        **output = pg;
1.34      chs       448:        (*output)++;
1.6       mrg       449:
                    450:        /* unlock anon and return success */
1.28      chuck     451:        if (pg->uobject)        /* XXXCDC: what if this is our uobj? bad */
1.6       mrg       452:                simple_unlock(&pg->uobject->vmobjlock);
                    453:        simple_unlock(&anon->an_lock);
1.52      yamt      454:        UVMHIST_LOG(loanhist, "->K done", 0,0,0,0);
1.34      chs       455:        return (1);
1.1       mrg       456: }
                    457:
                    458: /*
1.45      yamt      459:  * uvm_loanpage: loan out pages to kernel (->K)
1.42      yamt      460:  *
1.51      yamt      461:  * => pages should be object-owned and the object should be locked.
                    462:  * => in the case of error, the object might be unlocked and relocked.
                    463:  * => caller should busy the pages beforehand.
                    464:  * => pages will be unbusied.
                    465:  * => fail with EBUSY if meet a wired page.
1.42      yamt      466:  */
1.51      yamt      467: static int
1.45      yamt      468: uvm_loanpage(pgpp, npages)
1.42      yamt      469:        struct vm_page **pgpp;
                    470:        int npages;
                    471: {
                    472:        int i;
1.51      yamt      473:        int error = 0;
1.42      yamt      474:
1.52      yamt      475:        UVMHIST_FUNC(__func__); UVMHIST_CALLED(loanhist);
                    476:
1.42      yamt      477:        for (i = 0; i < npages; i++) {
                    478:                struct vm_page *pg = pgpp[i];
                    479:
                    480:                KASSERT(pg->uobject != NULL);
1.51      yamt      481:                KASSERT(pg->uobject == pgpp[0]->uobject);
1.42      yamt      482:                KASSERT(!(pg->flags & (PG_RELEASED|PG_PAGEOUT)));
                    483:                LOCK_ASSERT(simple_lock_held(&pg->uobject->vmobjlock));
                    484:                KASSERT(pg->flags & PG_BUSY);
                    485:
                    486:                uvm_lock_pageq();
1.51      yamt      487:                if (pg->wire_count > 0) {
                    488:                        uvm_unlock_pageq();
1.52      yamt      489:                        UVMHIST_LOG(loanhist, "wired %p", pg,0,0,0);
1.51      yamt      490:                        error = EBUSY;
                    491:                        break;
                    492:                }
1.42      yamt      493:                if (pg->loan_count == 0) {
                    494:                        pmap_page_protect(pg, VM_PROT_READ);
                    495:                }
                    496:                pg->loan_count++;
                    497:                uvm_pagedequeue(pg);
                    498:                uvm_unlock_pageq();
                    499:        }
1.51      yamt      500:
                    501:        uvm_page_unbusy(pgpp, npages);
                    502:
                    503:        if (error) {
                    504:                /*
                    505:                 * backout what we've done
                    506:                 */
                    507:                struct simplelock *slock = &pgpp[0]->uobject->vmobjlock;
                    508:
                    509:                simple_unlock(slock);
                    510:                uvm_unloan(pgpp, i, UVM_LOAN_TOPAGE);
                    511:                simple_lock(slock);
                    512:        }
                    513:
1.52      yamt      514:        UVMHIST_LOG(loanhist, "done %d", error,0,0,0);
1.51      yamt      515:        return error;
1.42      yamt      516: }
                    517:
                    518: /*
1.45      yamt      519:  * XXX UBC temp limit
                    520:  * number of pages to get at once.
                    521:  * should be <= MAX_READ_AHEAD in genfs_vnops.c
                    522:  */
                    523: #define        UVM_LOAN_GET_CHUNK      16
                    524:
                    525: /*
1.46      yamt      526:  * uvm_loanuobjpages: loan pages from a uobj out (O->K)
1.45      yamt      527:  *
1.46      yamt      528:  * => uobj shouldn't be locked.  (we'll lock it)
1.51      yamt      529:  * => fail with EBUSY if we meet a wired page.
1.45      yamt      530:  */
                    531: int
                    532: uvm_loanuobjpages(uobj, pgoff, orignpages, origpgpp)
                    533:        struct uvm_object *uobj;
                    534:        voff_t pgoff;
                    535:        int orignpages;
                    536:        struct vm_page **origpgpp;
                    537: {
1.51      yamt      538:        int ndone; /* # of pages loaned out */
1.45      yamt      539:        struct vm_page **pgpp;
                    540:        int error;
                    541:        int i;
                    542:        struct simplelock *slock;
                    543:
                    544:        pgpp = origpgpp;
                    545:        for (ndone = 0; ndone < orignpages; ) {
                    546:                int npages;
1.51      yamt      547:                /* npendloan: # of pages busied but not loand out yet. */
1.45      yamt      548:                int npendloan = 0xdead; /* XXX gcc */
                    549: reget:
                    550:                npages = MIN(UVM_LOAN_GET_CHUNK, orignpages - ndone);
                    551:                simple_lock(&uobj->vmobjlock);
                    552:                error = (*uobj->pgops->pgo_get)(uobj,
                    553:                    pgoff + (ndone << PAGE_SHIFT), pgpp, &npages, 0,
                    554:                    VM_PROT_READ, 0, PGO_SYNCIO);
                    555:                if (error == EAGAIN) {
                    556:                        tsleep(&lbolt, PVM, "nfsread", 0);
                    557:                        continue;
                    558:                }
1.51      yamt      559:                if (error)
                    560:                        goto fail;
1.45      yamt      561:
                    562:                KASSERT(npages > 0);
1.50      junyoung  563:
1.45      yamt      564:                /* loan and unbusy pages */
                    565:                slock = NULL;
                    566:                for (i = 0; i < npages; i++) {
                    567:                        struct simplelock *nextslock; /* slock for next page */
                    568:                        struct vm_page *pg = *pgpp;
                    569:
                    570:                        /* XXX assuming that the page is owned by uobj */
                    571:                        KASSERT(pg->uobject != NULL);
                    572:                        nextslock = &pg->uobject->vmobjlock;
                    573:
                    574:                        if (slock != nextslock) {
                    575:                                if (slock) {
                    576:                                        KASSERT(npendloan > 0);
1.51      yamt      577:                                        error = uvm_loanpage(pgpp - npendloan,
1.45      yamt      578:                                            npendloan);
                    579:                                        simple_unlock(slock);
1.51      yamt      580:                                        if (error)
                    581:                                                goto fail;
                    582:                                        ndone += npendloan;
                    583:                                        KASSERT(origpgpp + ndone == pgpp);
1.45      yamt      584:                                }
                    585:                                slock = nextslock;
1.51      yamt      586:                                npendloan = 0;
1.45      yamt      587:                                simple_lock(slock);
                    588:                        }
                    589:
1.51      yamt      590:                        if ((pg->flags & PG_RELEASED) != 0) {
1.45      yamt      591:                                /*
                    592:                                 * release pages and try again.
                    593:                                 */
                    594:                                simple_unlock(slock);
                    595:                                for (; i < npages; i++) {
                    596:                                        pg = pgpp[i];
                    597:                                        slock = &pg->uobject->vmobjlock;
                    598:
                    599:                                        simple_lock(slock);
                    600:                                        uvm_lock_pageq();
                    601:                                        uvm_page_unbusy(&pg, 1);
                    602:                                        uvm_unlock_pageq();
                    603:                                        simple_unlock(slock);
                    604:                                }
                    605:                                goto reget;
                    606:                        }
                    607:
                    608:                        npendloan++;
                    609:                        pgpp++;
1.51      yamt      610:                        KASSERT(origpgpp + ndone + npendloan == pgpp);
1.45      yamt      611:                }
                    612:                KASSERT(slock != NULL);
                    613:                KASSERT(npendloan > 0);
1.51      yamt      614:                error = uvm_loanpage(pgpp - npendloan, npendloan);
1.45      yamt      615:                simple_unlock(slock);
1.51      yamt      616:                if (error)
                    617:                        goto fail;
                    618:                ndone += npendloan;
                    619:                KASSERT(origpgpp + ndone == pgpp);
1.45      yamt      620:        }
                    621:
                    622:        return 0;
1.51      yamt      623:
                    624: fail:
                    625:        uvm_unloan(origpgpp, ndone, UVM_LOAN_TOPAGE);
                    626:
                    627:        return error;
1.45      yamt      628: }
                    629:
                    630: /*
1.1       mrg       631:  * uvm_loanuobj: loan a page from a uobj out
                    632:  *
1.28      chuck     633:  * => called with map, amap, uobj locked
1.1       mrg       634:  * => return value:
                    635:  *     -1 = fatal error, everything is unlocked, abort.
                    636:  *      0 = lookup in ufi went stale, everything unlocked, relookup and
                    637:  *             try again
                    638:  *      1 = got it, everything still locked
                    639:  */
                    640:
1.33      jdolecek  641: static int
1.6       mrg       642: uvm_loanuobj(ufi, output, flags, va)
                    643:        struct uvm_faultinfo *ufi;
                    644:        void ***output;
                    645:        int flags;
1.9       eeh       646:        vaddr_t va;
1.1       mrg       647: {
1.6       mrg       648:        struct vm_amap *amap = ufi->entry->aref.ar_amap;
                    649:        struct uvm_object *uobj = ufi->entry->object.uvm_obj;
                    650:        struct vm_page *pg;
                    651:        struct vm_anon *anon;
1.34      chs       652:        int error, npages;
1.6       mrg       653:        boolean_t locked;
                    654:
1.52      yamt      655:        UVMHIST_FUNC(__func__); UVMHIST_CALLED(loanhist);
                    656:
1.6       mrg       657:        /*
                    658:         * first we must make sure the page is resident.
                    659:         *
                    660:         * XXXCDC: duplicate code with uvm_fault().
                    661:         */
                    662:
1.28      chuck     663:        if (uobj->pgops->pgo_get) {     /* try locked pgo_get */
1.6       mrg       664:                npages = 1;
                    665:                pg = NULL;
1.37      enami     666:                error = (*uobj->pgops->pgo_get)(uobj,
                    667:                    va - ufi->entry->start + ufi->entry->offset,
1.6       mrg       668:                    &pg, &npages, 0, VM_PROT_READ, MADV_NORMAL, PGO_LOCKED);
                    669:        } else {
1.34      chs       670:                error = EIO;            /* must have pgo_get op */
1.6       mrg       671:        }
                    672:
                    673:        /*
                    674:         * check the result of the locked pgo_get.  if there is a problem,
                    675:         * then we fail the loan.
                    676:         */
                    677:
1.34      chs       678:        if (error && error != EBUSY) {
1.6       mrg       679:                uvmfault_unlockall(ufi, amap, uobj, NULL);
1.34      chs       680:                return (-1);
1.6       mrg       681:        }
                    682:
                    683:        /*
                    684:         * if we need to unlock for I/O, do so now.
                    685:         */
                    686:
1.34      chs       687:        if (error == EBUSY) {
1.6       mrg       688:                uvmfault_unlockall(ufi, amap, NULL, NULL);
1.34      chs       689:
                    690:                /* locked: uobj */
1.6       mrg       691:                npages = 1;
1.37      enami     692:                error = (*uobj->pgops->pgo_get)(uobj,
                    693:                    va - ufi->entry->start + ufi->entry->offset,
1.30      chs       694:                    &pg, &npages, 0, VM_PROT_READ, MADV_NORMAL, PGO_SYNCIO);
1.6       mrg       695:                /* locked: <nothing> */
1.29      chs       696:
1.34      chs       697:                if (error) {
1.43      yamt      698:                        if (error == EAGAIN) {
1.24      chs       699:                                tsleep(&lbolt, PVM, "fltagain2", 0);
1.34      chs       700:                                return (0);
1.29      chs       701:                        }
1.34      chs       702:                        return (-1);
1.6       mrg       703:                }
                    704:
                    705:                /*
                    706:                 * pgo_get was a success.   attempt to relock everything.
                    707:                 */
                    708:
                    709:                locked = uvmfault_relock(ufi);
                    710:                if (locked && amap)
1.13      chuck     711:                        amap_lock(amap);
1.6       mrg       712:                simple_lock(&uobj->vmobjlock);
                    713:
                    714:                /*
                    715:                 * verify that the page has not be released and re-verify
                    716:                 * that amap slot is still free.   if there is a problem we
                    717:                 * drop our lock (thus force a lookup refresh/retry).
                    718:                 */
1.29      chs       719:
1.6       mrg       720:                if ((pg->flags & PG_RELEASED) != 0 ||
                    721:                    (locked && amap && amap_lookup(&ufi->entry->aref,
1.10      chuck     722:                    ufi->orig_rvaddr - ufi->entry->start))) {
1.6       mrg       723:                        if (locked)
                    724:                                uvmfault_unlockall(ufi, amap, NULL, NULL);
                    725:                        locked = FALSE;
1.29      chs       726:                }
1.6       mrg       727:
                    728:                /*
                    729:                 * didn't get the lock?   release the page and retry.
                    730:                 */
                    731:
                    732:                if (locked == FALSE) {
1.32      chs       733:                        if (pg->flags & PG_WANTED) {
1.18      thorpej   734:                                wakeup(pg);
1.32      chs       735:                        }
1.6       mrg       736:                        if (pg->flags & PG_RELEASED) {
1.38      enami     737:                                uvm_lock_pageq();
1.32      chs       738:                                uvm_pagefree(pg);
1.38      enami     739:                                uvm_unlock_pageq();
1.6       mrg       740:                                return (0);
                    741:                        }
                    742:                        uvm_lock_pageq();
1.32      chs       743:                        uvm_pageactivate(pg);
1.6       mrg       744:                        uvm_unlock_pageq();
                    745:                        pg->flags &= ~(PG_BUSY|PG_WANTED);
                    746:                        UVM_PAGE_OWN(pg, NULL);
                    747:                        simple_unlock(&uobj->vmobjlock);
                    748:                        return (0);
                    749:                }
                    750:        }
                    751:
                    752:        /*
                    753:         * at this point we have the page we want ("pg") marked PG_BUSY for us
1.34      chs       754:         * and we have all data structures locked.  do the loanout.  page can
1.6       mrg       755:         * not be PG_RELEASED (we caught this above).
                    756:         */
                    757:
1.34      chs       758:        if ((flags & UVM_LOAN_TOANON) == 0) {
1.51      yamt      759:                if (uvm_loanpage(&pg, 1)) {
                    760:                        uvmfault_unlockall(ufi, amap, uobj, NULL);
                    761:                        return (-1);
                    762:                }
1.34      chs       763:                **output = pg;
                    764:                (*output)++;
                    765:                return (1);
1.6       mrg       766:        }
                    767:
                    768:        /*
                    769:         * must be a loan to an anon.   check to see if there is already
                    770:         * an anon associated with this page.  if so, then just return
1.29      chs       771:         * a reference to this object.   the page should already be
1.6       mrg       772:         * mapped read-only because it is already on loan.
                    773:         */
                    774:
                    775:        if (pg->uanon) {
                    776:                anon = pg->uanon;
                    777:                simple_lock(&anon->an_lock);
                    778:                anon->an_ref++;
                    779:                simple_unlock(&anon->an_lock);
1.34      chs       780:                if (pg->flags & PG_WANTED) {
1.18      thorpej   781:                        wakeup(pg);
1.34      chs       782:                }
1.6       mrg       783:                pg->flags &= ~(PG_WANTED|PG_BUSY);
                    784:                UVM_PAGE_OWN(pg, NULL);
1.34      chs       785:                **output = anon;
                    786:                (*output)++;
                    787:                return (1);
1.6       mrg       788:        }
1.29      chs       789:
1.6       mrg       790:        /*
                    791:         * need to allocate a new anon
                    792:         */
                    793:
                    794:        anon = uvm_analloc();
1.34      chs       795:        if (anon == NULL) {
1.51      yamt      796:                goto fail;
1.6       mrg       797:        }
1.53    ! yamt      798:        anon->an_page = pg;
1.6       mrg       799:        pg->uanon = anon;
                    800:        uvm_lock_pageq();
1.51      yamt      801:        if (pg->wire_count > 0) {
                    802:                uvm_unlock_pageq();
1.52      yamt      803:                UVMHIST_LOG(loanhist, "wired %p", pg,0,0,0);
1.51      yamt      804:                pg->uanon = NULL;
1.53    ! yamt      805:                anon->an_page = NULL;
1.51      yamt      806:                anon->an_ref--;
                    807:                simple_unlock(&anon->an_lock);
                    808:                uvm_anfree(anon);
                    809:                goto fail;
                    810:        }
1.34      chs       811:        if (pg->loan_count == 0) {
1.19      chs       812:                pmap_page_protect(pg, VM_PROT_READ);
1.34      chs       813:        }
1.6       mrg       814:        pg->loan_count++;
                    815:        uvm_pageactivate(pg);
                    816:        uvm_unlock_pageq();
1.34      chs       817:        if (pg->flags & PG_WANTED) {
1.18      thorpej   818:                wakeup(pg);
1.34      chs       819:        }
1.6       mrg       820:        pg->flags &= ~(PG_WANTED|PG_BUSY);
                    821:        UVM_PAGE_OWN(pg, NULL);
1.23      thorpej   822:        simple_unlock(&anon->an_lock);
1.34      chs       823:        **output = anon;
                    824:        (*output)++;
                    825:        return (1);
1.51      yamt      826:
                    827: fail:
1.52      yamt      828:        UVMHIST_LOG(loanhist, "fail", 0,0,0,0);
1.51      yamt      829:        /*
                    830:         * unlock everything and bail out.
                    831:         */
                    832:        if (pg->flags & PG_WANTED) {
                    833:                wakeup(pg);
                    834:        }
                    835:        pg->flags &= ~(PG_WANTED|PG_BUSY);
                    836:        UVM_PAGE_OWN(pg, NULL);
                    837:        uvmfault_unlockall(ufi, amap, uobj, NULL);
                    838:        return (-1);
1.1       mrg       839: }
                    840:
                    841: /*
1.40      thorpej   842:  * uvm_loanzero: loan a zero-fill page out
1.1       mrg       843:  *
1.28      chuck     844:  * => called with map, amap, uobj locked
1.1       mrg       845:  * => return value:
                    846:  *     -1 = fatal error, everything is unlocked, abort.
                    847:  *      0 = lookup in ufi went stale, everything unlocked, relookup and
                    848:  *             try again
                    849:  *      1 = got it, everything still locked
                    850:  */
                    851:
1.40      thorpej   852: static struct uvm_object uvm_loanzero_object;
                    853:
1.33      jdolecek  854: static int
1.6       mrg       855: uvm_loanzero(ufi, output, flags)
                    856:        struct uvm_faultinfo *ufi;
                    857:        void ***output;
                    858:        int flags;
1.1       mrg       859: {
1.6       mrg       860:        struct vm_anon *anon;
                    861:        struct vm_page *pg;
1.34      chs       862:        struct uvm_object *uobj = ufi->entry->object.uvm_obj;
                    863:        struct vm_amap *amap = ufi->entry->aref.ar_amap;
1.1       mrg       864:
1.52      yamt      865:        UVMHIST_FUNC(__func__); UVMHIST_CALLED(loanhist);
1.44      yamt      866: again:
1.40      thorpej   867:        simple_lock(&uvm_loanzero_object.vmobjlock);
                    868:
                    869:        /*
                    870:         * first, get ahold of our single zero page.
                    871:         */
                    872:
                    873:        if (__predict_false((pg =
                    874:                             TAILQ_FIRST(&uvm_loanzero_object.memq)) == NULL)) {
                    875:                while ((pg = uvm_pagealloc(&uvm_loanzero_object, 0, NULL,
                    876:                                           UVM_PGA_ZERO)) == NULL) {
                    877:                        simple_unlock(&uvm_loanzero_object.vmobjlock);
1.34      chs       878:                        uvmfault_unlockall(ufi, amap, uobj, NULL);
1.40      thorpej   879:                        uvm_wait("loanzero");
1.34      chs       880:                        if (!uvmfault_relock(ufi)) {
                    881:                                return (0);
                    882:                        }
                    883:                        if (amap) {
                    884:                                amap_lock(amap);
                    885:                        }
                    886:                        if (uobj) {
                    887:                                simple_lock(&uobj->vmobjlock);
                    888:                        }
1.44      yamt      889:                        goto again;
1.6       mrg       890:                }
1.29      chs       891:
1.40      thorpej   892:                /* got a zero'd page. */
                    893:                pg->flags &= ~(PG_WANTED|PG_BUSY|PG_FAKE);
                    894:                pg->flags |= PG_RDONLY;
1.44      yamt      895:                uvm_lock_pageq();
                    896:                uvm_pageactivate(pg);
                    897:                uvm_unlock_pageq();
1.6       mrg       898:                UVM_PAGE_OWN(pg, NULL);
1.40      thorpej   899:        }
                    900:
                    901:        if ((flags & UVM_LOAN_TOANON) == 0) {   /* loaning to kernel-page */
                    902:                uvm_lock_pageq();
                    903:                pg->loan_count++;
1.44      yamt      904:                uvm_pagedequeue(pg);
1.40      thorpej   905:                uvm_unlock_pageq();
                    906:                simple_unlock(&uvm_loanzero_object.vmobjlock);
1.6       mrg       907:                **output = pg;
1.34      chs       908:                (*output)++;
                    909:                return (1);
1.6       mrg       910:        }
                    911:
1.40      thorpej   912:        /*
                    913:         * loaning to an anon.  check to see if there is already an anon
                    914:         * associated with this page.  if so, then just return a reference
                    915:         * to this object.
                    916:         */
1.23      thorpej   917:
1.40      thorpej   918:        if (pg->uanon) {
                    919:                anon = pg->uanon;
                    920:                simple_lock(&anon->an_lock);
                    921:                anon->an_ref++;
                    922:                simple_unlock(&anon->an_lock);
                    923:                simple_unlock(&uvm_loanzero_object.vmobjlock);
                    924:                **output = anon;
                    925:                (*output)++;
                    926:                return (1);
                    927:        }
1.23      thorpej   928:
1.40      thorpej   929:        /*
                    930:         * need to allocate a new anon
                    931:         */
1.34      chs       932:
1.40      thorpej   933:        anon = uvm_analloc();
                    934:        if (anon == NULL) {
                    935:                /* out of swap causes us to fail */
                    936:                simple_unlock(&uvm_loanzero_object.vmobjlock);
                    937:                uvmfault_unlockall(ufi, amap, uobj, NULL);
                    938:                return (-1);
1.6       mrg       939:        }
1.53    ! yamt      940:        anon->an_page = pg;
1.40      thorpej   941:        pg->uanon = anon;
1.6       mrg       942:        uvm_lock_pageq();
1.40      thorpej   943:        pg->loan_count++;
1.6       mrg       944:        uvm_pageactivate(pg);
                    945:        uvm_unlock_pageq();
1.40      thorpej   946:        simple_unlock(&uvm_loanzero_object.vmobjlock);
1.6       mrg       947:        **output = anon;
1.34      chs       948:        (*output)++;
                    949:        return (1);
1.1       mrg       950: }
                    951:
                    952:
                    953: /*
                    954:  * uvm_unloananon: kill loans on anons (basically a normal ref drop)
                    955:  *
                    956:  * => we expect all our resources to be unlocked
                    957:  */
                    958:
1.33      jdolecek  959: static void
1.34      chs       960: uvm_unloananon(aloans, nanons)
1.6       mrg       961:        struct vm_anon **aloans;
1.34      chs       962:        int nanons;
1.1       mrg       963: {
1.6       mrg       964:        struct vm_anon *anon;
1.1       mrg       965:
1.6       mrg       966:        while (nanons-- > 0) {
1.12      chs       967:                int refs;
1.1       mrg       968:
1.6       mrg       969:                anon = *aloans++;
                    970:                simple_lock(&anon->an_lock);
1.12      chs       971:                refs = --anon->an_ref;
                    972:                simple_unlock(&anon->an_lock);
1.1       mrg       973:
1.12      chs       974:                if (refs == 0) {
1.34      chs       975:                        uvm_anfree(anon);
1.6       mrg       976:                }
                    977:        }
1.1       mrg       978: }
                    979:
                    980: /*
                    981:  * uvm_unloanpage: kill loans on pages loaned out to the kernel
                    982:  *
                    983:  * => we expect all our resources to be unlocked
                    984:  */
                    985:
1.33      jdolecek  986: static void
1.34      chs       987: uvm_unloanpage(ploans, npages)
1.6       mrg       988:        struct vm_page **ploans;
1.34      chs       989:        int npages;
1.1       mrg       990: {
1.6       mrg       991:        struct vm_page *pg;
1.36      chs       992:        struct simplelock *slock;
1.1       mrg       993:
1.6       mrg       994:        uvm_lock_pageq();
                    995:        while (npages-- > 0) {
                    996:                pg = *ploans++;
1.1       mrg       997:
1.6       mrg       998:                /*
1.36      chs       999:                 * do a little dance to acquire the object or anon lock
                   1000:                 * as appropriate.  we are locking in the wrong order,
                   1001:                 * so we have to do a try-lock here.
                   1002:                 */
                   1003:
                   1004:                slock = NULL;
                   1005:                while (pg->uobject != NULL || pg->uanon != NULL) {
                   1006:                        if (pg->uobject != NULL) {
                   1007:                                slock = &pg->uobject->vmobjlock;
                   1008:                        } else {
                   1009:                                slock = &pg->uanon->an_lock;
                   1010:                        }
                   1011:                        if (simple_lock_try(slock)) {
                   1012:                                break;
                   1013:                        }
                   1014:                        uvm_unlock_pageq();
                   1015:                        uvm_lock_pageq();
                   1016:                        slock = NULL;
                   1017:                }
                   1018:
                   1019:                /*
                   1020:                 * drop our loan.  if page is owned by an anon but
                   1021:                 * PQ_ANON is not set, the page was loaned to the anon
                   1022:                 * from an object which dropped ownership, so resolve
                   1023:                 * this by turning the anon's loan into real ownership
                   1024:                 * (ie. decrement loan_count again and set PQ_ANON).
                   1025:                 * after all this, if there are no loans left, put the
                   1026:                 * page back a paging queue (if the page is owned by
                   1027:                 * an anon) or free it (if the page is now unowned).
1.6       mrg      1028:                 */
1.1       mrg      1029:
1.34      chs      1030:                KASSERT(pg->loan_count > 0);
                   1031:                pg->loan_count--;
1.36      chs      1032:                if (pg->uobject == NULL && pg->uanon != NULL &&
                   1033:                    (pg->pqflags & PQ_ANON) == 0) {
                   1034:                        KASSERT(pg->loan_count > 0);
                   1035:                        pg->loan_count--;
                   1036:                        pg->pqflags |= PQ_ANON;
                   1037:                }
1.34      chs      1038:                if (pg->loan_count == 0) {
                   1039:                        if (pg->uobject == NULL && pg->uanon == NULL) {
                   1040:                                KASSERT((pg->flags & PG_BUSY) == 0);
                   1041:                                uvm_pagefree(pg);
                   1042:                        } else {
                   1043:                                uvm_pageactivate(pg);
                   1044:                        }
1.39      chs      1045:                } else if (pg->loan_count == 1 && pg->uobject != NULL &&
                   1046:                           pg->uanon != NULL) {
                   1047:                        uvm_pageactivate(pg);
1.36      chs      1048:                }
                   1049:                if (slock != NULL) {
                   1050:                        simple_unlock(slock);
1.6       mrg      1051:                }
                   1052:        }
                   1053:        uvm_unlock_pageq();
1.1       mrg      1054: }
                   1055:
1.33      jdolecek 1056: /*
1.34      chs      1057:  * uvm_unloan: kill loans on pages or anons.
1.33      jdolecek 1058:  */
1.34      chs      1059:
1.33      jdolecek 1060: void
1.34      chs      1061: uvm_unloan(void *v, int npages, int flags)
1.33      jdolecek 1062: {
1.34      chs      1063:        if (flags & UVM_LOAN_TOANON) {
                   1064:                uvm_unloananon(v, npages);
                   1065:        } else {
                   1066:                uvm_unloanpage(v, npages);
                   1067:        }
1.40      thorpej  1068: }
                   1069:
                   1070: /*
1.41      thorpej  1071:  * Minimal pager for uvm_loanzero_object.  We need to provide a "put"
                   1072:  * method, because the page can end up on a paging queue, and the
                   1073:  * page daemon will want to call pgo_put when it encounters the page
                   1074:  * on the inactive list.
                   1075:  */
                   1076:
                   1077: static int
                   1078: ulz_put(struct uvm_object *uobj, voff_t start, voff_t stop, int flags)
                   1079: {
                   1080:        struct vm_page *pg;
                   1081:
                   1082:        KDASSERT(uobj == &uvm_loanzero_object);
                   1083:
                   1084:        /*
                   1085:         * Don't need to do any work here if we're not freeing pages.
                   1086:         */
                   1087:
                   1088:        if ((flags & PGO_FREE) == 0) {
                   1089:                simple_unlock(&uobj->vmobjlock);
                   1090:                return 0;
                   1091:        }
                   1092:
                   1093:        /*
                   1094:         * we don't actually want to ever free the uvm_loanzero_page, so
                   1095:         * just reactivate or dequeue it.
                   1096:         */
                   1097:
                   1098:        pg = TAILQ_FIRST(&uobj->memq);
                   1099:        KASSERT(pg != NULL);
                   1100:        KASSERT(TAILQ_NEXT(pg, listq) == NULL);
                   1101:
                   1102:        uvm_lock_pageq();
                   1103:        if (pg->uanon)
                   1104:                uvm_pageactivate(pg);
                   1105:        else
                   1106:                uvm_pagedequeue(pg);
                   1107:        uvm_unlock_pageq();
                   1108:
                   1109:        simple_unlock(&uobj->vmobjlock);
                   1110:        return 0;
                   1111: }
                   1112:
                   1113: static struct uvm_pagerops ulz_pager = {
                   1114:        NULL,           /* init */
                   1115:        NULL,           /* reference */
                   1116:        NULL,           /* detach */
                   1117:        NULL,           /* fault */
                   1118:        NULL,           /* get */
                   1119:        ulz_put,        /* put */
                   1120: };
                   1121:
                   1122: /*
1.40      thorpej  1123:  * uvm_loan_init(): initialize the uvm_loan() facility.
                   1124:  */
                   1125:
                   1126: void
                   1127: uvm_loan_init(void)
                   1128: {
                   1129:
                   1130:        simple_lock_init(&uvm_loanzero_object.vmobjlock);
                   1131:        TAILQ_INIT(&uvm_loanzero_object.memq);
1.41      thorpej  1132:        uvm_loanzero_object.pgops = &ulz_pager;
1.52      yamt     1133:
                   1134:        UVMHIST_INIT(loanhist, 300);
1.42      yamt     1135: }
                   1136:
                   1137: /*
                   1138:  * uvm_loanbreak: break loan on a uobj page
                   1139:  *
                   1140:  * => called with uobj locked
                   1141:  * => the page should be busy
                   1142:  * => return value:
                   1143:  *     newly allocated page if succeeded
                   1144:  */
                   1145: struct vm_page *
                   1146: uvm_loanbreak(struct vm_page *uobjpage)
                   1147: {
                   1148:        struct vm_page *pg;
1.49      drochner 1149: #ifdef DIAGNOSTIC
1.42      yamt     1150:        struct uvm_object *uobj = uobjpage->uobject;
1.49      drochner 1151: #endif
1.42      yamt     1152:
                   1153:        KASSERT(uobj != NULL);
                   1154:        LOCK_ASSERT(simple_lock_held(&uobj->vmobjlock));
                   1155:        KASSERT(uobjpage->flags & PG_BUSY);
                   1156:
                   1157:        /* alloc new un-owned page */
                   1158:        pg = uvm_pagealloc(NULL, 0, NULL, 0);
                   1159:        if (pg == NULL)
                   1160:                return NULL;
                   1161:
                   1162:        /*
                   1163:         * copy the data from the old page to the new
                   1164:         * one and clear the fake/clean flags on the
                   1165:         * new page (keep it busy).  force a reload
                   1166:         * of the old page by clearing it from all
                   1167:         * pmaps.  then lock the page queues to
                   1168:         * rename the pages.
                   1169:         */
                   1170:
                   1171:        uvm_pagecopy(uobjpage, pg);     /* old -> new */
                   1172:        pg->flags &= ~(PG_FAKE|PG_CLEAN);
                   1173:        pmap_page_protect(uobjpage, VM_PROT_NONE);
                   1174:        if (uobjpage->flags & PG_WANTED)
                   1175:                wakeup(uobjpage);
                   1176:        /* uobj still locked */
                   1177:        uobjpage->flags &= ~(PG_WANTED|PG_BUSY);
                   1178:        UVM_PAGE_OWN(uobjpage, NULL);
                   1179:
                   1180:        uvm_lock_pageq();
1.48      yamt     1181:
                   1182:        /*
                   1183:         * replace uobjpage with new page.
                   1184:         */
                   1185:
                   1186:        uvm_pagereplace(uobjpage, pg);
1.42      yamt     1187:
                   1188:        /*
                   1189:         * if the page is no longer referenced by
                   1190:         * an anon (i.e. we are breaking an O->K
                   1191:         * loan), then remove it from any pageq's.
                   1192:         */
                   1193:        if (uobjpage->uanon == NULL)
                   1194:                uvm_pagedequeue(uobjpage);
                   1195:
                   1196:        /*
                   1197:         * at this point we have absolutely no
                   1198:         * control over uobjpage
                   1199:         */
                   1200:
                   1201:        /* install new page */
                   1202:        uvm_pageactivate(pg);
                   1203:        uvm_unlock_pageq();
                   1204:
                   1205:        /*
                   1206:         * done!  loan is broken and "pg" is
                   1207:         * PG_BUSY.   it can now replace uobjpage.
                   1208:         */
                   1209:
                   1210:        return pg;
1.33      jdolecek 1211: }

CVSweb <webmaster@jp.NetBSD.org>