[BACK]Return to uvm_fault.c CVS log [TXT][DIR] Up to [cvs.NetBSD.org] / src / sys / uvm

Annotation of src/sys/uvm/uvm_fault.c, Revision 1.180.4.2

1.180.4.2! bouyer      1: /*     $NetBSD: uvm_fault.c,v 1.182 2011/02/10 21:05:52 skrll Exp $    */
1.1       mrg         2:
                      3: /*
                      4:  * Copyright (c) 1997 Charles D. Cranor and Washington University.
                      5:  * All rights reserved.
                      6:  *
                      7:  * Redistribution and use in source and binary forms, with or without
                      8:  * modification, are permitted provided that the following conditions
                      9:  * are met:
                     10:  * 1. Redistributions of source code must retain the above copyright
                     11:  *    notice, this list of conditions and the following disclaimer.
                     12:  * 2. Redistributions in binary form must reproduce the above copyright
                     13:  *    notice, this list of conditions and the following disclaimer in the
                     14:  *    documentation and/or other materials provided with the distribution.
                     15:  *
                     16:  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
                     17:  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
                     18:  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
                     19:  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
                     20:  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
                     21:  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
                     22:  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
                     23:  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
                     24:  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
                     25:  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
1.4       mrg        26:  *
                     27:  * from: Id: uvm_fault.c,v 1.1.2.23 1998/02/06 05:29:05 chs Exp
1.1       mrg        28:  */
                     29:
                     30: /*
                     31:  * uvm_fault.c: fault handler
                     32:  */
1.71      lukem      33:
                     34: #include <sys/cdefs.h>
1.180.4.2! bouyer     35: __KERNEL_RCSID(0, "$NetBSD: uvm_fault.c,v 1.182 2011/02/10 21:05:52 skrll Exp $");
1.71      lukem      36:
                     37: #include "opt_uvmhist.h"
1.1       mrg        38:
                     39: #include <sys/param.h>
                     40: #include <sys/systm.h>
                     41: #include <sys/kernel.h>
                     42: #include <sys/proc.h>
                     43: #include <sys/malloc.h>
                     44: #include <sys/mman.h>
                     45:
                     46: #include <uvm/uvm.h>
                     47:
                     48: /*
                     49:  *
                     50:  * a word on page faults:
                     51:  *
                     52:  * types of page faults we handle:
                     53:  *
                     54:  * CASE 1: upper layer faults                   CASE 2: lower layer faults
                     55:  *
                     56:  *    CASE 1A         CASE 1B                  CASE 2A        CASE 2B
                     57:  *    read/write1     write>1                  read/write   +-cow_write/zero
1.63      chs        58:  *         |             |                         |        |
1.1       mrg        59:  *      +--|--+       +--|--+     +-----+       +  |  +     | +-----+
1.127     uebayasi   60:  * amap |  V  |       |  ---------> new |          |        | |  ^  |
1.1       mrg        61:  *      +-----+       +-----+     +-----+       +  |  +     | +--|--+
                     62:  *                                                 |        |    |
                     63:  *      +-----+       +-----+                   +--|--+     | +--|--+
1.127     uebayasi   64:  * uobj | d/c |       | d/c |                   |  V  |     +----+  |
1.1       mrg        65:  *      +-----+       +-----+                   +-----+       +-----+
                     66:  *
                     67:  * d/c = don't care
1.63      chs        68:  *
1.1       mrg        69:  *   case [0]: layerless fault
                     70:  *     no amap or uobj is present.   this is an error.
                     71:  *
                     72:  *   case [1]: upper layer fault [anon active]
                     73:  *     1A: [read] or [write with anon->an_ref == 1]
1.127     uebayasi   74:  *             I/O takes place in upper level anon and uobj is not touched.
1.1       mrg        75:  *     1B: [write with anon->an_ref > 1]
                     76:  *             new anon is alloc'd and data is copied off ["COW"]
                     77:  *
                     78:  *   case [2]: lower layer fault [uobj]
                     79:  *     2A: [read on non-NULL uobj] or [write to non-copy_on_write area]
                     80:  *             I/O takes place directly in object.
                     81:  *     2B: [write to copy_on_write] or [read on NULL uobj]
1.63      chs        82:  *             data is "promoted" from uobj to a new anon.
1.1       mrg        83:  *             if uobj is null, then we zero fill.
                     84:  *
                     85:  * we follow the standard UVM locking protocol ordering:
                     86:  *
1.63      chs        87:  * MAPS => AMAP => UOBJ => ANON => PAGE QUEUES (PQ)
1.1       mrg        88:  * we hold a PG_BUSY page if we unlock for I/O
                     89:  *
                     90:  *
                     91:  * the code is structured as follows:
1.63      chs        92:  *
1.1       mrg        93:  *     - init the "IN" params in the ufi structure
1.177     yamt       94:  *   ReFault: (ERESTART returned to the loop in uvm_fault_internal)
1.1       mrg        95:  *     - do lookups [locks maps], check protection, handle needs_copy
                     96:  *     - check for case 0 fault (error)
                     97:  *     - establish "range" of fault
                     98:  *     - if we have an amap lock it and extract the anons
                     99:  *     - if sequential advice deactivate pages behind us
                    100:  *     - at the same time check pmap for unmapped areas and anon for pages
                    101:  *      that we could map in (and do map it if found)
                    102:  *     - check object for resident pages that we could map in
                    103:  *     - if (case 2) goto Case2
                    104:  *     - >>> handle case 1
                    105:  *           - ensure source anon is resident in RAM
                    106:  *           - if case 1B alloc new anon and copy from source
                    107:  *           - map the correct page in
                    108:  *   Case2:
                    109:  *     - >>> handle case 2
                    110:  *           - ensure source page is resident (if uobj)
                    111:  *           - if case 2B alloc new anon and copy from source (could be zero
                    112:  *             fill if uobj == NULL)
                    113:  *           - map the correct page in
                    114:  *     - done!
                    115:  *
                    116:  * note on paging:
                    117:  *   if we have to do I/O we place a PG_BUSY page in the correct object,
                    118:  * unlock everything, and do the I/O.   when I/O is done we must reverify
                    119:  * the state of the world before assuming that our data structures are
                    120:  * valid.   [because mappings could change while the map is unlocked]
                    121:  *
                    122:  *  alternative 1: unbusy the page in question and restart the page fault
                    123:  *    from the top (ReFault).   this is easy but does not take advantage
1.63      chs       124:  *    of the information that we already have from our previous lookup,
1.1       mrg       125:  *    although it is possible that the "hints" in the vm_map will help here.
                    126:  *
                    127:  * alternative 2: the system already keeps track of a "version" number of
                    128:  *    a map.   [i.e. every time you write-lock a map (e.g. to change a
                    129:  *    mapping) you bump the version number up by one...]   so, we can save
                    130:  *    the version number of the map before we release the lock and start I/O.
                    131:  *    then when I/O is done we can relock and check the version numbers
                    132:  *    to see if anything changed.    this might save us some over 1 because
                    133:  *    we don't have to unbusy the page and may be less compares(?).
                    134:  *
                    135:  * alternative 3: put in backpointers or a way to "hold" part of a map
                    136:  *    in place while I/O is in progress.   this could be complex to
                    137:  *    implement (especially with structures like amap that can be referenced
                    138:  *    by multiple map entries, and figuring out what should wait could be
                    139:  *    complex as well...).
                    140:  *
1.125     ad        141:  * we use alternative 2.  given that we are multi-threaded now we may want
                    142:  * to reconsider the choice.
1.1       mrg       143:  */
                    144:
                    145: /*
                    146:  * local data structures
                    147:  */
                    148:
                    149: struct uvm_advice {
1.7       mrg       150:        int advice;
                    151:        int nback;
                    152:        int nforw;
1.1       mrg       153: };
                    154:
                    155: /*
                    156:  * page range array:
1.63      chs       157:  * note: index in array must match "advice" value
1.1       mrg       158:  * XXX: borrowed numbers from freebsd.   do they work well for us?
                    159:  */
                    160:
1.95      thorpej   161: static const struct uvm_advice uvmadvice[] = {
1.7       mrg       162:        { MADV_NORMAL, 3, 4 },
                    163:        { MADV_RANDOM, 0, 0 },
                    164:        { MADV_SEQUENTIAL, 8, 7},
1.1       mrg       165: };
                    166:
1.69      chs       167: #define UVM_MAXRANGE 16        /* must be MAX() of nback+nforw+1 */
1.1       mrg       168:
                    169: /*
                    170:  * private prototypes
                    171:  */
                    172:
                    173: /*
                    174:  * inline functions
                    175:  */
                    176:
                    177: /*
                    178:  * uvmfault_anonflush: try and deactivate pages in specified anons
                    179:  *
                    180:  * => does not have to deactivate page if it is busy
                    181:  */
                    182:
1.103     perry     183: static inline void
1.95      thorpej   184: uvmfault_anonflush(struct vm_anon **anons, int n)
1.1       mrg       185: {
1.7       mrg       186:        int lcv;
                    187:        struct vm_page *pg;
1.63      chs       188:
1.163     uebayasi  189:        for (lcv = 0; lcv < n; lcv++) {
1.7       mrg       190:                if (anons[lcv] == NULL)
                    191:                        continue;
1.122     ad        192:                mutex_enter(&anons[lcv]->an_lock);
1.94      yamt      193:                pg = anons[lcv]->an_page;
1.117     yamt      194:                if (pg && (pg->flags & PG_BUSY) == 0) {
1.122     ad        195:                        mutex_enter(&uvm_pageqlock);
1.7       mrg       196:                        if (pg->wire_count == 0) {
                    197:                                uvm_pagedeactivate(pg);
                    198:                        }
1.122     ad        199:                        mutex_exit(&uvm_pageqlock);
1.7       mrg       200:                }
1.122     ad        201:                mutex_exit(&anons[lcv]->an_lock);
1.7       mrg       202:        }
1.1       mrg       203: }
                    204:
                    205: /*
                    206:  * normal functions
                    207:  */
                    208:
                    209: /*
                    210:  * uvmfault_amapcopy: clear "needs_copy" in a map.
                    211:  *
                    212:  * => called with VM data structures unlocked (usually, see below)
                    213:  * => we get a write lock on the maps and clear needs_copy for a VA
                    214:  * => if we are out of RAM we sleep (waiting for more)
                    215:  */
                    216:
1.7       mrg       217: static void
1.95      thorpej   218: uvmfault_amapcopy(struct uvm_faultinfo *ufi)
1.1       mrg       219: {
1.69      chs       220:        for (;;) {
1.1       mrg       221:
1.7       mrg       222:                /*
                    223:                 * no mapping?  give up.
                    224:                 */
1.1       mrg       225:
1.119     thorpej   226:                if (uvmfault_lookup(ufi, true) == false)
1.7       mrg       227:                        return;
1.1       mrg       228:
1.7       mrg       229:                /*
                    230:                 * copy if needed.
                    231:                 */
1.1       mrg       232:
1.7       mrg       233:                if (UVM_ET_ISNEEDSCOPY(ufi->entry))
1.108     yamt      234:                        amap_copy(ufi->map, ufi->entry, AMAP_COPY_NOWAIT,
1.13      chuck     235:                                ufi->orig_rvaddr, ufi->orig_rvaddr + 1);
1.1       mrg       236:
1.7       mrg       237:                /*
                    238:                 * didn't work?  must be out of RAM.   unlock and sleep.
                    239:                 */
                    240:
                    241:                if (UVM_ET_ISNEEDSCOPY(ufi->entry)) {
1.119     thorpej   242:                        uvmfault_unlockmaps(ufi, true);
1.7       mrg       243:                        uvm_wait("fltamapcopy");
                    244:                        continue;
                    245:                }
                    246:
                    247:                /*
                    248:                 * got it!   unlock and return.
                    249:                 */
1.63      chs       250:
1.119     thorpej   251:                uvmfault_unlockmaps(ufi, true);
1.7       mrg       252:                return;
                    253:        }
                    254:        /*NOTREACHED*/
1.1       mrg       255: }
                    256:
                    257: /*
                    258:  * uvmfault_anonget: get data in an anon into a non-busy, non-released
                    259:  * page in that anon.
                    260:  *
                    261:  * => maps, amap, and anon locked by caller.
1.57      chs       262:  * => if we fail (result != 0) we unlock everything.
1.1       mrg       263:  * => if we are successful, we return with everything still locked.
                    264:  * => we don't move the page on the queues [gets moved later]
                    265:  * => if we allocate a new page [we_own], it gets put on the queues.
                    266:  *    either way, the result is that the page is on the queues at return time
                    267:  * => for pages which are on loan from a uvm_object (and thus are not
                    268:  *    owned by the anon): if successful, we return with the owning object
                    269:  *    locked.   the caller must unlock this object when it unlocks everything
                    270:  *    else.
                    271:  */
                    272:
1.47      chs       273: int
1.95      thorpej   274: uvmfault_anonget(struct uvm_faultinfo *ufi, struct vm_amap *amap,
                    275:     struct vm_anon *anon)
1.7       mrg       276: {
1.118     thorpej   277:        bool we_own;    /* we own anon's page? */
                    278:        bool locked;    /* did we relock? */
1.7       mrg       279:        struct vm_page *pg;
1.58      chs       280:        int error;
1.7       mrg       281:        UVMHIST_FUNC("uvmfault_anonget"); UVMHIST_CALLED(maphist);
                    282:
1.122     ad        283:        KASSERT(mutex_owned(&anon->an_lock));
1.53      thorpej   284:
1.58      chs       285:        error = 0;
1.9       chuck     286:        uvmexp.fltanget++;
                    287:         /* bump rusage counters */
1.94      yamt      288:        if (anon->an_page)
1.124     ad        289:                curlwp->l_ru.ru_minflt++;
1.9       chuck     290:        else
1.124     ad        291:                curlwp->l_ru.ru_majflt++;
1.7       mrg       292:
1.63      chs       293:        /*
1.7       mrg       294:         * loop until we get it, or fail.
                    295:         */
                    296:
1.69      chs       297:        for (;;) {
1.119     thorpej   298:                we_own = false;         /* true if we set PG_BUSY on a page */
1.94      yamt      299:                pg = anon->an_page;
1.1       mrg       300:
1.7       mrg       301:                /*
                    302:                 * if there is a resident page and it is loaned, then anon
                    303:                 * may not own it.   call out to uvm_anon_lockpage() to ensure
                    304:                 * the real owner of the page has been identified and locked.
                    305:                 */
                    306:
                    307:                if (pg && pg->loan_count)
1.13      chuck     308:                        pg = uvm_anon_lockloanpg(anon);
1.7       mrg       309:
                    310:                /*
                    311:                 * page there?   make sure it is not busy/released.
                    312:                 */
                    313:
                    314:                if (pg) {
                    315:
                    316:                        /*
                    317:                         * at this point, if the page has a uobject [meaning
                    318:                         * we have it on loan], then that uobject is locked
                    319:                         * by us!   if the page is busy, we drop all the
                    320:                         * locks (including uobject) and try again.
                    321:                         */
                    322:
1.69      chs       323:                        if ((pg->flags & PG_BUSY) == 0) {
1.7       mrg       324:                                UVMHIST_LOG(maphist, "<- OK",0,0,0,0);
1.57      chs       325:                                return (0);
1.7       mrg       326:                        }
                    327:                        pg->flags |= PG_WANTED;
                    328:                        uvmexp.fltpgwait++;
                    329:
                    330:                        /*
                    331:                         * the last unlock must be an atomic unlock+wait on
                    332:                         * the owner of page
                    333:                         */
1.69      chs       334:
1.7       mrg       335:                        if (pg->uobject) {      /* owner is uobject ? */
                    336:                                uvmfault_unlockall(ufi, amap, NULL, anon);
                    337:                                UVMHIST_LOG(maphist, " unlock+wait on uobj",0,
                    338:                                    0,0,0);
                    339:                                UVM_UNLOCK_AND_WAIT(pg,
                    340:                                    &pg->uobject->vmobjlock,
1.119     thorpej   341:                                    false, "anonget1",0);
1.7       mrg       342:                        } else {
                    343:                                /* anon owns page */
                    344:                                uvmfault_unlockall(ufi, amap, NULL, NULL);
                    345:                                UVMHIST_LOG(maphist, " unlock+wait on anon",0,
                    346:                                    0,0,0);
                    347:                                UVM_UNLOCK_AND_WAIT(pg,&anon->an_lock,0,
                    348:                                    "anonget2",0);
                    349:                        }
                    350:                } else {
1.101     yamt      351: #if defined(VMSWAP)
1.63      chs       352:
1.7       mrg       353:                        /*
                    354:                         * no page, we must try and bring it in.
                    355:                         */
1.69      chs       356:
1.180     enami     357:                        pg = uvm_pagealloc(NULL,
                    358:                            ufi != NULL ? ufi->orig_rvaddr : 0,
                    359:                            anon, UVM_FLAG_COLORMATCH);
1.7       mrg       360:                        if (pg == NULL) {               /* out of RAM.  */
                    361:                                uvmfault_unlockall(ufi, amap, NULL, anon);
                    362:                                uvmexp.fltnoram++;
                    363:                                UVMHIST_LOG(maphist, "  noram -- UVM_WAIT",0,
                    364:                                    0,0,0);
1.93      yamt      365:                                if (!uvm_reclaimable()) {
                    366:                                        return ENOMEM;
                    367:                                }
1.7       mrg       368:                                uvm_wait("flt_noram1");
                    369:                        } else {
                    370:                                /* we set the PG_BUSY bit */
1.119     thorpej   371:                                we_own = true;
1.7       mrg       372:                                uvmfault_unlockall(ufi, amap, NULL, anon);
                    373:
                    374:                                /*
                    375:                                 * we are passing a PG_BUSY+PG_FAKE+PG_CLEAN
                    376:                                 * page into the uvm_swap_get function with
1.18      chuck     377:                                 * all data structures unlocked.  note that
                    378:                                 * it is ok to read an_swslot here because
                    379:                                 * we hold PG_BUSY on the page.
1.7       mrg       380:                                 */
                    381:                                uvmexp.pageins++;
1.58      chs       382:                                error = uvm_swap_get(pg, anon->an_swslot,
1.7       mrg       383:                                    PGO_SYNCIO);
                    384:
                    385:                                /*
                    386:                                 * we clean up after the i/o below in the
                    387:                                 * "we_own" case
                    388:                                 */
                    389:                        }
1.101     yamt      390: #else /* defined(VMSWAP) */
                    391:                        panic("%s: no page", __func__);
                    392: #endif /* defined(VMSWAP) */
1.7       mrg       393:                }
                    394:
                    395:                /*
                    396:                 * now relock and try again
                    397:                 */
                    398:
                    399:                locked = uvmfault_relock(ufi);
1.47      chs       400:                if (locked && amap != NULL) {
1.19      chuck     401:                        amap_lock(amap);
1.7       mrg       402:                }
                    403:                if (locked || we_own)
1.122     ad        404:                        mutex_enter(&anon->an_lock);
1.7       mrg       405:
                    406:                /*
                    407:                 * if we own the page (i.e. we set PG_BUSY), then we need
                    408:                 * to clean up after the I/O. there are three cases to
                    409:                 * consider:
                    410:                 *   [1] page released during I/O: free anon and ReFault.
1.63      chs       411:                 *   [2] I/O not OK.   free the page and cause the fault
1.7       mrg       412:                 *       to fail.
                    413:                 *   [3] I/O OK!   activate the page and sync with the
                    414:                 *       non-we_own case (i.e. drop anon lock if not locked).
                    415:                 */
1.63      chs       416:
1.7       mrg       417:                if (we_own) {
1.101     yamt      418: #if defined(VMSWAP)
1.7       mrg       419:                        if (pg->flags & PG_WANTED) {
1.63      chs       420:                                wakeup(pg);
1.7       mrg       421:                        }
1.58      chs       422:                        if (error) {
1.1       mrg       423:
1.47      chs       424:                                /*
                    425:                                 * remove the swap slot from the anon
                    426:                                 * and mark the anon as having no real slot.
                    427:                                 * don't free the swap slot, thus preventing
                    428:                                 * it from being used again.
                    429:                                 */
1.69      chs       430:
1.84      pk        431:                                if (anon->an_swslot > 0)
                    432:                                        uvm_swap_markbad(anon->an_swslot, 1);
1.47      chs       433:                                anon->an_swslot = SWSLOT_BAD;
                    434:
1.88      yamt      435:                                if ((pg->flags & PG_RELEASED) != 0)
                    436:                                        goto released;
                    437:
1.47      chs       438:                                /*
1.7       mrg       439:                                 * note: page was never !PG_BUSY, so it
                    440:                                 * can't be mapped and thus no need to
                    441:                                 * pmap_page_protect it...
                    442:                                 */
1.69      chs       443:
1.122     ad        444:                                mutex_enter(&uvm_pageqlock);
1.7       mrg       445:                                uvm_pagefree(pg);
1.122     ad        446:                                mutex_exit(&uvm_pageqlock);
1.7       mrg       447:
                    448:                                if (locked)
                    449:                                        uvmfault_unlockall(ufi, amap, NULL,
                    450:                                            anon);
                    451:                                else
1.122     ad        452:                                        mutex_exit(&anon->an_lock);
1.7       mrg       453:                                UVMHIST_LOG(maphist, "<- ERROR", 0,0,0,0);
1.58      chs       454:                                return error;
1.7       mrg       455:                        }
1.63      chs       456:
1.88      yamt      457:                        if ((pg->flags & PG_RELEASED) != 0) {
                    458: released:
                    459:                                KASSERT(anon->an_ref == 0);
                    460:
                    461:                                /*
                    462:                                 * released while we unlocked amap.
                    463:                                 */
                    464:
                    465:                                if (locked)
                    466:                                        uvmfault_unlockall(ufi, amap, NULL,
                    467:                                            NULL);
                    468:
                    469:                                uvm_anon_release(anon);
                    470:
                    471:                                if (error) {
                    472:                                        UVMHIST_LOG(maphist,
                    473:                                            "<- ERROR/RELEASED", 0,0,0,0);
                    474:                                        return error;
                    475:                                }
                    476:
                    477:                                UVMHIST_LOG(maphist, "<- RELEASED", 0,0,0,0);
                    478:                                return ERESTART;
                    479:                        }
                    480:
1.7       mrg       481:                        /*
1.69      chs       482:                         * we've successfully read the page, activate it.
1.7       mrg       483:                         */
1.69      chs       484:
1.122     ad        485:                        mutex_enter(&uvm_pageqlock);
1.7       mrg       486:                        uvm_pageactivate(pg);
1.122     ad        487:                        mutex_exit(&uvm_pageqlock);
1.69      chs       488:                        pg->flags &= ~(PG_WANTED|PG_BUSY|PG_FAKE);
                    489:                        UVM_PAGE_OWN(pg, NULL);
1.7       mrg       490:                        if (!locked)
1.122     ad        491:                                mutex_exit(&anon->an_lock);
1.101     yamt      492: #else /* defined(VMSWAP) */
                    493:                        panic("%s: we_own", __func__);
                    494: #endif /* defined(VMSWAP) */
1.7       mrg       495:                }
                    496:
                    497:                /*
                    498:                 * we were not able to relock.   restart fault.
                    499:                 */
                    500:
                    501:                if (!locked) {
                    502:                        UVMHIST_LOG(maphist, "<- REFAULT", 0,0,0,0);
1.57      chs       503:                        return (ERESTART);
1.7       mrg       504:                }
                    505:
                    506:                /*
                    507:                 * verify no one has touched the amap and moved the anon on us.
                    508:                 */
1.1       mrg       509:
1.47      chs       510:                if (ufi != NULL &&
1.63      chs       511:                    amap_lookup(&ufi->entry->aref,
1.47      chs       512:                                ufi->orig_rvaddr - ufi->entry->start) != anon) {
1.63      chs       513:
1.7       mrg       514:                        uvmfault_unlockall(ufi, amap, NULL, anon);
                    515:                        UVMHIST_LOG(maphist, "<- REFAULT", 0,0,0,0);
1.57      chs       516:                        return (ERESTART);
1.7       mrg       517:                }
1.63      chs       518:
1.7       mrg       519:                /*
1.63      chs       520:                 * try it again!
1.7       mrg       521:                 */
1.1       mrg       522:
1.7       mrg       523:                uvmexp.fltanretry++;
                    524:                continue;
1.69      chs       525:        }
1.7       mrg       526:        /*NOTREACHED*/
1.1       mrg       527: }
                    528:
                    529: /*
1.106     yamt      530:  * uvmfault_promote: promote data to a new anon.  used for 1B and 2B.
                    531:  *
                    532:  *     1. allocate an anon and a page.
                    533:  *     2. fill its contents.
                    534:  *     3. put it into amap.
                    535:  *
                    536:  * => if we fail (result != 0) we unlock everything.
                    537:  * => on success, return a new locked anon via 'nanon'.
                    538:  *    (*nanon)->an_page will be a resident, locked, dirty page.
                    539:  */
                    540:
                    541: static int
                    542: uvmfault_promote(struct uvm_faultinfo *ufi,
                    543:     struct vm_anon *oanon,
                    544:     struct vm_page *uobjpage,
                    545:     struct vm_anon **nanon, /* OUT: allocated anon */
                    546:     struct vm_anon **spare)
                    547: {
                    548:        struct vm_amap *amap = ufi->entry->aref.ar_amap;
                    549:        struct uvm_object *uobj;
                    550:        struct vm_anon *anon;
                    551:        struct vm_page *pg;
                    552:        struct vm_page *opg;
                    553:        int error;
                    554:        UVMHIST_FUNC(__func__); UVMHIST_CALLED(maphist);
                    555:
                    556:        if (oanon) {
                    557:                /* anon COW */
                    558:                opg = oanon->an_page;
                    559:                KASSERT(opg != NULL);
                    560:                KASSERT(opg->uobject == NULL || opg->loan_count > 0);
                    561:        } else if (uobjpage != PGO_DONTCARE) {
                    562:                /* object-backed COW */
                    563:                opg = uobjpage;
                    564:        } else {
                    565:                /* ZFOD */
                    566:                opg = NULL;
                    567:        }
                    568:        if (opg != NULL) {
                    569:                uobj = opg->uobject;
                    570:        } else {
                    571:                uobj = NULL;
                    572:        }
                    573:
                    574:        KASSERT(amap != NULL);
                    575:        KASSERT(uobjpage != NULL);
                    576:        KASSERT(uobjpage == PGO_DONTCARE || (uobjpage->flags & PG_BUSY) != 0);
1.120     ad        577:        KASSERT(mutex_owned(&amap->am_l));
1.122     ad        578:        KASSERT(oanon == NULL || mutex_owned(&oanon->an_lock));
                    579:        KASSERT(uobj == NULL || mutex_owned(&uobj->vmobjlock));
                    580: #if 0
                    581:        KASSERT(*spare == NULL || !mutex_owned(&(*spare)->an_lock));
                    582: #endif
1.106     yamt      583:
                    584:        if (*spare != NULL) {
                    585:                anon = *spare;
                    586:                *spare = NULL;
1.122     ad        587:                mutex_enter(&anon->an_lock);
1.106     yamt      588:        } else if (ufi->map != kernel_map) {
                    589:                anon = uvm_analloc();
                    590:        } else {
                    591:                UVMHIST_LOG(maphist, "kernel_map, unlock and retry", 0,0,0,0);
                    592:
                    593:                /*
                    594:                 * we can't allocate anons with kernel_map locked.
                    595:                 */
                    596:
                    597:                uvm_page_unbusy(&uobjpage, 1);
                    598:                uvmfault_unlockall(ufi, amap, uobj, oanon);
                    599:
                    600:                *spare = uvm_analloc();
                    601:                if (*spare == NULL) {
                    602:                        goto nomem;
                    603:                }
1.122     ad        604:                mutex_exit(&(*spare)->an_lock);
1.106     yamt      605:                error = ERESTART;
                    606:                goto done;
                    607:        }
                    608:        if (anon) {
                    609:
                    610:                /*
                    611:                 * The new anon is locked.
                    612:                 *
                    613:                 * if opg == NULL, we want a zero'd, dirty page,
                    614:                 * so have uvm_pagealloc() do that for us.
                    615:                 */
                    616:
1.179     matt      617:                pg = uvm_pagealloc(NULL, ufi->orig_rvaddr, anon,
                    618:                    UVM_FLAG_COLORMATCH | (opg == NULL ? UVM_PGA_ZERO : 0));
1.106     yamt      619:        } else {
                    620:                pg = NULL;
                    621:        }
                    622:
                    623:        /*
                    624:         * out of memory resources?
                    625:         */
                    626:
                    627:        if (pg == NULL) {
                    628:                /* save anon for the next try. */
                    629:                if (anon != NULL) {
1.122     ad        630:                        mutex_exit(&anon->an_lock);
1.106     yamt      631:                        *spare = anon;
                    632:                }
                    633:
                    634:                /* unlock and fail ... */
                    635:                uvm_page_unbusy(&uobjpage, 1);
                    636:                uvmfault_unlockall(ufi, amap, uobj, oanon);
                    637: nomem:
                    638:                if (!uvm_reclaimable()) {
                    639:                        UVMHIST_LOG(maphist, "out of VM", 0,0,0,0);
                    640:                        uvmexp.fltnoanon++;
                    641:                        error = ENOMEM;
                    642:                        goto done;
                    643:                }
                    644:
                    645:                UVMHIST_LOG(maphist, "out of RAM, waiting for more", 0,0,0,0);
                    646:                uvmexp.fltnoram++;
                    647:                uvm_wait("flt_noram5");
                    648:                error = ERESTART;
                    649:                goto done;
                    650:        }
                    651:
                    652:        /* copy page [pg now dirty] */
                    653:        if (opg) {
                    654:                uvm_pagecopy(opg, pg);
                    655:        }
                    656:
                    657:        amap_add(&ufi->entry->aref, ufi->orig_rvaddr - ufi->entry->start, anon,
                    658:            oanon != NULL);
                    659:
                    660:        *nanon = anon;
                    661:        error = 0;
                    662: done:
                    663:        return error;
                    664: }
                    665:
                    666:
                    667: /*
1.1       mrg       668:  *   F A U L T   -   m a i n   e n t r y   p o i n t
                    669:  */
                    670:
                    671: /*
                    672:  * uvm_fault: page fault handler
                    673:  *
                    674:  * => called from MD code to resolve a page fault
1.63      chs       675:  * => VM data structures usually should be unlocked.   however, it is
1.1       mrg       676:  *     possible to call here with the main map locked if the caller
                    677:  *     gets a write lock, sets it recusive, and then calls us (c.f.
                    678:  *     uvm_map_pageable).   this should be avoided because it keeps
                    679:  *     the map locked off during I/O.
1.66      thorpej   680:  * => MUST NEVER BE CALLED IN INTERRUPT CONTEXT
1.1       mrg       681:  */
                    682:
1.24      mycroft   683: #define MASK(entry)     (UVM_ET_ISCOPYONWRITE(entry) ? \
                    684:                         ~VM_PROT_WRITE : VM_PROT_ALL)
                    685:
1.110     drochner  686: /* fault_flag values passed from uvm_fault_wire to uvm_fault_internal */
1.130     uebayasi  687: #define UVM_FAULT_WIRE         (1 << 0)
                    688: #define UVM_FAULT_MAXPROT      (1 << 1)
1.110     drochner  689:
1.140     uebayasi  690: struct uvm_faultctx {
                    691:        vm_prot_t access_type;
                    692:        vm_prot_t enter_prot;
1.150     uebayasi  693:        vaddr_t startva;
                    694:        int npages;
                    695:        int centeridx;
                    696:        struct vm_anon *anon_spare;
1.146     uebayasi  697:        bool wire_mapping;
1.140     uebayasi  698:        bool narrow;
1.146     uebayasi  699:        bool wire_paging;
1.140     uebayasi  700:        bool cow_now;
1.168     uebayasi  701:        bool promote;
1.140     uebayasi  702: };
                    703:
1.163     uebayasi  704: static inline int      uvm_fault_check(
                    705:                            struct uvm_faultinfo *, struct uvm_faultctx *,
1.177     yamt      706:                            struct vm_anon ***, bool);
1.163     uebayasi  707:
                    708: static int             uvm_fault_upper(
                    709:                            struct uvm_faultinfo *, struct uvm_faultctx *,
                    710:                            struct vm_anon **);
                    711: static inline int      uvm_fault_upper_lookup(
1.177     yamt      712:                            struct uvm_faultinfo *, const struct uvm_faultctx *,
1.163     uebayasi  713:                            struct vm_anon **, struct vm_page **);
                    714: static inline void     uvm_fault_upper_neighbor(
1.177     yamt      715:                            struct uvm_faultinfo *, const struct uvm_faultctx *,
1.163     uebayasi  716:                            vaddr_t, struct vm_page *, bool);
                    717: static inline int      uvm_fault_upper_loan(
                    718:                            struct uvm_faultinfo *, struct uvm_faultctx *,
                    719:                            struct vm_anon *, struct uvm_object **);
                    720: static inline int      uvm_fault_upper_promote(
                    721:                            struct uvm_faultinfo *, struct uvm_faultctx *,
                    722:                            struct uvm_object *, struct vm_anon *);
                    723: static inline int      uvm_fault_upper_direct(
                    724:                            struct uvm_faultinfo *, struct uvm_faultctx *,
                    725:                            struct uvm_object *, struct vm_anon *);
                    726: static int             uvm_fault_upper_enter(
1.177     yamt      727:                            struct uvm_faultinfo *, const struct uvm_faultctx *,
1.163     uebayasi  728:                            struct uvm_object *, struct vm_anon *,
                    729:                            struct vm_page *, struct vm_anon *);
1.169     uebayasi  730: static inline void     uvm_fault_upper_done(
1.177     yamt      731:                            struct uvm_faultinfo *, const struct uvm_faultctx *,
                    732:                            struct vm_anon *, struct vm_page *);
1.163     uebayasi  733:
                    734: static int             uvm_fault_lower(
                    735:                            struct uvm_faultinfo *, struct uvm_faultctx *,
                    736:                            struct vm_page **);
1.173     uebayasi  737: static inline void     uvm_fault_lower_lookup(
1.177     yamt      738:                            struct uvm_faultinfo *, const struct uvm_faultctx *,
1.163     uebayasi  739:                            struct vm_page **);
                    740: static inline void     uvm_fault_lower_neighbor(
1.177     yamt      741:                            struct uvm_faultinfo *, const struct uvm_faultctx *,
1.163     uebayasi  742:                            vaddr_t, struct vm_page *, bool);
                    743: static inline int      uvm_fault_lower_io(
1.177     yamt      744:                            struct uvm_faultinfo *, const struct uvm_faultctx *,
1.163     uebayasi  745:                            struct uvm_object **, struct vm_page **);
                    746: static inline int      uvm_fault_lower_direct(
                    747:                            struct uvm_faultinfo *, struct uvm_faultctx *,
                    748:                            struct uvm_object *, struct vm_page *);
                    749: static inline int      uvm_fault_lower_direct_loan(
                    750:                            struct uvm_faultinfo *, struct uvm_faultctx *,
                    751:                            struct uvm_object *, struct vm_page **,
                    752:                            struct vm_page **);
                    753: static inline int      uvm_fault_lower_promote(
                    754:                            struct uvm_faultinfo *, struct uvm_faultctx *,
                    755:                            struct uvm_object *, struct vm_page *);
                    756: static int             uvm_fault_lower_enter(
1.177     yamt      757:                            struct uvm_faultinfo *, const struct uvm_faultctx *,
1.163     uebayasi  758:                            struct uvm_object *,
                    759:                            struct vm_anon *, struct vm_page *,
                    760:                            struct vm_page *);
1.169     uebayasi  761: static inline void     uvm_fault_lower_done(
1.177     yamt      762:                            struct uvm_faultinfo *, const struct uvm_faultctx *,
                    763:                            struct uvm_object *, struct vm_page *);
1.138     uebayasi  764:
1.7       mrg       765: int
1.110     drochner  766: uvm_fault_internal(struct vm_map *orig_map, vaddr_t vaddr,
                    767:     vm_prot_t access_type, int fault_flag)
1.1       mrg       768: {
1.7       mrg       769:        struct uvm_faultinfo ufi;
1.140     uebayasi  770:        struct uvm_faultctx flt = {
                    771:                .access_type = access_type,
1.146     uebayasi  772:
                    773:                /* don't look for neighborhood * pages on "wire" fault */
                    774:                .narrow = (fault_flag & UVM_FAULT_WIRE) != 0,
                    775:
                    776:                /* "wire" fault causes wiring of both mapping and paging */
                    777:                .wire_mapping = (fault_flag & UVM_FAULT_WIRE) != 0,
                    778:                .wire_paging = (fault_flag & UVM_FAULT_WIRE) != 0,
1.140     uebayasi  779:        };
1.177     yamt      780:        const bool maxprot = (fault_flag & UVM_FAULT_MAXPROT) != 0;
1.137     uebayasi  781:        struct vm_anon *anons_store[UVM_MAXRANGE], **anons;
1.141     uebayasi  782:        struct vm_page *pages_store[UVM_MAXRANGE], **pages;
1.140     uebayasi  783:        int error;
1.7       mrg       784:        UVMHIST_FUNC("uvm_fault"); UVMHIST_CALLED(maphist);
1.1       mrg       785:
1.110     drochner  786:        UVMHIST_LOG(maphist, "(map=0x%x, vaddr=0x%x, at=%d, ff=%d)",
                    787:              orig_map, vaddr, access_type, fault_flag);
1.1       mrg       788:
1.178     matt      789:        curcpu()->ci_data.cpu_nfault++;
1.7       mrg       790:
                    791:        /*
                    792:         * init the IN parameters in the ufi
                    793:         */
1.1       mrg       794:
1.7       mrg       795:        ufi.orig_map = orig_map;
                    796:        ufi.orig_rvaddr = trunc_page(vaddr);
                    797:        ufi.orig_size = PAGE_SIZE;      /* can't get any smaller than this */
                    798:
1.142     uebayasi  799:        error = ERESTART;
                    800:        while (error == ERESTART) {
1.143     uebayasi  801:                anons = anons_store;
                    802:                pages = pages_store;
1.1       mrg       803:
1.177     yamt      804:                error = uvm_fault_check(&ufi, &flt, &anons, maxprot);
1.143     uebayasi  805:                if (error != 0)
                    806:                        continue;
1.141     uebayasi  807:
1.143     uebayasi  808:                error = uvm_fault_upper_lookup(&ufi, &flt, anons, pages);
                    809:                if (error != 0)
                    810:                        continue;
1.138     uebayasi  811:
1.144     uebayasi  812:                if (pages[flt.centeridx] == PGO_DONTCARE)
1.148     uebayasi  813:                        error = uvm_fault_upper(&ufi, &flt, anons);
1.167     uebayasi  814:                else {
1.177     yamt      815:                        struct uvm_object * const uobj =
                    816:                            ufi.entry->object.uvm_obj;
1.167     uebayasi  817:
                    818:                        if (uobj && uobj->pgops->pgo_fault != NULL) {
1.173     uebayasi  819:                                /*
                    820:                                 * invoke "special" fault routine.
                    821:                                 */
1.167     uebayasi  822:                                mutex_enter(&uobj->vmobjlock);
1.173     uebayasi  823:                                /* locked: maps(read), amap(if there), uobj */
                    824:                                error = uobj->pgops->pgo_fault(&ufi,
                    825:                                    flt.startva, pages, flt.npages,
                    826:                                    flt.centeridx, flt.access_type,
                    827:                                    PGO_LOCKED|PGO_SYNCIO);
1.167     uebayasi  828:
1.177     yamt      829:                                /*
                    830:                                 * locked: nothing, pgo_fault has unlocked
                    831:                                 * everything
                    832:                                 */
1.167     uebayasi  833:
                    834:                                /*
1.177     yamt      835:                                 * object fault routine responsible for
                    836:                                 * pmap_update().
1.167     uebayasi  837:                                 */
                    838:                        } else {
                    839:                                error = uvm_fault_lower(&ufi, &flt, pages);
                    840:                        }
                    841:                }
1.142     uebayasi  842:        }
1.138     uebayasi  843:
1.140     uebayasi  844:        if (flt.anon_spare != NULL) {
                    845:                flt.anon_spare->an_ref--;
                    846:                uvm_anfree(flt.anon_spare);
1.138     uebayasi  847:        }
                    848:        return error;
1.141     uebayasi  849: }
1.138     uebayasi  850:
1.173     uebayasi  851: /*
                    852:  * uvm_fault_check: check prot, handle needs-copy, etc.
                    853:  *
                    854:  *     1. lookup entry.
                    855:  *     2. check protection.
                    856:  *     3. adjust fault condition (mainly for simulated fault).
                    857:  *     4. handle needs-copy (lazy amap copy).
                    858:  *     5. establish range of interest for neighbor fault (aka pre-fault).
                    859:  *     6. look up anons (if amap exists).
                    860:  *     7. flush pages (if MADV_SEQUENTIAL)
                    861:  *
                    862:  * => called with nothing locked.
                    863:  * => if we fail (result != 0) we unlock everything.
1.177     yamt      864:  * => initialize/adjust many members of flt.
1.173     uebayasi  865:  */
                    866:
1.144     uebayasi  867: static int
1.141     uebayasi  868: uvm_fault_check(
                    869:        struct uvm_faultinfo *ufi, struct uvm_faultctx *flt,
1.177     yamt      870:        struct vm_anon ***ranons, bool maxprot)
1.141     uebayasi  871: {
                    872:        struct vm_amap *amap;
                    873:        struct uvm_object *uobj;
1.137     uebayasi  874:        vm_prot_t check_prot;
                    875:        int nback, nforw;
1.164     mlelstv   876:        UVMHIST_FUNC("uvm_fault_check"); UVMHIST_CALLED(maphist);
1.137     uebayasi  877:
1.7       mrg       878:        /*
                    879:         * lookup and lock the maps
                    880:         */
                    881:
1.141     uebayasi  882:        if (uvmfault_lookup(ufi, false) == false) {
1.177     yamt      883:                UVMHIST_LOG(maphist, "<- no mapping @ 0x%x", ufi->orig_rvaddr,
                    884:                    0,0,0);
1.141     uebayasi  885:                return EFAULT;
1.7       mrg       886:        }
                    887:        /* locked: maps(read) */
                    888:
1.61      thorpej   889: #ifdef DIAGNOSTIC
1.141     uebayasi  890:        if ((ufi->map->flags & VM_MAP_PAGEABLE) == 0) {
1.61      thorpej   891:                printf("Page fault on non-pageable map:\n");
1.141     uebayasi  892:                printf("ufi->map = %p\n", ufi->map);
                    893:                printf("ufi->orig_map = %p\n", ufi->orig_map);
                    894:                printf("ufi->orig_rvaddr = 0x%lx\n", (u_long) ufi->orig_rvaddr);
                    895:                panic("uvm_fault: (ufi->map->flags & VM_MAP_PAGEABLE) == 0");
1.61      thorpej   896:        }
                    897: #endif
1.58      chs       898:
1.7       mrg       899:        /*
                    900:         * check protection
                    901:         */
                    902:
1.177     yamt      903:        check_prot = maxprot ?
1.141     uebayasi  904:            ufi->entry->max_protection : ufi->entry->protection;
                    905:        if ((check_prot & flt->access_type) != flt->access_type) {
1.7       mrg       906:                UVMHIST_LOG(maphist,
                    907:                    "<- protection failure (prot=0x%x, access=0x%x)",
1.141     uebayasi  908:                    ufi->entry->protection, flt->access_type, 0, 0);
                    909:                uvmfault_unlockmaps(ufi, false);
                    910:                return EACCES;
1.7       mrg       911:        }
                    912:
                    913:        /*
                    914:         * "enter_prot" is the protection we want to enter the page in at.
                    915:         * for certain pages (e.g. copy-on-write pages) this protection can
1.141     uebayasi  916:         * be more strict than ufi->entry->protection.  "wired" means either
1.7       mrg       917:         * the entry is wired or we are fault-wiring the pg.
                    918:         */
                    919:
1.141     uebayasi  920:        flt->enter_prot = ufi->entry->protection;
1.146     uebayasi  921:        if (VM_MAPENT_ISWIRED(ufi->entry))
                    922:                flt->wire_mapping = true;
                    923:
                    924:        if (flt->wire_mapping) {
1.141     uebayasi  925:                flt->access_type = flt->enter_prot; /* full access for wired */
                    926:                flt->cow_now = (check_prot & VM_PROT_WRITE) != 0;
1.73      chs       927:        } else {
1.141     uebayasi  928:                flt->cow_now = (flt->access_type & VM_PROT_WRITE) != 0;
1.73      chs       929:        }
1.7       mrg       930:
1.168     uebayasi  931:        flt->promote = false;
                    932:
1.7       mrg       933:        /*
                    934:         * handle "needs_copy" case.   if we need to copy the amap we will
                    935:         * have to drop our readlock and relock it with a write lock.  (we
                    936:         * need a write lock to change anything in a map entry [e.g.
                    937:         * needs_copy]).
                    938:         */
                    939:
1.141     uebayasi  940:        if (UVM_ET_ISNEEDSCOPY(ufi->entry)) {
                    941:                if (flt->cow_now || (ufi->entry->object.uvm_obj == NULL)) {
1.177     yamt      942:                        KASSERT(!maxprot);
1.7       mrg       943:                        /* need to clear */
                    944:                        UVMHIST_LOG(maphist,
                    945:                            "  need to clear needs_copy and refault",0,0,0,0);
1.141     uebayasi  946:                        uvmfault_unlockmaps(ufi, false);
                    947:                        uvmfault_amapcopy(ufi);
1.7       mrg       948:                        uvmexp.fltamcopy++;
1.141     uebayasi  949:                        return ERESTART;
1.7       mrg       950:
                    951:                } else {
                    952:
                    953:                        /*
                    954:                         * ensure that we pmap_enter page R/O since
                    955:                         * needs_copy is still true
                    956:                         */
1.72      chs       957:
1.141     uebayasi  958:                        flt->enter_prot &= ~VM_PROT_WRITE;
1.7       mrg       959:                }
                    960:        }
                    961:
                    962:        /*
                    963:         * identify the players
                    964:         */
                    965:
1.141     uebayasi  966:        amap = ufi->entry->aref.ar_amap;        /* upper layer */
                    967:        uobj = ufi->entry->object.uvm_obj;      /* lower layer */
1.7       mrg       968:
                    969:        /*
                    970:         * check for a case 0 fault.  if nothing backing the entry then
                    971:         * error now.
                    972:         */
                    973:
                    974:        if (amap == NULL && uobj == NULL) {
1.141     uebayasi  975:                uvmfault_unlockmaps(ufi, false);
1.7       mrg       976:                UVMHIST_LOG(maphist,"<- no backing store, no overlay",0,0,0,0);
1.141     uebayasi  977:                return EFAULT;
1.7       mrg       978:        }
1.1       mrg       979:
1.7       mrg       980:        /*
                    981:         * establish range of interest based on advice from mapper
                    982:         * and then clip to fit map entry.   note that we only want
1.63      chs       983:         * to do this the first time through the fault.   if we
1.7       mrg       984:         * ReFault we will disable this by setting "narrow" to true.
                    985:         */
1.1       mrg       986:
1.141     uebayasi  987:        if (flt->narrow == false) {
1.7       mrg       988:
                    989:                /* wide fault (!narrow) */
1.141     uebayasi  990:                KASSERT(uvmadvice[ufi->entry->advice].advice ==
                    991:                         ufi->entry->advice);
                    992:                nback = MIN(uvmadvice[ufi->entry->advice].nback,
1.177     yamt      993:                    (ufi->orig_rvaddr - ufi->entry->start) >> PAGE_SHIFT);
1.141     uebayasi  994:                flt->startva = ufi->orig_rvaddr - (nback << PAGE_SHIFT);
1.7       mrg       995:                /*
                    996:                 * note: "-1" because we don't want to count the
                    997:                 * faulting page as forw
                    998:                 */
1.177     yamt      999:                nforw = MIN(uvmadvice[ufi->entry->advice].nforw,
                   1000:                            ((ufi->entry->end - ufi->orig_rvaddr) >>
                   1001:                             PAGE_SHIFT) - 1);
1.141     uebayasi 1002:                flt->npages = nback + nforw + 1;
                   1003:                flt->centeridx = nback;
1.7       mrg      1004:
1.141     uebayasi 1005:                flt->narrow = true;     /* ensure only once per-fault */
1.7       mrg      1006:
                   1007:        } else {
1.63      chs      1008:
1.7       mrg      1009:                /* narrow fault! */
                   1010:                nback = nforw = 0;
1.141     uebayasi 1011:                flt->startva = ufi->orig_rvaddr;
                   1012:                flt->npages = 1;
                   1013:                flt->centeridx = 0;
1.1       mrg      1014:
1.7       mrg      1015:        }
1.131     uebayasi 1016:        /* offset from entry's start to pgs' start */
1.141     uebayasi 1017:        const voff_t eoff = flt->startva - ufi->entry->start;
1.1       mrg      1018:
1.7       mrg      1019:        /* locked: maps(read) */
1.13      chuck    1020:        UVMHIST_LOG(maphist, "  narrow=%d, back=%d, forw=%d, startva=0x%x",
1.141     uebayasi 1021:                    flt->narrow, nback, nforw, flt->startva);
                   1022:        UVMHIST_LOG(maphist, "  entry=0x%x, amap=0x%x, obj=0x%x", ufi->entry,
1.16      chs      1023:                    amap, uobj, 0);
1.1       mrg      1024:
1.7       mrg      1025:        /*
                   1026:         * if we've got an amap, lock it and extract current anons.
                   1027:         */
                   1028:
                   1029:        if (amap) {
1.19      chuck    1030:                amap_lock(amap);
1.141     uebayasi 1031:                amap_lookups(&ufi->entry->aref, eoff, *ranons, flt->npages);
1.7       mrg      1032:        } else {
1.141     uebayasi 1033:                *ranons = NULL; /* to be safe */
1.7       mrg      1034:        }
                   1035:
                   1036:        /* locked: maps(read), amap(if there) */
1.120     ad       1037:        KASSERT(amap == NULL || mutex_owned(&amap->am_l));
1.7       mrg      1038:
                   1039:        /*
                   1040:         * for MADV_SEQUENTIAL mappings we want to deactivate the back pages
                   1041:         * now and then forget about them (for the rest of the fault).
                   1042:         */
                   1043:
1.141     uebayasi 1044:        if (ufi->entry->advice == MADV_SEQUENTIAL && nback != 0) {
1.7       mrg      1045:
                   1046:                UVMHIST_LOG(maphist, "  MADV_SEQUENTIAL: flushing backpages",
                   1047:                    0,0,0,0);
                   1048:                /* flush back-page anons? */
1.63      chs      1049:                if (amap)
1.141     uebayasi 1050:                        uvmfault_anonflush(*ranons, nback);
1.7       mrg      1051:
                   1052:                /* flush object? */
                   1053:                if (uobj) {
1.137     uebayasi 1054:                        voff_t uoff;
                   1055:
1.141     uebayasi 1056:                        uoff = ufi->entry->offset + eoff;
1.122     ad       1057:                        mutex_enter(&uobj->vmobjlock);
1.90      yamt     1058:                        (void) (uobj->pgops->pgo_put)(uobj, uoff, uoff +
1.15      chs      1059:                                    (nback << PAGE_SHIFT), PGO_DEACTIVATE);
1.7       mrg      1060:                }
                   1061:
                   1062:                /* now forget about the backpages */
                   1063:                if (amap)
1.141     uebayasi 1064:                        *ranons += nback;
                   1065:                flt->startva += (nback << PAGE_SHIFT);
                   1066:                flt->npages -= nback;
                   1067:                flt->centeridx = 0;
1.7       mrg      1068:        }
1.137     uebayasi 1069:        /*
                   1070:         * => startva is fixed
                   1071:         * => npages is fixed
                   1072:         */
1.177     yamt     1073:        KASSERT(flt->startva <= ufi->orig_rvaddr);
                   1074:        KASSERT(ufi->orig_rvaddr + ufi->orig_size <=
                   1075:            flt->startva + (flt->npages << PAGE_SHIFT));
1.141     uebayasi 1076:        return 0;
                   1077: }
                   1078:
1.173     uebayasi 1079: /*
                   1080:  * uvm_fault_upper_lookup: look up existing h/w mapping and amap.
                   1081:  *
                   1082:  * iterate range of interest:
                   1083:  *     1. check if h/w mapping exists.  if yes, we don't care
                   1084:  *     2. check if anon exists.  if not, page is lower.
                   1085:  *     3. if anon exists, enter h/w mapping for neighbors.
                   1086:  *
                   1087:  * => called with amap locked (if exists).
                   1088:  */
                   1089:
1.144     uebayasi 1090: static int
1.141     uebayasi 1091: uvm_fault_upper_lookup(
1.177     yamt     1092:        struct uvm_faultinfo *ufi, const struct uvm_faultctx *flt,
1.141     uebayasi 1093:        struct vm_anon **anons, struct vm_page **pages)
                   1094: {
                   1095:        struct vm_amap *amap = ufi->entry->aref.ar_amap;
1.137     uebayasi 1096:        int lcv;
                   1097:        vaddr_t currva;
1.144     uebayasi 1098:        bool shadowed;
1.164     mlelstv  1099:        UVMHIST_FUNC("uvm_fault_upper_lookup"); UVMHIST_CALLED(maphist);
1.7       mrg      1100:
                   1101:        /* locked: maps(read), amap(if there) */
1.120     ad       1102:        KASSERT(amap == NULL || mutex_owned(&amap->am_l));
1.1       mrg      1103:
1.7       mrg      1104:        /*
                   1105:         * map in the backpages and frontpages we found in the amap in hopes
                   1106:         * of preventing future faults.    we also init the pages[] array as
                   1107:         * we go.
                   1108:         */
                   1109:
1.141     uebayasi 1110:        currva = flt->startva;
1.144     uebayasi 1111:        shadowed = false;
1.163     uebayasi 1112:        for (lcv = 0; lcv < flt->npages; lcv++, currva += PAGE_SIZE) {
1.7       mrg      1113:                /*
1.177     yamt     1114:                 * don't play with VAs that are already mapped
                   1115:                 * (except for center)
1.7       mrg      1116:                 */
1.141     uebayasi 1117:                if (lcv != flt->centeridx &&
                   1118:                    pmap_extract(ufi->orig_map->pmap, currva, NULL)) {
1.52      chs      1119:                        pages[lcv] = PGO_DONTCARE;
                   1120:                        continue;
1.7       mrg      1121:                }
                   1122:
                   1123:                /*
                   1124:                 * unmapped or center page.   check if any anon at this level.
                   1125:                 */
                   1126:                if (amap == NULL || anons[lcv] == NULL) {
                   1127:                        pages[lcv] = NULL;
                   1128:                        continue;
                   1129:                }
                   1130:
                   1131:                /*
                   1132:                 * check for present page and map if possible.   re-activate it.
                   1133:                 */
                   1134:
                   1135:                pages[lcv] = PGO_DONTCARE;
1.177     yamt     1136:                if (lcv == flt->centeridx) {    /* save center for later! */
1.144     uebayasi 1137:                        shadowed = true;
1.151     uebayasi 1138:                } else {
1.161     uebayasi 1139:                        struct vm_anon *anon = anons[lcv];
                   1140:
                   1141:                        mutex_enter(&anon->an_lock);
1.172     uebayasi 1142:                        struct vm_page *pg = anon->an_page;
                   1143:
                   1144:                        /* ignore loaned and busy pages */
                   1145:                        if (pg != NULL && pg->loan_count == 0 &&
                   1146:                            (pg->flags & PG_BUSY) == 0)
                   1147:                                uvm_fault_upper_neighbor(ufi, flt, currva,
                   1148:                                    pg, anon->an_ref > 1);
1.161     uebayasi 1149:                        mutex_exit(&anon->an_lock);
1.7       mrg      1150:                }
1.151     uebayasi 1151:        }
                   1152:
1.160     uebayasi 1153:        /* locked: maps(read), amap(if there) */
                   1154:        KASSERT(amap == NULL || mutex_owned(&amap->am_l));
                   1155:        /* (shadowed == true) if there is an anon at the faulting address */
                   1156:        UVMHIST_LOG(maphist, "  shadowed=%d, will_get=%d", shadowed,
1.164     mlelstv  1157:            (ufi->entry->object.uvm_obj && shadowed != false),0,0);
1.160     uebayasi 1158:
                   1159:        /*
                   1160:         * note that if we are really short of RAM we could sleep in the above
                   1161:         * call to pmap_enter with everything locked.   bad?
                   1162:         *
                   1163:         * XXX Actually, that is bad; pmap_enter() should just fail in that
                   1164:         * XXX case.  --thorpej
                   1165:         */
1.151     uebayasi 1166:
                   1167:        return 0;
                   1168: }
                   1169:
1.173     uebayasi 1170: /*
                   1171:  * uvm_fault_upper_neighbor: enter single lower neighbor page.
                   1172:  *
                   1173:  * => called with amap and anon locked.
                   1174:  */
                   1175:
1.151     uebayasi 1176: static void
1.163     uebayasi 1177: uvm_fault_upper_neighbor(
1.177     yamt     1178:        struct uvm_faultinfo *ufi, const struct uvm_faultctx *flt,
1.161     uebayasi 1179:        vaddr_t currva, struct vm_page *pg, bool readonly)
1.151     uebayasi 1180: {
1.164     mlelstv  1181:        UVMHIST_FUNC("uvm_fault_upper_neighbor"); UVMHIST_CALLED(maphist);
1.151     uebayasi 1182:
1.173     uebayasi 1183:        /* locked: amap, anon */
                   1184:
1.152     uebayasi 1185:        mutex_enter(&uvm_pageqlock);
1.161     uebayasi 1186:        uvm_pageenqueue(pg);
1.152     uebayasi 1187:        mutex_exit(&uvm_pageqlock);
                   1188:        UVMHIST_LOG(maphist,
                   1189:            "  MAPPING: n anon: pm=0x%x, va=0x%x, pg=0x%x",
1.161     uebayasi 1190:            ufi->orig_map->pmap, currva, pg, 0);
1.152     uebayasi 1191:        uvmexp.fltnamap++;
                   1192:
                   1193:        /*
1.161     uebayasi 1194:         * Since this page isn't the page that's actually faulting,
                   1195:         * ignore pmap_enter() failures; it's not critical that we
                   1196:         * enter these right now.
1.152     uebayasi 1197:         */
                   1198:
                   1199:        (void) pmap_enter(ufi->orig_map->pmap, currva,
1.161     uebayasi 1200:            VM_PAGE_TO_PHYS(pg),
                   1201:            readonly ? (flt->enter_prot & ~VM_PROT_WRITE) :
1.152     uebayasi 1202:            flt->enter_prot,
1.154     uebayasi 1203:            PMAP_CANFAIL | (flt->wire_mapping ? PMAP_WIRED : 0));
1.52      chs      1204:
1.152     uebayasi 1205:        pmap_update(ufi->orig_map->pmap);
1.151     uebayasi 1206: }
                   1207:
1.173     uebayasi 1208: /*
                   1209:  * uvm_fault_upper: handle upper fault.
                   1210:  *
                   1211:  *     1. acquire anon lock.
                   1212:  *     2. get anon.  let uvmfault_anonget do the dirty work.
                   1213:  *     3. handle loan.
                   1214:  *     4. dispatch direct or promote handlers.
                   1215:  */
1.134     uebayasi 1216:
1.138     uebayasi 1217: static int
                   1218: uvm_fault_upper(
1.140     uebayasi 1219:        struct uvm_faultinfo *ufi, struct uvm_faultctx *flt,
1.148     uebayasi 1220:        struct vm_anon **anons)
1.138     uebayasi 1221: {
1.148     uebayasi 1222:        struct vm_amap * const amap = ufi->entry->aref.ar_amap;
                   1223:        struct vm_anon * const anon = anons[flt->centeridx];
                   1224:        struct uvm_object *uobj;
1.138     uebayasi 1225:        int error;
1.164     mlelstv  1226:        UVMHIST_FUNC("uvm_fault_upper"); UVMHIST_CALLED(maphist);
1.137     uebayasi 1227:
1.7       mrg      1228:        /* locked: maps(read), amap */
1.133     uebayasi 1229:        KASSERT(mutex_owned(&amap->am_l));
1.7       mrg      1230:
                   1231:        /*
                   1232:         * handle case 1: fault on an anon in our amap
                   1233:         */
                   1234:
                   1235:        UVMHIST_LOG(maphist, "  case 1 fault: anon=0x%x", anon, 0,0,0);
1.122     ad       1236:        mutex_enter(&anon->an_lock);
1.7       mrg      1237:
                   1238:        /* locked: maps(read), amap, anon */
1.120     ad       1239:        KASSERT(mutex_owned(&amap->am_l));
1.122     ad       1240:        KASSERT(mutex_owned(&anon->an_lock));
1.7       mrg      1241:
                   1242:        /*
                   1243:         * no matter if we have case 1A or case 1B we are going to need to
                   1244:         * have the anon's memory resident.   ensure that now.
                   1245:         */
                   1246:
                   1247:        /*
1.47      chs      1248:         * let uvmfault_anonget do the dirty work.
1.51      thorpej  1249:         * if it fails (!OK) it will unlock everything for us.
1.47      chs      1250:         * if it succeeds, locks are still valid and locked.
1.7       mrg      1251:         * also, if it is OK, then the anon's page is on the queues.
                   1252:         * if the page is on loan from a uvm_object, then anonget will
                   1253:         * lock that object for us if it does not fail.
                   1254:         */
                   1255:
1.138     uebayasi 1256:        error = uvmfault_anonget(ufi, amap, anon);
1.58      chs      1257:        switch (error) {
1.57      chs      1258:        case 0:
1.63      chs      1259:                break;
1.7       mrg      1260:
1.57      chs      1261:        case ERESTART:
1.139     uebayasi 1262:                return ERESTART;
1.7       mrg      1263:
1.57      chs      1264:        case EAGAIN:
1.128     pooka    1265:                kpause("fltagain1", false, hz/2, NULL);
1.139     uebayasi 1266:                return ERESTART;
1.51      thorpej  1267:
                   1268:        default:
1.138     uebayasi 1269:                return error;
1.1       mrg      1270:        }
1.7       mrg      1271:
                   1272:        /*
                   1273:         * uobj is non null if the page is on loan from an object (i.e. uobj)
                   1274:         */
                   1275:
1.94      yamt     1276:        uobj = anon->an_page->uobject;  /* locked by anonget if !NULL */
1.7       mrg      1277:
                   1278:        /* locked: maps(read), amap, anon, uobj(if one) */
1.120     ad       1279:        KASSERT(mutex_owned(&amap->am_l));
1.122     ad       1280:        KASSERT(mutex_owned(&anon->an_lock));
                   1281:        KASSERT(uobj == NULL || mutex_owned(&uobj->vmobjlock));
1.7       mrg      1282:
                   1283:        /*
1.63      chs      1284:         * special handling for loaned pages
1.7       mrg      1285:         */
1.52      chs      1286:
1.94      yamt     1287:        if (anon->an_page->loan_count) {
1.148     uebayasi 1288:                error = uvm_fault_upper_loan(ufi, flt, anon, &uobj);
                   1289:                if (error != 0)
                   1290:                        return error;
                   1291:        }
1.160     uebayasi 1292:
                   1293:        /*
                   1294:         * if we are case 1B then we will need to allocate a new blank
                   1295:         * anon to transfer the data into.   note that we have a lock
                   1296:         * on anon, so no one can busy or release the page until we are done.
                   1297:         * also note that the ref count can't drop to zero here because
                   1298:         * it is > 1 and we are only dropping one ref.
                   1299:         *
                   1300:         * in the (hopefully very rare) case that we are out of RAM we
                   1301:         * will unlock, wait for more RAM, and refault.
                   1302:         *
                   1303:         * if we are out of anon VM we kill the process (XXX: could wait?).
                   1304:         */
                   1305:
                   1306:        if (flt->cow_now && anon->an_ref > 1) {
1.168     uebayasi 1307:                flt->promote = true;
1.160     uebayasi 1308:                error = uvm_fault_upper_promote(ufi, flt, uobj, anon);
                   1309:        } else {
                   1310:                error = uvm_fault_upper_direct(ufi, flt, uobj, anon);
                   1311:        }
                   1312:        return error;
1.148     uebayasi 1313: }
                   1314:
1.173     uebayasi 1315: /*
                   1316:  * uvm_fault_upper_loan: handle loaned upper page.
                   1317:  *
1.177     yamt     1318:  *     1. if not cow'ing now, simply adjust flt->enter_prot.
1.173     uebayasi 1319:  *     2. if cow'ing now, and if ref count is 1, break loan.
                   1320:  */
                   1321:
1.148     uebayasi 1322: static int
                   1323: uvm_fault_upper_loan(
                   1324:        struct uvm_faultinfo *ufi, struct uvm_faultctx *flt,
                   1325:        struct vm_anon *anon, struct uvm_object **ruobj)
                   1326: {
1.149     uebayasi 1327:        struct vm_amap * const amap = ufi->entry->aref.ar_amap;
1.151     uebayasi 1328:        int error = 0;
1.173     uebayasi 1329:        UVMHIST_FUNC("uvm_fault_upper_loan"); UVMHIST_CALLED(maphist);
1.149     uebayasi 1330:
                   1331:        if (!flt->cow_now) {
1.7       mrg      1332:
1.149     uebayasi 1333:                /*
                   1334:                 * for read faults on loaned pages we just cap the
                   1335:                 * protection at read-only.
                   1336:                 */
1.63      chs      1337:
1.149     uebayasi 1338:                flt->enter_prot = flt->enter_prot & ~VM_PROT_WRITE;
1.7       mrg      1339:
1.149     uebayasi 1340:        } else {
                   1341:                /*
                   1342:                 * note that we can't allow writes into a loaned page!
                   1343:                 *
                   1344:                 * if we have a write fault on a loaned page in an
                   1345:                 * anon then we need to look at the anon's ref count.
                   1346:                 * if it is greater than one then we are going to do
                   1347:                 * a normal copy-on-write fault into a new anon (this
                   1348:                 * is not a problem).  however, if the reference count
                   1349:                 * is one (a case where we would normally allow a
                   1350:                 * write directly to the page) then we need to kill
                   1351:                 * the loan before we continue.
                   1352:                 */
                   1353:
                   1354:                /* >1 case is already ok */
                   1355:                if (anon->an_ref == 1) {
1.155     uebayasi 1356:                        error = uvm_loanbreak_anon(anon, *ruobj);
1.151     uebayasi 1357:                        if (error != 0) {
                   1358:                                uvmfault_unlockall(ufi, amap, *ruobj, anon);
                   1359:                                uvm_wait("flt_noram2");
                   1360:                                return ERESTART;
                   1361:                        }
1.155     uebayasi 1362:                        /* if we were a loan reciever uobj is gone */
                   1363:                        if (*ruobj)
                   1364:                                *ruobj = NULL;
1.151     uebayasi 1365:                }
                   1366:        }
                   1367:        return error;
                   1368: }
                   1369:
1.173     uebayasi 1370: /*
                   1371:  * uvm_fault_upper_promote: promote upper page.
                   1372:  *
                   1373:  *     1. call uvmfault_promote.
                   1374:  *     2. enqueue page.
                   1375:  *     3. deref.
                   1376:  *     4. pass page to uvm_fault_upper_enter.
                   1377:  */
                   1378:
1.148     uebayasi 1379: static int
                   1380: uvm_fault_upper_promote(
                   1381:        struct uvm_faultinfo *ufi, struct uvm_faultctx *flt,
                   1382:        struct uvm_object *uobj, struct vm_anon *anon)
                   1383: {
1.149     uebayasi 1384:        struct vm_anon * const oanon = anon;
                   1385:        struct vm_page *pg;
                   1386:        int error;
1.164     mlelstv  1387:        UVMHIST_FUNC("uvm_fault_upper_promote"); UVMHIST_CALLED(maphist);
1.149     uebayasi 1388:
                   1389:        UVMHIST_LOG(maphist, "  case 1B: COW fault",0,0,0,0);
                   1390:        uvmexp.flt_acow++;
                   1391:
1.177     yamt     1392:        error = uvmfault_promote(ufi, oanon, PGO_DONTCARE, &anon,
                   1393:            &flt->anon_spare);
1.149     uebayasi 1394:        switch (error) {
                   1395:        case 0:
                   1396:                break;
                   1397:        case ERESTART:
                   1398:                return ERESTART;
                   1399:        default:
                   1400:                return error;
                   1401:        }
1.7       mrg      1402:
1.149     uebayasi 1403:        pg = anon->an_page;
                   1404:        mutex_enter(&uvm_pageqlock);
                   1405:        uvm_pageactivate(pg);
                   1406:        mutex_exit(&uvm_pageqlock);
                   1407:        pg->flags &= ~(PG_BUSY|PG_FAKE);
                   1408:        UVM_PAGE_OWN(pg, NULL);
1.7       mrg      1409:
1.149     uebayasi 1410:        /* deref: can not drop to zero here by defn! */
                   1411:        oanon->an_ref--;
1.53      thorpej  1412:
1.149     uebayasi 1413:        /*
                   1414:         * note: oanon is still locked, as is the new anon.  we
                   1415:         * need to check for this later when we unlock oanon; if
                   1416:         * oanon != anon, we'll have to unlock anon, too.
                   1417:         */
1.7       mrg      1418:
1.149     uebayasi 1419:        return uvm_fault_upper_enter(ufi, flt, uobj, anon, pg, oanon);
1.148     uebayasi 1420: }
                   1421:
1.173     uebayasi 1422: /*
                   1423:  * uvm_fault_upper_direct: handle direct fault.
                   1424:  */
                   1425:
1.148     uebayasi 1426: static int
                   1427: uvm_fault_upper_direct(
                   1428:        struct uvm_faultinfo *ufi, struct uvm_faultctx *flt,
                   1429:        struct uvm_object *uobj, struct vm_anon *anon)
                   1430: {
1.149     uebayasi 1431:        struct vm_anon * const oanon = anon;
                   1432:        struct vm_page *pg;
1.173     uebayasi 1433:        UVMHIST_FUNC("uvm_fault_upper_direct"); UVMHIST_CALLED(maphist);
1.52      chs      1434:
1.149     uebayasi 1435:        uvmexp.flt_anon++;
                   1436:        pg = anon->an_page;
                   1437:        if (anon->an_ref > 1)     /* disallow writes to ref > 1 anons */
                   1438:                flt->enter_prot = flt->enter_prot & ~VM_PROT_WRITE;
1.7       mrg      1439:
1.149     uebayasi 1440:        return uvm_fault_upper_enter(ufi, flt, uobj, anon, pg, oanon);
1.148     uebayasi 1441: }
                   1442:
1.173     uebayasi 1443: /*
                   1444:  * uvm_fault_upper_enter: enter h/w mapping of upper page.
                   1445:  */
                   1446:
1.148     uebayasi 1447: static int
                   1448: uvm_fault_upper_enter(
1.177     yamt     1449:        struct uvm_faultinfo *ufi, const struct uvm_faultctx *flt,
1.148     uebayasi 1450:        struct uvm_object *uobj, struct vm_anon *anon, struct vm_page *pg,
                   1451:        struct vm_anon *oanon)
                   1452: {
                   1453:        struct vm_amap * const amap = ufi->entry->aref.ar_amap;
1.164     mlelstv  1454:        UVMHIST_FUNC("uvm_fault_upper_enter"); UVMHIST_CALLED(maphist);
1.7       mrg      1455:
1.173     uebayasi 1456:        /* locked: maps(read), amap, oanon, anon(if different from oanon) */
1.120     ad       1457:        KASSERT(mutex_owned(&amap->am_l));
1.122     ad       1458:        KASSERT(mutex_owned(&anon->an_lock));
                   1459:        KASSERT(mutex_owned(&oanon->an_lock));
1.7       mrg      1460:
                   1461:        /*
1.69      chs      1462:         * now map the page in.
1.7       mrg      1463:         */
                   1464:
1.177     yamt     1465:        UVMHIST_LOG(maphist,
                   1466:            "  MAPPING: anon: pm=0x%x, va=0x%x, pg=0x%x, promote=%d",
1.168     uebayasi 1467:            ufi->orig_map->pmap, ufi->orig_rvaddr, pg, flt->promote);
1.177     yamt     1468:        if (pmap_enter(ufi->orig_map->pmap, ufi->orig_rvaddr,
                   1469:            VM_PAGE_TO_PHYS(pg),
                   1470:            flt->enter_prot, flt->access_type | PMAP_CANFAIL |
                   1471:            (flt->wire_mapping ? PMAP_WIRED : 0)) != 0) {
1.69      chs      1472:
1.46      thorpej  1473:                /*
                   1474:                 * No need to undo what we did; we can simply think of
                   1475:                 * this as the pmap throwing away the mapping information.
                   1476:                 *
                   1477:                 * We do, however, have to go through the ReFault path,
                   1478:                 * as the map may change while we're asleep.
                   1479:                 */
1.69      chs      1480:
1.53      thorpej  1481:                if (anon != oanon)
1.122     ad       1482:                        mutex_exit(&anon->an_lock);
1.138     uebayasi 1483:                uvmfault_unlockall(ufi, amap, uobj, oanon);
1.92      yamt     1484:                if (!uvm_reclaimable()) {
1.46      thorpej  1485:                        UVMHIST_LOG(maphist,
                   1486:                            "<- failed.  out of VM",0,0,0,0);
                   1487:                        /* XXX instrumentation */
1.148     uebayasi 1488:                        return ENOMEM;
1.46      thorpej  1489:                }
                   1490:                /* XXX instrumentation */
                   1491:                uvm_wait("flt_pmfail1");
1.139     uebayasi 1492:                return ERESTART;
1.46      thorpej  1493:        }
1.7       mrg      1494:
1.177     yamt     1495:        uvm_fault_upper_done(ufi, flt, anon, pg);
1.169     uebayasi 1496:
                   1497:        /*
                   1498:         * done case 1!  finish up by unlocking everything and returning success
                   1499:         */
                   1500:
1.175     rmind    1501:        if (anon != oanon) {
1.169     uebayasi 1502:                mutex_exit(&anon->an_lock);
1.175     rmind    1503:        }
                   1504:        pmap_update(ufi->orig_map->pmap);
1.169     uebayasi 1505:        uvmfault_unlockall(ufi, amap, uobj, oanon);
                   1506:        return 0;
1.148     uebayasi 1507: }
                   1508:
1.173     uebayasi 1509: /*
                   1510:  * uvm_fault_upper_done: queue upper center page.
                   1511:  */
                   1512:
1.169     uebayasi 1513: static void
1.148     uebayasi 1514: uvm_fault_upper_done(
1.177     yamt     1515:        struct uvm_faultinfo *ufi, const struct uvm_faultctx *flt,
                   1516:        struct vm_anon *anon, struct vm_page *pg)
1.148     uebayasi 1517: {
1.174     rmind    1518:        const bool wire_paging = flt->wire_paging;
                   1519:
1.173     uebayasi 1520:        UVMHIST_FUNC("uvm_fault_upper_done"); UVMHIST_CALLED(maphist);
1.148     uebayasi 1521:
1.7       mrg      1522:        /*
1.46      thorpej  1523:         * ... update the page queues.
1.7       mrg      1524:         */
                   1525:
1.122     ad       1526:        mutex_enter(&uvm_pageqlock);
1.174     rmind    1527:        if (wire_paging) {
1.8       chuck    1528:                uvm_pagewire(pg);
1.29      chs      1529:
                   1530:                /*
                   1531:                 * since the now-wired page cannot be paged out,
                   1532:                 * release its swap resources for others to use.
                   1533:                 * since an anon with no swap cannot be PG_CLEAN,
                   1534:                 * clear its clean flag now.
                   1535:                 */
                   1536:
                   1537:                pg->flags &= ~(PG_CLEAN);
1.174     rmind    1538:
1.7       mrg      1539:        } else {
                   1540:                uvm_pageactivate(pg);
                   1541:        }
1.122     ad       1542:        mutex_exit(&uvm_pageqlock);
1.174     rmind    1543:
                   1544:        if (wire_paging) {
                   1545:                uvm_anon_dropswap(anon);
                   1546:        }
1.138     uebayasi 1547: }
1.1       mrg      1548:
1.173     uebayasi 1549: /*
                   1550:  * uvm_fault_lower: handle lower fault.
                   1551:  *
                   1552:  *     1. check uobj
                   1553:  *     1.1. if null, ZFOD.
                   1554:  *     1.2. if not null, look up unnmapped neighbor pages.
                   1555:  *     2. for center page, check if promote.
                   1556:  *     2.1. ZFOD always needs promotion.
                   1557:  *     2.2. other uobjs, when entry is marked COW (usually MAP_PRIVATE vnode).
                   1558:  *     3. if uobj is not ZFOD and page is not found, do i/o.
                   1559:  *     4. dispatch either direct / promote fault.
                   1560:  */
                   1561:
1.138     uebayasi 1562: static int
1.173     uebayasi 1563: uvm_fault_lower(
1.140     uebayasi 1564:        struct uvm_faultinfo *ufi, struct uvm_faultctx *flt,
1.173     uebayasi 1565:        struct vm_page **pages)
1.138     uebayasi 1566: {
1.148     uebayasi 1567: #ifdef DIAGNOSTIC
1.173     uebayasi 1568:        struct vm_amap *amap = ufi->entry->aref.ar_amap;
1.148     uebayasi 1569: #endif
1.173     uebayasi 1570:        struct uvm_object *uobj = ufi->entry->object.uvm_obj;
                   1571:        struct vm_page *uobjpage;
1.138     uebayasi 1572:        int error;
1.173     uebayasi 1573:        UVMHIST_FUNC("uvm_fault_lower"); UVMHIST_CALLED(maphist);
                   1574:
                   1575:        /* locked: maps(read), amap(if there), uobj(if !null) */
1.137     uebayasi 1576:
1.7       mrg      1577:        /*
1.173     uebayasi 1578:         * now, if the desired page is not shadowed by the amap and we have
                   1579:         * a backing object that does not have a special fault routine, then
                   1580:         * we ask (with pgo_get) the object for resident pages that we care
                   1581:         * about and attempt to map them in.  we do not let pgo_get block
                   1582:         * (PGO_LOCKED).
                   1583:         */
                   1584:
                   1585:        if (uobj == NULL) {
                   1586:                /* zero fill; don't care neighbor pages */
                   1587:                uobjpage = NULL;
                   1588:        } else {
                   1589:                uvm_fault_lower_lookup(ufi, flt, pages);
                   1590:                uobjpage = pages[flt->centeridx];
                   1591:        }
                   1592:
                   1593:        /*
                   1594:         * note that at this point we are done with any front or back pages.
                   1595:         * we are now going to focus on the center page (i.e. the one we've
                   1596:         * faulted on).  if we have faulted on the upper (anon) layer
                   1597:         * [i.e. case 1], then the anon we want is anons[centeridx] (we have
                   1598:         * not touched it yet).  if we have faulted on the bottom (uobj)
                   1599:         * layer [i.e. case 2] and the page was both present and available,
                   1600:         * then we've got a pointer to it as "uobjpage" and we've already
                   1601:         * made it BUSY.
1.7       mrg      1602:         */
                   1603:
                   1604:        /*
                   1605:         * locked:
                   1606:         * maps(read), amap(if there), uobj(if !null), uobjpage(if !null)
                   1607:         */
1.120     ad       1608:        KASSERT(amap == NULL || mutex_owned(&amap->am_l));
1.122     ad       1609:        KASSERT(uobj == NULL || mutex_owned(&uobj->vmobjlock));
1.120     ad       1610:        KASSERT(uobjpage == NULL || (uobjpage->flags & PG_BUSY) != 0);
1.7       mrg      1611:
                   1612:        /*
                   1613:         * note that uobjpage can not be PGO_DONTCARE at this point.  we now
                   1614:         * set uobjpage to PGO_DONTCARE if we are doing a zero fill.  if we
                   1615:         * have a backing object, check and see if we are going to promote
                   1616:         * the data up to an anon during the fault.
                   1617:         */
                   1618:
                   1619:        if (uobj == NULL) {
1.63      chs      1620:                uobjpage = PGO_DONTCARE;
1.168     uebayasi 1621:                flt->promote = true;            /* always need anon here */
1.7       mrg      1622:        } else {
1.52      chs      1623:                KASSERT(uobjpage != PGO_DONTCARE);
1.168     uebayasi 1624:                flt->promote = flt->cow_now && UVM_ET_ISCOPYONWRITE(ufi->entry);
1.7       mrg      1625:        }
                   1626:        UVMHIST_LOG(maphist, "  case 2 fault: promote=%d, zfill=%d",
1.168     uebayasi 1627:            flt->promote, (uobj == NULL), 0,0);
1.1       mrg      1628:
1.7       mrg      1629:        /*
1.9       chuck    1630:         * if uobjpage is not null then we do not need to do I/O to get the
                   1631:         * uobjpage.
                   1632:         *
1.63      chs      1633:         * if uobjpage is null, then we need to unlock and ask the pager to
1.7       mrg      1634:         * get the data for us.   once we have the data, we need to reverify
                   1635:         * the state the world.   we are currently not holding any resources.
                   1636:         */
1.1       mrg      1637:
1.9       chuck    1638:        if (uobjpage) {
                   1639:                /* update rusage counters */
1.124     ad       1640:                curlwp->l_ru.ru_minflt++;
1.9       chuck    1641:        } else {
1.163     uebayasi 1642:                error = uvm_fault_lower_io(ufi, flt, &uobj, &uobjpage);
1.148     uebayasi 1643:                if (error != 0)
                   1644:                        return error;
                   1645:        }
1.160     uebayasi 1646:
                   1647:        /*
                   1648:         * locked:
                   1649:         * maps(read), amap(if !null), uobj(if !null), uobjpage(if uobj)
                   1650:         */
                   1651:        KASSERT(amap == NULL || mutex_owned(&amap->am_l));
                   1652:        KASSERT(uobj == NULL || mutex_owned(&uobj->vmobjlock));
                   1653:        KASSERT(uobj == NULL || (uobjpage->flags & PG_BUSY) != 0);
                   1654:
                   1655:        /*
                   1656:         * notes:
                   1657:         *  - at this point uobjpage can not be NULL
                   1658:         *  - at this point uobjpage can not be PG_RELEASED (since we checked
                   1659:         *  for it above)
                   1660:         *  - at this point uobjpage could be PG_WANTED (handle later)
                   1661:         */
                   1662:
1.177     yamt     1663:        KASSERT(uobjpage != NULL);
1.160     uebayasi 1664:        KASSERT(uobj == NULL || uobj == uobjpage->uobject);
                   1665:        KASSERT(uobj == NULL || !UVM_OBJ_IS_CLEAN(uobjpage->uobject) ||
                   1666:            (uobjpage->flags & PG_CLEAN) != 0);
                   1667:
1.177     yamt     1668:        if (!flt->promote) {
1.163     uebayasi 1669:                error = uvm_fault_lower_direct(ufi, flt, uobj, uobjpage);
1.160     uebayasi 1670:        } else {
1.163     uebayasi 1671:                error = uvm_fault_lower_promote(ufi, flt, uobj, uobjpage);
1.160     uebayasi 1672:        }
                   1673:        return error;
1.148     uebayasi 1674: }
                   1675:
1.173     uebayasi 1676: /*
                   1677:  * uvm_fault_lower_lookup: look up on-memory uobj pages.
                   1678:  *
                   1679:  *     1. get on-memory pages.
                   1680:  *     2. if failed, give up (get only center page later).
                   1681:  *     3. if succeeded, enter h/w mapping of neighbor pages.
                   1682:  */
                   1683:
                   1684: static void
                   1685: uvm_fault_lower_lookup(
1.177     yamt     1686:        struct uvm_faultinfo *ufi, const struct uvm_faultctx *flt,
1.173     uebayasi 1687:        struct vm_page **pages)
                   1688: {
                   1689:        struct uvm_object *uobj = ufi->entry->object.uvm_obj;
                   1690:        int lcv, gotpages;
                   1691:        vaddr_t currva;
                   1692:        UVMHIST_FUNC("uvm_fault_lower_lookup"); UVMHIST_CALLED(maphist);
                   1693:
                   1694:        mutex_enter(&uobj->vmobjlock);
                   1695:        /* locked: maps(read), amap(if there), uobj */
                   1696:        /*
                   1697:         * the following call to pgo_get does _not_ change locking state
                   1698:         */
                   1699:
                   1700:        uvmexp.fltlget++;
                   1701:        gotpages = flt->npages;
                   1702:        (void) uobj->pgops->pgo_get(uobj,
                   1703:            ufi->entry->offset + flt->startva - ufi->entry->start,
                   1704:            pages, &gotpages, flt->centeridx,
                   1705:            flt->access_type & MASK(ufi->entry), ufi->entry->advice, PGO_LOCKED);
                   1706:
                   1707:        /*
                   1708:         * check for pages to map, if we got any
                   1709:         */
                   1710:
                   1711:        if (gotpages == 0) {
                   1712:                pages[flt->centeridx] = NULL;
                   1713:                return;
                   1714:        }
                   1715:
                   1716:        currva = flt->startva;
                   1717:        for (lcv = 0; lcv < flt->npages; lcv++, currva += PAGE_SIZE) {
                   1718:                struct vm_page *curpg;
                   1719:
                   1720:                curpg = pages[lcv];
                   1721:                if (curpg == NULL || curpg == PGO_DONTCARE) {
                   1722:                        continue;
                   1723:                }
                   1724:                KASSERT(curpg->uobject == uobj);
                   1725:
                   1726:                /*
                   1727:                 * if center page is resident and not PG_BUSY|PG_RELEASED
                   1728:                 * then pgo_get made it PG_BUSY for us and gave us a handle
                   1729:                 * to it.
                   1730:                 */
                   1731:
                   1732:                if (lcv == flt->centeridx) {
                   1733:                        UVMHIST_LOG(maphist, "  got uobjpage "
                   1734:                            "(0x%x) with locked get",
                   1735:                            curpg, 0,0,0);
                   1736:                } else {
                   1737:                        bool readonly = (curpg->flags & PG_RDONLY)
                   1738:                            || (curpg->loan_count > 0)
                   1739:                            || UVM_OBJ_NEEDS_WRITEFAULT(curpg->uobject);
                   1740:
                   1741:                        uvm_fault_lower_neighbor(ufi, flt,
                   1742:                            currva, curpg, readonly);
                   1743:                }
                   1744:        }
                   1745:        pmap_update(ufi->orig_map->pmap);
                   1746: }
                   1747:
                   1748: /*
                   1749:  * uvm_fault_lower_neighbor: enter h/w mapping of lower neighbor page.
                   1750:  */
                   1751:
                   1752: static void
                   1753: uvm_fault_lower_neighbor(
1.177     yamt     1754:        struct uvm_faultinfo *ufi, const struct uvm_faultctx *flt,
1.173     uebayasi 1755:        vaddr_t currva, struct vm_page *pg, bool readonly)
                   1756: {
1.180.4.2! bouyer   1757:        UVMHIST_FUNC(__func__); UVMHIST_CALLED(maphist);
1.173     uebayasi 1758:
                   1759:        /* locked: maps(read), amap(if there), uobj */
                   1760:
                   1761:        /*
                   1762:         * calling pgo_get with PGO_LOCKED returns us pages which
                   1763:         * are neither busy nor released, so we don't need to check
                   1764:         * for this.  we can just directly enter the pages.
                   1765:         */
                   1766:
                   1767:        mutex_enter(&uvm_pageqlock);
                   1768:        uvm_pageenqueue(pg);
                   1769:        mutex_exit(&uvm_pageqlock);
                   1770:        UVMHIST_LOG(maphist,
                   1771:            "  MAPPING: n obj: pm=0x%x, va=0x%x, pg=0x%x",
                   1772:            ufi->orig_map->pmap, currva, pg, 0);
                   1773:        uvmexp.fltnomap++;
                   1774:
                   1775:        /*
                   1776:         * Since this page isn't the page that's actually faulting,
                   1777:         * ignore pmap_enter() failures; it's not critical that we
                   1778:         * enter these right now.
                   1779:         * NOTE: page can't be PG_WANTED or PG_RELEASED because we've
                   1780:         * held the lock the whole time we've had the handle.
                   1781:         */
                   1782:        KASSERT((pg->flags & PG_PAGEOUT) == 0);
                   1783:        KASSERT((pg->flags & PG_RELEASED) == 0);
                   1784:        KASSERT((pg->flags & PG_WANTED) == 0);
                   1785:        KASSERT(!UVM_OBJ_IS_CLEAN(pg->uobject) ||
                   1786:            (pg->flags & PG_CLEAN) != 0);
                   1787:        pg->flags &= ~(PG_BUSY);
                   1788:        UVM_PAGE_OWN(pg, NULL);
                   1789:
                   1790:        (void) pmap_enter(ufi->orig_map->pmap, currva,
                   1791:            VM_PAGE_TO_PHYS(pg),
                   1792:            readonly ? (flt->enter_prot & ~VM_PROT_WRITE) :
                   1793:            flt->enter_prot & MASK(ufi->entry),
                   1794:            PMAP_CANFAIL | (flt->wire_mapping ? PMAP_WIRED : 0));
                   1795: }
                   1796:
                   1797: /*
                   1798:  * uvm_fault_lower_io: get lower page from backing store.
                   1799:  *
                   1800:  *     1. unlock everything, because i/o will block.
                   1801:  *     2. call pgo_get.
                   1802:  *     3. if failed, recover.
                   1803:  *     4. if succeeded, relock everything and verify things.
                   1804:  */
                   1805:
1.148     uebayasi 1806: static int
1.163     uebayasi 1807: uvm_fault_lower_io(
1.177     yamt     1808:        struct uvm_faultinfo *ufi, const struct uvm_faultctx *flt,
1.156     uebayasi 1809:        struct uvm_object **ruobj, struct vm_page **ruobjpage)
1.148     uebayasi 1810: {
1.149     uebayasi 1811:        struct vm_amap * const amap = ufi->entry->aref.ar_amap;
1.156     uebayasi 1812:        struct uvm_object *uobj = *ruobj;
1.158     uebayasi 1813:        struct vm_page *pg;
1.149     uebayasi 1814:        bool locked;
                   1815:        int gotpages;
                   1816:        int error;
                   1817:        voff_t uoff;
1.164     mlelstv  1818:        UVMHIST_FUNC("uvm_fault_lower_io"); UVMHIST_CALLED(maphist);
1.149     uebayasi 1819:
                   1820:        /* update rusage counters */
                   1821:        curlwp->l_ru.ru_majflt++;
1.137     uebayasi 1822:
1.149     uebayasi 1823:        /* locked: maps(read), amap(if there), uobj */
                   1824:        uvmfault_unlockall(ufi, amap, NULL, NULL);
                   1825:        /* locked: uobj */
1.63      chs      1826:
1.149     uebayasi 1827:        uvmexp.fltget++;
                   1828:        gotpages = 1;
1.166     mlelstv  1829:        pg = NULL;
1.149     uebayasi 1830:        uoff = (ufi->orig_rvaddr - ufi->entry->start) + ufi->entry->offset;
1.158     uebayasi 1831:        error = uobj->pgops->pgo_get(uobj, uoff, &pg, &gotpages,
1.149     uebayasi 1832:            0, flt->access_type & MASK(ufi->entry), ufi->entry->advice,
                   1833:            PGO_SYNCIO);
1.158     uebayasi 1834:        /* locked: pg(if no error) */
1.52      chs      1835:
1.149     uebayasi 1836:        /*
                   1837:         * recover from I/O
                   1838:         */
1.1       mrg      1839:
1.149     uebayasi 1840:        if (error) {
                   1841:                if (error == EAGAIN) {
                   1842:                        UVMHIST_LOG(maphist,
                   1843:                            "  pgo_get says TRY AGAIN!",0,0,0,0);
                   1844:                        kpause("fltagain2", false, hz/2, NULL);
                   1845:                        return ERESTART;
                   1846:                }
1.1       mrg      1847:
1.139     uebayasi 1848: #if 0
1.149     uebayasi 1849:                KASSERT(error != ERESTART);
1.139     uebayasi 1850: #else
1.149     uebayasi 1851:                /* XXXUEBS don't re-fault? */
                   1852:                if (error == ERESTART)
                   1853:                        error = EIO;
1.139     uebayasi 1854: #endif
                   1855:
1.149     uebayasi 1856:                UVMHIST_LOG(maphist, "<- pgo_get failed (code %d)",
                   1857:                    error, 0,0,0);
                   1858:                return error;
                   1859:        }
1.7       mrg      1860:
1.158     uebayasi 1861:        /* locked: pg */
1.7       mrg      1862:
1.165     mlelstv  1863:        KASSERT((pg->flags & PG_BUSY) != 0);
                   1864:
1.149     uebayasi 1865:        mutex_enter(&uvm_pageqlock);
1.158     uebayasi 1866:        uvm_pageactivate(pg);
1.149     uebayasi 1867:        mutex_exit(&uvm_pageqlock);
1.69      chs      1868:
1.149     uebayasi 1869:        /*
                   1870:         * re-verify the state of the world by first trying to relock
                   1871:         * the maps.  always relock the object.
                   1872:         */
1.7       mrg      1873:
1.149     uebayasi 1874:        locked = uvmfault_relock(ufi);
                   1875:        if (locked && amap)
                   1876:                amap_lock(amap);
1.156     uebayasi 1877:
                   1878:        /* might be changed */
1.158     uebayasi 1879:        uobj = pg->uobject;
1.156     uebayasi 1880:
1.149     uebayasi 1881:        mutex_enter(&uobj->vmobjlock);
1.63      chs      1882:
1.158     uebayasi 1883:        /* locked(locked): maps(read), amap(if !null), uobj, pg */
                   1884:        /* locked(!locked): uobj, pg */
1.7       mrg      1885:
1.149     uebayasi 1886:        /*
                   1887:         * verify that the page has not be released and re-verify
                   1888:         * that amap slot is still free.   if there is a problem,
                   1889:         * we unlock and clean up.
                   1890:         */
1.7       mrg      1891:
1.158     uebayasi 1892:        if ((pg->flags & PG_RELEASED) != 0 ||
                   1893:            (locked && amap && amap_lookup(&ufi->entry->aref,
1.149     uebayasi 1894:              ufi->orig_rvaddr - ufi->entry->start))) {
                   1895:                if (locked)
                   1896:                        uvmfault_unlockall(ufi, amap, NULL, NULL);
                   1897:                locked = false;
                   1898:        }
1.7       mrg      1899:
1.149     uebayasi 1900:        /*
                   1901:         * didn't get the lock?   release the page and retry.
                   1902:         */
1.7       mrg      1903:
1.149     uebayasi 1904:        if (locked == false) {
                   1905:                UVMHIST_LOG(maphist,
                   1906:                    "  wasn't able to relock after fault: retry",
                   1907:                    0,0,0,0);
1.158     uebayasi 1908:                if (pg->flags & PG_WANTED) {
                   1909:                        wakeup(pg);
                   1910:                }
                   1911:                if (pg->flags & PG_RELEASED) {
1.149     uebayasi 1912:                        uvmexp.fltpgrele++;
1.158     uebayasi 1913:                        uvm_pagefree(pg);
1.157     uebayasi 1914:                        mutex_exit(&uobj->vmobjlock);
1.139     uebayasi 1915:                        return ERESTART;
1.7       mrg      1916:                }
1.158     uebayasi 1917:                pg->flags &= ~(PG_BUSY|PG_WANTED);
                   1918:                UVM_PAGE_OWN(pg, NULL);
1.149     uebayasi 1919:                mutex_exit(&uobj->vmobjlock);
                   1920:                return ERESTART;
                   1921:        }
1.7       mrg      1922:
1.149     uebayasi 1923:        /*
1.158     uebayasi 1924:         * we have the data in pg which is busy and
1.149     uebayasi 1925:         * not released.  we are holding object lock (so the page
                   1926:         * can't be released on us).
                   1927:         */
1.7       mrg      1928:
1.158     uebayasi 1929:        /* locked: maps(read), amap(if !null), uobj, pg */
1.148     uebayasi 1930:
1.156     uebayasi 1931:        *ruobj = uobj;
1.158     uebayasi 1932:        *ruobjpage = pg;
1.148     uebayasi 1933:        return 0;
                   1934: }
                   1935:
1.173     uebayasi 1936: /*
                   1937:  * uvm_fault_lower_direct: fault lower center page
                   1938:  *
1.177     yamt     1939:  *     1. adjust flt->enter_prot.
1.173     uebayasi 1940:  *     2. if page is loaned, resolve.
                   1941:  */
                   1942:
1.148     uebayasi 1943: int
1.163     uebayasi 1944: uvm_fault_lower_direct(
1.148     uebayasi 1945:        struct uvm_faultinfo *ufi, struct uvm_faultctx *flt,
1.156     uebayasi 1946:        struct uvm_object *uobj, struct vm_page *uobjpage)
1.148     uebayasi 1947: {
1.149     uebayasi 1948:        struct vm_page *pg;
1.173     uebayasi 1949:        UVMHIST_FUNC("uvm_fault_lower_direct"); UVMHIST_CALLED(maphist);
1.149     uebayasi 1950:
                   1951:        /*
                   1952:         * we are not promoting.   if the mapping is COW ensure that we
                   1953:         * don't give more access than we should (e.g. when doing a read
                   1954:         * fault on a COPYONWRITE mapping we want to map the COW page in
                   1955:         * R/O even though the entry protection could be R/W).
                   1956:         *
                   1957:         * set "pg" to the page we want to map in (uobjpage, usually)
                   1958:         */
1.1       mrg      1959:
1.149     uebayasi 1960:        uvmexp.flt_obj++;
                   1961:        if (UVM_ET_ISCOPYONWRITE(ufi->entry) ||
                   1962:            UVM_OBJ_NEEDS_WRITEFAULT(uobjpage->uobject))
                   1963:                flt->enter_prot &= ~VM_PROT_WRITE;
                   1964:        pg = uobjpage;          /* map in the actual object */
1.7       mrg      1965:
1.149     uebayasi 1966:        KASSERT(uobjpage != PGO_DONTCARE);
1.7       mrg      1967:
1.149     uebayasi 1968:        /*
                   1969:         * we are faulting directly on the page.   be careful
                   1970:         * about writing to loaned pages...
                   1971:         */
                   1972:
                   1973:        if (uobjpage->loan_count) {
1.163     uebayasi 1974:                uvm_fault_lower_direct_loan(ufi, flt, uobj, &pg, &uobjpage);
1.151     uebayasi 1975:        }
                   1976:        KASSERT(pg == uobjpage);
                   1977:
1.163     uebayasi 1978:        return uvm_fault_lower_enter(ufi, flt, uobj, NULL, pg, uobjpage);
1.151     uebayasi 1979: }
                   1980:
1.173     uebayasi 1981: /*
                   1982:  * uvm_fault_lower_direct_loan: resolve loaned page.
                   1983:  *
1.177     yamt     1984:  *     1. if not cow'ing, adjust flt->enter_prot.
1.173     uebayasi 1985:  *     2. if cow'ing, break loan.
                   1986:  */
                   1987:
1.151     uebayasi 1988: static int
1.163     uebayasi 1989: uvm_fault_lower_direct_loan(
1.151     uebayasi 1990:        struct uvm_faultinfo *ufi, struct uvm_faultctx *flt,
1.177     yamt     1991:        struct uvm_object *uobj, struct vm_page **rpg,
                   1992:        struct vm_page **ruobjpage)
1.151     uebayasi 1993: {
1.152     uebayasi 1994:        struct vm_amap * const amap = ufi->entry->aref.ar_amap;
                   1995:        struct vm_page *pg;
                   1996:        struct vm_page *uobjpage = *ruobjpage;
1.164     mlelstv  1997:        UVMHIST_FUNC("uvm_fault_lower_direct_loan"); UVMHIST_CALLED(maphist);
1.152     uebayasi 1998:
                   1999:        if (!flt->cow_now) {
                   2000:                /* read fault: cap the protection at readonly */
                   2001:                /* cap! */
                   2002:                flt->enter_prot = flt->enter_prot & ~VM_PROT_WRITE;
                   2003:        } else {
                   2004:                /* write fault: must break the loan here */
                   2005:
                   2006:                pg = uvm_loanbreak(uobjpage);
                   2007:                if (pg == NULL) {
                   2008:
                   2009:                        /*
                   2010:                         * drop ownership of page, it can't be released
                   2011:                         */
                   2012:
                   2013:                        if (uobjpage->flags & PG_WANTED)
                   2014:                                wakeup(uobjpage);
                   2015:                        uobjpage->flags &= ~(PG_BUSY|PG_WANTED);
                   2016:                        UVM_PAGE_OWN(uobjpage, NULL);
                   2017:
                   2018:                        uvmfault_unlockall(ufi, amap, uobj, NULL);
                   2019:                        UVMHIST_LOG(maphist,
                   2020:                          "  out of RAM breaking loan, waiting",
                   2021:                          0,0,0,0);
                   2022:                        uvmexp.fltnoram++;
                   2023:                        uvm_wait("flt_noram4");
                   2024:                        return ERESTART;
1.69      chs      2025:                }
1.152     uebayasi 2026:                *rpg = pg;
                   2027:                *ruobjpage = pg;
                   2028:        }
                   2029:        return 0;
1.148     uebayasi 2030: }
                   2031:
1.173     uebayasi 2032: /*
                   2033:  * uvm_fault_lower_promote: promote lower page.
                   2034:  *
                   2035:  *     1. call uvmfault_promote.
                   2036:  *     2. fill in data.
                   2037:  *     3. if not ZFOD, dispose old page.
                   2038:  */
                   2039:
1.148     uebayasi 2040: int
1.163     uebayasi 2041: uvm_fault_lower_promote(
1.148     uebayasi 2042:        struct uvm_faultinfo *ufi, struct uvm_faultctx *flt,
1.156     uebayasi 2043:        struct uvm_object *uobj, struct vm_page *uobjpage)
1.148     uebayasi 2044: {
1.149     uebayasi 2045:        struct vm_amap * const amap = ufi->entry->aref.ar_amap;
                   2046:        struct vm_anon *anon;
                   2047:        struct vm_page *pg;
                   2048:        int error;
1.164     mlelstv  2049:        UVMHIST_FUNC("uvm_fault_lower_promote"); UVMHIST_CALLED(maphist);
1.63      chs      2050:
1.149     uebayasi 2051:        /*
                   2052:         * if we are going to promote the data to an anon we
                   2053:         * allocate a blank anon here and plug it into our amap.
                   2054:         */
1.1       mrg      2055: #if DIAGNOSTIC
1.149     uebayasi 2056:        if (amap == NULL)
                   2057:                panic("uvm_fault: want to promote data, but no anon");
1.1       mrg      2058: #endif
1.149     uebayasi 2059:        error = uvmfault_promote(ufi, NULL, uobjpage,
                   2060:            &anon, &flt->anon_spare);
                   2061:        switch (error) {
                   2062:        case 0:
                   2063:                break;
                   2064:        case ERESTART:
                   2065:                return ERESTART;
                   2066:        default:
                   2067:                return error;
                   2068:        }
                   2069:
                   2070:        pg = anon->an_page;
                   2071:
                   2072:        /*
                   2073:         * fill in the data
                   2074:         */
1.105     yamt     2075:
1.149     uebayasi 2076:        if (uobjpage != PGO_DONTCARE) {
                   2077:                uvmexp.flt_prcopy++;
1.1       mrg      2078:
1.7       mrg      2079:                /*
1.149     uebayasi 2080:                 * promote to shared amap?  make sure all sharing
                   2081:                 * procs see it
1.7       mrg      2082:                 */
                   2083:
1.149     uebayasi 2084:                if ((amap_flags(amap) & AMAP_SHARED) != 0) {
                   2085:                        pmap_page_protect(uobjpage, VM_PROT_NONE);
1.7       mrg      2086:                        /*
1.149     uebayasi 2087:                         * XXX: PAGE MIGHT BE WIRED!
1.7       mrg      2088:                         */
1.149     uebayasi 2089:                }
1.69      chs      2090:
1.149     uebayasi 2091:                /*
                   2092:                 * dispose of uobjpage.  it can't be PG_RELEASED
                   2093:                 * since we still hold the object lock.
                   2094:                 * drop handle to uobj as well.
                   2095:                 */
                   2096:
                   2097:                if (uobjpage->flags & PG_WANTED)
                   2098:                        /* still have the obj lock */
                   2099:                        wakeup(uobjpage);
                   2100:                uobjpage->flags &= ~(PG_BUSY|PG_WANTED);
                   2101:                UVM_PAGE_OWN(uobjpage, NULL);
                   2102:                mutex_exit(&uobj->vmobjlock);
                   2103:                uobj = NULL;
                   2104:
                   2105:                UVMHIST_LOG(maphist,
                   2106:                    "  promote uobjpage 0x%x to anon/page 0x%x/0x%x",
                   2107:                    uobjpage, anon, pg, 0);
1.63      chs      2108:
1.149     uebayasi 2109:        } else {
                   2110:                uvmexp.flt_przero++;
1.7       mrg      2111:
1.149     uebayasi 2112:                /*
                   2113:                 * Page is zero'd and marked dirty by
                   2114:                 * uvmfault_promote().
                   2115:                 */
1.52      chs      2116:
1.149     uebayasi 2117:                UVMHIST_LOG(maphist,"  zero fill anon/page 0x%x/0%x",
                   2118:                    anon, pg, 0, 0);
                   2119:        }
1.148     uebayasi 2120:
1.163     uebayasi 2121:        return uvm_fault_lower_enter(ufi, flt, uobj, anon, pg, uobjpage);
1.148     uebayasi 2122: }
                   2123:
1.173     uebayasi 2124: /*
                   2125:  * uvm_fault_lower_enter: enter h/w mapping of lower page.
                   2126:  */
                   2127:
1.148     uebayasi 2128: int
1.163     uebayasi 2129: uvm_fault_lower_enter(
1.177     yamt     2130:        struct uvm_faultinfo *ufi, const struct uvm_faultctx *flt,
1.148     uebayasi 2131:        struct uvm_object *uobj,
                   2132:        struct vm_anon *anon, struct vm_page *pg, struct vm_page *uobjpage)
                   2133: {
                   2134:        struct vm_amap * const amap = ufi->entry->aref.ar_amap;
                   2135:        int error;
1.164     mlelstv  2136:        UVMHIST_FUNC("uvm_fault_lower_enter"); UVMHIST_CALLED(maphist);
1.7       mrg      2137:
                   2138:        /*
                   2139:         * locked:
1.53      thorpej  2140:         * maps(read), amap(if !null), uobj(if !null), uobjpage(if uobj),
                   2141:         *   anon(if !null), pg(if anon)
1.7       mrg      2142:         *
                   2143:         * note: pg is either the uobjpage or the new page in the new anon
                   2144:         */
1.120     ad       2145:        KASSERT(amap == NULL || mutex_owned(&amap->am_l));
1.122     ad       2146:        KASSERT(uobj == NULL || mutex_owned(&uobj->vmobjlock));
1.120     ad       2147:        KASSERT(uobj == NULL || (uobjpage->flags & PG_BUSY) != 0);
1.122     ad       2148:        KASSERT(anon == NULL || mutex_owned(&anon->an_lock));
1.120     ad       2149:        KASSERT((pg->flags & PG_BUSY) != 0);
1.7       mrg      2150:
                   2151:        /*
                   2152:         * all resources are present.   we can now map it in and free our
                   2153:         * resources.
                   2154:         */
                   2155:
                   2156:        UVMHIST_LOG(maphist,
1.168     uebayasi 2157:            "  MAPPING: case2: pm=0x%x, va=0x%x, pg=0x%x, promote=%d",
                   2158:            ufi->orig_map->pmap, ufi->orig_rvaddr, pg, flt->promote);
1.140     uebayasi 2159:        KASSERT((flt->access_type & VM_PROT_WRITE) == 0 ||
1.75      chs      2160:                (pg->flags & PG_RDONLY) == 0);
1.177     yamt     2161:        if (pmap_enter(ufi->orig_map->pmap, ufi->orig_rvaddr,
                   2162:            VM_PAGE_TO_PHYS(pg),
                   2163:            (pg->flags & PG_RDONLY) != 0 ?
                   2164:            flt->enter_prot & ~VM_PROT_WRITE : flt->enter_prot,
                   2165:            flt->access_type | PMAP_CANFAIL |
                   2166:            (flt->wire_mapping ? PMAP_WIRED : 0)) != 0) {
1.52      chs      2167:
1.46      thorpej  2168:                /*
                   2169:                 * No need to undo what we did; we can simply think of
                   2170:                 * this as the pmap throwing away the mapping information.
                   2171:                 *
                   2172:                 * We do, however, have to go through the ReFault path,
                   2173:                 * as the map may change while we're asleep.
                   2174:                 */
1.52      chs      2175:
1.46      thorpej  2176:                if (pg->flags & PG_WANTED)
1.69      chs      2177:                        wakeup(pg);
1.46      thorpej  2178:
1.63      chs      2179:                /*
1.46      thorpej  2180:                 * note that pg can't be PG_RELEASED since we did not drop
                   2181:                 * the object lock since the last time we checked.
                   2182:                 */
1.111     yamt     2183:                KASSERT((pg->flags & PG_RELEASED) == 0);
1.63      chs      2184:
1.46      thorpej  2185:                pg->flags &= ~(PG_BUSY|PG_FAKE|PG_WANTED);
                   2186:                UVM_PAGE_OWN(pg, NULL);
1.171     uebayasi 2187:
1.138     uebayasi 2188:                uvmfault_unlockall(ufi, amap, uobj, anon);
1.92      yamt     2189:                if (!uvm_reclaimable()) {
1.46      thorpej  2190:                        UVMHIST_LOG(maphist,
                   2191:                            "<- failed.  out of VM",0,0,0,0);
                   2192:                        /* XXX instrumentation */
1.106     yamt     2193:                        error = ENOMEM;
1.138     uebayasi 2194:                        return error;
1.46      thorpej  2195:                }
                   2196:                /* XXX instrumentation */
                   2197:                uvm_wait("flt_pmfail2");
1.139     uebayasi 2198:                return ERESTART;
1.46      thorpej  2199:        }
1.1       mrg      2200:
1.177     yamt     2201:        uvm_fault_lower_done(ufi, flt, uobj, pg);
                   2202:
                   2203:        /*
                   2204:         * note that pg can't be PG_RELEASED since we did not drop the object
                   2205:         * lock since the last time we checked.
                   2206:         */
                   2207:        KASSERT((pg->flags & PG_RELEASED) == 0);
                   2208:        if (pg->flags & PG_WANTED)
                   2209:                wakeup(pg);
                   2210:        pg->flags &= ~(PG_BUSY|PG_FAKE|PG_WANTED);
                   2211:        UVM_PAGE_OWN(pg, NULL);
1.169     uebayasi 2212:
1.175     rmind    2213:        pmap_update(ufi->orig_map->pmap);
1.169     uebayasi 2214:        uvmfault_unlockall(ufi, amap, uobj, anon);
1.175     rmind    2215:
1.169     uebayasi 2216:        UVMHIST_LOG(maphist, "<- done (SUCCESS!)",0,0,0,0);
                   2217:        return 0;
1.148     uebayasi 2218: }
                   2219:
1.173     uebayasi 2220: /*
                   2221:  * uvm_fault_lower_done: queue lower center page.
                   2222:  */
                   2223:
1.169     uebayasi 2224: void
1.163     uebayasi 2225: uvm_fault_lower_done(
1.177     yamt     2226:        struct uvm_faultinfo *ufi, const struct uvm_faultctx *flt,
                   2227:        struct uvm_object *uobj, struct vm_page *pg)
1.148     uebayasi 2228: {
1.174     rmind    2229:        bool dropswap = false;
                   2230:
1.164     mlelstv  2231:        UVMHIST_FUNC("uvm_fault_lower_done"); UVMHIST_CALLED(maphist);
1.148     uebayasi 2232:
1.122     ad       2233:        mutex_enter(&uvm_pageqlock);
1.146     uebayasi 2234:        if (flt->wire_paging) {
1.8       chuck    2235:                uvm_pagewire(pg);
1.22      chs      2236:                if (pg->pqflags & PQ_AOBJ) {
1.29      chs      2237:
                   2238:                        /*
                   2239:                         * since the now-wired page cannot be paged out,
                   2240:                         * release its swap resources for others to use.
                   2241:                         * since an aobj page with no swap cannot be PG_CLEAN,
                   2242:                         * clear its clean flag now.
                   2243:                         */
                   2244:
1.113     christos 2245:                        KASSERT(uobj != NULL);
1.29      chs      2246:                        pg->flags &= ~(PG_CLEAN);
1.174     rmind    2247:                        dropswap = true;
1.22      chs      2248:                }
1.7       mrg      2249:        } else {
                   2250:                uvm_pageactivate(pg);
                   2251:        }
1.122     ad       2252:        mutex_exit(&uvm_pageqlock);
1.171     uebayasi 2253:
1.174     rmind    2254:        if (dropswap) {
                   2255:                uao_dropswap(uobj, pg->offset >> PAGE_SHIFT);
                   2256:        }
1.1       mrg      2257: }
                   2258:
1.110     drochner 2259:
1.1       mrg      2260: /*
                   2261:  * uvm_fault_wire: wire down a range of virtual addresses in a map.
                   2262:  *
1.36      thorpej  2263:  * => map may be read-locked by caller, but MUST NOT be write-locked.
                   2264:  * => if map is read-locked, any operations which may cause map to
                   2265:  *     be write-locked in uvm_fault() must be taken care of by
                   2266:  *     the caller.  See uvm_map_pageable().
1.1       mrg      2267:  */
                   2268:
1.7       mrg      2269: int
1.95      thorpej  2270: uvm_fault_wire(struct vm_map *map, vaddr_t start, vaddr_t end,
1.130     uebayasi 2271:     vm_prot_t access_type, int maxprot)
1.7       mrg      2272: {
1.12      eeh      2273:        vaddr_t va;
1.58      chs      2274:        int error;
1.7       mrg      2275:
                   2276:        /*
1.47      chs      2277:         * now fault it in a page at a time.   if the fault fails then we have
1.63      chs      2278:         * to undo what we have done.   note that in uvm_fault VM_PROT_NONE
1.47      chs      2279:         * is replaced with the max protection if fault_type is VM_FAULT_WIRE.
1.7       mrg      2280:         */
1.1       mrg      2281:
1.65      chs      2282:        /*
                   2283:         * XXX work around overflowing a vaddr_t.  this prevents us from
                   2284:         * wiring the last page in the address space, though.
                   2285:         */
                   2286:        if (start > end) {
                   2287:                return EFAULT;
                   2288:        }
                   2289:
1.163     uebayasi 2290:        for (va = start; va < end; va += PAGE_SIZE) {
1.110     drochner 2291:                error = uvm_fault_internal(map, va, access_type,
1.177     yamt     2292:                    (maxprot ? UVM_FAULT_MAXPROT : 0) | UVM_FAULT_WIRE);
1.58      chs      2293:                if (error) {
1.7       mrg      2294:                        if (va != start) {
1.31      thorpej  2295:                                uvm_fault_unwire(map, start, va);
1.7       mrg      2296:                        }
1.58      chs      2297:                        return error;
1.7       mrg      2298:                }
                   2299:        }
1.58      chs      2300:        return 0;
1.1       mrg      2301: }
                   2302:
                   2303: /*
                   2304:  * uvm_fault_unwire(): unwire range of virtual space.
                   2305:  */
                   2306:
1.7       mrg      2307: void
1.95      thorpej  2308: uvm_fault_unwire(struct vm_map *map, vaddr_t start, vaddr_t end)
1.36      thorpej  2309: {
                   2310:        vm_map_lock_read(map);
                   2311:        uvm_fault_unwire_locked(map, start, end);
                   2312:        vm_map_unlock_read(map);
                   2313: }
                   2314:
                   2315: /*
                   2316:  * uvm_fault_unwire_locked(): the guts of uvm_fault_unwire().
                   2317:  *
                   2318:  * => map must be at least read-locked.
                   2319:  */
                   2320:
                   2321: void
1.95      thorpej  2322: uvm_fault_unwire_locked(struct vm_map *map, vaddr_t start, vaddr_t end)
1.7       mrg      2323: {
1.64      chs      2324:        struct vm_map_entry *entry;
1.31      thorpej  2325:        pmap_t pmap = vm_map_pmap(map);
1.42      thorpej  2326:        vaddr_t va;
1.12      eeh      2327:        paddr_t pa;
1.42      thorpej  2328:        struct vm_page *pg;
1.31      thorpej  2329:
1.52      chs      2330:        KASSERT((map->flags & VM_MAP_INTRSAFE) == 0);
1.7       mrg      2331:
                   2332:        /*
                   2333:         * we assume that the area we are unwiring has actually been wired
                   2334:         * in the first place.   this means that we should be able to extract
                   2335:         * the PAs from the pmap.   we also lock out the page daemon so that
                   2336:         * we can call uvm_pageunwire.
                   2337:         */
1.37      thorpej  2338:
1.122     ad       2339:        mutex_enter(&uvm_pageqlock);
1.7       mrg      2340:
1.37      thorpej  2341:        /*
                   2342:         * find the beginning map entry for the region.
                   2343:         */
1.74      chs      2344:
1.56      chs      2345:        KASSERT(start >= vm_map_min(map) && end <= vm_map_max(map));
1.119     thorpej  2346:        if (uvm_map_lookup_entry(map, start, &entry) == false)
1.37      thorpej  2347:                panic("uvm_fault_unwire_locked: address not in map");
                   2348:
1.69      chs      2349:        for (va = start; va < end; va += PAGE_SIZE) {
1.119     thorpej  2350:                if (pmap_extract(pmap, va, &pa) == false)
1.74      chs      2351:                        continue;
1.42      thorpej  2352:
                   2353:                /*
1.74      chs      2354:                 * find the map entry for the current address.
1.42      thorpej  2355:                 */
1.56      chs      2356:
                   2357:                KASSERT(va >= entry->start);
1.74      chs      2358:                while (va >= entry->end) {
1.56      chs      2359:                        KASSERT(entry->next != &map->header &&
                   2360:                                entry->next->start <= entry->end);
1.42      thorpej  2361:                        entry = entry->next;
                   2362:                }
1.37      thorpej  2363:
1.42      thorpej  2364:                /*
                   2365:                 * if the entry is no longer wired, tell the pmap.
                   2366:                 */
1.74      chs      2367:
1.42      thorpej  2368:                if (VM_MAPENT_ISWIRED(entry) == 0)
                   2369:                        pmap_unwire(pmap, va);
                   2370:
                   2371:                pg = PHYS_TO_VM_PAGE(pa);
                   2372:                if (pg)
                   2373:                        uvm_pageunwire(pg);
1.7       mrg      2374:        }
1.1       mrg      2375:
1.122     ad       2376:        mutex_exit(&uvm_pageqlock);
1.1       mrg      2377: }

CVSweb <webmaster@jp.NetBSD.org>