[BACK]Return to uvm_fault.c CVS log [TXT][DIR] Up to [cvs.NetBSD.org] / src / sys / uvm

Annotation of src/sys/uvm/uvm_fault.c, Revision 1.123.6.2

1.123.6.1  mjf         1: /*     $NetBSD$        */
1.1       mrg         2:
                      3: /*
                      4:  *
                      5:  * Copyright (c) 1997 Charles D. Cranor and Washington University.
                      6:  * All rights reserved.
                      7:  *
                      8:  * Redistribution and use in source and binary forms, with or without
                      9:  * modification, are permitted provided that the following conditions
                     10:  * are met:
                     11:  * 1. Redistributions of source code must retain the above copyright
                     12:  *    notice, this list of conditions and the following disclaimer.
                     13:  * 2. Redistributions in binary form must reproduce the above copyright
                     14:  *    notice, this list of conditions and the following disclaimer in the
                     15:  *    documentation and/or other materials provided with the distribution.
                     16:  * 3. All advertising materials mentioning features or use of this software
                     17:  *    must display the following acknowledgement:
                     18:  *      This product includes software developed by Charles D. Cranor and
                     19:  *      Washington University.
                     20:  * 4. The name of the author may not be used to endorse or promote products
                     21:  *    derived from this software without specific prior written permission.
                     22:  *
                     23:  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
                     24:  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
                     25:  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
                     26:  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
                     27:  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
                     28:  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
                     29:  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
                     30:  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
                     31:  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
                     32:  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
1.4       mrg        33:  *
                     34:  * from: Id: uvm_fault.c,v 1.1.2.23 1998/02/06 05:29:05 chs Exp
1.1       mrg        35:  */
                     36:
                     37: /*
                     38:  * uvm_fault.c: fault handler
                     39:  */
1.71      lukem      40:
                     41: #include <sys/cdefs.h>
1.123.6.1  mjf        42: __KERNEL_RCSID(0, "$NetBSD$");
1.71      lukem      43:
                     44: #include "opt_uvmhist.h"
1.1       mrg        45:
                     46: #include <sys/param.h>
                     47: #include <sys/systm.h>
                     48: #include <sys/kernel.h>
                     49: #include <sys/proc.h>
                     50: #include <sys/malloc.h>
                     51: #include <sys/mman.h>
1.9       chuck      52: #include <sys/user.h>
1.1       mrg        53:
                     54: #include <uvm/uvm.h>
                     55:
                     56: /*
                     57:  *
                     58:  * a word on page faults:
                     59:  *
                     60:  * types of page faults we handle:
                     61:  *
                     62:  * CASE 1: upper layer faults                   CASE 2: lower layer faults
                     63:  *
                     64:  *    CASE 1A         CASE 1B                  CASE 2A        CASE 2B
                     65:  *    read/write1     write>1                  read/write   +-cow_write/zero
1.63      chs        66:  *         |             |                         |        |
1.1       mrg        67:  *      +--|--+       +--|--+     +-----+       +  |  +     | +-----+
                     68:  * amap |  V  |       |  ----------->new|          |        | |  ^  |
                     69:  *      +-----+       +-----+     +-----+       +  |  +     | +--|--+
                     70:  *                                                 |        |    |
                     71:  *      +-----+       +-----+                   +--|--+     | +--|--+
                     72:  * uobj | d/c |       | d/c |                   |  V  |     +----|  |
                     73:  *      +-----+       +-----+                   +-----+       +-----+
                     74:  *
                     75:  * d/c = don't care
1.63      chs        76:  *
1.1       mrg        77:  *   case [0]: layerless fault
                     78:  *     no amap or uobj is present.   this is an error.
                     79:  *
                     80:  *   case [1]: upper layer fault [anon active]
                     81:  *     1A: [read] or [write with anon->an_ref == 1]
                     82:  *             I/O takes place in top level anon and uobj is not touched.
                     83:  *     1B: [write with anon->an_ref > 1]
                     84:  *             new anon is alloc'd and data is copied off ["COW"]
                     85:  *
                     86:  *   case [2]: lower layer fault [uobj]
                     87:  *     2A: [read on non-NULL uobj] or [write to non-copy_on_write area]
                     88:  *             I/O takes place directly in object.
                     89:  *     2B: [write to copy_on_write] or [read on NULL uobj]
1.63      chs        90:  *             data is "promoted" from uobj to a new anon.
1.1       mrg        91:  *             if uobj is null, then we zero fill.
                     92:  *
                     93:  * we follow the standard UVM locking protocol ordering:
                     94:  *
1.63      chs        95:  * MAPS => AMAP => UOBJ => ANON => PAGE QUEUES (PQ)
1.1       mrg        96:  * we hold a PG_BUSY page if we unlock for I/O
                     97:  *
                     98:  *
                     99:  * the code is structured as follows:
1.63      chs       100:  *
1.1       mrg       101:  *     - init the "IN" params in the ufi structure
                    102:  *   ReFault:
                    103:  *     - do lookups [locks maps], check protection, handle needs_copy
                    104:  *     - check for case 0 fault (error)
                    105:  *     - establish "range" of fault
                    106:  *     - if we have an amap lock it and extract the anons
                    107:  *     - if sequential advice deactivate pages behind us
                    108:  *     - at the same time check pmap for unmapped areas and anon for pages
                    109:  *      that we could map in (and do map it if found)
                    110:  *     - check object for resident pages that we could map in
                    111:  *     - if (case 2) goto Case2
                    112:  *     - >>> handle case 1
                    113:  *           - ensure source anon is resident in RAM
                    114:  *           - if case 1B alloc new anon and copy from source
                    115:  *           - map the correct page in
                    116:  *   Case2:
                    117:  *     - >>> handle case 2
                    118:  *           - ensure source page is resident (if uobj)
                    119:  *           - if case 2B alloc new anon and copy from source (could be zero
                    120:  *             fill if uobj == NULL)
                    121:  *           - map the correct page in
                    122:  *     - done!
                    123:  *
                    124:  * note on paging:
                    125:  *   if we have to do I/O we place a PG_BUSY page in the correct object,
                    126:  * unlock everything, and do the I/O.   when I/O is done we must reverify
                    127:  * the state of the world before assuming that our data structures are
                    128:  * valid.   [because mappings could change while the map is unlocked]
                    129:  *
                    130:  *  alternative 1: unbusy the page in question and restart the page fault
                    131:  *    from the top (ReFault).   this is easy but does not take advantage
1.63      chs       132:  *    of the information that we already have from our previous lookup,
1.1       mrg       133:  *    although it is possible that the "hints" in the vm_map will help here.
                    134:  *
                    135:  * alternative 2: the system already keeps track of a "version" number of
                    136:  *    a map.   [i.e. every time you write-lock a map (e.g. to change a
                    137:  *    mapping) you bump the version number up by one...]   so, we can save
                    138:  *    the version number of the map before we release the lock and start I/O.
                    139:  *    then when I/O is done we can relock and check the version numbers
                    140:  *    to see if anything changed.    this might save us some over 1 because
                    141:  *    we don't have to unbusy the page and may be less compares(?).
                    142:  *
                    143:  * alternative 3: put in backpointers or a way to "hold" part of a map
                    144:  *    in place while I/O is in progress.   this could be complex to
                    145:  *    implement (especially with structures like amap that can be referenced
                    146:  *    by multiple map entries, and figuring out what should wait could be
                    147:  *    complex as well...).
                    148:  *
1.123.6.2! mjf       149:  * we use alternative 2.  given that we are multi-threaded now we may want
        !           150:  * to reconsider the choice.
1.1       mrg       151:  */
                    152:
                    153: /*
                    154:  * local data structures
                    155:  */
                    156:
                    157: struct uvm_advice {
1.7       mrg       158:        int advice;
                    159:        int nback;
                    160:        int nforw;
1.1       mrg       161: };
                    162:
                    163: /*
                    164:  * page range array:
1.63      chs       165:  * note: index in array must match "advice" value
1.1       mrg       166:  * XXX: borrowed numbers from freebsd.   do they work well for us?
                    167:  */
                    168:
1.95      thorpej   169: static const struct uvm_advice uvmadvice[] = {
1.7       mrg       170:        { MADV_NORMAL, 3, 4 },
                    171:        { MADV_RANDOM, 0, 0 },
                    172:        { MADV_SEQUENTIAL, 8, 7},
1.1       mrg       173: };
                    174:
1.69      chs       175: #define UVM_MAXRANGE 16        /* must be MAX() of nback+nforw+1 */
1.1       mrg       176:
                    177: /*
                    178:  * private prototypes
                    179:  */
                    180:
                    181: /*
                    182:  * inline functions
                    183:  */
                    184:
                    185: /*
                    186:  * uvmfault_anonflush: try and deactivate pages in specified anons
                    187:  *
                    188:  * => does not have to deactivate page if it is busy
                    189:  */
                    190:
1.103     perry     191: static inline void
1.95      thorpej   192: uvmfault_anonflush(struct vm_anon **anons, int n)
1.1       mrg       193: {
1.7       mrg       194:        int lcv;
                    195:        struct vm_page *pg;
1.63      chs       196:
1.7       mrg       197:        for (lcv = 0 ; lcv < n ; lcv++) {
                    198:                if (anons[lcv] == NULL)
                    199:                        continue;
1.122     ad        200:                mutex_enter(&anons[lcv]->an_lock);
1.94      yamt      201:                pg = anons[lcv]->an_page;
1.117     yamt      202:                if (pg && (pg->flags & PG_BUSY) == 0) {
1.122     ad        203:                        mutex_enter(&uvm_pageqlock);
1.7       mrg       204:                        if (pg->wire_count == 0) {
                    205:                                uvm_pagedeactivate(pg);
                    206:                        }
1.122     ad        207:                        mutex_exit(&uvm_pageqlock);
1.7       mrg       208:                }
1.122     ad        209:                mutex_exit(&anons[lcv]->an_lock);
1.7       mrg       210:        }
1.1       mrg       211: }
                    212:
                    213: /*
                    214:  * normal functions
                    215:  */
                    216:
                    217: /*
                    218:  * uvmfault_amapcopy: clear "needs_copy" in a map.
                    219:  *
                    220:  * => called with VM data structures unlocked (usually, see below)
                    221:  * => we get a write lock on the maps and clear needs_copy for a VA
                    222:  * => if we are out of RAM we sleep (waiting for more)
                    223:  */
                    224:
1.7       mrg       225: static void
1.95      thorpej   226: uvmfault_amapcopy(struct uvm_faultinfo *ufi)
1.1       mrg       227: {
1.69      chs       228:        for (;;) {
1.1       mrg       229:
1.7       mrg       230:                /*
                    231:                 * no mapping?  give up.
                    232:                 */
1.1       mrg       233:
1.119     thorpej   234:                if (uvmfault_lookup(ufi, true) == false)
1.7       mrg       235:                        return;
1.1       mrg       236:
1.7       mrg       237:                /*
                    238:                 * copy if needed.
                    239:                 */
1.1       mrg       240:
1.7       mrg       241:                if (UVM_ET_ISNEEDSCOPY(ufi->entry))
1.108     yamt      242:                        amap_copy(ufi->map, ufi->entry, AMAP_COPY_NOWAIT,
1.13      chuck     243:                                ufi->orig_rvaddr, ufi->orig_rvaddr + 1);
1.1       mrg       244:
1.7       mrg       245:                /*
                    246:                 * didn't work?  must be out of RAM.   unlock and sleep.
                    247:                 */
                    248:
                    249:                if (UVM_ET_ISNEEDSCOPY(ufi->entry)) {
1.119     thorpej   250:                        uvmfault_unlockmaps(ufi, true);
1.7       mrg       251:                        uvm_wait("fltamapcopy");
                    252:                        continue;
                    253:                }
                    254:
                    255:                /*
                    256:                 * got it!   unlock and return.
                    257:                 */
1.63      chs       258:
1.119     thorpej   259:                uvmfault_unlockmaps(ufi, true);
1.7       mrg       260:                return;
                    261:        }
                    262:        /*NOTREACHED*/
1.1       mrg       263: }
                    264:
                    265: /*
                    266:  * uvmfault_anonget: get data in an anon into a non-busy, non-released
                    267:  * page in that anon.
                    268:  *
                    269:  * => maps, amap, and anon locked by caller.
1.57      chs       270:  * => if we fail (result != 0) we unlock everything.
1.1       mrg       271:  * => if we are successful, we return with everything still locked.
                    272:  * => we don't move the page on the queues [gets moved later]
                    273:  * => if we allocate a new page [we_own], it gets put on the queues.
                    274:  *    either way, the result is that the page is on the queues at return time
                    275:  * => for pages which are on loan from a uvm_object (and thus are not
                    276:  *    owned by the anon): if successful, we return with the owning object
                    277:  *    locked.   the caller must unlock this object when it unlocks everything
                    278:  *    else.
                    279:  */
                    280:
1.47      chs       281: int
1.95      thorpej   282: uvmfault_anonget(struct uvm_faultinfo *ufi, struct vm_amap *amap,
                    283:     struct vm_anon *anon)
1.7       mrg       284: {
1.118     thorpej   285:        bool we_own;    /* we own anon's page? */
                    286:        bool locked;    /* did we relock? */
1.7       mrg       287:        struct vm_page *pg;
1.58      chs       288:        int error;
1.7       mrg       289:        UVMHIST_FUNC("uvmfault_anonget"); UVMHIST_CALLED(maphist);
                    290:
1.122     ad        291:        KASSERT(mutex_owned(&anon->an_lock));
1.53      thorpej   292:
1.58      chs       293:        error = 0;
1.9       chuck     294:        uvmexp.fltanget++;
                    295:         /* bump rusage counters */
1.94      yamt      296:        if (anon->an_page)
1.123.6.1  mjf       297:                curlwp->l_ru.ru_minflt++;
1.9       chuck     298:        else
1.123.6.1  mjf       299:                curlwp->l_ru.ru_majflt++;
1.7       mrg       300:
1.63      chs       301:        /*
1.7       mrg       302:         * loop until we get it, or fail.
                    303:         */
                    304:
1.69      chs       305:        for (;;) {
1.119     thorpej   306:                we_own = false;         /* true if we set PG_BUSY on a page */
1.94      yamt      307:                pg = anon->an_page;
1.1       mrg       308:
1.7       mrg       309:                /*
                    310:                 * if there is a resident page and it is loaned, then anon
                    311:                 * may not own it.   call out to uvm_anon_lockpage() to ensure
                    312:                 * the real owner of the page has been identified and locked.
                    313:                 */
                    314:
                    315:                if (pg && pg->loan_count)
1.13      chuck     316:                        pg = uvm_anon_lockloanpg(anon);
1.7       mrg       317:
                    318:                /*
                    319:                 * page there?   make sure it is not busy/released.
                    320:                 */
                    321:
                    322:                if (pg) {
                    323:
                    324:                        /*
                    325:                         * at this point, if the page has a uobject [meaning
                    326:                         * we have it on loan], then that uobject is locked
                    327:                         * by us!   if the page is busy, we drop all the
                    328:                         * locks (including uobject) and try again.
                    329:                         */
                    330:
1.69      chs       331:                        if ((pg->flags & PG_BUSY) == 0) {
1.7       mrg       332:                                UVMHIST_LOG(maphist, "<- OK",0,0,0,0);
1.57      chs       333:                                return (0);
1.7       mrg       334:                        }
                    335:                        pg->flags |= PG_WANTED;
                    336:                        uvmexp.fltpgwait++;
                    337:
                    338:                        /*
                    339:                         * the last unlock must be an atomic unlock+wait on
                    340:                         * the owner of page
                    341:                         */
1.69      chs       342:
1.7       mrg       343:                        if (pg->uobject) {      /* owner is uobject ? */
                    344:                                uvmfault_unlockall(ufi, amap, NULL, anon);
                    345:                                UVMHIST_LOG(maphist, " unlock+wait on uobj",0,
                    346:                                    0,0,0);
                    347:                                UVM_UNLOCK_AND_WAIT(pg,
                    348:                                    &pg->uobject->vmobjlock,
1.119     thorpej   349:                                    false, "anonget1",0);
1.7       mrg       350:                        } else {
                    351:                                /* anon owns page */
                    352:                                uvmfault_unlockall(ufi, amap, NULL, NULL);
                    353:                                UVMHIST_LOG(maphist, " unlock+wait on anon",0,
                    354:                                    0,0,0);
                    355:                                UVM_UNLOCK_AND_WAIT(pg,&anon->an_lock,0,
                    356:                                    "anonget2",0);
                    357:                        }
                    358:                } else {
1.101     yamt      359: #if defined(VMSWAP)
1.63      chs       360:
1.7       mrg       361:                        /*
                    362:                         * no page, we must try and bring it in.
                    363:                         */
1.69      chs       364:
1.28      chs       365:                        pg = uvm_pagealloc(NULL, 0, anon, 0);
1.7       mrg       366:                        if (pg == NULL) {               /* out of RAM.  */
                    367:                                uvmfault_unlockall(ufi, amap, NULL, anon);
                    368:                                uvmexp.fltnoram++;
                    369:                                UVMHIST_LOG(maphist, "  noram -- UVM_WAIT",0,
                    370:                                    0,0,0);
1.93      yamt      371:                                if (!uvm_reclaimable()) {
                    372:                                        return ENOMEM;
                    373:                                }
1.7       mrg       374:                                uvm_wait("flt_noram1");
                    375:                        } else {
                    376:                                /* we set the PG_BUSY bit */
1.119     thorpej   377:                                we_own = true;
1.7       mrg       378:                                uvmfault_unlockall(ufi, amap, NULL, anon);
                    379:
                    380:                                /*
                    381:                                 * we are passing a PG_BUSY+PG_FAKE+PG_CLEAN
                    382:                                 * page into the uvm_swap_get function with
1.18      chuck     383:                                 * all data structures unlocked.  note that
                    384:                                 * it is ok to read an_swslot here because
                    385:                                 * we hold PG_BUSY on the page.
1.7       mrg       386:                                 */
                    387:                                uvmexp.pageins++;
1.58      chs       388:                                error = uvm_swap_get(pg, anon->an_swslot,
1.7       mrg       389:                                    PGO_SYNCIO);
                    390:
                    391:                                /*
                    392:                                 * we clean up after the i/o below in the
                    393:                                 * "we_own" case
                    394:                                 */
                    395:                        }
1.101     yamt      396: #else /* defined(VMSWAP) */
                    397:                        panic("%s: no page", __func__);
                    398: #endif /* defined(VMSWAP) */
1.7       mrg       399:                }
                    400:
                    401:                /*
                    402:                 * now relock and try again
                    403:                 */
                    404:
                    405:                locked = uvmfault_relock(ufi);
1.47      chs       406:                if (locked && amap != NULL) {
1.19      chuck     407:                        amap_lock(amap);
1.7       mrg       408:                }
                    409:                if (locked || we_own)
1.122     ad        410:                        mutex_enter(&anon->an_lock);
1.7       mrg       411:
                    412:                /*
                    413:                 * if we own the page (i.e. we set PG_BUSY), then we need
                    414:                 * to clean up after the I/O. there are three cases to
                    415:                 * consider:
                    416:                 *   [1] page released during I/O: free anon and ReFault.
1.63      chs       417:                 *   [2] I/O not OK.   free the page and cause the fault
1.7       mrg       418:                 *       to fail.
                    419:                 *   [3] I/O OK!   activate the page and sync with the
                    420:                 *       non-we_own case (i.e. drop anon lock if not locked).
                    421:                 */
1.63      chs       422:
1.7       mrg       423:                if (we_own) {
1.101     yamt      424: #if defined(VMSWAP)
1.7       mrg       425:                        if (pg->flags & PG_WANTED) {
1.63      chs       426:                                wakeup(pg);
1.7       mrg       427:                        }
1.58      chs       428:                        if (error) {
1.1       mrg       429:
1.47      chs       430:                                /*
                    431:                                 * remove the swap slot from the anon
                    432:                                 * and mark the anon as having no real slot.
                    433:                                 * don't free the swap slot, thus preventing
                    434:                                 * it from being used again.
                    435:                                 */
1.69      chs       436:
1.84      pk        437:                                if (anon->an_swslot > 0)
                    438:                                        uvm_swap_markbad(anon->an_swslot, 1);
1.47      chs       439:                                anon->an_swslot = SWSLOT_BAD;
                    440:
1.88      yamt      441:                                if ((pg->flags & PG_RELEASED) != 0)
                    442:                                        goto released;
                    443:
1.47      chs       444:                                /*
1.7       mrg       445:                                 * note: page was never !PG_BUSY, so it
                    446:                                 * can't be mapped and thus no need to
                    447:                                 * pmap_page_protect it...
                    448:                                 */
1.69      chs       449:
1.122     ad        450:                                mutex_enter(&uvm_pageqlock);
1.7       mrg       451:                                uvm_pagefree(pg);
1.122     ad        452:                                mutex_exit(&uvm_pageqlock);
1.7       mrg       453:
                    454:                                if (locked)
                    455:                                        uvmfault_unlockall(ufi, amap, NULL,
                    456:                                            anon);
                    457:                                else
1.122     ad        458:                                        mutex_exit(&anon->an_lock);
1.7       mrg       459:                                UVMHIST_LOG(maphist, "<- ERROR", 0,0,0,0);
1.58      chs       460:                                return error;
1.7       mrg       461:                        }
1.63      chs       462:
1.88      yamt      463:                        if ((pg->flags & PG_RELEASED) != 0) {
                    464: released:
                    465:                                KASSERT(anon->an_ref == 0);
                    466:
                    467:                                /*
                    468:                                 * released while we unlocked amap.
                    469:                                 */
                    470:
                    471:                                if (locked)
                    472:                                        uvmfault_unlockall(ufi, amap, NULL,
                    473:                                            NULL);
                    474:
                    475:                                uvm_anon_release(anon);
                    476:
                    477:                                if (error) {
                    478:                                        UVMHIST_LOG(maphist,
                    479:                                            "<- ERROR/RELEASED", 0,0,0,0);
                    480:                                        return error;
                    481:                                }
                    482:
                    483:                                UVMHIST_LOG(maphist, "<- RELEASED", 0,0,0,0);
                    484:                                return ERESTART;
                    485:                        }
                    486:
1.7       mrg       487:                        /*
1.69      chs       488:                         * we've successfully read the page, activate it.
1.7       mrg       489:                         */
1.69      chs       490:
1.122     ad        491:                        mutex_enter(&uvm_pageqlock);
1.7       mrg       492:                        uvm_pageactivate(pg);
1.122     ad        493:                        mutex_exit(&uvm_pageqlock);
1.69      chs       494:                        pg->flags &= ~(PG_WANTED|PG_BUSY|PG_FAKE);
                    495:                        UVM_PAGE_OWN(pg, NULL);
1.7       mrg       496:                        if (!locked)
1.122     ad        497:                                mutex_exit(&anon->an_lock);
1.101     yamt      498: #else /* defined(VMSWAP) */
                    499:                        panic("%s: we_own", __func__);
                    500: #endif /* defined(VMSWAP) */
1.7       mrg       501:                }
                    502:
                    503:                /*
                    504:                 * we were not able to relock.   restart fault.
                    505:                 */
                    506:
                    507:                if (!locked) {
                    508:                        UVMHIST_LOG(maphist, "<- REFAULT", 0,0,0,0);
1.57      chs       509:                        return (ERESTART);
1.7       mrg       510:                }
                    511:
                    512:                /*
                    513:                 * verify no one has touched the amap and moved the anon on us.
                    514:                 */
1.1       mrg       515:
1.47      chs       516:                if (ufi != NULL &&
1.63      chs       517:                    amap_lookup(&ufi->entry->aref,
1.47      chs       518:                                ufi->orig_rvaddr - ufi->entry->start) != anon) {
1.63      chs       519:
1.7       mrg       520:                        uvmfault_unlockall(ufi, amap, NULL, anon);
                    521:                        UVMHIST_LOG(maphist, "<- REFAULT", 0,0,0,0);
1.57      chs       522:                        return (ERESTART);
1.7       mrg       523:                }
1.63      chs       524:
1.7       mrg       525:                /*
1.63      chs       526:                 * try it again!
1.7       mrg       527:                 */
1.1       mrg       528:
1.7       mrg       529:                uvmexp.fltanretry++;
                    530:                continue;
1.69      chs       531:        }
1.7       mrg       532:        /*NOTREACHED*/
1.1       mrg       533: }
                    534:
                    535: /*
1.106     yamt      536:  * uvmfault_promote: promote data to a new anon.  used for 1B and 2B.
                    537:  *
                    538:  *     1. allocate an anon and a page.
                    539:  *     2. fill its contents.
                    540:  *     3. put it into amap.
                    541:  *
                    542:  * => if we fail (result != 0) we unlock everything.
                    543:  * => on success, return a new locked anon via 'nanon'.
                    544:  *    (*nanon)->an_page will be a resident, locked, dirty page.
                    545:  */
                    546:
                    547: static int
                    548: uvmfault_promote(struct uvm_faultinfo *ufi,
                    549:     struct vm_anon *oanon,
                    550:     struct vm_page *uobjpage,
                    551:     struct vm_anon **nanon, /* OUT: allocated anon */
                    552:     struct vm_anon **spare)
                    553: {
                    554:        struct vm_amap *amap = ufi->entry->aref.ar_amap;
                    555:        struct uvm_object *uobj;
                    556:        struct vm_anon *anon;
                    557:        struct vm_page *pg;
                    558:        struct vm_page *opg;
                    559:        int error;
                    560:        UVMHIST_FUNC(__func__); UVMHIST_CALLED(maphist);
                    561:
                    562:        if (oanon) {
                    563:                /* anon COW */
                    564:                opg = oanon->an_page;
                    565:                KASSERT(opg != NULL);
                    566:                KASSERT(opg->uobject == NULL || opg->loan_count > 0);
                    567:        } else if (uobjpage != PGO_DONTCARE) {
                    568:                /* object-backed COW */
                    569:                opg = uobjpage;
                    570:        } else {
                    571:                /* ZFOD */
                    572:                opg = NULL;
                    573:        }
                    574:        if (opg != NULL) {
                    575:                uobj = opg->uobject;
                    576:        } else {
                    577:                uobj = NULL;
                    578:        }
                    579:
                    580:        KASSERT(amap != NULL);
                    581:        KASSERT(uobjpage != NULL);
                    582:        KASSERT(uobjpage == PGO_DONTCARE || (uobjpage->flags & PG_BUSY) != 0);
1.120     ad        583:        KASSERT(mutex_owned(&amap->am_l));
1.122     ad        584:        KASSERT(oanon == NULL || mutex_owned(&oanon->an_lock));
                    585:        KASSERT(uobj == NULL || mutex_owned(&uobj->vmobjlock));
                    586: #if 0
                    587:        KASSERT(*spare == NULL || !mutex_owned(&(*spare)->an_lock));
                    588: #endif
1.106     yamt      589:
                    590:        if (*spare != NULL) {
                    591:                anon = *spare;
                    592:                *spare = NULL;
1.122     ad        593:                mutex_enter(&anon->an_lock);
1.106     yamt      594:        } else if (ufi->map != kernel_map) {
                    595:                anon = uvm_analloc();
                    596:        } else {
                    597:                UVMHIST_LOG(maphist, "kernel_map, unlock and retry", 0,0,0,0);
                    598:
                    599:                /*
                    600:                 * we can't allocate anons with kernel_map locked.
                    601:                 */
                    602:
                    603:                uvm_page_unbusy(&uobjpage, 1);
                    604:                uvmfault_unlockall(ufi, amap, uobj, oanon);
                    605:
                    606:                *spare = uvm_analloc();
                    607:                if (*spare == NULL) {
                    608:                        goto nomem;
                    609:                }
1.122     ad        610:                mutex_exit(&(*spare)->an_lock);
1.106     yamt      611:                error = ERESTART;
                    612:                goto done;
                    613:        }
                    614:        if (anon) {
                    615:
                    616:                /*
                    617:                 * The new anon is locked.
                    618:                 *
                    619:                 * if opg == NULL, we want a zero'd, dirty page,
                    620:                 * so have uvm_pagealloc() do that for us.
                    621:                 */
                    622:
                    623:                pg = uvm_pagealloc(NULL, 0, anon,
                    624:                    (opg == NULL) ? UVM_PGA_ZERO : 0);
                    625:        } else {
                    626:                pg = NULL;
                    627:        }
                    628:
                    629:        /*
                    630:         * out of memory resources?
                    631:         */
                    632:
                    633:        if (pg == NULL) {
                    634:                /* save anon for the next try. */
                    635:                if (anon != NULL) {
1.122     ad        636:                        mutex_exit(&anon->an_lock);
1.106     yamt      637:                        *spare = anon;
                    638:                }
                    639:
                    640:                /* unlock and fail ... */
                    641:                uvm_page_unbusy(&uobjpage, 1);
                    642:                uvmfault_unlockall(ufi, amap, uobj, oanon);
                    643: nomem:
                    644:                if (!uvm_reclaimable()) {
                    645:                        UVMHIST_LOG(maphist, "out of VM", 0,0,0,0);
                    646:                        uvmexp.fltnoanon++;
                    647:                        error = ENOMEM;
                    648:                        goto done;
                    649:                }
                    650:
                    651:                UVMHIST_LOG(maphist, "out of RAM, waiting for more", 0,0,0,0);
                    652:                uvmexp.fltnoram++;
                    653:                uvm_wait("flt_noram5");
                    654:                error = ERESTART;
                    655:                goto done;
                    656:        }
                    657:
                    658:        /* copy page [pg now dirty] */
                    659:        if (opg) {
                    660:                uvm_pagecopy(opg, pg);
                    661:        }
                    662:
                    663:        amap_add(&ufi->entry->aref, ufi->orig_rvaddr - ufi->entry->start, anon,
                    664:            oanon != NULL);
                    665:
                    666:        *nanon = anon;
                    667:        error = 0;
                    668: done:
                    669:        return error;
                    670: }
                    671:
                    672:
                    673: /*
1.1       mrg       674:  *   F A U L T   -   m a i n   e n t r y   p o i n t
                    675:  */
                    676:
                    677: /*
                    678:  * uvm_fault: page fault handler
                    679:  *
                    680:  * => called from MD code to resolve a page fault
1.63      chs       681:  * => VM data structures usually should be unlocked.   however, it is
1.1       mrg       682:  *     possible to call here with the main map locked if the caller
                    683:  *     gets a write lock, sets it recusive, and then calls us (c.f.
                    684:  *     uvm_map_pageable).   this should be avoided because it keeps
                    685:  *     the map locked off during I/O.
1.66      thorpej   686:  * => MUST NEVER BE CALLED IN INTERRUPT CONTEXT
1.1       mrg       687:  */
                    688:
1.24      mycroft   689: #define MASK(entry)     (UVM_ET_ISCOPYONWRITE(entry) ? \
                    690:                         ~VM_PROT_WRITE : VM_PROT_ALL)
                    691:
1.110     drochner  692: /* fault_flag values passed from uvm_fault_wire to uvm_fault_internal */
                    693: #define UVM_FAULT_WIRE 1
                    694: #define UVM_FAULT_WIREMAX 2
                    695:
1.7       mrg       696: int
1.110     drochner  697: uvm_fault_internal(struct vm_map *orig_map, vaddr_t vaddr,
                    698:     vm_prot_t access_type, int fault_flag)
1.1       mrg       699: {
1.7       mrg       700:        struct uvm_faultinfo ufi;
1.72      chs       701:        vm_prot_t enter_prot, check_prot;
1.118     thorpej   702:        bool wired, narrow, promote, locked, shadowed, wire_fault, cow_now;
1.58      chs       703:        int npages, nback, nforw, centeridx, error, lcv, gotpages;
1.90      yamt      704:        vaddr_t startva, currva;
1.79      yamt      705:        voff_t uoff;
1.7       mrg       706:        struct vm_amap *amap;
                    707:        struct uvm_object *uobj;
                    708:        struct vm_anon *anons_store[UVM_MAXRANGE], **anons, *anon, *oanon;
1.106     yamt      709:        struct vm_anon *anon_spare;
1.7       mrg       710:        struct vm_page *pages[UVM_MAXRANGE], *pg, *uobjpage;
                    711:        UVMHIST_FUNC("uvm_fault"); UVMHIST_CALLED(maphist);
1.1       mrg       712:
1.110     drochner  713:        UVMHIST_LOG(maphist, "(map=0x%x, vaddr=0x%x, at=%d, ff=%d)",
                    714:              orig_map, vaddr, access_type, fault_flag);
1.1       mrg       715:
1.106     yamt      716:        anon = anon_spare = NULL;
1.52      chs       717:        pg = NULL;
1.1       mrg       718:
1.7       mrg       719:        uvmexp.faults++;        /* XXX: locking? */
                    720:
                    721:        /*
                    722:         * init the IN parameters in the ufi
                    723:         */
1.1       mrg       724:
1.7       mrg       725:        ufi.orig_map = orig_map;
                    726:        ufi.orig_rvaddr = trunc_page(vaddr);
                    727:        ufi.orig_size = PAGE_SIZE;      /* can't get any smaller than this */
1.110     drochner  728:        wire_fault = (fault_flag > 0);
1.72      chs       729:        if (wire_fault)
1.119     thorpej   730:                narrow = true;          /* don't look for neighborhood
1.7       mrg       731:                                         * pages on wire */
                    732:        else
1.119     thorpej   733:                narrow = false;         /* normal fault */
1.7       mrg       734:
                    735:        /*
                    736:         * "goto ReFault" means restart the page fault from ground zero.
                    737:         */
1.1       mrg       738: ReFault:
                    739:
1.7       mrg       740:        /*
                    741:         * lookup and lock the maps
                    742:         */
                    743:
1.119     thorpej   744:        if (uvmfault_lookup(&ufi, false) == false) {
1.7       mrg       745:                UVMHIST_LOG(maphist, "<- no mapping @ 0x%x", vaddr, 0,0,0);
1.106     yamt      746:                error = EFAULT;
                    747:                goto done;
1.7       mrg       748:        }
                    749:        /* locked: maps(read) */
                    750:
1.61      thorpej   751: #ifdef DIAGNOSTIC
                    752:        if ((ufi.map->flags & VM_MAP_PAGEABLE) == 0) {
                    753:                printf("Page fault on non-pageable map:\n");
                    754:                printf("ufi.map = %p\n", ufi.map);
                    755:                printf("ufi.orig_map = %p\n", ufi.orig_map);
                    756:                printf("ufi.orig_rvaddr = 0x%lx\n", (u_long) ufi.orig_rvaddr);
                    757:                panic("uvm_fault: (ufi.map->flags & VM_MAP_PAGEABLE) == 0");
                    758:        }
                    759: #endif
1.58      chs       760:
1.7       mrg       761:        /*
                    762:         * check protection
                    763:         */
                    764:
1.110     drochner  765:        check_prot = fault_flag == UVM_FAULT_WIREMAX ?
1.72      chs       766:            ufi.entry->max_protection : ufi.entry->protection;
                    767:        if ((check_prot & access_type) != access_type) {
1.7       mrg       768:                UVMHIST_LOG(maphist,
                    769:                    "<- protection failure (prot=0x%x, access=0x%x)",
                    770:                    ufi.entry->protection, access_type, 0, 0);
1.119     thorpej   771:                uvmfault_unlockmaps(&ufi, false);
1.106     yamt      772:                error = EACCES;
                    773:                goto done;
1.7       mrg       774:        }
                    775:
                    776:        /*
                    777:         * "enter_prot" is the protection we want to enter the page in at.
                    778:         * for certain pages (e.g. copy-on-write pages) this protection can
                    779:         * be more strict than ufi.entry->protection.  "wired" means either
                    780:         * the entry is wired or we are fault-wiring the pg.
                    781:         */
                    782:
                    783:        enter_prot = ufi.entry->protection;
1.72      chs       784:        wired = VM_MAPENT_ISWIRED(ufi.entry) || wire_fault;
1.73      chs       785:        if (wired) {
1.7       mrg       786:                access_type = enter_prot; /* full access for wired */
1.73      chs       787:                cow_now = (check_prot & VM_PROT_WRITE) != 0;
                    788:        } else {
                    789:                cow_now = (access_type & VM_PROT_WRITE) != 0;
                    790:        }
1.7       mrg       791:
                    792:        /*
                    793:         * handle "needs_copy" case.   if we need to copy the amap we will
                    794:         * have to drop our readlock and relock it with a write lock.  (we
                    795:         * need a write lock to change anything in a map entry [e.g.
                    796:         * needs_copy]).
                    797:         */
                    798:
                    799:        if (UVM_ET_ISNEEDSCOPY(ufi.entry)) {
1.73      chs       800:                if (cow_now || (ufi.entry->object.uvm_obj == NULL)) {
1.116     yamt      801:                        KASSERT(fault_flag != UVM_FAULT_WIREMAX);
1.7       mrg       802:                        /* need to clear */
                    803:                        UVMHIST_LOG(maphist,
                    804:                            "  need to clear needs_copy and refault",0,0,0,0);
1.119     thorpej   805:                        uvmfault_unlockmaps(&ufi, false);
1.7       mrg       806:                        uvmfault_amapcopy(&ufi);
                    807:                        uvmexp.fltamcopy++;
                    808:                        goto ReFault;
                    809:
                    810:                } else {
                    811:
                    812:                        /*
                    813:                         * ensure that we pmap_enter page R/O since
                    814:                         * needs_copy is still true
                    815:                         */
1.72      chs       816:
1.63      chs       817:                        enter_prot &= ~VM_PROT_WRITE;
1.7       mrg       818:                }
                    819:        }
                    820:
                    821:        /*
                    822:         * identify the players
                    823:         */
                    824:
1.46      thorpej   825:        amap = ufi.entry->aref.ar_amap;         /* top layer */
1.7       mrg       826:        uobj = ufi.entry->object.uvm_obj;       /* bottom layer */
                    827:
                    828:        /*
                    829:         * check for a case 0 fault.  if nothing backing the entry then
                    830:         * error now.
                    831:         */
                    832:
                    833:        if (amap == NULL && uobj == NULL) {
1.119     thorpej   834:                uvmfault_unlockmaps(&ufi, false);
1.7       mrg       835:                UVMHIST_LOG(maphist,"<- no backing store, no overlay",0,0,0,0);
1.106     yamt      836:                error = EFAULT;
                    837:                goto done;
1.7       mrg       838:        }
1.1       mrg       839:
1.7       mrg       840:        /*
                    841:         * establish range of interest based on advice from mapper
                    842:         * and then clip to fit map entry.   note that we only want
1.63      chs       843:         * to do this the first time through the fault.   if we
1.7       mrg       844:         * ReFault we will disable this by setting "narrow" to true.
                    845:         */
1.1       mrg       846:
1.119     thorpej   847:        if (narrow == false) {
1.7       mrg       848:
                    849:                /* wide fault (!narrow) */
1.52      chs       850:                KASSERT(uvmadvice[ufi.entry->advice].advice ==
                    851:                         ufi.entry->advice);
1.69      chs       852:                nback = MIN(uvmadvice[ufi.entry->advice].nback,
1.15      chs       853:                            (ufi.orig_rvaddr - ufi.entry->start) >> PAGE_SHIFT);
                    854:                startva = ufi.orig_rvaddr - (nback << PAGE_SHIFT);
1.69      chs       855:                nforw = MIN(uvmadvice[ufi.entry->advice].nforw,
1.15      chs       856:                            ((ufi.entry->end - ufi.orig_rvaddr) >>
                    857:                             PAGE_SHIFT) - 1);
1.7       mrg       858:                /*
                    859:                 * note: "-1" because we don't want to count the
                    860:                 * faulting page as forw
                    861:                 */
                    862:                npages = nback + nforw + 1;
                    863:                centeridx = nback;
                    864:
1.119     thorpej   865:                narrow = true;  /* ensure only once per-fault */
1.7       mrg       866:
                    867:        } else {
1.63      chs       868:
1.7       mrg       869:                /* narrow fault! */
                    870:                nback = nforw = 0;
1.13      chuck     871:                startva = ufi.orig_rvaddr;
1.7       mrg       872:                npages = 1;
                    873:                centeridx = 0;
1.1       mrg       874:
1.7       mrg       875:        }
1.1       mrg       876:
1.7       mrg       877:        /* locked: maps(read) */
1.13      chuck     878:        UVMHIST_LOG(maphist, "  narrow=%d, back=%d, forw=%d, startva=0x%x",
1.16      chs       879:                    narrow, nback, nforw, startva);
1.7       mrg       880:        UVMHIST_LOG(maphist, "  entry=0x%x, amap=0x%x, obj=0x%x", ufi.entry,
1.16      chs       881:                    amap, uobj, 0);
1.1       mrg       882:
1.7       mrg       883:        /*
                    884:         * if we've got an amap, lock it and extract current anons.
                    885:         */
                    886:
                    887:        if (amap) {
1.19      chuck     888:                amap_lock(amap);
1.7       mrg       889:                anons = anons_store;
                    890:                amap_lookups(&ufi.entry->aref, startva - ufi.entry->start,
                    891:                    anons, npages);
                    892:        } else {
                    893:                anons = NULL;   /* to be safe */
                    894:        }
                    895:
                    896:        /* locked: maps(read), amap(if there) */
1.120     ad        897:        KASSERT(amap == NULL || mutex_owned(&amap->am_l));
1.7       mrg       898:
                    899:        /*
                    900:         * for MADV_SEQUENTIAL mappings we want to deactivate the back pages
                    901:         * now and then forget about them (for the rest of the fault).
                    902:         */
                    903:
1.70      chs       904:        if (ufi.entry->advice == MADV_SEQUENTIAL && nback != 0) {
1.7       mrg       905:
                    906:                UVMHIST_LOG(maphist, "  MADV_SEQUENTIAL: flushing backpages",
                    907:                    0,0,0,0);
                    908:                /* flush back-page anons? */
1.63      chs       909:                if (amap)
1.7       mrg       910:                        uvmfault_anonflush(anons, nback);
                    911:
                    912:                /* flush object? */
                    913:                if (uobj) {
1.90      yamt      914:                        uoff = (startva - ufi.entry->start) + ufi.entry->offset;
1.122     ad        915:                        mutex_enter(&uobj->vmobjlock);
1.90      yamt      916:                        (void) (uobj->pgops->pgo_put)(uobj, uoff, uoff +
1.15      chs       917:                                    (nback << PAGE_SHIFT), PGO_DEACTIVATE);
1.7       mrg       918:                }
                    919:
                    920:                /* now forget about the backpages */
                    921:                if (amap)
                    922:                        anons += nback;
1.52      chs       923:                startva += (nback << PAGE_SHIFT);
1.7       mrg       924:                npages -= nback;
                    925:                nback = centeridx = 0;
                    926:        }
                    927:
                    928:        /* locked: maps(read), amap(if there) */
1.120     ad        929:        KASSERT(amap == NULL || mutex_owned(&amap->am_l));
1.1       mrg       930:
1.7       mrg       931:        /*
                    932:         * map in the backpages and frontpages we found in the amap in hopes
                    933:         * of preventing future faults.    we also init the pages[] array as
                    934:         * we go.
                    935:         */
                    936:
1.13      chuck     937:        currva = startva;
1.119     thorpej   938:        shadowed = false;
1.7       mrg       939:        for (lcv = 0 ; lcv < npages ; lcv++, currva += PAGE_SIZE) {
                    940:
                    941:                /*
                    942:                 * dont play with VAs that are already mapped
1.13      chuck     943:                 * except for center)
1.7       mrg       944:                 */
1.52      chs       945:                if (lcv != centeridx &&
1.89      yamt      946:                    pmap_extract(ufi.orig_map->pmap, currva, NULL)) {
1.52      chs       947:                        pages[lcv] = PGO_DONTCARE;
                    948:                        continue;
1.7       mrg       949:                }
                    950:
                    951:                /*
                    952:                 * unmapped or center page.   check if any anon at this level.
                    953:                 */
                    954:                if (amap == NULL || anons[lcv] == NULL) {
                    955:                        pages[lcv] = NULL;
                    956:                        continue;
                    957:                }
                    958:
                    959:                /*
                    960:                 * check for present page and map if possible.   re-activate it.
                    961:                 */
                    962:
                    963:                pages[lcv] = PGO_DONTCARE;
                    964:                if (lcv == centeridx) {         /* save center for later! */
1.119     thorpej   965:                        shadowed = true;
1.7       mrg       966:                        continue;
                    967:                }
                    968:                anon = anons[lcv];
1.122     ad        969:                mutex_enter(&anon->an_lock);
1.7       mrg       970:                /* ignore loaned pages */
1.94      yamt      971:                if (anon->an_page && anon->an_page->loan_count == 0 &&
                    972:                    (anon->an_page->flags & PG_BUSY) == 0) {
1.122     ad        973:                        mutex_enter(&uvm_pageqlock);
1.112     yamt      974:                        uvm_pageenqueue(anon->an_page);
1.122     ad        975:                        mutex_exit(&uvm_pageqlock);
1.7       mrg       976:                        UVMHIST_LOG(maphist,
                    977:                            "  MAPPING: n anon: pm=0x%x, va=0x%x, pg=0x%x",
1.94      yamt      978:                            ufi.orig_map->pmap, currva, anon->an_page, 0);
1.7       mrg       979:                        uvmexp.fltnamap++;
1.52      chs       980:
1.46      thorpej   981:                        /*
                    982:                         * Since this isn't the page that's actually faulting,
                    983:                         * ignore pmap_enter() failures; it's not critical
                    984:                         * that we enter these right now.
                    985:                         */
1.52      chs       986:
1.46      thorpej   987:                        (void) pmap_enter(ufi.orig_map->pmap, currva,
1.94      yamt      988:                            VM_PAGE_TO_PHYS(anon->an_page),
1.25      mycroft   989:                            (anon->an_ref > 1) ? (enter_prot & ~VM_PROT_WRITE) :
1.46      thorpej   990:                            enter_prot,
                    991:                            PMAP_CANFAIL |
                    992:                             (VM_MAPENT_ISWIRED(ufi.entry) ? PMAP_WIRED : 0));
1.7       mrg       993:                }
1.122     ad        994:                mutex_exit(&anon->an_lock);
1.68      chris     995:                pmap_update(ufi.orig_map->pmap);
1.7       mrg       996:        }
                    997:
                    998:        /* locked: maps(read), amap(if there) */
1.120     ad        999:        KASSERT(amap == NULL || mutex_owned(&amap->am_l));
1.119     thorpej  1000:        /* (shadowed == true) if there is an anon at the faulting address */
1.63      chs      1001:        UVMHIST_LOG(maphist, "  shadowed=%d, will_get=%d", shadowed,
1.119     thorpej  1002:            (uobj && shadowed == false),0,0);
1.1       mrg      1003:
1.7       mrg      1004:        /*
                   1005:         * note that if we are really short of RAM we could sleep in the above
                   1006:         * call to pmap_enter with everything locked.   bad?
1.46      thorpej  1007:         *
                   1008:         * XXX Actually, that is bad; pmap_enter() should just fail in that
                   1009:         * XXX case.  --thorpej
1.7       mrg      1010:         */
1.63      chs      1011:
1.7       mrg      1012:        /*
                   1013:         * if the desired page is not shadowed by the amap and we have a
                   1014:         * backing object, then we check to see if the backing object would
                   1015:         * prefer to handle the fault itself (rather than letting us do it
                   1016:         * with the usual pgo_get hook).  the backing object signals this by
                   1017:         * providing a pgo_fault routine.
                   1018:         */
1.1       mrg      1019:
1.119     thorpej  1020:        if (uobj && shadowed == false && uobj->pgops->pgo_fault != NULL) {
1.122     ad       1021:                mutex_enter(&uobj->vmobjlock);
1.7       mrg      1022:                /* locked: maps(read), amap (if there), uobj */
1.58      chs      1023:                error = uobj->pgops->pgo_fault(&ufi, startva, pages, npages,
1.109     drochner 1024:                    centeridx, access_type, PGO_LOCKED|PGO_SYNCIO);
1.52      chs      1025:
1.7       mrg      1026:                /* locked: nothing, pgo_fault has unlocked everything */
                   1027:
1.59      chs      1028:                if (error == ERESTART)
1.7       mrg      1029:                        goto ReFault;           /* try again! */
1.61      thorpej  1030:                /*
                   1031:                 * object fault routine responsible for pmap_update().
                   1032:                 */
1.106     yamt     1033:                goto done;
1.7       mrg      1034:        }
                   1035:
                   1036:        /*
                   1037:         * now, if the desired page is not shadowed by the amap and we have
                   1038:         * a backing object that does not have a special fault routine, then
                   1039:         * we ask (with pgo_get) the object for resident pages that we care
                   1040:         * about and attempt to map them in.  we do not let pgo_get block
                   1041:         * (PGO_LOCKED).
                   1042:         */
                   1043:
1.119     thorpej  1044:        if (uobj && shadowed == false) {
1.122     ad       1045:                mutex_enter(&uobj->vmobjlock);
1.7       mrg      1046:                /* locked (!shadowed): maps(read), amap (if there), uobj */
                   1047:                /*
                   1048:                 * the following call to pgo_get does _not_ change locking state
                   1049:                 */
                   1050:
                   1051:                uvmexp.fltlget++;
                   1052:                gotpages = npages;
1.52      chs      1053:                (void) uobj->pgops->pgo_get(uobj, ufi.entry->offset +
1.1       mrg      1054:                                (startva - ufi.entry->start),
                   1055:                                pages, &gotpages, centeridx,
1.24      mycroft  1056:                                access_type & MASK(ufi.entry),
1.1       mrg      1057:                                ufi.entry->advice, PGO_LOCKED);
                   1058:
1.7       mrg      1059:                /*
                   1060:                 * check for pages to map, if we got any
                   1061:                 */
                   1062:
                   1063:                uobjpage = NULL;
                   1064:
                   1065:                if (gotpages) {
1.13      chuck    1066:                        currva = startva;
1.69      chs      1067:                        for (lcv = 0; lcv < npages;
                   1068:                             lcv++, currva += PAGE_SIZE) {
1.86      yamt     1069:                                struct vm_page *curpg;
1.118     thorpej  1070:                                bool readonly;
1.86      yamt     1071:
                   1072:                                curpg = pages[lcv];
                   1073:                                if (curpg == NULL || curpg == PGO_DONTCARE) {
1.7       mrg      1074:                                        continue;
1.69      chs      1075:                                }
1.107     yamt     1076:                                KASSERT(curpg->uobject == uobj);
1.1       mrg      1077:
1.52      chs      1078:                                /*
                   1079:                                 * if center page is resident and not
                   1080:                                 * PG_BUSY|PG_RELEASED then pgo_get
                   1081:                                 * made it PG_BUSY for us and gave
                   1082:                                 * us a handle to it.   remember this
                   1083:                                 * page as "uobjpage." (for later use).
                   1084:                                 */
1.63      chs      1085:
1.52      chs      1086:                                if (lcv == centeridx) {
1.86      yamt     1087:                                        uobjpage = curpg;
1.52      chs      1088:                                        UVMHIST_LOG(maphist, "  got uobjpage "
1.63      chs      1089:                                            "(0x%x) with locked get",
1.7       mrg      1090:                                            uobjpage, 0,0,0);
1.52      chs      1091:                                        continue;
1.7       mrg      1092:                                }
1.63      chs      1093:
                   1094:                                /*
1.69      chs      1095:                                 * calling pgo_get with PGO_LOCKED returns us
                   1096:                                 * pages which are neither busy nor released,
                   1097:                                 * so we don't need to check for this.
                   1098:                                 * we can just directly enter the pages.
1.7       mrg      1099:                                 */
                   1100:
1.122     ad       1101:                                mutex_enter(&uvm_pageqlock);
1.112     yamt     1102:                                uvm_pageenqueue(curpg);
1.122     ad       1103:                                mutex_exit(&uvm_pageqlock);
1.7       mrg      1104:                                UVMHIST_LOG(maphist,
                   1105:                                  "  MAPPING: n obj: pm=0x%x, va=0x%x, pg=0x%x",
1.86      yamt     1106:                                  ufi.orig_map->pmap, currva, curpg, 0);
1.7       mrg      1107:                                uvmexp.fltnomap++;
1.52      chs      1108:
1.46      thorpej  1109:                                /*
                   1110:                                 * Since this page isn't the page that's
1.85      dbj      1111:                                 * actually faulting, ignore pmap_enter()
1.46      thorpej  1112:                                 * failures; it's not critical that we
                   1113:                                 * enter these right now.
                   1114:                                 */
1.86      yamt     1115:                                KASSERT((curpg->flags & PG_PAGEOUT) == 0);
                   1116:                                KASSERT((curpg->flags & PG_RELEASED) == 0);
1.97      yamt     1117:                                KASSERT(!UVM_OBJ_IS_CLEAN(curpg->uobject) ||
1.96      yamt     1118:                                    (curpg->flags & PG_CLEAN) != 0);
1.86      yamt     1119:                                readonly = (curpg->flags & PG_RDONLY)
1.96      yamt     1120:                                    || (curpg->loan_count > 0)
1.98      yamt     1121:                                    || UVM_OBJ_NEEDS_WRITEFAULT(curpg->uobject);
1.52      chs      1122:
1.46      thorpej  1123:                                (void) pmap_enter(ufi.orig_map->pmap, currva,
1.86      yamt     1124:                                    VM_PAGE_TO_PHYS(curpg),
                   1125:                                    readonly ?
1.76      chs      1126:                                    enter_prot & ~VM_PROT_WRITE :
                   1127:                                    enter_prot & MASK(ufi.entry),
1.46      thorpej  1128:                                    PMAP_CANFAIL |
                   1129:                                     (wired ? PMAP_WIRED : 0));
1.7       mrg      1130:
1.63      chs      1131:                                /*
1.7       mrg      1132:                                 * NOTE: page can't be PG_WANTED or PG_RELEASED
                   1133:                                 * because we've held the lock the whole time
                   1134:                                 * we've had the handle.
                   1135:                                 */
1.111     yamt     1136:                                KASSERT((curpg->flags & PG_WANTED) == 0);
                   1137:                                KASSERT((curpg->flags & PG_RELEASED) == 0);
1.52      chs      1138:
1.86      yamt     1139:                                curpg->flags &= ~(PG_BUSY);
                   1140:                                UVM_PAGE_OWN(curpg, NULL);
1.69      chs      1141:                        }
1.68      chris    1142:                        pmap_update(ufi.orig_map->pmap);
1.69      chs      1143:                }
1.7       mrg      1144:        } else {
                   1145:                uobjpage = NULL;
                   1146:        }
                   1147:
                   1148:        /* locked (shadowed): maps(read), amap */
1.63      chs      1149:        /* locked (!shadowed): maps(read), amap(if there),
1.7       mrg      1150:                 uobj(if !null), uobjpage(if !null) */
1.111     yamt     1151:        if (shadowed) {
1.120     ad       1152:                KASSERT(mutex_owned(&amap->am_l));
1.111     yamt     1153:        } else {
1.120     ad       1154:                KASSERT(amap == NULL || mutex_owned(&amap->am_l));
1.122     ad       1155:                KASSERT(uobj == NULL || mutex_owned(&uobj->vmobjlock));
1.111     yamt     1156:                KASSERT(uobjpage == NULL || (uobjpage->flags & PG_BUSY) != 0);
                   1157:        }
1.7       mrg      1158:
                   1159:        /*
                   1160:         * note that at this point we are done with any front or back pages.
                   1161:         * we are now going to focus on the center page (i.e. the one we've
                   1162:         * faulted on).  if we have faulted on the top (anon) layer
                   1163:         * [i.e. case 1], then the anon we want is anons[centeridx] (we have
                   1164:         * not touched it yet).  if we have faulted on the bottom (uobj)
                   1165:         * layer [i.e. case 2] and the page was both present and available,
                   1166:         * then we've got a pointer to it as "uobjpage" and we've already
1.8       chuck    1167:         * made it BUSY.
1.7       mrg      1168:         */
                   1169:
                   1170:        /*
                   1171:         * there are four possible cases we must address: 1A, 1B, 2A, and 2B
                   1172:         */
                   1173:
                   1174:        /*
                   1175:         * redirect case 2: if we are not shadowed, go to case 2.
                   1176:         */
                   1177:
1.119     thorpej  1178:        if (shadowed == false)
1.7       mrg      1179:                goto Case2;
                   1180:
                   1181:        /* locked: maps(read), amap */
                   1182:
                   1183:        /*
                   1184:         * handle case 1: fault on an anon in our amap
                   1185:         */
                   1186:
                   1187:        anon = anons[centeridx];
                   1188:        UVMHIST_LOG(maphist, "  case 1 fault: anon=0x%x", anon, 0,0,0);
1.122     ad       1189:        mutex_enter(&anon->an_lock);
1.7       mrg      1190:
                   1191:        /* locked: maps(read), amap, anon */
1.120     ad       1192:        KASSERT(mutex_owned(&amap->am_l));
1.122     ad       1193:        KASSERT(mutex_owned(&anon->an_lock));
1.7       mrg      1194:
                   1195:        /*
                   1196:         * no matter if we have case 1A or case 1B we are going to need to
                   1197:         * have the anon's memory resident.   ensure that now.
                   1198:         */
                   1199:
                   1200:        /*
1.47      chs      1201:         * let uvmfault_anonget do the dirty work.
1.51      thorpej  1202:         * if it fails (!OK) it will unlock everything for us.
1.47      chs      1203:         * if it succeeds, locks are still valid and locked.
1.7       mrg      1204:         * also, if it is OK, then the anon's page is on the queues.
                   1205:         * if the page is on loan from a uvm_object, then anonget will
                   1206:         * lock that object for us if it does not fail.
                   1207:         */
                   1208:
1.58      chs      1209:        error = uvmfault_anonget(&ufi, amap, anon);
                   1210:        switch (error) {
1.57      chs      1211:        case 0:
1.63      chs      1212:                break;
1.7       mrg      1213:
1.57      chs      1214:        case ERESTART:
1.7       mrg      1215:                goto ReFault;
                   1216:
1.57      chs      1217:        case EAGAIN:
1.52      chs      1218:                tsleep(&lbolt, PVM, "fltagain1", 0);
                   1219:                goto ReFault;
1.51      thorpej  1220:
                   1221:        default:
1.106     yamt     1222:                goto done;
1.1       mrg      1223:        }
1.7       mrg      1224:
                   1225:        /*
                   1226:         * uobj is non null if the page is on loan from an object (i.e. uobj)
                   1227:         */
                   1228:
1.94      yamt     1229:        uobj = anon->an_page->uobject;  /* locked by anonget if !NULL */
1.7       mrg      1230:
                   1231:        /* locked: maps(read), amap, anon, uobj(if one) */
1.120     ad       1232:        KASSERT(mutex_owned(&amap->am_l));
1.122     ad       1233:        KASSERT(mutex_owned(&anon->an_lock));
                   1234:        KASSERT(uobj == NULL || mutex_owned(&uobj->vmobjlock));
1.7       mrg      1235:
                   1236:        /*
1.63      chs      1237:         * special handling for loaned pages
1.7       mrg      1238:         */
1.52      chs      1239:
1.94      yamt     1240:        if (anon->an_page->loan_count) {
1.7       mrg      1241:
1.73      chs      1242:                if (!cow_now) {
1.63      chs      1243:
1.7       mrg      1244:                        /*
                   1245:                         * for read faults on loaned pages we just cap the
                   1246:                         * protection at read-only.
                   1247:                         */
                   1248:
                   1249:                        enter_prot = enter_prot & ~VM_PROT_WRITE;
                   1250:
                   1251:                } else {
                   1252:                        /*
                   1253:                         * note that we can't allow writes into a loaned page!
                   1254:                         *
                   1255:                         * if we have a write fault on a loaned page in an
                   1256:                         * anon then we need to look at the anon's ref count.
                   1257:                         * if it is greater than one then we are going to do
                   1258:                         * a normal copy-on-write fault into a new anon (this
                   1259:                         * is not a problem).  however, if the reference count
                   1260:                         * is one (a case where we would normally allow a
                   1261:                         * write directly to the page) then we need to kill
                   1262:                         * the loan before we continue.
                   1263:                         */
                   1264:
                   1265:                        /* >1 case is already ok */
                   1266:                        if (anon->an_ref == 1) {
                   1267:
                   1268:                                /* get new un-owned replacement page */
1.28      chs      1269:                                pg = uvm_pagealloc(NULL, 0, NULL, 0);
1.7       mrg      1270:                                if (pg == NULL) {
                   1271:                                        uvmfault_unlockall(&ufi, amap, uobj,
                   1272:                                            anon);
                   1273:                                        uvm_wait("flt_noram2");
                   1274:                                        goto ReFault;
                   1275:                                }
                   1276:
                   1277:                                /*
                   1278:                                 * copy data, kill loan, and drop uobj lock
                   1279:                                 * (if any)
                   1280:                                 */
                   1281:                                /* copy old -> new */
1.94      yamt     1282:                                uvm_pagecopy(anon->an_page, pg);
1.7       mrg      1283:
                   1284:                                /* force reload */
1.115     yamt     1285:                                pmap_page_protect(anon->an_page, VM_PROT_NONE);
1.122     ad       1286:                                mutex_enter(&uvm_pageqlock);      /* KILL loan */
1.78      thorpej  1287:
1.94      yamt     1288:                                anon->an_page->uanon = NULL;
1.7       mrg      1289:                                /* in case we owned */
1.94      yamt     1290:                                anon->an_page->pqflags &= ~PQ_ANON;
1.78      thorpej  1291:
                   1292:                                if (uobj) {
                   1293:                                        /* if we were receiver of loan */
1.94      yamt     1294:                                        anon->an_page->loan_count--;
1.78      thorpej  1295:                                } else {
                   1296:                                        /*
                   1297:                                         * we were the lender (A->K); need
                   1298:                                         * to remove the page from pageq's.
                   1299:                                         */
1.94      yamt     1300:                                        uvm_pagedequeue(anon->an_page);
1.78      thorpej  1301:                                }
                   1302:
1.7       mrg      1303:                                if (uobj) {
1.122     ad       1304:                                        mutex_exit(&uobj->vmobjlock);
1.7       mrg      1305:                                        uobj = NULL;
                   1306:                                }
                   1307:
                   1308:                                /* install new page in anon */
1.94      yamt     1309:                                anon->an_page = pg;
1.7       mrg      1310:                                pg->uanon = anon;
                   1311:                                pg->pqflags |= PQ_ANON;
1.112     yamt     1312:
                   1313:                                uvm_pageactivate(pg);
1.122     ad       1314:                                mutex_exit(&uvm_pageqlock);
1.112     yamt     1315:
1.7       mrg      1316:                                pg->flags &= ~(PG_BUSY|PG_FAKE);
                   1317:                                UVM_PAGE_OWN(pg, NULL);
                   1318:
                   1319:                                /* done! */
                   1320:                        }     /* ref == 1 */
                   1321:                }       /* write fault */
                   1322:        }         /* loan count */
                   1323:
                   1324:        /*
                   1325:         * if we are case 1B then we will need to allocate a new blank
                   1326:         * anon to transfer the data into.   note that we have a lock
                   1327:         * on anon, so no one can busy or release the page until we are done.
                   1328:         * also note that the ref count can't drop to zero here because
                   1329:         * it is > 1 and we are only dropping one ref.
                   1330:         *
1.63      chs      1331:         * in the (hopefully very rare) case that we are out of RAM we
                   1332:         * will unlock, wait for more RAM, and refault.
1.7       mrg      1333:         *
                   1334:         * if we are out of anon VM we kill the process (XXX: could wait?).
                   1335:         */
                   1336:
1.73      chs      1337:        if (cow_now && anon->an_ref > 1) {
1.7       mrg      1338:
                   1339:                UVMHIST_LOG(maphist, "  case 1B: COW fault",0,0,0,0);
                   1340:                uvmexp.flt_acow++;
                   1341:                oanon = anon;           /* oanon = old, locked anon */
1.22      chs      1342:
1.106     yamt     1343:                error = uvmfault_promote(&ufi, oanon, PGO_DONTCARE,
                   1344:                    &anon, &anon_spare);
                   1345:                switch (error) {
                   1346:                case 0:
                   1347:                        break;
                   1348:                case ERESTART:
1.7       mrg      1349:                        goto ReFault;
1.106     yamt     1350:                default:
                   1351:                        goto done;
1.7       mrg      1352:                }
                   1353:
1.106     yamt     1354:                pg = anon->an_page;
1.122     ad       1355:                mutex_enter(&uvm_pageqlock);
1.69      chs      1356:                uvm_pageactivate(pg);
1.122     ad       1357:                mutex_exit(&uvm_pageqlock);
1.105     yamt     1358:                pg->flags &= ~(PG_BUSY|PG_FAKE);
1.7       mrg      1359:                UVM_PAGE_OWN(pg, NULL);
                   1360:
                   1361:                /* deref: can not drop to zero here by defn! */
                   1362:                oanon->an_ref--;
1.53      thorpej  1363:
1.7       mrg      1364:                /*
1.53      thorpej  1365:                 * note: oanon is still locked, as is the new anon.  we
                   1366:                 * need to check for this later when we unlock oanon; if
                   1367:                 * oanon != anon, we'll have to unlock anon, too.
1.7       mrg      1368:                 */
                   1369:
                   1370:        } else {
1.52      chs      1371:
1.7       mrg      1372:                uvmexp.flt_anon++;
                   1373:                oanon = anon;           /* old, locked anon is same as anon */
1.94      yamt     1374:                pg = anon->an_page;
1.7       mrg      1375:                if (anon->an_ref > 1)     /* disallow writes to ref > 1 anons */
                   1376:                        enter_prot = enter_prot & ~VM_PROT_WRITE;
                   1377:
                   1378:        }
                   1379:
1.53      thorpej  1380:        /* locked: maps(read), amap, oanon, anon (if different from oanon) */
1.120     ad       1381:        KASSERT(mutex_owned(&amap->am_l));
1.122     ad       1382:        KASSERT(mutex_owned(&anon->an_lock));
                   1383:        KASSERT(mutex_owned(&oanon->an_lock));
1.7       mrg      1384:
                   1385:        /*
1.69      chs      1386:         * now map the page in.
1.7       mrg      1387:         */
                   1388:
                   1389:        UVMHIST_LOG(maphist, "  MAPPING: anon: pm=0x%x, va=0x%x, pg=0x%x",
                   1390:            ufi.orig_map->pmap, ufi.orig_rvaddr, pg, 0);
1.46      thorpej  1391:        if (pmap_enter(ufi.orig_map->pmap, ufi.orig_rvaddr, VM_PAGE_TO_PHYS(pg),
                   1392:            enter_prot, access_type | PMAP_CANFAIL | (wired ? PMAP_WIRED : 0))
1.58      chs      1393:            != 0) {
1.69      chs      1394:
1.46      thorpej  1395:                /*
                   1396:                 * No need to undo what we did; we can simply think of
                   1397:                 * this as the pmap throwing away the mapping information.
                   1398:                 *
                   1399:                 * We do, however, have to go through the ReFault path,
                   1400:                 * as the map may change while we're asleep.
                   1401:                 */
1.69      chs      1402:
1.53      thorpej  1403:                if (anon != oanon)
1.122     ad       1404:                        mutex_exit(&anon->an_lock);
1.46      thorpej  1405:                uvmfault_unlockall(&ufi, amap, uobj, oanon);
1.92      yamt     1406:                if (!uvm_reclaimable()) {
1.46      thorpej  1407:                        UVMHIST_LOG(maphist,
                   1408:                            "<- failed.  out of VM",0,0,0,0);
                   1409:                        /* XXX instrumentation */
1.106     yamt     1410:                        error = ENOMEM;
                   1411:                        goto done;
1.46      thorpej  1412:                }
                   1413:                /* XXX instrumentation */
                   1414:                uvm_wait("flt_pmfail1");
                   1415:                goto ReFault;
                   1416:        }
1.7       mrg      1417:
                   1418:        /*
1.46      thorpej  1419:         * ... update the page queues.
1.7       mrg      1420:         */
                   1421:
1.122     ad       1422:        mutex_enter(&uvm_pageqlock);
1.72      chs      1423:        if (wire_fault) {
1.8       chuck    1424:                uvm_pagewire(pg);
1.29      chs      1425:
                   1426:                /*
                   1427:                 * since the now-wired page cannot be paged out,
                   1428:                 * release its swap resources for others to use.
                   1429:                 * since an anon with no swap cannot be PG_CLEAN,
                   1430:                 * clear its clean flag now.
                   1431:                 */
                   1432:
                   1433:                pg->flags &= ~(PG_CLEAN);
1.22      chs      1434:                uvm_anon_dropswap(anon);
1.7       mrg      1435:        } else {
                   1436:                uvm_pageactivate(pg);
                   1437:        }
1.122     ad       1438:        mutex_exit(&uvm_pageqlock);
1.7       mrg      1439:
                   1440:        /*
                   1441:         * done case 1!  finish up by unlocking everything and returning success
                   1442:         */
1.1       mrg      1443:
1.53      thorpej  1444:        if (anon != oanon)
1.122     ad       1445:                mutex_exit(&anon->an_lock);
1.7       mrg      1446:        uvmfault_unlockall(&ufi, amap, uobj, oanon);
1.68      chris    1447:        pmap_update(ufi.orig_map->pmap);
1.106     yamt     1448:        error = 0;
                   1449:        goto done;
1.1       mrg      1450:
                   1451: Case2:
1.7       mrg      1452:        /*
                   1453:         * handle case 2: faulting on backing object or zero fill
                   1454:         */
                   1455:
                   1456:        /*
                   1457:         * locked:
                   1458:         * maps(read), amap(if there), uobj(if !null), uobjpage(if !null)
                   1459:         */
1.120     ad       1460:        KASSERT(amap == NULL || mutex_owned(&amap->am_l));
1.122     ad       1461:        KASSERT(uobj == NULL || mutex_owned(&uobj->vmobjlock));
1.120     ad       1462:        KASSERT(uobjpage == NULL || (uobjpage->flags & PG_BUSY) != 0);
1.7       mrg      1463:
                   1464:        /*
                   1465:         * note that uobjpage can not be PGO_DONTCARE at this point.  we now
                   1466:         * set uobjpage to PGO_DONTCARE if we are doing a zero fill.  if we
                   1467:         * have a backing object, check and see if we are going to promote
                   1468:         * the data up to an anon during the fault.
                   1469:         */
                   1470:
                   1471:        if (uobj == NULL) {
1.63      chs      1472:                uobjpage = PGO_DONTCARE;
1.119     thorpej  1473:                promote = true;         /* always need anon here */
1.7       mrg      1474:        } else {
1.52      chs      1475:                KASSERT(uobjpage != PGO_DONTCARE);
1.73      chs      1476:                promote = cow_now && UVM_ET_ISCOPYONWRITE(ufi.entry);
1.7       mrg      1477:        }
                   1478:        UVMHIST_LOG(maphist, "  case 2 fault: promote=%d, zfill=%d",
1.46      thorpej  1479:            promote, (uobj == NULL), 0,0);
1.1       mrg      1480:
1.7       mrg      1481:        /*
1.9       chuck    1482:         * if uobjpage is not null then we do not need to do I/O to get the
                   1483:         * uobjpage.
                   1484:         *
1.63      chs      1485:         * if uobjpage is null, then we need to unlock and ask the pager to
1.7       mrg      1486:         * get the data for us.   once we have the data, we need to reverify
                   1487:         * the state the world.   we are currently not holding any resources.
                   1488:         */
1.1       mrg      1489:
1.9       chuck    1490:        if (uobjpage) {
                   1491:                /* update rusage counters */
1.123.6.1  mjf      1492:                curlwp->l_ru.ru_minflt++;
1.9       chuck    1493:        } else {
                   1494:                /* update rusage counters */
1.123.6.1  mjf      1495:                curlwp->l_ru.ru_majflt++;
1.63      chs      1496:
1.7       mrg      1497:                /* locked: maps(read), amap(if there), uobj */
                   1498:                uvmfault_unlockall(&ufi, amap, NULL, NULL);
                   1499:                /* locked: uobj */
                   1500:
                   1501:                uvmexp.fltget++;
                   1502:                gotpages = 1;
1.52      chs      1503:                uoff = (ufi.orig_rvaddr - ufi.entry->start) + ufi.entry->offset;
1.58      chs      1504:                error = uobj->pgops->pgo_get(uobj, uoff, &uobjpage, &gotpages,
1.52      chs      1505:                    0, access_type & MASK(ufi.entry), ufi.entry->advice,
                   1506:                    PGO_SYNCIO);
1.58      chs      1507:                /* locked: uobjpage(if no error) */
1.120     ad       1508:                KASSERT(error != 0 || (uobjpage->flags & PG_BUSY) != 0);
1.52      chs      1509:
1.7       mrg      1510:                /*
                   1511:                 * recover from I/O
                   1512:                 */
1.1       mrg      1513:
1.58      chs      1514:                if (error) {
                   1515:                        if (error == EAGAIN) {
1.46      thorpej  1516:                                UVMHIST_LOG(maphist,
                   1517:                                    "  pgo_get says TRY AGAIN!",0,0,0,0);
1.57      chs      1518:                                tsleep(&lbolt, PVM, "fltagain2", 0);
1.46      thorpej  1519:                                goto ReFault;
1.7       mrg      1520:                        }
1.1       mrg      1521:
1.7       mrg      1522:                        UVMHIST_LOG(maphist, "<- pgo_get failed (code %d)",
1.58      chs      1523:                            error, 0,0,0);
1.106     yamt     1524:                        goto done;
1.7       mrg      1525:                }
                   1526:
                   1527:                /* locked: uobjpage */
                   1528:
1.122     ad       1529:                mutex_enter(&uvm_pageqlock);
1.69      chs      1530:                uvm_pageactivate(uobjpage);
1.122     ad       1531:                mutex_exit(&uvm_pageqlock);
1.69      chs      1532:
1.7       mrg      1533:                /*
                   1534:                 * re-verify the state of the world by first trying to relock
                   1535:                 * the maps.  always relock the object.
                   1536:                 */
                   1537:
                   1538:                locked = uvmfault_relock(&ufi);
                   1539:                if (locked && amap)
1.19      chuck    1540:                        amap_lock(amap);
1.107     yamt     1541:                uobj = uobjpage->uobject;
1.122     ad       1542:                mutex_enter(&uobj->vmobjlock);
1.63      chs      1543:
1.7       mrg      1544:                /* locked(locked): maps(read), amap(if !null), uobj, uobjpage */
                   1545:                /* locked(!locked): uobj, uobjpage */
                   1546:
                   1547:                /*
                   1548:                 * verify that the page has not be released and re-verify
                   1549:                 * that amap slot is still free.   if there is a problem,
                   1550:                 * we unlock and clean up.
                   1551:                 */
                   1552:
                   1553:                if ((uobjpage->flags & PG_RELEASED) != 0 ||
1.63      chs      1554:                    (locked && amap &&
1.7       mrg      1555:                    amap_lookup(&ufi.entry->aref,
1.13      chuck    1556:                      ufi.orig_rvaddr - ufi.entry->start))) {
1.63      chs      1557:                        if (locked)
1.7       mrg      1558:                                uvmfault_unlockall(&ufi, amap, NULL, NULL);
1.119     thorpej  1559:                        locked = false;
1.7       mrg      1560:                }
                   1561:
                   1562:                /*
                   1563:                 * didn't get the lock?   release the page and retry.
                   1564:                 */
                   1565:
1.119     thorpej  1566:                if (locked == false) {
1.7       mrg      1567:                        UVMHIST_LOG(maphist,
1.63      chs      1568:                            "  wasn't able to relock after fault: retry",
1.7       mrg      1569:                            0,0,0,0);
                   1570:                        if (uobjpage->flags & PG_WANTED)
1.44      thorpej  1571:                                wakeup(uobjpage);
1.7       mrg      1572:                        if (uobjpage->flags & PG_RELEASED) {
                   1573:                                uvmexp.fltpgrele++;
1.69      chs      1574:                                uvm_pagefree(uobjpage);
1.7       mrg      1575:                                goto ReFault;
                   1576:                        }
                   1577:                        uobjpage->flags &= ~(PG_BUSY|PG_WANTED);
                   1578:                        UVM_PAGE_OWN(uobjpage, NULL);
1.122     ad       1579:                        mutex_exit(&uobj->vmobjlock);
1.7       mrg      1580:                        goto ReFault;
                   1581:                }
                   1582:
                   1583:                /*
1.69      chs      1584:                 * we have the data in uobjpage which is busy and
                   1585:                 * not released.  we are holding object lock (so the page
1.7       mrg      1586:                 * can't be released on us).
                   1587:                 */
                   1588:
                   1589:                /* locked: maps(read), amap(if !null), uobj, uobjpage */
                   1590:        }
1.1       mrg      1591:
                   1592:        /*
1.7       mrg      1593:         * locked:
                   1594:         * maps(read), amap(if !null), uobj(if !null), uobjpage(if uobj)
1.1       mrg      1595:         */
1.120     ad       1596:        KASSERT(amap == NULL || mutex_owned(&amap->am_l));
1.122     ad       1597:        KASSERT(uobj == NULL || mutex_owned(&uobj->vmobjlock));
1.120     ad       1598:        KASSERT(uobj == NULL || (uobjpage->flags & PG_BUSY) != 0);
1.1       mrg      1599:
1.7       mrg      1600:        /*
                   1601:         * notes:
                   1602:         *  - at this point uobjpage can not be NULL
                   1603:         *  - at this point uobjpage can not be PG_RELEASED (since we checked
                   1604:         *  for it above)
                   1605:         *  - at this point uobjpage could be PG_WANTED (handle later)
                   1606:         */
1.63      chs      1607:
1.107     yamt     1608:        KASSERT(uobj == NULL || uobj == uobjpage->uobject);
1.97      yamt     1609:        KASSERT(uobj == NULL || !UVM_OBJ_IS_CLEAN(uobjpage->uobject) ||
1.96      yamt     1610:            (uobjpage->flags & PG_CLEAN) != 0);
1.119     thorpej  1611:        if (promote == false) {
1.1       mrg      1612:
1.7       mrg      1613:                /*
                   1614:                 * we are not promoting.   if the mapping is COW ensure that we
                   1615:                 * don't give more access than we should (e.g. when doing a read
                   1616:                 * fault on a COPYONWRITE mapping we want to map the COW page in
                   1617:                 * R/O even though the entry protection could be R/W).
                   1618:                 *
                   1619:                 * set "pg" to the page we want to map in (uobjpage, usually)
                   1620:                 */
                   1621:
1.53      thorpej  1622:                /* no anon in this case. */
                   1623:                anon = NULL;
                   1624:
1.7       mrg      1625:                uvmexp.flt_obj++;
1.96      yamt     1626:                if (UVM_ET_ISCOPYONWRITE(ufi.entry) ||
1.98      yamt     1627:                    UVM_OBJ_NEEDS_WRITEFAULT(uobjpage->uobject))
1.24      mycroft  1628:                        enter_prot &= ~VM_PROT_WRITE;
1.7       mrg      1629:                pg = uobjpage;          /* map in the actual object */
                   1630:
1.111     yamt     1631:                KASSERT(uobjpage != PGO_DONTCARE);
1.7       mrg      1632:
                   1633:                /*
                   1634:                 * we are faulting directly on the page.   be careful
                   1635:                 * about writing to loaned pages...
                   1636:                 */
1.69      chs      1637:
1.7       mrg      1638:                if (uobjpage->loan_count) {
1.73      chs      1639:                        if (!cow_now) {
1.7       mrg      1640:                                /* read fault: cap the protection at readonly */
                   1641:                                /* cap! */
                   1642:                                enter_prot = enter_prot & ~VM_PROT_WRITE;
                   1643:                        } else {
                   1644:                                /* write fault: must break the loan here */
                   1645:
1.82      yamt     1646:                                pg = uvm_loanbreak(uobjpage);
1.7       mrg      1647:                                if (pg == NULL) {
1.69      chs      1648:
1.7       mrg      1649:                                        /*
                   1650:                                         * drop ownership of page, it can't
                   1651:                                         * be released
1.46      thorpej  1652:                                         */
1.69      chs      1653:
1.7       mrg      1654:                                        if (uobjpage->flags & PG_WANTED)
1.44      thorpej  1655:                                                wakeup(uobjpage);
1.7       mrg      1656:                                        uobjpage->flags &= ~(PG_BUSY|PG_WANTED);
                   1657:                                        UVM_PAGE_OWN(uobjpage, NULL);
                   1658:
                   1659:                                        uvmfault_unlockall(&ufi, amap, uobj,
                   1660:                                          NULL);
                   1661:                                        UVMHIST_LOG(maphist,
1.20      mrg      1662:                                          "  out of RAM breaking loan, waiting",
                   1663:                                          0,0,0,0);
1.7       mrg      1664:                                        uvmexp.fltnoram++;
                   1665:                                        uvm_wait("flt_noram4");
                   1666:                                        goto ReFault;
                   1667:                                }
                   1668:                                uobjpage = pg;
1.69      chs      1669:                        }
                   1670:                }
1.7       mrg      1671:        } else {
1.63      chs      1672:
1.7       mrg      1673:                /*
                   1674:                 * if we are going to promote the data to an anon we
                   1675:                 * allocate a blank anon here and plug it into our amap.
                   1676:                 */
1.1       mrg      1677: #if DIAGNOSTIC
1.7       mrg      1678:                if (amap == NULL)
                   1679:                        panic("uvm_fault: want to promote data, but no anon");
1.1       mrg      1680: #endif
1.106     yamt     1681:                error = uvmfault_promote(&ufi, NULL, uobjpage,
                   1682:                    &anon, &anon_spare);
                   1683:                switch (error) {
                   1684:                case 0:
                   1685:                        break;
                   1686:                case ERESTART:
                   1687:                        goto ReFault;
                   1688:                default:
                   1689:                        goto done;
1.105     yamt     1690:                }
                   1691:
1.106     yamt     1692:                pg = anon->an_page;
1.1       mrg      1693:
1.7       mrg      1694:                /*
                   1695:                 * fill in the data
                   1696:                 */
                   1697:
                   1698:                if (uobjpage != PGO_DONTCARE) {
                   1699:                        uvmexp.flt_prcopy++;
                   1700:
                   1701:                        /*
                   1702:                         * promote to shared amap?  make sure all sharing
                   1703:                         * procs see it
                   1704:                         */
1.69      chs      1705:
1.19      chuck    1706:                        if ((amap_flags(amap) & AMAP_SHARED) != 0) {
1.45      chs      1707:                                pmap_page_protect(uobjpage, VM_PROT_NONE);
1.62      thorpej  1708:                                /*
                   1709:                                 * XXX: PAGE MIGHT BE WIRED!
                   1710:                                 */
1.7       mrg      1711:                        }
1.63      chs      1712:
1.7       mrg      1713:                        /*
                   1714:                         * dispose of uobjpage.  it can't be PG_RELEASED
1.52      chs      1715:                         * since we still hold the object lock.
                   1716:                         * drop handle to uobj as well.
1.7       mrg      1717:                         */
                   1718:
                   1719:                        if (uobjpage->flags & PG_WANTED)
                   1720:                                /* still have the obj lock */
1.44      thorpej  1721:                                wakeup(uobjpage);
1.7       mrg      1722:                        uobjpage->flags &= ~(PG_BUSY|PG_WANTED);
                   1723:                        UVM_PAGE_OWN(uobjpage, NULL);
1.122     ad       1724:                        mutex_exit(&uobj->vmobjlock);
1.7       mrg      1725:                        uobj = NULL;
1.52      chs      1726:
1.7       mrg      1727:                        UVMHIST_LOG(maphist,
                   1728:                            "  promote uobjpage 0x%x to anon/page 0x%x/0x%x",
                   1729:                            uobjpage, anon, pg, 0);
                   1730:
                   1731:                } else {
                   1732:                        uvmexp.flt_przero++;
1.69      chs      1733:
1.48      thorpej  1734:                        /*
1.106     yamt     1735:                         * Page is zero'd and marked dirty by
                   1736:                         * uvmfault_promote().
1.48      thorpej  1737:                         */
1.69      chs      1738:
1.7       mrg      1739:                        UVMHIST_LOG(maphist,"  zero fill anon/page 0x%x/0%x",
                   1740:                            anon, pg, 0, 0);
                   1741:                }
                   1742:        }
                   1743:
                   1744:        /*
                   1745:         * locked:
1.53      thorpej  1746:         * maps(read), amap(if !null), uobj(if !null), uobjpage(if uobj),
                   1747:         *   anon(if !null), pg(if anon)
1.7       mrg      1748:         *
                   1749:         * note: pg is either the uobjpage or the new page in the new anon
                   1750:         */
1.120     ad       1751:        KASSERT(amap == NULL || mutex_owned(&amap->am_l));
1.122     ad       1752:        KASSERT(uobj == NULL || mutex_owned(&uobj->vmobjlock));
1.120     ad       1753:        KASSERT(uobj == NULL || (uobjpage->flags & PG_BUSY) != 0);
1.122     ad       1754:        KASSERT(anon == NULL || mutex_owned(&anon->an_lock));
1.120     ad       1755:        KASSERT((pg->flags & PG_BUSY) != 0);
1.7       mrg      1756:
                   1757:        /*
                   1758:         * all resources are present.   we can now map it in and free our
                   1759:         * resources.
                   1760:         */
                   1761:
                   1762:        UVMHIST_LOG(maphist,
                   1763:            "  MAPPING: case2: pm=0x%x, va=0x%x, pg=0x%x, promote=%d",
                   1764:            ufi.orig_map->pmap, ufi.orig_rvaddr, pg, promote);
1.75      chs      1765:        KASSERT((access_type & VM_PROT_WRITE) == 0 ||
                   1766:                (pg->flags & PG_RDONLY) == 0);
1.46      thorpej  1767:        if (pmap_enter(ufi.orig_map->pmap, ufi.orig_rvaddr, VM_PAGE_TO_PHYS(pg),
1.76      chs      1768:            pg->flags & PG_RDONLY ? enter_prot & ~VM_PROT_WRITE : enter_prot,
1.58      chs      1769:            access_type | PMAP_CANFAIL | (wired ? PMAP_WIRED : 0)) != 0) {
1.52      chs      1770:
1.46      thorpej  1771:                /*
                   1772:                 * No need to undo what we did; we can simply think of
                   1773:                 * this as the pmap throwing away the mapping information.
                   1774:                 *
                   1775:                 * We do, however, have to go through the ReFault path,
                   1776:                 * as the map may change while we're asleep.
                   1777:                 */
1.52      chs      1778:
1.46      thorpej  1779:                if (pg->flags & PG_WANTED)
1.69      chs      1780:                        wakeup(pg);
1.46      thorpej  1781:
1.63      chs      1782:                /*
1.46      thorpej  1783:                 * note that pg can't be PG_RELEASED since we did not drop
                   1784:                 * the object lock since the last time we checked.
                   1785:                 */
1.111     yamt     1786:                KASSERT((pg->flags & PG_RELEASED) == 0);
1.63      chs      1787:
1.46      thorpej  1788:                pg->flags &= ~(PG_BUSY|PG_FAKE|PG_WANTED);
                   1789:                UVM_PAGE_OWN(pg, NULL);
1.53      thorpej  1790:                uvmfault_unlockall(&ufi, amap, uobj, anon);
1.92      yamt     1791:                if (!uvm_reclaimable()) {
1.46      thorpej  1792:                        UVMHIST_LOG(maphist,
                   1793:                            "<- failed.  out of VM",0,0,0,0);
                   1794:                        /* XXX instrumentation */
1.106     yamt     1795:                        error = ENOMEM;
                   1796:                        goto done;
1.46      thorpej  1797:                }
                   1798:                /* XXX instrumentation */
                   1799:                uvm_wait("flt_pmfail2");
                   1800:                goto ReFault;
                   1801:        }
1.1       mrg      1802:
1.122     ad       1803:        mutex_enter(&uvm_pageqlock);
1.72      chs      1804:        if (wire_fault) {
1.8       chuck    1805:                uvm_pagewire(pg);
1.22      chs      1806:                if (pg->pqflags & PQ_AOBJ) {
1.29      chs      1807:
                   1808:                        /*
                   1809:                         * since the now-wired page cannot be paged out,
                   1810:                         * release its swap resources for others to use.
                   1811:                         * since an aobj page with no swap cannot be PG_CLEAN,
                   1812:                         * clear its clean flag now.
                   1813:                         */
                   1814:
1.113     christos 1815:                        KASSERT(uobj != NULL);
1.29      chs      1816:                        pg->flags &= ~(PG_CLEAN);
1.22      chs      1817:                        uao_dropswap(uobj, pg->offset >> PAGE_SHIFT);
                   1818:                }
1.7       mrg      1819:        } else {
                   1820:                uvm_pageactivate(pg);
                   1821:        }
1.122     ad       1822:        mutex_exit(&uvm_pageqlock);
1.7       mrg      1823:        if (pg->flags & PG_WANTED)
1.69      chs      1824:                wakeup(pg);
1.7       mrg      1825:
1.63      chs      1826:        /*
                   1827:         * note that pg can't be PG_RELEASED since we did not drop the object
1.7       mrg      1828:         * lock since the last time we checked.
                   1829:         */
1.111     yamt     1830:        KASSERT((pg->flags & PG_RELEASED) == 0);
1.63      chs      1831:
1.7       mrg      1832:        pg->flags &= ~(PG_BUSY|PG_FAKE|PG_WANTED);
                   1833:        UVM_PAGE_OWN(pg, NULL);
1.53      thorpej  1834:        uvmfault_unlockall(&ufi, amap, uobj, anon);
1.68      chris    1835:        pmap_update(ufi.orig_map->pmap);
1.7       mrg      1836:        UVMHIST_LOG(maphist, "<- done (SUCCESS!)",0,0,0,0);
1.106     yamt     1837:        error = 0;
                   1838: done:
                   1839:        if (anon_spare != NULL) {
                   1840:                anon_spare->an_ref--;
                   1841:                uvm_anfree(anon_spare);
                   1842:        }
                   1843:        return error;
1.1       mrg      1844: }
                   1845:
1.110     drochner 1846:
1.1       mrg      1847: /*
                   1848:  * uvm_fault_wire: wire down a range of virtual addresses in a map.
                   1849:  *
1.36      thorpej  1850:  * => map may be read-locked by caller, but MUST NOT be write-locked.
                   1851:  * => if map is read-locked, any operations which may cause map to
                   1852:  *     be write-locked in uvm_fault() must be taken care of by
                   1853:  *     the caller.  See uvm_map_pageable().
1.1       mrg      1854:  */
                   1855:
1.7       mrg      1856: int
1.95      thorpej  1857: uvm_fault_wire(struct vm_map *map, vaddr_t start, vaddr_t end,
1.110     drochner 1858:     vm_prot_t access_type, int wiremax)
1.7       mrg      1859: {
1.12      eeh      1860:        vaddr_t va;
1.58      chs      1861:        int error;
1.7       mrg      1862:
                   1863:        /*
1.47      chs      1864:         * now fault it in a page at a time.   if the fault fails then we have
1.63      chs      1865:         * to undo what we have done.   note that in uvm_fault VM_PROT_NONE
1.47      chs      1866:         * is replaced with the max protection if fault_type is VM_FAULT_WIRE.
1.7       mrg      1867:         */
1.1       mrg      1868:
1.65      chs      1869:        /*
                   1870:         * XXX work around overflowing a vaddr_t.  this prevents us from
                   1871:         * wiring the last page in the address space, though.
                   1872:         */
                   1873:        if (start > end) {
                   1874:                return EFAULT;
                   1875:        }
                   1876:
1.7       mrg      1877:        for (va = start ; va < end ; va += PAGE_SIZE) {
1.110     drochner 1878:                error = uvm_fault_internal(map, va, access_type,
                   1879:                                wiremax ? UVM_FAULT_WIREMAX : UVM_FAULT_WIRE);
1.58      chs      1880:                if (error) {
1.7       mrg      1881:                        if (va != start) {
1.31      thorpej  1882:                                uvm_fault_unwire(map, start, va);
1.7       mrg      1883:                        }
1.58      chs      1884:                        return error;
1.7       mrg      1885:                }
                   1886:        }
1.58      chs      1887:        return 0;
1.1       mrg      1888: }
                   1889:
                   1890: /*
                   1891:  * uvm_fault_unwire(): unwire range of virtual space.
                   1892:  */
                   1893:
1.7       mrg      1894: void
1.95      thorpej  1895: uvm_fault_unwire(struct vm_map *map, vaddr_t start, vaddr_t end)
1.36      thorpej  1896: {
                   1897:        vm_map_lock_read(map);
                   1898:        uvm_fault_unwire_locked(map, start, end);
                   1899:        vm_map_unlock_read(map);
                   1900: }
                   1901:
                   1902: /*
                   1903:  * uvm_fault_unwire_locked(): the guts of uvm_fault_unwire().
                   1904:  *
                   1905:  * => map must be at least read-locked.
                   1906:  */
                   1907:
                   1908: void
1.95      thorpej  1909: uvm_fault_unwire_locked(struct vm_map *map, vaddr_t start, vaddr_t end)
1.7       mrg      1910: {
1.64      chs      1911:        struct vm_map_entry *entry;
1.31      thorpej  1912:        pmap_t pmap = vm_map_pmap(map);
1.42      thorpej  1913:        vaddr_t va;
1.12      eeh      1914:        paddr_t pa;
1.42      thorpej  1915:        struct vm_page *pg;
1.31      thorpej  1916:
1.52      chs      1917:        KASSERT((map->flags & VM_MAP_INTRSAFE) == 0);
1.7       mrg      1918:
                   1919:        /*
                   1920:         * we assume that the area we are unwiring has actually been wired
                   1921:         * in the first place.   this means that we should be able to extract
                   1922:         * the PAs from the pmap.   we also lock out the page daemon so that
                   1923:         * we can call uvm_pageunwire.
                   1924:         */
1.37      thorpej  1925:
1.122     ad       1926:        mutex_enter(&uvm_pageqlock);
1.7       mrg      1927:
1.37      thorpej  1928:        /*
                   1929:         * find the beginning map entry for the region.
                   1930:         */
1.74      chs      1931:
1.56      chs      1932:        KASSERT(start >= vm_map_min(map) && end <= vm_map_max(map));
1.119     thorpej  1933:        if (uvm_map_lookup_entry(map, start, &entry) == false)
1.37      thorpej  1934:                panic("uvm_fault_unwire_locked: address not in map");
                   1935:
1.69      chs      1936:        for (va = start; va < end; va += PAGE_SIZE) {
1.119     thorpej  1937:                if (pmap_extract(pmap, va, &pa) == false)
1.74      chs      1938:                        continue;
1.42      thorpej  1939:
                   1940:                /*
1.74      chs      1941:                 * find the map entry for the current address.
1.42      thorpej  1942:                 */
1.56      chs      1943:
                   1944:                KASSERT(va >= entry->start);
1.74      chs      1945:                while (va >= entry->end) {
1.56      chs      1946:                        KASSERT(entry->next != &map->header &&
                   1947:                                entry->next->start <= entry->end);
1.42      thorpej  1948:                        entry = entry->next;
                   1949:                }
1.37      thorpej  1950:
1.42      thorpej  1951:                /*
                   1952:                 * if the entry is no longer wired, tell the pmap.
                   1953:                 */
1.74      chs      1954:
1.42      thorpej  1955:                if (VM_MAPENT_ISWIRED(entry) == 0)
                   1956:                        pmap_unwire(pmap, va);
                   1957:
                   1958:                pg = PHYS_TO_VM_PAGE(pa);
                   1959:                if (pg)
                   1960:                        uvm_pageunwire(pg);
1.7       mrg      1961:        }
1.1       mrg      1962:
1.122     ad       1963:        mutex_exit(&uvm_pageqlock);
1.1       mrg      1964: }

CVSweb <webmaster@jp.NetBSD.org>