[BACK]Return to uvm_map.c CVS log [TXT][DIR] Up to [cvs.NetBSD.org] / src / sys / uvm

Annotation of src/sys/uvm/uvm_map.c, Revision 1.77.2.1

1.77.2.1! fvdl        1: /*     $NetBSD: uvm_map.c,v 1.77 2000/06/13 04:10:47 chs Exp $ */
1.1       mrg         2:
                      3: /*
                      4:  * Copyright (c) 1997 Charles D. Cranor and Washington University.
                      5:  * Copyright (c) 1991, 1993, The Regents of the University of California.
                      6:  *
                      7:  * All rights reserved.
                      8:  *
                      9:  * This code is derived from software contributed to Berkeley by
                     10:  * The Mach Operating System project at Carnegie-Mellon University.
                     11:  *
                     12:  * Redistribution and use in source and binary forms, with or without
                     13:  * modification, are permitted provided that the following conditions
                     14:  * are met:
                     15:  * 1. Redistributions of source code must retain the above copyright
                     16:  *    notice, this list of conditions and the following disclaimer.
                     17:  * 2. Redistributions in binary form must reproduce the above copyright
                     18:  *    notice, this list of conditions and the following disclaimer in the
                     19:  *    documentation and/or other materials provided with the distribution.
                     20:  * 3. All advertising materials mentioning features or use of this software
                     21:  *    must display the following acknowledgement:
                     22:  *     This product includes software developed by Charles D. Cranor,
                     23:  *      Washington University, the University of California, Berkeley and
                     24:  *      its contributors.
                     25:  * 4. Neither the name of the University nor the names of its contributors
                     26:  *    may be used to endorse or promote products derived from this software
                     27:  *    without specific prior written permission.
                     28:  *
                     29:  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
                     30:  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
                     31:  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
                     32:  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
                     33:  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
                     34:  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
                     35:  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
                     36:  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
                     37:  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
                     38:  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
                     39:  * SUCH DAMAGE.
                     40:  *
                     41:  *     @(#)vm_map.c    8.3 (Berkeley) 1/12/94
1.3       mrg        42:  * from: Id: uvm_map.c,v 1.1.2.27 1998/02/07 01:16:54 chs Exp
1.1       mrg        43:  *
                     44:  *
                     45:  * Copyright (c) 1987, 1990 Carnegie-Mellon University.
                     46:  * All rights reserved.
                     47:  *
                     48:  * Permission to use, copy, modify and distribute this software and
                     49:  * its documentation is hereby granted, provided that both the copyright
                     50:  * notice and this permission notice appear in all copies of the
                     51:  * software, derivative works or modified versions, and any portions
                     52:  * thereof, and that both notices appear in supporting documentation.
                     53:  *
                     54:  * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
                     55:  * CONDITION.  CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND
                     56:  * FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
                     57:  *
                     58:  * Carnegie Mellon requests users of this software to return to
                     59:  *
                     60:  *  Software Distribution Coordinator  or  Software.Distribution@CS.CMU.EDU
                     61:  *  School of Computer Science
                     62:  *  Carnegie Mellon University
                     63:  *  Pittsburgh PA 15213-3890
                     64:  *
                     65:  * any improvements or extensions that they make and grant Carnegie the
                     66:  * rights to redistribute these changes.
                     67:  */
                     68:
1.21      jonathan   69: #include "opt_ddb.h"
1.6       mrg        70: #include "opt_uvmhist.h"
1.31      tron       71: #include "opt_sysv.h"
1.6       mrg        72:
1.1       mrg        73: /*
                     74:  * uvm_map.c: uvm map operations
                     75:  */
                     76:
                     77: #include <sys/param.h>
                     78: #include <sys/systm.h>
                     79: #include <sys/mman.h>
                     80: #include <sys/proc.h>
                     81: #include <sys/malloc.h>
1.25      thorpej    82: #include <sys/pool.h>
1.1       mrg        83:
                     84: #ifdef SYSVSHM
                     85: #include <sys/shm.h>
                     86: #endif
                     87:
                     88: #include <vm/vm.h>
                     89: #include <vm/vm_page.h>
                     90: #include <vm/vm_kern.h>
                     91:
                     92: #define UVM_MAP
                     93: #include <uvm/uvm.h>
1.21      jonathan   94:
                     95: #ifdef DDB
                     96: #include <uvm/uvm_ddb.h>
                     97: #endif
                     98:
1.1       mrg        99:
                    100: struct uvm_cnt uvm_map_call, map_backmerge, map_forwmerge;
                    101: struct uvm_cnt uvm_mlk_call, uvm_mlk_hint;
                    102:
                    103: /*
1.25      thorpej   104:  * pool for vmspace structures.
                    105:  */
                    106:
                    107: struct pool uvm_vmspace_pool;
                    108:
1.26      thorpej   109: /*
                    110:  * pool for dynamically-allocated map entries.
                    111:  */
                    112:
                    113: struct pool uvm_map_entry_pool;
1.25      thorpej   114:
1.40      thorpej   115: #ifdef PMAP_GROWKERNEL
                    116: /*
                    117:  * This global represents the end of the kernel virtual address
                    118:  * space.  If we want to exceed this, we must grow the kernel
                    119:  * virtual address space dynamically.
                    120:  *
                    121:  * Note, this variable is locked by kernel_map's lock.
                    122:  */
                    123: vaddr_t uvm_maxkaddr;
                    124: #endif
                    125:
1.25      thorpej   126: /*
1.1       mrg       127:  * macros
                    128:  */
                    129:
                    130: /*
                    131:  * uvm_map_entry_link: insert entry into a map
                    132:  *
                    133:  * => map must be locked
                    134:  */
1.10      mrg       135: #define uvm_map_entry_link(map, after_where, entry) do { \
                    136:        (map)->nentries++; \
                    137:        (entry)->prev = (after_where); \
                    138:        (entry)->next = (after_where)->next; \
                    139:        (entry)->prev->next = (entry); \
                    140:        (entry)->next->prev = (entry); \
                    141: } while (0)
                    142:
1.1       mrg       143: /*
                    144:  * uvm_map_entry_unlink: remove entry from a map
                    145:  *
                    146:  * => map must be locked
                    147:  */
1.10      mrg       148: #define uvm_map_entry_unlink(map, entry) do { \
                    149:        (map)->nentries--; \
                    150:        (entry)->next->prev = (entry)->prev; \
                    151:        (entry)->prev->next = (entry)->next; \
                    152: } while (0)
1.1       mrg       153:
                    154: /*
                    155:  * SAVE_HINT: saves the specified entry as the hint for future lookups.
                    156:  *
                    157:  * => map need not be locked (protected by hint_lock).
                    158:  */
1.10      mrg       159: #define SAVE_HINT(map,value) do { \
                    160:        simple_lock(&(map)->hint_lock); \
                    161:        (map)->hint = (value); \
                    162:        simple_unlock(&(map)->hint_lock); \
                    163: } while (0)
1.1       mrg       164:
                    165: /*
                    166:  * VM_MAP_RANGE_CHECK: check and correct range
                    167:  *
                    168:  * => map must at least be read locked
                    169:  */
                    170:
1.10      mrg       171: #define VM_MAP_RANGE_CHECK(map, start, end) do { \
                    172:        if (start < vm_map_min(map))            \
                    173:                start = vm_map_min(map);        \
                    174:        if (end > vm_map_max(map))              \
                    175:                end = vm_map_max(map);          \
                    176:        if (start > end)                        \
                    177:                start = end;                    \
                    178: } while (0)
1.1       mrg       179:
                    180: /*
                    181:  * local prototypes
                    182:  */
                    183:
                    184: static vm_map_entry_t  uvm_mapent_alloc __P((vm_map_t));
                    185: static void            uvm_mapent_copy __P((vm_map_entry_t,vm_map_entry_t));
                    186: static void            uvm_mapent_free __P((vm_map_entry_t));
                    187: static void            uvm_map_entry_unwire __P((vm_map_t, vm_map_entry_t));
                    188:
                    189: /*
                    190:  * local inlines
                    191:  */
                    192:
                    193: /*
                    194:  * uvm_mapent_alloc: allocate a map entry
                    195:  *
                    196:  * => XXX: static pool for kernel map?
                    197:  */
                    198:
1.10      mrg       199: static __inline vm_map_entry_t
                    200: uvm_mapent_alloc(map)
                    201:        vm_map_t map;
                    202: {
                    203:        vm_map_entry_t me;
                    204:        int s;
                    205:        UVMHIST_FUNC("uvm_mapent_alloc");
                    206:        UVMHIST_CALLED(maphist);
1.1       mrg       207:
1.45      thorpej   208:        if ((map->flags & VM_MAP_INTRSAFE) == 0 &&
                    209:            map != kernel_map && kernel_map != NULL /* XXX */) {
1.26      thorpej   210:                me = pool_get(&uvm_map_entry_pool, PR_WAITOK);
1.10      mrg       211:                me->flags = 0;
                    212:                /* me can't be null, wait ok */
                    213:        } else {
                    214:                s = splimp();   /* protect kentry_free list with splimp */
                    215:                simple_lock(&uvm.kentry_lock);
                    216:                me = uvm.kentry_free;
                    217:                if (me) uvm.kentry_free = me->next;
                    218:                simple_unlock(&uvm.kentry_lock);
                    219:                splx(s);
                    220:                if (!me)
1.45      thorpej   221:        panic("mapent_alloc: out of static map entries, check MAX_KMAPENT");
1.10      mrg       222:                me->flags = UVM_MAP_STATIC;
                    223:        }
1.1       mrg       224:
1.45      thorpej   225:        UVMHIST_LOG(maphist, "<- new entry=0x%x [kentry=%d]",
                    226:                me, ((map->flags & VM_MAP_INTRSAFE) != 0 || map == kernel_map)
                    227:                ? TRUE : FALSE, 0, 0);
1.10      mrg       228:        return(me);
1.1       mrg       229: }
                    230:
                    231: /*
                    232:  * uvm_mapent_free: free map entry
                    233:  *
                    234:  * => XXX: static pool for kernel map?
                    235:  */
                    236:
1.10      mrg       237: static __inline void
                    238: uvm_mapent_free(me)
                    239:        vm_map_entry_t me;
1.1       mrg       240: {
1.10      mrg       241:        int s;
                    242:        UVMHIST_FUNC("uvm_mapent_free");
                    243:        UVMHIST_CALLED(maphist);
                    244:        UVMHIST_LOG(maphist,"<- freeing map entry=0x%x [flags=%d]",
1.1       mrg       245:                me, me->flags, 0, 0);
1.10      mrg       246:        if ((me->flags & UVM_MAP_STATIC) == 0) {
1.26      thorpej   247:                pool_put(&uvm_map_entry_pool, me);
1.10      mrg       248:        } else {
                    249:                s = splimp();   /* protect kentry_free list with splimp */
                    250:                simple_lock(&uvm.kentry_lock);
                    251:                me->next = uvm.kentry_free;
                    252:                uvm.kentry_free = me;
                    253:                simple_unlock(&uvm.kentry_lock);
                    254:                splx(s);
                    255:        }
1.1       mrg       256: }
                    257:
                    258: /*
                    259:  * uvm_mapent_copy: copy a map entry, preserving flags
                    260:  */
                    261:
1.10      mrg       262: static __inline void
                    263: uvm_mapent_copy(src, dst)
                    264:        vm_map_entry_t src;
                    265:        vm_map_entry_t dst;
                    266: {
1.1       mrg       267:
1.23      perry     268:        memcpy(dst, src, ((char *)&src->uvm_map_entry_stop_copy) - ((char*)src));
1.1       mrg       269: }
                    270:
                    271: /*
                    272:  * uvm_map_entry_unwire: unwire a map entry
                    273:  *
                    274:  * => map should be locked by caller
                    275:  */
                    276:
1.10      mrg       277: static __inline void
                    278: uvm_map_entry_unwire(map, entry)
                    279:        vm_map_t map;
                    280:        vm_map_entry_t entry;
                    281: {
1.1       mrg       282:
1.10      mrg       283:        entry->wired_count = 0;
1.57      thorpej   284:        uvm_fault_unwire_locked(map, entry->start, entry->end);
1.1       mrg       285: }
                    286:
                    287: /*
                    288:  * uvm_map_init: init mapping system at boot time.   note that we allocate
                    289:  * and init the static pool of vm_map_entry_t's for the kernel here.
                    290:  */
                    291:
1.10      mrg       292: void
                    293: uvm_map_init()
1.1       mrg       294: {
1.10      mrg       295:        static struct vm_map_entry kernel_map_entry[MAX_KMAPENT];
1.1       mrg       296: #if defined(UVMHIST)
1.10      mrg       297:        static struct uvm_history_ent maphistbuf[100];
                    298:        static struct uvm_history_ent pdhistbuf[100];
1.1       mrg       299: #endif
1.10      mrg       300:        int lcv;
                    301:
                    302:        /*
                    303:         * first, init logging system.
                    304:         */
1.1       mrg       305:
1.10      mrg       306:        UVMHIST_FUNC("uvm_map_init");
                    307:        UVMHIST_INIT_STATIC(maphist, maphistbuf);
                    308:        UVMHIST_INIT_STATIC(pdhist, pdhistbuf);
                    309:        UVMHIST_CALLED(maphist);
                    310:        UVMHIST_LOG(maphist,"<starting uvm map system>", 0, 0, 0, 0);
                    311:        UVMCNT_INIT(uvm_map_call,  UVMCNT_CNT, 0,
                    312:            "# uvm_map() successful calls", 0);
                    313:        UVMCNT_INIT(map_backmerge, UVMCNT_CNT, 0, "# uvm_map() back merges", 0);
                    314:        UVMCNT_INIT(map_forwmerge, UVMCNT_CNT, 0, "# uvm_map() missed forward",
                    315:            0);
                    316:        UVMCNT_INIT(uvm_mlk_call,  UVMCNT_CNT, 0, "# map lookup calls", 0);
                    317:        UVMCNT_INIT(uvm_mlk_hint,  UVMCNT_CNT, 0, "# map lookup hint hits", 0);
                    318:
                    319:        /*
                    320:         * now set up static pool of kernel map entrys ...
                    321:         */
                    322:
                    323:        simple_lock_init(&uvm.kentry_lock);
                    324:        uvm.kentry_free = NULL;
                    325:        for (lcv = 0 ; lcv < MAX_KMAPENT ; lcv++) {
                    326:                kernel_map_entry[lcv].next = uvm.kentry_free;
                    327:                uvm.kentry_free = &kernel_map_entry[lcv];
                    328:        }
1.1       mrg       329:
1.25      thorpej   330:        /*
                    331:         * initialize the map-related pools.
                    332:         */
                    333:        pool_init(&uvm_vmspace_pool, sizeof(struct vmspace),
                    334:            0, 0, 0, "vmsppl", 0,
1.26      thorpej   335:            pool_page_alloc_nointr, pool_page_free_nointr, M_VMMAP);
                    336:        pool_init(&uvm_map_entry_pool, sizeof(struct vm_map_entry),
                    337:            0, 0, 0, "vmmpepl", 0,
1.25      thorpej   338:            pool_page_alloc_nointr, pool_page_free_nointr, M_VMMAP);
1.1       mrg       339: }
                    340:
                    341: /*
                    342:  * clippers
                    343:  */
                    344:
                    345: /*
                    346:  * uvm_map_clip_start: ensure that the entry begins at or after
                    347:  *     the starting address, if it doesn't we split the entry.
                    348:  *
                    349:  * => caller should use UVM_MAP_CLIP_START macro rather than calling
                    350:  *    this directly
                    351:  * => map must be locked by caller
                    352:  */
                    353:
                    354: void uvm_map_clip_start(map, entry, start)
1.32      mrg       355:        vm_map_t       map;
                    356:        vm_map_entry_t entry;
                    357:        vaddr_t    start;
1.1       mrg       358: {
1.32      mrg       359:        vm_map_entry_t new_entry;
1.24      eeh       360:        vaddr_t new_adj;
1.1       mrg       361:
                    362:        /* uvm_map_simplify_entry(map, entry); */ /* XXX */
                    363:
1.10      mrg       364:        /*
                    365:         * Split off the front portion.  note that we must insert the new
                    366:         * entry BEFORE this one, so that this entry has the specified
1.1       mrg       367:         * starting address.
1.10      mrg       368:         */
1.1       mrg       369:
1.10      mrg       370:        new_entry = uvm_mapent_alloc(map);
1.1       mrg       371:        uvm_mapent_copy(entry, new_entry); /* entry -> new_entry */
1.10      mrg       372:
                    373:        new_entry->end = start;
1.1       mrg       374:        new_adj = start - new_entry->start;
                    375:        if (entry->object.uvm_obj)
1.10      mrg       376:                entry->offset += new_adj;       /* shift start over */
                    377:        entry->start = start;
1.1       mrg       378:
                    379:        if (new_entry->aref.ar_amap) {
1.10      mrg       380:                amap_splitref(&new_entry->aref, &entry->aref, new_adj);
1.1       mrg       381:        }
                    382:
1.10      mrg       383:        uvm_map_entry_link(map, entry->prev, new_entry);
                    384:
1.29      chuck     385:        if (UVM_ET_ISSUBMAP(entry)) {
                    386:                /* ... unlikely to happen, but play it safe */
                    387:                 uvm_map_reference(new_entry->object.sub_map);
1.1       mrg       388:        } else {
1.10      mrg       389:                if (UVM_ET_ISOBJ(entry) &&
                    390:                    entry->object.uvm_obj->pgops &&
                    391:                    entry->object.uvm_obj->pgops->pgo_reference)
                    392:                        entry->object.uvm_obj->pgops->pgo_reference(
                    393:                            entry->object.uvm_obj);
1.1       mrg       394:        }
                    395: }
                    396:
                    397: /*
                    398:  * uvm_map_clip_end: ensure that the entry ends at or before
                    399:  *     the ending address, if it does't we split the reference
                    400:  *
                    401:  * => caller should use UVM_MAP_CLIP_END macro rather than calling
                    402:  *    this directly
                    403:  * => map must be locked by caller
                    404:  */
                    405:
1.10      mrg       406: void
                    407: uvm_map_clip_end(map, entry, end)
                    408:        vm_map_t        map;
                    409:        vm_map_entry_t  entry;
1.24      eeh       410:        vaddr_t end;
1.1       mrg       411: {
1.10      mrg       412:        vm_map_entry_t  new_entry;
1.24      eeh       413:        vaddr_t new_adj; /* #bytes we move start forward */
1.1       mrg       414:
                    415:        /*
                    416:         *      Create a new entry and insert it
                    417:         *      AFTER the specified entry
                    418:         */
                    419:
                    420:        new_entry = uvm_mapent_alloc(map);
                    421:        uvm_mapent_copy(entry, new_entry); /* entry -> new_entry */
                    422:
                    423:        new_entry->start = entry->end = end;
                    424:        new_adj = end - entry->start;
                    425:        if (new_entry->object.uvm_obj)
                    426:                new_entry->offset += new_adj;
                    427:
1.10      mrg       428:        if (entry->aref.ar_amap)
                    429:                amap_splitref(&entry->aref, &new_entry->aref, new_adj);
1.1       mrg       430:
                    431:        uvm_map_entry_link(map, entry, new_entry);
                    432:
1.29      chuck     433:        if (UVM_ET_ISSUBMAP(entry)) {
                    434:                /* ... unlikely to happen, but play it safe */
                    435:                uvm_map_reference(new_entry->object.sub_map);
1.1       mrg       436:        } else {
1.10      mrg       437:                if (UVM_ET_ISOBJ(entry) &&
                    438:                    entry->object.uvm_obj->pgops &&
                    439:                    entry->object.uvm_obj->pgops->pgo_reference)
                    440:                        entry->object.uvm_obj->pgops->pgo_reference(
                    441:                            entry->object.uvm_obj);
1.1       mrg       442:        }
                    443: }
                    444:
                    445:
                    446: /*
                    447:  *   M A P   -   m a i n   e n t r y   p o i n t
                    448:  */
                    449: /*
                    450:  * uvm_map: establish a valid mapping in a map
                    451:  *
                    452:  * => assume startp is page aligned.
                    453:  * => assume size is a multiple of PAGE_SIZE.
                    454:  * => assume sys_mmap provides enough of a "hint" to have us skip
                    455:  *     over text/data/bss area.
                    456:  * => map must be unlocked (we will lock it)
                    457:  * => <uobj,uoffset> value meanings (4 cases):
                    458:  *      [1] <NULL,uoffset>             == uoffset is a hint for PMAP_PREFER
                    459:  *      [2] <NULL,UVM_UNKNOWN_OFFSET>  == don't PMAP_PREFER
                    460:  *      [3] <uobj,uoffset>             == normal mapping
                    461:  *      [4] <uobj,UVM_UNKNOWN_OFFSET>  == uvm_map finds offset based on VA
                    462:  *
                    463:  *    case [4] is for kernel mappings where we don't know the offset until
1.8       chuck     464:  *    we've found a virtual address.   note that kernel object offsets are
                    465:  *    always relative to vm_map_min(kernel_map).
1.1       mrg       466:  * => XXXCDC: need way to map in external amap?
                    467:  */
                    468:
1.10      mrg       469: int
                    470: uvm_map(map, startp, size, uobj, uoffset, flags)
                    471:        vm_map_t map;
1.24      eeh       472:        vaddr_t *startp;        /* IN/OUT */
                    473:        vsize_t size;
1.10      mrg       474:        struct uvm_object *uobj;
1.70      kleink    475:        voff_t uoffset;
1.10      mrg       476:        uvm_flag_t flags;
                    477: {
                    478:        vm_map_entry_t prev_entry, new_entry;
                    479:        vm_prot_t prot = UVM_PROTECTION(flags), maxprot =
                    480:            UVM_MAXPROTECTION(flags);
                    481:        vm_inherit_t inherit = UVM_INHERIT(flags);
                    482:        int advice = UVM_ADVICE(flags);
                    483:        UVMHIST_FUNC("uvm_map");
                    484:        UVMHIST_CALLED(maphist);
1.1       mrg       485:
1.10      mrg       486:        UVMHIST_LOG(maphist, "(map=0x%x, *startp=0x%x, size=%d, flags=0x%x)",
                    487:            map, *startp, size, flags);
                    488:        UVMHIST_LOG(maphist, "  uobj/offset 0x%x/%d", uobj, uoffset,0,0);
1.1       mrg       489:
1.10      mrg       490:        /*
                    491:         * step 0: sanity check of protection code
                    492:         */
1.1       mrg       493:
1.10      mrg       494:        if ((prot & maxprot) != prot) {
                    495:                UVMHIST_LOG(maphist, "<- prot. failure:  prot=0x%x, max=0x%x",
                    496:                prot, maxprot,0,0);
                    497:                return(KERN_PROTECTION_FAILURE);
                    498:        }
1.1       mrg       499:
1.10      mrg       500:        /*
                    501:         * step 1: figure out where to put new VM range
                    502:         */
1.1       mrg       503:
1.10      mrg       504:        if (vm_map_lock_try(map) == FALSE) {
                    505:                if (flags & UVM_FLAG_TRYLOCK)
                    506:                        return(KERN_FAILURE);
                    507:                vm_map_lock(map); /* could sleep here */
                    508:        }
                    509:        if ((prev_entry = uvm_map_findspace(map, *startp, size, startp,
                    510:            uobj, uoffset, flags & UVM_FLAG_FIXED)) == NULL) {
                    511:                UVMHIST_LOG(maphist,"<- uvm_map_findspace failed!",0,0,0,0);
                    512:                vm_map_unlock(map);
                    513:                return (KERN_NO_SPACE);
                    514:        }
1.1       mrg       515:
1.40      thorpej   516: #ifdef PMAP_GROWKERNEL
1.10      mrg       517:        {
                    518:                /*
1.40      thorpej   519:                 * If the kernel pmap can't map the requested space,
                    520:                 * then allocate more resources for it.
1.10      mrg       521:                 */
1.40      thorpej   522:                if (map == kernel_map && uvm_maxkaddr < (*startp + size))
                    523:                        uvm_maxkaddr = pmap_growkernel(*startp + size);
1.10      mrg       524:        }
                    525: #endif
                    526:
                    527:        UVMCNT_INCR(uvm_map_call);
                    528:
                    529:        /*
                    530:         * if uobj is null, then uoffset is either a VAC hint for PMAP_PREFER
                    531:         * [typically from uvm_map_reserve] or it is UVM_UNKNOWN_OFFSET.   in
                    532:         * either case we want to zero it  before storing it in the map entry
                    533:         * (because it looks strange and confusing when debugging...)
                    534:         *
                    535:         * if uobj is not null
                    536:         *   if uoffset is not UVM_UNKNOWN_OFFSET then we have a normal mapping
                    537:         *      and we do not need to change uoffset.
                    538:         *   if uoffset is UVM_UNKNOWN_OFFSET then we need to find the offset
                    539:         *      now (based on the starting address of the map).   this case is
                    540:         *      for kernel object mappings where we don't know the offset until
                    541:         *      the virtual address is found (with uvm_map_findspace).   the
                    542:         *      offset is the distance we are from the start of the map.
                    543:         */
                    544:
                    545:        if (uobj == NULL) {
                    546:                uoffset = 0;
                    547:        } else {
                    548:                if (uoffset == UVM_UNKNOWN_OFFSET) {
1.8       chuck     549: #ifdef DIAGNOSTIC
1.42      thorpej   550:                        if (UVM_OBJ_IS_KERN_OBJECT(uobj) == 0)
                    551:                                panic("uvm_map: unknown offset with "
                    552:                                    "non-kernel object");
1.8       chuck     553: #endif
1.10      mrg       554:                        uoffset = *startp - vm_map_min(kernel_map);
                    555:                }
                    556:        }
                    557:
                    558:        /*
                    559:         * step 2: try and insert in map by extending previous entry, if
                    560:         * possible
                    561:         * XXX: we don't try and pull back the next entry.   might be useful
                    562:         * for a stack, but we are currently allocating our stack in advance.
                    563:         */
                    564:
                    565:        if ((flags & UVM_FLAG_NOMERGE) == 0 &&
                    566:            prev_entry->end == *startp && prev_entry != &map->header &&
                    567:            prev_entry->object.uvm_obj == uobj) {
                    568:
                    569:                if (uobj && prev_entry->offset +
                    570:                    (prev_entry->end - prev_entry->start) != uoffset)
                    571:                        goto step3;
                    572:
1.29      chuck     573:                if (UVM_ET_ISSUBMAP(prev_entry))
1.10      mrg       574:                        goto step3;
                    575:
                    576:                if (prev_entry->protection != prot ||
                    577:                    prev_entry->max_protection != maxprot)
                    578:                        goto step3;
                    579:
                    580:                if (prev_entry->inheritance != inherit ||
                    581:                    prev_entry->advice != advice)
                    582:                        goto step3;
                    583:
1.56      thorpej   584:                /* wiring status must match (new area is unwired) */
1.55      thorpej   585:                if (VM_MAPENT_ISWIRED(prev_entry))
1.10      mrg       586:                        goto step3;
                    587:
                    588:                /*
                    589:                 * can't extend a shared amap.  note: no need to lock amap to
1.34      chuck     590:                 * look at refs since we don't care about its exact value.
1.10      mrg       591:                 * if it is one (i.e. we have only reference) it will stay there
                    592:                 */
                    593:
                    594:                if (prev_entry->aref.ar_amap &&
1.34      chuck     595:                    amap_refs(prev_entry->aref.ar_amap) != 1) {
1.10      mrg       596:                        goto step3;
                    597:                }
                    598:
                    599:                /* got it! */
                    600:
                    601:                UVMCNT_INCR(map_backmerge);
                    602:                UVMHIST_LOG(maphist,"  starting back merge", 0, 0, 0, 0);
                    603:
                    604:                /*
                    605:                 * drop our reference to uobj since we are extending a reference
                    606:                 * that we already have (the ref count can not drop to zero).
                    607:                 */
                    608:                if (uobj && uobj->pgops->pgo_detach)
                    609:                        uobj->pgops->pgo_detach(uobj);
                    610:
                    611:                if (prev_entry->aref.ar_amap) {
                    612:                        amap_extend(prev_entry, size);
                    613:                }
                    614:
                    615:                prev_entry->end += size;
                    616:                map->size += size;
                    617:
                    618:                UVMHIST_LOG(maphist,"<- done (via backmerge)!", 0, 0, 0, 0);
                    619:                vm_map_unlock(map);
                    620:                return (KERN_SUCCESS);
                    621:
1.1       mrg       622:        }
1.10      mrg       623: step3:
                    624:        UVMHIST_LOG(maphist,"  allocating new map entry", 0, 0, 0, 0);
                    625:
                    626:        /*
                    627:         * check for possible forward merge (which we don't do) and count
                    628:         * the number of times we missed a *possible* chance to merge more
                    629:         */
1.1       mrg       630:
1.10      mrg       631:        if ((flags & UVM_FLAG_NOMERGE) == 0 &&
                    632:            prev_entry->next != &map->header &&
                    633:            prev_entry->next->start == (*startp + size))
                    634:                UVMCNT_INCR(map_forwmerge);
1.1       mrg       635:
                    636:        /*
1.10      mrg       637:         * step 3: allocate new entry and link it in
1.1       mrg       638:         */
                    639:
1.10      mrg       640:        new_entry = uvm_mapent_alloc(map);
                    641:        new_entry->start = *startp;
                    642:        new_entry->end = new_entry->start + size;
                    643:        new_entry->object.uvm_obj = uobj;
                    644:        new_entry->offset = uoffset;
                    645:
                    646:        if (uobj)
                    647:                new_entry->etype = UVM_ET_OBJ;
                    648:        else
                    649:                new_entry->etype = 0;
                    650:
                    651:        if (flags & UVM_FLAG_COPYONW) {
                    652:                new_entry->etype |= UVM_ET_COPYONWRITE;
                    653:                if ((flags & UVM_FLAG_OVERLAY) == 0)
                    654:                        new_entry->etype |= UVM_ET_NEEDSCOPY;
                    655:        }
                    656:
                    657:        new_entry->protection = prot;
                    658:        new_entry->max_protection = maxprot;
                    659:        new_entry->inheritance = inherit;
                    660:        new_entry->wired_count = 0;
                    661:        new_entry->advice = advice;
                    662:        if (flags & UVM_FLAG_OVERLAY) {
                    663:                /*
                    664:                 * to_add: for BSS we overallocate a little since we
                    665:                 * are likely to extend
                    666:                 */
1.24      eeh       667:                vaddr_t to_add = (flags & UVM_FLAG_AMAPPAD) ?
1.30      chs       668:                        UVM_AMAP_CHUNK << PAGE_SHIFT : 0;
1.10      mrg       669:                struct vm_amap *amap = amap_alloc(size, to_add, M_WAITOK);
1.34      chuck     670:                new_entry->aref.ar_pageoff = 0;
1.10      mrg       671:                new_entry->aref.ar_amap = amap;
                    672:        } else {
1.77      chs       673:                new_entry->aref.ar_pageoff = 0;
1.10      mrg       674:                new_entry->aref.ar_amap = NULL;
1.1       mrg       675:        }
                    676:
1.10      mrg       677:        uvm_map_entry_link(map, prev_entry, new_entry);
                    678:
1.1       mrg       679:        map->size += size;
                    680:
1.10      mrg       681:        /*
                    682:         *      Update the free space hint
                    683:         */
                    684:
                    685:        if ((map->first_free == prev_entry) &&
                    686:            (prev_entry->end >= new_entry->start))
                    687:                map->first_free = new_entry;
                    688:
                    689:        UVMHIST_LOG(maphist,"<- done!", 0, 0, 0, 0);
1.1       mrg       690:        vm_map_unlock(map);
                    691:        return(KERN_SUCCESS);
                    692: }
                    693:
                    694: /*
                    695:  * uvm_map_lookup_entry: find map entry at or before an address
                    696:  *
                    697:  * => map must at least be read-locked by caller
                    698:  * => entry is returned in "entry"
                    699:  * => return value is true if address is in the returned entry
                    700:  */
                    701:
1.10      mrg       702: boolean_t
                    703: uvm_map_lookup_entry(map, address, entry)
1.32      mrg       704:        vm_map_t        map;
                    705:        vaddr_t address;
1.10      mrg       706:        vm_map_entry_t          *entry;         /* OUT */
1.1       mrg       707: {
1.32      mrg       708:        vm_map_entry_t          cur;
                    709:        vm_map_entry_t          last;
1.1       mrg       710:        UVMHIST_FUNC("uvm_map_lookup_entry");
                    711:        UVMHIST_CALLED(maphist);
                    712:
                    713:        UVMHIST_LOG(maphist,"(map=0x%x,addr=0x%x,ent=0x%x)",
1.10      mrg       714:            map, address, entry, 0);
1.1       mrg       715:
                    716:        /*
1.10      mrg       717:         * start looking either from the head of the
                    718:         * list, or from the hint.
1.1       mrg       719:         */
                    720:
                    721:        simple_lock(&map->hint_lock);
                    722:        cur = map->hint;
                    723:        simple_unlock(&map->hint_lock);
                    724:
                    725:        if (cur == &map->header)
                    726:                cur = cur->next;
                    727:
                    728:        UVMCNT_INCR(uvm_mlk_call);
                    729:        if (address >= cur->start) {
                    730:                /*
1.10      mrg       731:                 * go from hint to end of list.
1.1       mrg       732:                 *
1.10      mrg       733:                 * but first, make a quick check to see if
                    734:                 * we are already looking at the entry we
                    735:                 * want (which is usually the case).
                    736:                 * note also that we don't need to save the hint
                    737:                 * here... it is the same hint (unless we are
                    738:                 * at the header, in which case the hint didn't
                    739:                 * buy us anything anyway).
1.1       mrg       740:                 */
                    741:                last = &map->header;
                    742:                if ((cur != last) && (cur->end > address)) {
                    743:                        UVMCNT_INCR(uvm_mlk_hint);
                    744:                        *entry = cur;
                    745:                        UVMHIST_LOG(maphist,"<- got it via hint (0x%x)",
1.10      mrg       746:                            cur, 0, 0, 0);
                    747:                        return (TRUE);
1.1       mrg       748:                }
1.10      mrg       749:        } else {
1.1       mrg       750:                /*
1.10      mrg       751:                 * go from start to hint, *inclusively*
1.1       mrg       752:                 */
                    753:                last = cur->next;
                    754:                cur = map->header.next;
                    755:        }
                    756:
                    757:        /*
1.10      mrg       758:         * search linearly
1.1       mrg       759:         */
                    760:
                    761:        while (cur != last) {
                    762:                if (cur->end > address) {
                    763:                        if (address >= cur->start) {
                    764:                                /*
1.10      mrg       765:                                 * save this lookup for future
                    766:                                 * hints, and return
1.1       mrg       767:                                 */
                    768:
                    769:                                *entry = cur;
                    770:                                SAVE_HINT(map, cur);
                    771:                                UVMHIST_LOG(maphist,"<- search got it (0x%x)",
1.10      mrg       772:                                        cur, 0, 0, 0);
                    773:                                return (TRUE);
1.1       mrg       774:                        }
                    775:                        break;
                    776:                }
                    777:                cur = cur->next;
                    778:        }
                    779:        *entry = cur->prev;
                    780:        SAVE_HINT(map, *entry);
                    781:        UVMHIST_LOG(maphist,"<- failed!",0,0,0,0);
1.10      mrg       782:        return (FALSE);
1.1       mrg       783: }
                    784:
                    785:
                    786: /*
                    787:  * uvm_map_findspace: find "length" sized space in "map".
                    788:  *
                    789:  * => "hint" is a hint about where we want it, unless fixed is true
                    790:  *     (in which case we insist on using "hint").
                    791:  * => "result" is VA returned
                    792:  * => uobj/uoffset are to be used to handle VAC alignment, if required
                    793:  * => caller must at least have read-locked map
                    794:  * => returns NULL on failure, or pointer to prev. map entry if success
                    795:  * => note this is a cross between the old vm_map_findspace and vm_map_find
                    796:  */
                    797:
1.10      mrg       798: vm_map_entry_t
                    799: uvm_map_findspace(map, hint, length, result, uobj, uoffset, fixed)
                    800:        vm_map_t map;
1.24      eeh       801:        vaddr_t hint;
                    802:        vsize_t length;
                    803:        vaddr_t *result; /* OUT */
1.10      mrg       804:        struct uvm_object *uobj;
1.70      kleink    805:        voff_t uoffset;
1.10      mrg       806:        boolean_t fixed;
1.1       mrg       807: {
                    808:        vm_map_entry_t entry, next, tmp;
1.24      eeh       809:        vaddr_t end;
1.1       mrg       810:        UVMHIST_FUNC("uvm_map_findspace");
                    811:        UVMHIST_CALLED(maphist);
                    812:
                    813:        UVMHIST_LOG(maphist, "(map=0x%x, hint=0x%x, len=%d, fixed=%d)",
                    814:                map, hint, length, fixed);
                    815:
                    816:        if (hint < map->min_offset) {   /* check ranges ... */
                    817:                if (fixed) {
                    818:                        UVMHIST_LOG(maphist,"<- VA below map range",0,0,0,0);
                    819:                        return(NULL);
                    820:                }
                    821:                hint = map->min_offset;
                    822:        }
                    823:        if (hint > map->max_offset) {
                    824:                UVMHIST_LOG(maphist,"<- VA 0x%x > range [0x%x->0x%x]",
                    825:                                hint, map->min_offset, map->max_offset, 0);
                    826:                return(NULL);
                    827:        }
                    828:
                    829:        /*
                    830:         * Look for the first possible address; if there's already
                    831:         * something at this address, we have to start after it.
                    832:         */
                    833:
                    834:        if (!fixed && hint == map->min_offset) {
                    835:                if ((entry = map->first_free) != &map->header)
                    836:                        hint = entry->end;
                    837:        } else {
                    838:                if (uvm_map_lookup_entry(map, hint, &tmp)) {
                    839:                        /* "hint" address already in use ... */
                    840:                        if (fixed) {
                    841:                                UVMHIST_LOG(maphist,"<- fixed & VA in use",
1.10      mrg       842:                                    0, 0, 0, 0);
1.1       mrg       843:                                return(NULL);
                    844:                        }
                    845:                        hint = tmp->end;
                    846:                }
                    847:                entry = tmp;
                    848:        }
                    849:
                    850:        /*
                    851:         * Look through the rest of the map, trying to fit a new region in
                    852:         * the gap between existing regions, or after the very last region.
                    853:         * note: entry->end   = base VA of current gap,
                    854:         *       next->start  = VA of end of current gap
                    855:         */
                    856:        for (;; hint = (entry = next)->end) {
                    857:                /*
                    858:                 * Find the end of the proposed new region.  Be sure we didn't
                    859:                 * go beyond the end of the map, or wrap around the address;
                    860:                 * if so, we lose.  Otherwise, if this is the last entry, or
                    861:                 * if the proposed new region fits before the next entry, we
                    862:                 * win.
                    863:                 */
                    864:
                    865: #ifdef PMAP_PREFER
                    866:                /*
                    867:                 * push hint forward as needed to avoid VAC alias problems.
                    868:                 * we only do this if a valid offset is specified.
                    869:                 */
                    870:                if (!fixed && uoffset != UVM_UNKNOWN_OFFSET)
                    871:                  PMAP_PREFER(uoffset, &hint);
                    872: #endif
                    873:                end = hint + length;
                    874:                if (end > map->max_offset || end < hint) {
                    875:                        UVMHIST_LOG(maphist,"<- failed (off end)", 0,0,0,0);
                    876:                        return (NULL);
                    877:                }
                    878:                next = entry->next;
                    879:                if (next == &map->header || next->start >= end)
                    880:                        break;
                    881:                if (fixed) {
                    882:                        UVMHIST_LOG(maphist,"<- fixed mapping failed", 0,0,0,0);
                    883:                        return(NULL); /* only one shot at it ... */
                    884:                }
                    885:        }
                    886:        SAVE_HINT(map, entry);
                    887:        *result = hint;
                    888:        UVMHIST_LOG(maphist,"<- got it!  (result=0x%x)", hint, 0,0,0);
                    889:        return (entry);
                    890: }
                    891:
                    892: /*
                    893:  *   U N M A P   -   m a i n   h e l p e r   f u n c t i o n s
                    894:  */
                    895:
                    896: /*
                    897:  * uvm_unmap_remove: remove mappings from a vm_map (from "start" up to "stop")
                    898:  *
                    899:  * => caller must check alignment and size
                    900:  * => map must be locked by caller
                    901:  * => we return a list of map entries that we've remove from the map
                    902:  *    in "entry_list"
                    903:  */
                    904:
1.10      mrg       905: int
1.29      chuck     906: uvm_unmap_remove(map, start, end, entry_list)
1.10      mrg       907:        vm_map_t map;
1.24      eeh       908:        vaddr_t start,end;
1.10      mrg       909:        vm_map_entry_t *entry_list;     /* OUT */
                    910: {
                    911:        vm_map_entry_t entry, first_entry, next;
1.24      eeh       912:        vaddr_t len;
1.10      mrg       913:        UVMHIST_FUNC("uvm_unmap_remove");
                    914:        UVMHIST_CALLED(maphist);
                    915:
                    916:        UVMHIST_LOG(maphist,"(map=0x%x, start=0x%x, end=0x%x)",
                    917:            map, start, end, 0);
                    918:
                    919:        VM_MAP_RANGE_CHECK(map, start, end);
                    920:
                    921:        /*
                    922:         * find first entry
                    923:         */
                    924:        if (uvm_map_lookup_entry(map, start, &first_entry) == TRUE) {
1.29      chuck     925:                /* clip and go... */
1.10      mrg       926:                entry = first_entry;
                    927:                UVM_MAP_CLIP_START(map, entry, start);
                    928:                /* critical!  prevents stale hint */
                    929:                SAVE_HINT(map, entry->prev);
                    930:
                    931:        } else {
                    932:                entry = first_entry->next;
                    933:        }
                    934:
                    935:        /*
                    936:         * Save the free space hint
                    937:         */
                    938:
                    939:        if (map->first_free->start >= start)
                    940:                map->first_free = entry->prev;
                    941:
                    942:        /*
                    943:         * note: we now re-use first_entry for a different task.  we remove
                    944:         * a number of map entries from the map and save them in a linked
                    945:         * list headed by "first_entry".  once we remove them from the map
                    946:         * the caller should unlock the map and drop the references to the
                    947:         * backing objects [c.f. uvm_unmap_detach].  the object is to
                    948:         * seperate unmapping from reference dropping.  why?
                    949:         *   [1] the map has to be locked for unmapping
                    950:         *   [2] the map need not be locked for reference dropping
                    951:         *   [3] dropping references may trigger pager I/O, and if we hit
                    952:         *       a pager that does synchronous I/O we may have to wait for it.
                    953:         *   [4] we would like all waiting for I/O to occur with maps unlocked
                    954:         *       so that we don't block other threads.
                    955:         */
                    956:        first_entry = NULL;
                    957:        *entry_list = NULL;             /* to be safe */
                    958:
                    959:        /*
                    960:         * break up the area into map entry sized regions and unmap.  note
                    961:         * that all mappings have to be removed before we can even consider
                    962:         * dropping references to amaps or VM objects (otherwise we could end
                    963:         * up with a mapping to a page on the free list which would be very bad)
                    964:         */
                    965:
                    966:        while ((entry != &map->header) && (entry->start < end)) {
                    967:
                    968:                UVM_MAP_CLIP_END(map, entry, end);
                    969:                next = entry->next;
                    970:                len = entry->end - entry->start;
                    971:
                    972:                /*
                    973:                 * unwire before removing addresses from the pmap; otherwise
                    974:                 * unwiring will put the entries back into the pmap (XXX).
                    975:                 */
1.1       mrg       976:
1.55      thorpej   977:                if (VM_MAPENT_ISWIRED(entry))
1.10      mrg       978:                        uvm_map_entry_unwire(map, entry);
                    979:
                    980:                /*
                    981:                 * special case: handle mappings to anonymous kernel objects.
                    982:                 * we want to free these pages right away...
                    983:                 */
                    984:                if (UVM_ET_ISOBJ(entry) &&
1.42      thorpej   985:                    UVM_OBJ_IS_KERN_OBJECT(entry->object.uvm_obj)) {
1.1       mrg       986: #ifdef DIAGNOSTIC
1.10      mrg       987:                        if (vm_map_pmap(map) != pmap_kernel())
1.42      thorpej   988:                                panic("uvm_unmap_remove: kernel object "
                    989:                                    "mapped by non-kernel map");
1.1       mrg       990: #endif
                    991:
1.10      mrg       992:                        /*
                    993:                         * note: kernel object mappings are currently used in
                    994:                         * two ways:
                    995:                         *  [1] "normal" mappings of pages in the kernel object
                    996:                         *  [2] uvm_km_valloc'd allocations in which we
                    997:                         *      pmap_enter in some non-kernel-object page
                    998:                         *      (e.g. vmapbuf).
                    999:                         *
                   1000:                         * for case [1], we need to remove the mapping from
                   1001:                         * the pmap and then remove the page from the kernel
                   1002:                         * object (because, once pages in a kernel object are
                   1003:                         * unmapped they are no longer needed, unlike, say,
                   1004:                         * a vnode where you might want the data to persist
                   1005:                         * until flushed out of a queue).
                   1006:                         *
                   1007:                         * for case [2], we need to remove the mapping from
                   1008:                         * the pmap.  there shouldn't be any pages at the
                   1009:                         * specified offset in the kernel object [but it
                   1010:                         * doesn't hurt to call uvm_km_pgremove just to be
                   1011:                         * safe?]
                   1012:                         *
                   1013:                         * uvm_km_pgremove currently does the following:
                   1014:                         *   for pages in the kernel object in range:
1.43      thorpej  1015:                         *     - drops the swap slot
1.10      mrg      1016:                         *     - uvm_pagefree the page
                   1017:                         *
1.43      thorpej  1018:                         * note there is version of uvm_km_pgremove() that
                   1019:                         * is used for "intrsafe" objects.
1.10      mrg      1020:                         */
                   1021:
                   1022:                        /*
1.43      thorpej  1023:                         * remove mappings from pmap and drop the pages
                   1024:                         * from the object.  offsets are always relative
                   1025:                         * to vm_map_min(kernel_map).
1.10      mrg      1026:                         */
1.43      thorpej  1027:                        if (UVM_OBJ_IS_INTRSAFE_OBJECT(entry->object.uvm_obj)) {
                   1028:                                pmap_kremove(entry->start, len);
                   1029:                                uvm_km_pgremove_intrsafe(entry->object.uvm_obj,
                   1030:                                    entry->start - vm_map_min(kernel_map),
                   1031:                                    entry->end - vm_map_min(kernel_map));
                   1032:                        } else {
                   1033:                                pmap_remove(pmap_kernel(), entry->start,
                   1034:                                    entry->start + len);
                   1035:                                uvm_km_pgremove(entry->object.uvm_obj,
                   1036:                                    entry->start - vm_map_min(kernel_map),
                   1037:                                    entry->end - vm_map_min(kernel_map));
                   1038:                        }
1.10      mrg      1039:
                   1040:                        /*
                   1041:                         * null out kernel_object reference, we've just
                   1042:                         * dropped it
                   1043:                         */
                   1044:                        entry->etype &= ~UVM_ET_OBJ;
                   1045:                        entry->object.uvm_obj = NULL;   /* to be safe */
                   1046:
                   1047:                } else {
1.29      chuck    1048:                        /*
                   1049:                         * remove mappings the standard way.
                   1050:                         */
                   1051:                        pmap_remove(map->pmap, entry->start, entry->end);
1.10      mrg      1052:                }
                   1053:
                   1054:                /*
1.29      chuck    1055:                 * remove entry from map and put it on our list of entries
                   1056:                 * that we've nuked.  then go do next entry.
1.10      mrg      1057:                 */
                   1058:                UVMHIST_LOG(maphist, "  removed map entry 0x%x", entry, 0, 0,0);
                   1059:                uvm_map_entry_unlink(map, entry);
                   1060:                map->size -= len;
                   1061:                entry->next = first_entry;
                   1062:                first_entry = entry;
                   1063:                entry = next;           /* next entry, please */
                   1064:        }
                   1065:
                   1066:        /*
                   1067:         * now we've cleaned up the map and are ready for the caller to drop
                   1068:         * references to the mapped objects.
                   1069:         */
                   1070:
                   1071:        *entry_list = first_entry;
                   1072:        UVMHIST_LOG(maphist,"<- done!", 0, 0, 0, 0);
                   1073:        return(KERN_SUCCESS);
1.1       mrg      1074: }
                   1075:
                   1076: /*
                   1077:  * uvm_unmap_detach: drop references in a chain of map entries
                   1078:  *
                   1079:  * => we will free the map entries as we traverse the list.
                   1080:  */
                   1081:
1.10      mrg      1082: void
                   1083: uvm_unmap_detach(first_entry, amap_unref_flags)
                   1084:        vm_map_entry_t first_entry;
                   1085:        int amap_unref_flags;
1.1       mrg      1086: {
1.10      mrg      1087:        vm_map_entry_t next_entry;
                   1088:        UVMHIST_FUNC("uvm_unmap_detach"); UVMHIST_CALLED(maphist);
1.1       mrg      1089:
1.10      mrg      1090:        while (first_entry) {
1.1       mrg      1091:
                   1092: #ifdef DIAGNOSTIC
1.10      mrg      1093:                /*
                   1094:                 * sanity check
                   1095:                 */
                   1096:                /* was part of vm_map_entry_delete() */
1.55      thorpej  1097:                if (VM_MAPENT_ISWIRED(first_entry))
1.10      mrg      1098:                        panic("unmap: still wired!");
1.1       mrg      1099: #endif
                   1100:
1.10      mrg      1101:                UVMHIST_LOG(maphist,
1.29      chuck    1102:                    "  detach 0x%x: amap=0x%x, obj=0x%x, submap?=%d",
                   1103:                    first_entry, first_entry->aref.ar_amap,
                   1104:                    first_entry->object.uvm_obj,
                   1105:                    UVM_ET_ISSUBMAP(first_entry));
1.1       mrg      1106:
1.10      mrg      1107:                /*
                   1108:                 * drop reference to amap, if we've got one
                   1109:                 */
                   1110:
                   1111:                if (first_entry->aref.ar_amap)
                   1112:                        amap_unref(first_entry, amap_unref_flags);
                   1113:
                   1114:                /*
                   1115:                 * drop reference to our backing object, if we've got one
                   1116:                 */
                   1117:
1.29      chuck    1118:                if (UVM_ET_ISSUBMAP(first_entry)) {
                   1119:                        /* ... unlikely to happen, but play it safe */
                   1120:                        uvm_map_deallocate(first_entry->object.sub_map);
1.10      mrg      1121:                } else {
                   1122:                        if (UVM_ET_ISOBJ(first_entry) &&
                   1123:                            first_entry->object.uvm_obj->pgops->pgo_detach)
                   1124:                                first_entry->object.uvm_obj->pgops->
                   1125:                                    pgo_detach(first_entry->object.uvm_obj);
                   1126:                }
                   1127:
                   1128:                /*
                   1129:                 * next entry
                   1130:                 */
                   1131:                next_entry = first_entry->next;
                   1132:                uvm_mapent_free(first_entry);
                   1133:                first_entry = next_entry;
                   1134:        }
                   1135:
                   1136:        /*
                   1137:         * done!
                   1138:         */
                   1139:        UVMHIST_LOG(maphist, "<- done", 0,0,0,0);
                   1140:        return;
1.1       mrg      1141: }
                   1142:
                   1143: /*
                   1144:  *   E X T R A C T I O N   F U N C T I O N S
                   1145:  */
                   1146:
                   1147: /*
                   1148:  * uvm_map_reserve: reserve space in a vm_map for future use.
                   1149:  *
                   1150:  * => we reserve space in a map by putting a dummy map entry in the
                   1151:  *    map (dummy means obj=NULL, amap=NULL, prot=VM_PROT_NONE)
                   1152:  * => map should be unlocked (we will write lock it)
                   1153:  * => we return true if we were able to reserve space
                   1154:  * => XXXCDC: should be inline?
                   1155:  */
                   1156:
1.10      mrg      1157: int
                   1158: uvm_map_reserve(map, size, offset, raddr)
                   1159:        vm_map_t map;
1.24      eeh      1160:        vsize_t size;
                   1161:        vaddr_t offset;    /* hint for pmap_prefer */
1.75      pk       1162:        vaddr_t *raddr; /* IN:hint, OUT: reserved VA */
1.1       mrg      1163: {
1.10      mrg      1164:        UVMHIST_FUNC("uvm_map_reserve"); UVMHIST_CALLED(maphist);
1.1       mrg      1165:
1.10      mrg      1166:        UVMHIST_LOG(maphist, "(map=0x%x, size=0x%x, offset=0x%x,addr=0x%x)",
1.1       mrg      1167:              map,size,offset,raddr);
                   1168:
1.10      mrg      1169:        size = round_page(size);
                   1170:        if (*raddr < vm_map_min(map))
                   1171:                *raddr = vm_map_min(map);                /* hint */
1.1       mrg      1172:
1.10      mrg      1173:        /*
                   1174:         * reserve some virtual space.
                   1175:         */
1.1       mrg      1176:
1.10      mrg      1177:        if (uvm_map(map, raddr, size, NULL, offset,
                   1178:            UVM_MAPFLAG(UVM_PROT_NONE, UVM_PROT_NONE, UVM_INH_NONE,
                   1179:            UVM_ADV_RANDOM, UVM_FLAG_NOMERGE)) != KERN_SUCCESS) {
                   1180:            UVMHIST_LOG(maphist, "<- done (no VM)", 0,0,0,0);
                   1181:                return (FALSE);
                   1182:        }
                   1183:
                   1184:        UVMHIST_LOG(maphist, "<- done (*raddr=0x%x)", *raddr,0,0,0);
                   1185:        return (TRUE);
1.1       mrg      1186: }
                   1187:
                   1188: /*
                   1189:  * uvm_map_replace: replace a reserved (blank) area of memory with
                   1190:  * real mappings.
                   1191:  *
                   1192:  * => caller must WRITE-LOCK the map
                   1193:  * => we return TRUE if replacement was a success
                   1194:  * => we expect the newents chain to have nnewents entrys on it and
                   1195:  *    we expect newents->prev to point to the last entry on the list
                   1196:  * => note newents is allowed to be NULL
                   1197:  */
                   1198:
1.10      mrg      1199: int
                   1200: uvm_map_replace(map, start, end, newents, nnewents)
                   1201:        struct vm_map *map;
1.24      eeh      1202:        vaddr_t start, end;
1.10      mrg      1203:        vm_map_entry_t newents;
                   1204:        int nnewents;
                   1205: {
                   1206:        vm_map_entry_t oldent, last;
                   1207:        UVMHIST_FUNC("uvm_map_replace");
                   1208:        UVMHIST_CALLED(maphist);
1.1       mrg      1209:
1.10      mrg      1210:        /*
                   1211:         * first find the blank map entry at the specified address
                   1212:         */
                   1213:
                   1214:        if (!uvm_map_lookup_entry(map, start, &oldent)) {
                   1215:                return(FALSE);
                   1216:        }
                   1217:
                   1218:        /*
                   1219:         * check to make sure we have a proper blank entry
                   1220:         */
1.1       mrg      1221:
1.10      mrg      1222:        if (oldent->start != start || oldent->end != end ||
                   1223:            oldent->object.uvm_obj != NULL || oldent->aref.ar_amap != NULL) {
                   1224:                return (FALSE);
                   1225:        }
1.1       mrg      1226:
                   1227: #ifdef DIAGNOSTIC
1.10      mrg      1228:        /*
                   1229:         * sanity check the newents chain
                   1230:         */
                   1231:        {
                   1232:                vm_map_entry_t tmpent = newents;
                   1233:                int nent = 0;
1.24      eeh      1234:                vaddr_t cur = start;
1.10      mrg      1235:
                   1236:                while (tmpent) {
                   1237:                        nent++;
                   1238:                        if (tmpent->start < cur)
                   1239:                                panic("uvm_map_replace1");
                   1240:                        if (tmpent->start > tmpent->end || tmpent->end > end) {
                   1241:                printf("tmpent->start=0x%lx, tmpent->end=0x%lx, end=0x%lx\n",
                   1242:                            tmpent->start, tmpent->end, end);
                   1243:                                panic("uvm_map_replace2");
                   1244:                        }
                   1245:                        cur = tmpent->end;
                   1246:                        if (tmpent->next) {
                   1247:                                if (tmpent->next->prev != tmpent)
                   1248:                                        panic("uvm_map_replace3");
                   1249:                        } else {
                   1250:                                if (newents->prev != tmpent)
                   1251:                                        panic("uvm_map_replace4");
                   1252:                        }
                   1253:                        tmpent = tmpent->next;
                   1254:                }
                   1255:                if (nent != nnewents)
                   1256:                        panic("uvm_map_replace5");
                   1257:        }
                   1258: #endif
                   1259:
                   1260:        /*
                   1261:         * map entry is a valid blank!   replace it.   (this does all the
                   1262:         * work of map entry link/unlink...).
                   1263:         */
                   1264:
                   1265:        if (newents) {
                   1266:
                   1267:                last = newents->prev;           /* we expect this */
                   1268:
                   1269:                /* critical: flush stale hints out of map */
                   1270:                SAVE_HINT(map, newents);
                   1271:                if (map->first_free == oldent)
                   1272:                        map->first_free = last;
                   1273:
                   1274:                last->next = oldent->next;
                   1275:                last->next->prev = last;
                   1276:                newents->prev = oldent->prev;
                   1277:                newents->prev->next = newents;
                   1278:                map->nentries = map->nentries + (nnewents - 1);
                   1279:
                   1280:        } else {
                   1281:
                   1282:                /* critical: flush stale hints out of map */
                   1283:                SAVE_HINT(map, oldent->prev);
                   1284:                if (map->first_free == oldent)
                   1285:                        map->first_free = oldent->prev;
                   1286:
                   1287:                /* NULL list of new entries: just remove the old one */
                   1288:                uvm_map_entry_unlink(map, oldent);
                   1289:        }
                   1290:
                   1291:
                   1292:        /*
                   1293:         * now we can free the old blank entry, unlock the map and return.
                   1294:         */
1.1       mrg      1295:
1.10      mrg      1296:        uvm_mapent_free(oldent);
                   1297:        return(TRUE);
1.1       mrg      1298: }
                   1299:
                   1300: /*
                   1301:  * uvm_map_extract: extract a mapping from a map and put it somewhere
                   1302:  *     (maybe removing the old mapping)
                   1303:  *
                   1304:  * => maps should be unlocked (we will write lock them)
                   1305:  * => returns 0 on success, error code otherwise
                   1306:  * => start must be page aligned
                   1307:  * => len must be page sized
                   1308:  * => flags:
                   1309:  *      UVM_EXTRACT_REMOVE: remove mappings from srcmap
                   1310:  *      UVM_EXTRACT_CONTIG: abort if unmapped area (advisory only)
                   1311:  *      UVM_EXTRACT_QREF: for a temporary extraction do quick obj refs
                   1312:  *      UVM_EXTRACT_FIXPROT: set prot to maxprot as we go
                   1313:  *    >>>NOTE: if you set REMOVE, you are not allowed to use CONTIG or QREF!<<<
                   1314:  *    >>>NOTE: QREF's must be unmapped via the QREF path, thus should only
                   1315:  *             be used from within the kernel in a kernel level map <<<
                   1316:  */
                   1317:
1.10      mrg      1318: int
                   1319: uvm_map_extract(srcmap, start, len, dstmap, dstaddrp, flags)
                   1320:        vm_map_t srcmap, dstmap;
1.24      eeh      1321:        vaddr_t start, *dstaddrp;
                   1322:        vsize_t len;
1.10      mrg      1323:        int flags;
                   1324: {
1.24      eeh      1325:        vaddr_t dstaddr, end, newend, oldoffset, fudge, orig_fudge,
1.10      mrg      1326:            oldstart;
                   1327:        vm_map_entry_t chain, endchain, entry, orig_entry, newentry, deadentry;
1.20      chuck    1328:        vm_map_entry_t oldentry;
1.24      eeh      1329:        vsize_t elen;
1.10      mrg      1330:        int nchain, error, copy_ok;
                   1331:        UVMHIST_FUNC("uvm_map_extract"); UVMHIST_CALLED(maphist);
                   1332:        UVMHIST_LOG(maphist,"(srcmap=0x%x,start=0x%x, len=0x%x", srcmap, start,
                   1333:            len,0);
                   1334:        UVMHIST_LOG(maphist," ...,dstmap=0x%x, flags=0x%x)", dstmap,flags,0,0);
                   1335:
                   1336: #ifdef DIAGNOSTIC
                   1337:        /*
                   1338:         * step 0: sanity check: start must be on a page boundary, length
                   1339:         * must be page sized.  can't ask for CONTIG/QREF if you asked for
                   1340:         * REMOVE.
                   1341:         */
                   1342:        if ((start & PAGE_MASK) || (len & PAGE_MASK))
                   1343:                panic("uvm_map_extract1");
                   1344:        if (flags & UVM_EXTRACT_REMOVE)
                   1345:                if (flags & (UVM_EXTRACT_CONTIG|UVM_EXTRACT_QREF))
                   1346:                        panic("uvm_map_extract2");
                   1347: #endif
                   1348:
                   1349:
                   1350:        /*
                   1351:         * step 1: reserve space in the target map for the extracted area
                   1352:         */
                   1353:
1.76      pk       1354:        dstaddr = vm_map_min(dstmap);
1.10      mrg      1355:        if (uvm_map_reserve(dstmap, len, start, &dstaddr) == FALSE)
                   1356:                return(ENOMEM);
                   1357:        *dstaddrp = dstaddr;    /* pass address back to caller */
                   1358:        UVMHIST_LOG(maphist, "  dstaddr=0x%x", dstaddr,0,0,0);
                   1359:
                   1360:
                   1361:        /*
                   1362:         * step 2: setup for the extraction process loop by init'ing the
                   1363:         * map entry chain, locking src map, and looking up the first useful
                   1364:         * entry in the map.
                   1365:         */
1.1       mrg      1366:
1.10      mrg      1367:        end = start + len;
                   1368:        newend = dstaddr + len;
                   1369:        chain = endchain = NULL;
                   1370:        nchain = 0;
                   1371:        vm_map_lock(srcmap);
                   1372:
                   1373:        if (uvm_map_lookup_entry(srcmap, start, &entry)) {
                   1374:
                   1375:                /* "start" is within an entry */
                   1376:                if (flags & UVM_EXTRACT_QREF) {
                   1377:                        /*
                   1378:                         * for quick references we don't clip the entry, so
                   1379:                         * the entry may map space "before" the starting
                   1380:                         * virtual address... this is the "fudge" factor
                   1381:                         * (which can be non-zero only the first time
                   1382:                         * through the "while" loop in step 3).
                   1383:                         */
                   1384:                        fudge = start - entry->start;
                   1385:                } else {
                   1386:                        /*
                   1387:                         * normal reference: we clip the map to fit (thus
                   1388:                         * fudge is zero)
                   1389:                         */
                   1390:                        UVM_MAP_CLIP_START(srcmap, entry, start);
                   1391:                        SAVE_HINT(srcmap, entry->prev);
                   1392:                        fudge = 0;
                   1393:                }
1.1       mrg      1394:
1.10      mrg      1395:        } else {
                   1396:
                   1397:                /* "start" is not within an entry ... skip to next entry */
                   1398:                if (flags & UVM_EXTRACT_CONTIG) {
                   1399:                        error = EINVAL;
                   1400:                        goto bad;    /* definite hole here ... */
                   1401:                }
1.1       mrg      1402:
1.10      mrg      1403:                entry = entry->next;
                   1404:                fudge = 0;
                   1405:        }
                   1406:        /* save values from srcmap for step 6 */
                   1407:        orig_entry = entry;
                   1408:        orig_fudge = fudge;
1.1       mrg      1409:
                   1410:
1.10      mrg      1411:        /*
                   1412:         * step 3: now start looping through the map entries, extracting
                   1413:         * as we go.
                   1414:         */
1.1       mrg      1415:
1.10      mrg      1416:        while (entry->start < end && entry != &srcmap->header) {
                   1417:
                   1418:                /* if we are not doing a quick reference, clip it */
                   1419:                if ((flags & UVM_EXTRACT_QREF) == 0)
                   1420:                        UVM_MAP_CLIP_END(srcmap, entry, end);
                   1421:
                   1422:                /* clear needs_copy (allow chunking) */
                   1423:                if (UVM_ET_ISNEEDSCOPY(entry)) {
                   1424:                        if (fudge)
                   1425:                                oldstart = entry->start;
                   1426:                        else
                   1427:                                oldstart = 0;   /* XXX: gcc */
                   1428:                        amap_copy(srcmap, entry, M_NOWAIT, TRUE, start, end);
                   1429:                        if (UVM_ET_ISNEEDSCOPY(entry)) {  /* failed? */
                   1430:                                error = ENOMEM;
                   1431:                                goto bad;
                   1432:                        }
                   1433:                        /* amap_copy could clip (during chunk)!  update fudge */
                   1434:                        if (fudge) {
                   1435:                                fudge = fudge - (entry->start - oldstart);
                   1436:                                orig_fudge = fudge;
                   1437:                        }
                   1438:                }
1.1       mrg      1439:
1.10      mrg      1440:                /* calculate the offset of this from "start" */
                   1441:                oldoffset = (entry->start + fudge) - start;
1.1       mrg      1442:
1.10      mrg      1443:                /* allocate a new map entry */
                   1444:                newentry = uvm_mapent_alloc(dstmap);
                   1445:                if (newentry == NULL) {
                   1446:                        error = ENOMEM;
                   1447:                        goto bad;
                   1448:                }
                   1449:
                   1450:                /* set up new map entry */
                   1451:                newentry->next = NULL;
                   1452:                newentry->prev = endchain;
                   1453:                newentry->start = dstaddr + oldoffset;
                   1454:                newentry->end =
                   1455:                    newentry->start + (entry->end - (entry->start + fudge));
1.37      chs      1456:                if (newentry->end > newend || newentry->end < newentry->start)
1.10      mrg      1457:                        newentry->end = newend;
                   1458:                newentry->object.uvm_obj = entry->object.uvm_obj;
                   1459:                if (newentry->object.uvm_obj) {
                   1460:                        if (newentry->object.uvm_obj->pgops->pgo_reference)
                   1461:                                newentry->object.uvm_obj->pgops->
                   1462:                                    pgo_reference(newentry->object.uvm_obj);
                   1463:                                newentry->offset = entry->offset + fudge;
                   1464:                } else {
                   1465:                        newentry->offset = 0;
                   1466:                }
                   1467:                newentry->etype = entry->etype;
                   1468:                newentry->protection = (flags & UVM_EXTRACT_FIXPROT) ?
                   1469:                        entry->max_protection : entry->protection;
                   1470:                newentry->max_protection = entry->max_protection;
                   1471:                newentry->inheritance = entry->inheritance;
                   1472:                newentry->wired_count = 0;
                   1473:                newentry->aref.ar_amap = entry->aref.ar_amap;
                   1474:                if (newentry->aref.ar_amap) {
1.34      chuck    1475:                        newentry->aref.ar_pageoff =
                   1476:                            entry->aref.ar_pageoff + (fudge >> PAGE_SHIFT);
1.10      mrg      1477:                        amap_ref(newentry, AMAP_SHARED |
                   1478:                            ((flags & UVM_EXTRACT_QREF) ? AMAP_REFALL : 0));
                   1479:                } else {
1.34      chuck    1480:                        newentry->aref.ar_pageoff = 0;
1.10      mrg      1481:                }
                   1482:                newentry->advice = entry->advice;
                   1483:
                   1484:                /* now link it on the chain */
                   1485:                nchain++;
                   1486:                if (endchain == NULL) {
                   1487:                        chain = endchain = newentry;
                   1488:                } else {
                   1489:                        endchain->next = newentry;
                   1490:                        endchain = newentry;
                   1491:                }
                   1492:
                   1493:                /* end of 'while' loop! */
                   1494:                if ((flags & UVM_EXTRACT_CONTIG) && entry->end < end &&
                   1495:                    (entry->next == &srcmap->header ||
                   1496:                    entry->next->start != entry->end)) {
                   1497:                        error = EINVAL;
                   1498:                        goto bad;
                   1499:                }
                   1500:                entry = entry->next;
                   1501:                fudge = 0;
                   1502:        }
                   1503:
                   1504:
                   1505:        /*
                   1506:         * step 4: close off chain (in format expected by uvm_map_replace)
                   1507:         */
                   1508:
                   1509:        if (chain)
                   1510:                chain->prev = endchain;
                   1511:
                   1512:
                   1513:        /*
                   1514:         * step 5: attempt to lock the dest map so we can pmap_copy.
                   1515:         * note usage of copy_ok:
                   1516:         *   1 => dstmap locked, pmap_copy ok, and we "replace" here (step 5)
                   1517:         *   0 => dstmap unlocked, NO pmap_copy, and we will "replace" in step 7
                   1518:         */
                   1519:
                   1520:        if (srcmap == dstmap || vm_map_lock_try(dstmap) == TRUE) {
                   1521:
                   1522:                copy_ok = 1;
                   1523:                if (!uvm_map_replace(dstmap, dstaddr, dstaddr+len, chain,
                   1524:                    nchain)) {
                   1525:                        if (srcmap != dstmap)
                   1526:                                vm_map_unlock(dstmap);
                   1527:                        error = EIO;
                   1528:                        goto bad;
                   1529:                }
                   1530:
                   1531:        } else {
                   1532:
                   1533:                copy_ok = 0;
                   1534:                /* replace defered until step 7 */
                   1535:
                   1536:        }
                   1537:
                   1538:
                   1539:        /*
                   1540:         * step 6: traverse the srcmap a second time to do the following:
                   1541:         *  - if we got a lock on the dstmap do pmap_copy
                   1542:         *  - if UVM_EXTRACT_REMOVE remove the entries
                   1543:         * we make use of orig_entry and orig_fudge (saved in step 2)
                   1544:         */
                   1545:
                   1546:        if (copy_ok || (flags & UVM_EXTRACT_REMOVE)) {
                   1547:
                   1548:                /* purge possible stale hints from srcmap */
                   1549:                if (flags & UVM_EXTRACT_REMOVE) {
                   1550:                        SAVE_HINT(srcmap, orig_entry->prev);
                   1551:                        if (srcmap->first_free->start >= start)
                   1552:                                srcmap->first_free = orig_entry->prev;
                   1553:                }
                   1554:
                   1555:                entry = orig_entry;
                   1556:                fudge = orig_fudge;
                   1557:                deadentry = NULL;       /* for UVM_EXTRACT_REMOVE */
                   1558:
                   1559:                while (entry->start < end && entry != &srcmap->header) {
                   1560:
                   1561:                        if (copy_ok) {
1.74      thorpej  1562:                                oldoffset = (entry->start + fudge) - start;
                   1563:                                elen = min(end, entry->end) -
                   1564:                                    (entry->start + fudge);
                   1565:                                pmap_copy(dstmap->pmap, srcmap->pmap,
                   1566:                                    dstaddr + oldoffset, elen,
                   1567:                                    entry->start + fudge);
1.10      mrg      1568:                        }
                   1569:
1.74      thorpej  1570:                        /* we advance "entry" in the following if statement */
1.10      mrg      1571:                        if (flags & UVM_EXTRACT_REMOVE) {
1.20      chuck    1572:                                pmap_remove(srcmap->pmap, entry->start,
                   1573:                                                entry->end);
                   1574:                                oldentry = entry;       /* save entry */
                   1575:                                entry = entry->next;    /* advance */
                   1576:                                uvm_map_entry_unlink(srcmap, oldentry);
                   1577:                                                        /* add to dead list */
                   1578:                                oldentry->next = deadentry;
                   1579:                                deadentry = oldentry;
                   1580:                        } else {
                   1581:                                entry = entry->next;            /* advance */
1.10      mrg      1582:                        }
                   1583:
                   1584:                        /* end of 'while' loop */
                   1585:                        fudge = 0;
                   1586:                }
                   1587:
                   1588:                /*
                   1589:                 * unlock dstmap.  we will dispose of deadentry in
                   1590:                 * step 7 if needed
                   1591:                 */
                   1592:                if (copy_ok && srcmap != dstmap)
                   1593:                        vm_map_unlock(dstmap);
                   1594:
                   1595:        }
                   1596:        else
                   1597:                deadentry = NULL; /* XXX: gcc */
                   1598:
                   1599:        /*
                   1600:         * step 7: we are done with the source map, unlock.   if copy_ok
                   1601:         * is 0 then we have not replaced the dummy mapping in dstmap yet
                   1602:         * and we need to do so now.
                   1603:         */
                   1604:
                   1605:        vm_map_unlock(srcmap);
                   1606:        if ((flags & UVM_EXTRACT_REMOVE) && deadentry)
                   1607:                uvm_unmap_detach(deadentry, 0);   /* dispose of old entries */
                   1608:
                   1609:        /* now do the replacement if we didn't do it in step 5 */
                   1610:        if (copy_ok == 0) {
                   1611:                vm_map_lock(dstmap);
                   1612:                error = uvm_map_replace(dstmap, dstaddr, dstaddr+len, chain,
                   1613:                    nchain);
                   1614:                vm_map_unlock(dstmap);
                   1615:
                   1616:                if (error == FALSE) {
                   1617:                        error = EIO;
                   1618:                        goto bad2;
                   1619:                }
                   1620:        }
                   1621:
                   1622:        /*
                   1623:         * done!
                   1624:         */
                   1625:        return(0);
                   1626:
                   1627:        /*
                   1628:         * bad: failure recovery
                   1629:         */
                   1630: bad:
                   1631:        vm_map_unlock(srcmap);
                   1632: bad2:                  /* src already unlocked */
                   1633:        if (chain)
                   1634:                uvm_unmap_detach(chain,
                   1635:                    (flags & UVM_EXTRACT_QREF) ? AMAP_REFALL : 0);
1.29      chuck    1636:        uvm_unmap(dstmap, dstaddr, dstaddr+len);   /* ??? */
1.10      mrg      1637:        return(error);
                   1638: }
                   1639:
                   1640: /* end of extraction functions */
1.1       mrg      1641:
                   1642: /*
                   1643:  * uvm_map_submap: punch down part of a map into a submap
                   1644:  *
                   1645:  * => only the kernel_map is allowed to be submapped
                   1646:  * => the purpose of submapping is to break up the locking granularity
                   1647:  *     of a larger map
                   1648:  * => the range specified must have been mapped previously with a uvm_map()
                   1649:  *     call [with uobj==NULL] to create a blank map entry in the main map.
                   1650:  *     [And it had better still be blank!]
                   1651:  * => maps which contain submaps should never be copied or forked.
                   1652:  * => to remove a submap, use uvm_unmap() on the main map
                   1653:  *     and then uvm_map_deallocate() the submap.
                   1654:  * => main map must be unlocked.
                   1655:  * => submap must have been init'd and have a zero reference count.
                   1656:  *     [need not be locked as we don't actually reference it]
                   1657:  */
1.10      mrg      1658:
                   1659: int
                   1660: uvm_map_submap(map, start, end, submap)
                   1661:        vm_map_t map, submap;
1.24      eeh      1662:        vaddr_t start, end;
1.10      mrg      1663: {
                   1664:        vm_map_entry_t entry;
                   1665:        int result;
                   1666:        UVMHIST_FUNC("uvm_map_submap"); UVMHIST_CALLED(maphist);
1.1       mrg      1667:
1.10      mrg      1668:        vm_map_lock(map);
1.1       mrg      1669:
1.10      mrg      1670:        VM_MAP_RANGE_CHECK(map, start, end);
                   1671:
                   1672:        if (uvm_map_lookup_entry(map, start, &entry)) {
                   1673:                UVM_MAP_CLIP_START(map, entry, start);
                   1674:                UVM_MAP_CLIP_END(map, entry, end);              /* to be safe */
                   1675:        }
                   1676:        else {
                   1677:                entry = NULL;
                   1678:        }
1.1       mrg      1679:
1.10      mrg      1680:        if (entry != NULL &&
                   1681:            entry->start == start && entry->end == end &&
                   1682:            entry->object.uvm_obj == NULL && entry->aref.ar_amap == NULL &&
                   1683:            !UVM_ET_ISCOPYONWRITE(entry) && !UVM_ET_ISNEEDSCOPY(entry)) {
                   1684:
                   1685:                /*
                   1686:                 * doit!
                   1687:                 */
1.29      chuck    1688:                entry->etype |= UVM_ET_SUBMAP;
1.10      mrg      1689:                entry->object.sub_map = submap;
                   1690:                entry->offset = 0;
                   1691:                uvm_map_reference(submap);
                   1692:                result = KERN_SUCCESS;
                   1693:        } else {
                   1694:                result = KERN_INVALID_ARGUMENT;
                   1695:        }
                   1696:        vm_map_unlock(map);
1.1       mrg      1697:
1.10      mrg      1698:        return(result);
1.1       mrg      1699: }
                   1700:
                   1701:
                   1702: /*
                   1703:  * uvm_map_protect: change map protection
                   1704:  *
                   1705:  * => set_max means set max_protection.
                   1706:  * => map must be unlocked.
                   1707:  */
                   1708:
1.36      mycroft  1709: #define MASK(entry)     (UVM_ET_ISCOPYONWRITE(entry) ? \
                   1710:                         ~VM_PROT_WRITE : VM_PROT_ALL)
1.1       mrg      1711: #define max(a,b)        ((a) > (b) ? (a) : (b))
                   1712:
1.10      mrg      1713: int
                   1714: uvm_map_protect(map, start, end, new_prot, set_max)
                   1715:        vm_map_t map;
1.24      eeh      1716:        vaddr_t start, end;
1.10      mrg      1717:        vm_prot_t new_prot;
                   1718:        boolean_t set_max;
                   1719: {
                   1720:        vm_map_entry_t current, entry;
1.65      thorpej  1721:        int rv = KERN_SUCCESS;
1.10      mrg      1722:        UVMHIST_FUNC("uvm_map_protect"); UVMHIST_CALLED(maphist);
                   1723:        UVMHIST_LOG(maphist,"(map=0x%x,start=0x%x,end=0x%x,new_prot=0x%x)",
1.1       mrg      1724:        map, start, end, new_prot);
1.10      mrg      1725:
                   1726:        vm_map_lock(map);
1.1       mrg      1727:
1.10      mrg      1728:        VM_MAP_RANGE_CHECK(map, start, end);
1.1       mrg      1729:
1.10      mrg      1730:        if (uvm_map_lookup_entry(map, start, &entry)) {
                   1731:                UVM_MAP_CLIP_START(map, entry, start);
                   1732:        } else {
                   1733:                entry = entry->next;
                   1734:        }
                   1735:
1.1       mrg      1736:        /*
1.10      mrg      1737:         * make a first pass to check for protection violations.
1.1       mrg      1738:         */
                   1739:
1.10      mrg      1740:        current = entry;
                   1741:        while ((current != &map->header) && (current->start < end)) {
1.65      thorpej  1742:                if (UVM_ET_ISSUBMAP(current)) {
                   1743:                        rv = KERN_INVALID_ARGUMENT;
                   1744:                        goto out;
                   1745:                }
1.10      mrg      1746:                if ((new_prot & current->max_protection) != new_prot) {
1.65      thorpej  1747:                        rv = KERN_PROTECTION_FAILURE;
                   1748:                        goto out;
1.10      mrg      1749:                }
1.65      thorpej  1750:                current = current->next;
1.10      mrg      1751:        }
                   1752:
                   1753:        /* go back and fix up protections (no need to clip this time). */
                   1754:
                   1755:        current = entry;
                   1756:
                   1757:        while ((current != &map->header) && (current->start < end)) {
                   1758:                vm_prot_t old_prot;
                   1759:
                   1760:                UVM_MAP_CLIP_END(map, current, end);
                   1761:
                   1762:                old_prot = current->protection;
                   1763:                if (set_max)
                   1764:                        current->protection =
                   1765:                            (current->max_protection = new_prot) & old_prot;
                   1766:                else
                   1767:                        current->protection = new_prot;
                   1768:
                   1769:                /*
                   1770:                 * update physical map if necessary.  worry about copy-on-write
                   1771:                 * here -- CHECK THIS XXX
                   1772:                 */
                   1773:
                   1774:                if (current->protection != old_prot) {
1.29      chuck    1775:                        /* update pmap! */
                   1776:                        pmap_protect(map->pmap, current->start, current->end,
                   1777:                            current->protection & MASK(entry));
1.65      thorpej  1778:                }
1.10      mrg      1779:
1.65      thorpej  1780:                /*
                   1781:                 * If the map is configured to lock any future mappings,
                   1782:                 * wire this entry now if the old protection was VM_PROT_NONE
                   1783:                 * and the new protection is not VM_PROT_NONE.
                   1784:                 */
                   1785:
                   1786:                if ((map->flags & VM_MAP_WIREFUTURE) != 0 &&
                   1787:                    VM_MAPENT_ISWIRED(entry) == 0 &&
                   1788:                    old_prot == VM_PROT_NONE &&
                   1789:                    new_prot != VM_PROT_NONE) {
                   1790:                        if (uvm_map_pageable(map, entry->start,
                   1791:                            entry->end, FALSE,
                   1792:                            UVM_LK_ENTER|UVM_LK_EXIT) != KERN_SUCCESS) {
                   1793:                                /*
                   1794:                                 * If locking the entry fails, remember the
                   1795:                                 * error if it's the first one.  Note we
                   1796:                                 * still continue setting the protection in
                   1797:                                 * the map, but will return the resource
                   1798:                                 * shortage condition regardless.
                   1799:                                 *
                   1800:                                 * XXX Ignore what the actual error is,
                   1801:                                 * XXX just call it a resource shortage
                   1802:                                 * XXX so that it doesn't get confused
                   1803:                                 * XXX what uvm_map_protect() itself would
                   1804:                                 * XXX normally return.
                   1805:                                 */
                   1806:                                rv = KERN_RESOURCE_SHORTAGE;
                   1807:                        }
1.10      mrg      1808:                }
1.65      thorpej  1809:
1.10      mrg      1810:                current = current->next;
                   1811:        }
                   1812:
1.65      thorpej  1813:  out:
1.10      mrg      1814:        vm_map_unlock(map);
1.65      thorpej  1815:        UVMHIST_LOG(maphist, "<- done, rv=%d",rv,0,0,0);
                   1816:        return (rv);
1.1       mrg      1817: }
                   1818:
                   1819: #undef  max
                   1820: #undef  MASK
                   1821:
                   1822: /*
                   1823:  * uvm_map_inherit: set inheritance code for range of addrs in map.
                   1824:  *
                   1825:  * => map must be unlocked
                   1826:  * => note that the inherit code is used during a "fork".  see fork
                   1827:  *     code for details.
                   1828:  */
                   1829:
1.10      mrg      1830: int
                   1831: uvm_map_inherit(map, start, end, new_inheritance)
                   1832:        vm_map_t map;
1.24      eeh      1833:        vaddr_t start;
                   1834:        vaddr_t end;
1.10      mrg      1835:        vm_inherit_t new_inheritance;
                   1836: {
                   1837:        vm_map_entry_t entry, temp_entry;
                   1838:        UVMHIST_FUNC("uvm_map_inherit"); UVMHIST_CALLED(maphist);
                   1839:        UVMHIST_LOG(maphist,"(map=0x%x,start=0x%x,end=0x%x,new_inh=0x%x)",
                   1840:            map, start, end, new_inheritance);
                   1841:
                   1842:        switch (new_inheritance) {
1.77.2.1! fvdl     1843:        case MAP_INHERIT_NONE:
        !          1844:        case MAP_INHERIT_COPY:
        !          1845:        case MAP_INHERIT_SHARE:
1.10      mrg      1846:                break;
                   1847:        default:
                   1848:                UVMHIST_LOG(maphist,"<- done (INVALID ARG)",0,0,0,0);
1.38      mrg      1849:                return (KERN_INVALID_ARGUMENT);
1.10      mrg      1850:        }
1.1       mrg      1851:
1.10      mrg      1852:        vm_map_lock(map);
                   1853:
                   1854:        VM_MAP_RANGE_CHECK(map, start, end);
                   1855:
                   1856:        if (uvm_map_lookup_entry(map, start, &temp_entry)) {
                   1857:                entry = temp_entry;
                   1858:                UVM_MAP_CLIP_START(map, entry, start);
                   1859:        }  else {
                   1860:                entry = temp_entry->next;
                   1861:        }
                   1862:
                   1863:        while ((entry != &map->header) && (entry->start < end)) {
                   1864:                UVM_MAP_CLIP_END(map, entry, end);
                   1865:
                   1866:                entry->inheritance = new_inheritance;
                   1867:
                   1868:                entry = entry->next;
                   1869:        }
                   1870:
                   1871:        vm_map_unlock(map);
                   1872:        UVMHIST_LOG(maphist,"<- done (OK)",0,0,0,0);
                   1873:        return(KERN_SUCCESS);
1.41      mrg      1874: }
                   1875:
                   1876: /*
                   1877:  * uvm_map_advice: set advice code for range of addrs in map.
                   1878:  *
                   1879:  * => map must be unlocked
                   1880:  */
                   1881:
                   1882: int
                   1883: uvm_map_advice(map, start, end, new_advice)
                   1884:        vm_map_t map;
                   1885:        vaddr_t start;
                   1886:        vaddr_t end;
                   1887:        int new_advice;
                   1888: {
                   1889:        vm_map_entry_t entry, temp_entry;
                   1890:        UVMHIST_FUNC("uvm_map_advice"); UVMHIST_CALLED(maphist);
                   1891:        UVMHIST_LOG(maphist,"(map=0x%x,start=0x%x,end=0x%x,new_adv=0x%x)",
                   1892:            map, start, end, new_advice);
                   1893:
                   1894:        vm_map_lock(map);
                   1895:
                   1896:        VM_MAP_RANGE_CHECK(map, start, end);
                   1897:
                   1898:        if (uvm_map_lookup_entry(map, start, &temp_entry)) {
                   1899:                entry = temp_entry;
                   1900:                UVM_MAP_CLIP_START(map, entry, start);
                   1901:        } else {
                   1902:                entry = temp_entry->next;
                   1903:        }
1.61      thorpej  1904:
                   1905:        /*
                   1906:         * XXXJRT: disallow holes?
                   1907:         */
                   1908:
1.41      mrg      1909:        while ((entry != &map->header) && (entry->start < end)) {
                   1910:                UVM_MAP_CLIP_END(map, entry, end);
                   1911:
                   1912:                switch (new_advice) {
                   1913:                case MADV_NORMAL:
                   1914:                case MADV_RANDOM:
                   1915:                case MADV_SEQUENTIAL:
                   1916:                        /* nothing special here */
                   1917:                        break;
                   1918:
                   1919:                default:
1.50      mrg      1920:                        vm_map_unlock(map);
1.41      mrg      1921:                        UVMHIST_LOG(maphist,"<- done (INVALID ARG)",0,0,0,0);
                   1922:                        return (KERN_INVALID_ARGUMENT);
                   1923:                }
                   1924:
                   1925:
                   1926:                entry->advice = new_advice;
                   1927:
                   1928:                entry = entry->next;
                   1929:        }
                   1930:
                   1931:        vm_map_unlock(map);
                   1932:        UVMHIST_LOG(maphist,"<- done (OK)",0,0,0,0);
                   1933:        return (KERN_SUCCESS);
1.1       mrg      1934: }
                   1935:
                   1936: /*
                   1937:  * uvm_map_pageable: sets the pageability of a range in a map.
                   1938:  *
1.56      thorpej  1939:  * => wires map entries.  should not be used for transient page locking.
                   1940:  *     for that, use uvm_fault_wire()/uvm_fault_unwire() (see uvm_vslock()).
1.1       mrg      1941:  * => regions sepcified as not pageable require lock-down (wired) memory
                   1942:  *     and page tables.
1.59      thorpej  1943:  * => map must never be read-locked
                   1944:  * => if islocked is TRUE, map is already write-locked
                   1945:  * => we always unlock the map, since we must downgrade to a read-lock
                   1946:  *     to call uvm_fault_wire()
1.1       mrg      1947:  * => XXXCDC: check this and try and clean it up.
                   1948:  */
                   1949:
1.19      kleink   1950: int
1.64      thorpej  1951: uvm_map_pageable(map, start, end, new_pageable, lockflags)
1.11      mrg      1952:        vm_map_t map;
1.24      eeh      1953:        vaddr_t start, end;
1.64      thorpej  1954:        boolean_t new_pageable;
                   1955:        int lockflags;
1.1       mrg      1956: {
1.54      thorpej  1957:        vm_map_entry_t entry, start_entry, failed_entry;
1.10      mrg      1958:        int rv;
1.60      thorpej  1959: #ifdef DIAGNOSTIC
                   1960:        u_int timestamp_save;
                   1961: #endif
1.10      mrg      1962:        UVMHIST_FUNC("uvm_map_pageable"); UVMHIST_CALLED(maphist);
                   1963:        UVMHIST_LOG(maphist,"(map=0x%x,start=0x%x,end=0x%x,new_pageable=0x%x)",
1.1       mrg      1964:        map, start, end, new_pageable);
                   1965:
1.45      thorpej  1966: #ifdef DIAGNOSTIC
                   1967:        if ((map->flags & VM_MAP_PAGEABLE) == 0)
                   1968:                panic("uvm_map_pageable: map %p not pageable", map);
                   1969: #endif
                   1970:
1.64      thorpej  1971:        if ((lockflags & UVM_LK_ENTER) == 0)
1.59      thorpej  1972:                vm_map_lock(map);
1.64      thorpej  1973:
1.10      mrg      1974:        VM_MAP_RANGE_CHECK(map, start, end);
                   1975:
                   1976:        /*
                   1977:         * only one pageability change may take place at one time, since
                   1978:         * uvm_fault_wire assumes it will be called only once for each
                   1979:         * wiring/unwiring.  therefore, we have to make sure we're actually
                   1980:         * changing the pageability for the entire region.  we do so before
                   1981:         * making any changes.
                   1982:         */
                   1983:
                   1984:        if (uvm_map_lookup_entry(map, start, &start_entry) == FALSE) {
1.64      thorpej  1985:                if ((lockflags & UVM_LK_EXIT) == 0)
                   1986:                        vm_map_unlock(map);
1.10      mrg      1987:
                   1988:                UVMHIST_LOG(maphist,"<- done (INVALID ARG)",0,0,0,0);
                   1989:                return (KERN_INVALID_ADDRESS);
                   1990:        }
                   1991:        entry = start_entry;
                   1992:
                   1993:        /*
                   1994:         * handle wiring and unwiring seperately.
                   1995:         */
1.1       mrg      1996:
1.56      thorpej  1997:        if (new_pageable) {             /* unwire */
1.10      mrg      1998:                UVM_MAP_CLIP_START(map, entry, start);
                   1999:                /*
                   2000:                 * unwiring.  first ensure that the range to be unwired is
                   2001:                 * really wired down and that there are no holes.
                   2002:                 */
                   2003:                while ((entry != &map->header) && (entry->start < end)) {
                   2004:                        if (entry->wired_count == 0 ||
                   2005:                            (entry->end < end &&
1.55      thorpej  2006:                             (entry->next == &map->header ||
                   2007:                              entry->next->start > entry->end))) {
1.64      thorpej  2008:                                if ((lockflags & UVM_LK_EXIT) == 0)
                   2009:                                        vm_map_unlock(map);
1.10      mrg      2010:                                UVMHIST_LOG(maphist,
                   2011:                                    "<- done (INVALID UNWIRE ARG)",0,0,0,0);
                   2012:                                return (KERN_INVALID_ARGUMENT);
                   2013:                        }
                   2014:                        entry = entry->next;
                   2015:                }
                   2016:
                   2017:                /*
1.56      thorpej  2018:                 * POSIX 1003.1b - a single munlock call unlocks a region,
                   2019:                 * regardless of the number of mlock calls made on that
                   2020:                 * region.
1.10      mrg      2021:                 */
                   2022:                entry = start_entry;
                   2023:                while ((entry != &map->header) && (entry->start < end)) {
                   2024:                        UVM_MAP_CLIP_END(map, entry, end);
1.56      thorpej  2025:                        if (VM_MAPENT_ISWIRED(entry))
1.10      mrg      2026:                                uvm_map_entry_unwire(map, entry);
                   2027:                        entry = entry->next;
                   2028:                }
1.64      thorpej  2029:                if ((lockflags & UVM_LK_EXIT) == 0)
                   2030:                        vm_map_unlock(map);
1.10      mrg      2031:                UVMHIST_LOG(maphist,"<- done (OK UNWIRE)",0,0,0,0);
                   2032:                return(KERN_SUCCESS);
                   2033:
                   2034:                /*
                   2035:                 * end of unwire case!
                   2036:                 */
                   2037:        }
                   2038:
                   2039:        /*
                   2040:         * wire case: in two passes [XXXCDC: ugly block of code here]
                   2041:         *
                   2042:         * 1: holding the write lock, we create any anonymous maps that need
                   2043:         *    to be created.  then we clip each map entry to the region to
                   2044:         *    be wired and increment its wiring count.
                   2045:         *
                   2046:         * 2: we downgrade to a read lock, and call uvm_fault_wire to fault
1.56      thorpej  2047:         *    in the pages for any newly wired area (wired_count == 1).
1.10      mrg      2048:         *
                   2049:         *    downgrading to a read lock for uvm_fault_wire avoids a possible
                   2050:         *    deadlock with another thread that may have faulted on one of
                   2051:         *    the pages to be wired (it would mark the page busy, blocking
                   2052:         *    us, then in turn block on the map lock that we hold).  because
                   2053:         *    of problems in the recursive lock package, we cannot upgrade
                   2054:         *    to a write lock in vm_map_lookup.  thus, any actions that
                   2055:         *    require the write lock must be done beforehand.  because we
                   2056:         *    keep the read lock on the map, the copy-on-write status of the
                   2057:         *    entries we modify here cannot change.
                   2058:         */
                   2059:
                   2060:        while ((entry != &map->header) && (entry->start < end)) {
1.55      thorpej  2061:                if (VM_MAPENT_ISWIRED(entry) == 0) { /* not already wired? */
1.10      mrg      2062:                        /*
                   2063:                         * perform actions of vm_map_lookup that need the
                   2064:                         * write lock on the map: create an anonymous map
                   2065:                         * for a copy-on-write region, or an anonymous map
1.29      chuck    2066:                         * for a zero-fill region.  (XXXCDC: submap case
                   2067:                         * ok?)
1.10      mrg      2068:                         */
1.29      chuck    2069:                        if (!UVM_ET_ISSUBMAP(entry)) {  /* not submap */
1.54      thorpej  2070:                                if (UVM_ET_ISNEEDSCOPY(entry) &&
                   2071:                                    ((entry->protection & VM_PROT_WRITE) ||
                   2072:                                     (entry->object.uvm_obj == NULL))) {
1.10      mrg      2073:                                        amap_copy(map, entry, M_WAITOK, TRUE,
                   2074:                                            start, end);
                   2075:                                        /* XXXCDC: wait OK? */
                   2076:                                }
                   2077:                        }
1.55      thorpej  2078:                }
1.10      mrg      2079:                UVM_MAP_CLIP_START(map, entry, start);
                   2080:                UVM_MAP_CLIP_END(map, entry, end);
                   2081:                entry->wired_count++;
                   2082:
                   2083:                /*
                   2084:                 * Check for holes
                   2085:                 */
1.54      thorpej  2086:                if (entry->protection == VM_PROT_NONE ||
                   2087:                    (entry->end < end &&
                   2088:                     (entry->next == &map->header ||
                   2089:                      entry->next->start > entry->end))) {
1.10      mrg      2090:                        /*
                   2091:                         * found one.  amap creation actions do not need to
                   2092:                         * be undone, but the wired counts need to be restored.
                   2093:                         */
                   2094:                        while (entry != &map->header && entry->end > start) {
                   2095:                                entry->wired_count--;
                   2096:                                entry = entry->prev;
                   2097:                        }
1.64      thorpej  2098:                        if ((lockflags & UVM_LK_EXIT) == 0)
                   2099:                                vm_map_unlock(map);
1.10      mrg      2100:                        UVMHIST_LOG(maphist,"<- done (INVALID WIRE)",0,0,0,0);
1.38      mrg      2101:                        return (KERN_INVALID_ARGUMENT);
1.10      mrg      2102:                }
                   2103:                entry = entry->next;
                   2104:        }
                   2105:
                   2106:        /*
                   2107:         * Pass 2.
                   2108:         */
1.51      thorpej  2109:
1.60      thorpej  2110: #ifdef DIAGNOSTIC
                   2111:        timestamp_save = map->timestamp;
                   2112: #endif
                   2113:        vm_map_busy(map);
1.51      thorpej  2114:        vm_map_downgrade(map);
1.10      mrg      2115:
                   2116:        rv = 0;
                   2117:        entry = start_entry;
                   2118:        while (entry != &map->header && entry->start < end) {
1.51      thorpej  2119:                if (entry->wired_count == 1) {
1.44      thorpej  2120:                        rv = uvm_fault_wire(map, entry->start, entry->end,
1.46      thorpej  2121:                            entry->protection);
1.10      mrg      2122:                        if (rv) {
1.51      thorpej  2123:                                /*
                   2124:                                 * wiring failed.  break out of the loop.
                   2125:                                 * we'll clean up the map below, once we
                   2126:                                 * have a write lock again.
                   2127:                                 */
                   2128:                                break;
1.10      mrg      2129:                        }
                   2130:                }
                   2131:                entry = entry->next;
                   2132:        }
                   2133:
1.52      thorpej  2134:        if (rv) {        /* failed? */
                   2135:                /*
                   2136:                 * Get back to an exclusive (write) lock.
                   2137:                 */
                   2138:                vm_map_upgrade(map);
1.60      thorpej  2139:                vm_map_unbusy(map);
                   2140:
                   2141: #ifdef DIAGNOSTIC
                   2142:                if (timestamp_save != map->timestamp)
                   2143:                        panic("uvm_map_pageable: stale map");
                   2144: #endif
1.10      mrg      2145:
1.51      thorpej  2146:                /*
                   2147:                 * first drop the wiring count on all the entries
                   2148:                 * which haven't actually been wired yet.
                   2149:                 */
1.54      thorpej  2150:                failed_entry = entry;
                   2151:                while (entry != &map->header && entry->start < end) {
1.51      thorpej  2152:                        entry->wired_count--;
1.54      thorpej  2153:                        entry = entry->next;
                   2154:                }
1.51      thorpej  2155:
                   2156:                /*
1.54      thorpej  2157:                 * now, unwire all the entries that were successfully
                   2158:                 * wired above.
1.51      thorpej  2159:                 */
1.54      thorpej  2160:                entry = start_entry;
                   2161:                while (entry != failed_entry) {
                   2162:                        entry->wired_count--;
1.55      thorpej  2163:                        if (VM_MAPENT_ISWIRED(entry) == 0)
1.54      thorpej  2164:                                uvm_map_entry_unwire(map, entry);
                   2165:                        entry = entry->next;
                   2166:                }
1.64      thorpej  2167:                if ((lockflags & UVM_LK_EXIT) == 0)
                   2168:                        vm_map_unlock(map);
1.10      mrg      2169:                UVMHIST_LOG(maphist, "<- done (RV=%d)", rv,0,0,0);
                   2170:                return(rv);
                   2171:        }
1.51      thorpej  2172:
1.52      thorpej  2173:        /* We are holding a read lock here. */
1.64      thorpej  2174:        if ((lockflags & UVM_LK_EXIT) == 0) {
                   2175:                vm_map_unbusy(map);
                   2176:                vm_map_unlock_read(map);
                   2177:        } else {
                   2178:                /*
                   2179:                 * Get back to an exclusive (write) lock.
                   2180:                 */
                   2181:                vm_map_upgrade(map);
                   2182:                vm_map_unbusy(map);
                   2183:        }
                   2184:
1.10      mrg      2185:        UVMHIST_LOG(maphist,"<- done (OK WIRE)",0,0,0,0);
                   2186:        return(KERN_SUCCESS);
1.1       mrg      2187: }
                   2188:
                   2189: /*
1.54      thorpej  2190:  * uvm_map_pageable_all: special case of uvm_map_pageable - affects
                   2191:  * all mapped regions.
                   2192:  *
                   2193:  * => map must not be locked.
                   2194:  * => if no flags are specified, all regions are unwired.
                   2195:  * => XXXJRT: has some of the same problems as uvm_map_pageable() above.
                   2196:  */
                   2197:
                   2198: int
                   2199: uvm_map_pageable_all(map, flags, limit)
                   2200:        vm_map_t map;
                   2201:        int flags;
                   2202:        vsize_t limit;
                   2203: {
                   2204:        vm_map_entry_t entry, failed_entry;
                   2205:        vsize_t size;
                   2206:        int rv;
1.60      thorpej  2207: #ifdef DIAGNOSTIC
                   2208:        u_int timestamp_save;
                   2209: #endif
1.54      thorpej  2210:        UVMHIST_FUNC("uvm_map_pageable_all"); UVMHIST_CALLED(maphist);
                   2211:        UVMHIST_LOG(maphist,"(map=0x%x,flags=0x%x)", map, flags, 0, 0);
                   2212:
                   2213: #ifdef DIAGNOSTIC
                   2214:        if ((map->flags & VM_MAP_PAGEABLE) == 0)
                   2215:                panic("uvm_map_pageable_all: map %p not pageable", map);
                   2216: #endif
                   2217:
                   2218:        vm_map_lock(map);
                   2219:
                   2220:        /*
                   2221:         * handle wiring and unwiring separately.
                   2222:         */
                   2223:
                   2224:        if (flags == 0) {                       /* unwire */
                   2225:                /*
1.56      thorpej  2226:                 * POSIX 1003.1b -- munlockall unlocks all regions,
                   2227:                 * regardless of how many times mlockall has been called.
1.54      thorpej  2228:                 */
                   2229:                for (entry = map->header.next; entry != &map->header;
                   2230:                     entry = entry->next) {
1.56      thorpej  2231:                        if (VM_MAPENT_ISWIRED(entry))
                   2232:                                uvm_map_entry_unwire(map, entry);
1.54      thorpej  2233:                }
1.61      thorpej  2234:                vm_map_modflags(map, 0, VM_MAP_WIREFUTURE);
1.54      thorpej  2235:                vm_map_unlock(map);
                   2236:                UVMHIST_LOG(maphist,"<- done (OK UNWIRE)",0,0,0,0);
                   2237:                return (KERN_SUCCESS);
                   2238:
                   2239:                /*
                   2240:                 * end of unwire case!
                   2241:                 */
                   2242:        }
                   2243:
                   2244:        if (flags & MCL_FUTURE) {
                   2245:                /*
                   2246:                 * must wire all future mappings; remember this.
                   2247:                 */
1.61      thorpej  2248:                vm_map_modflags(map, VM_MAP_WIREFUTURE, 0);
1.54      thorpej  2249:        }
                   2250:
                   2251:        if ((flags & MCL_CURRENT) == 0) {
                   2252:                /*
                   2253:                 * no more work to do!
                   2254:                 */
                   2255:                UVMHIST_LOG(maphist,"<- done (OK no wire)",0,0,0,0);
                   2256:                vm_map_unlock(map);
                   2257:                return (KERN_SUCCESS);
                   2258:        }
                   2259:
                   2260:        /*
                   2261:         * wire case: in three passes [XXXCDC: ugly block of code here]
                   2262:         *
                   2263:         * 1: holding the write lock, count all pages mapped by non-wired
                   2264:         *    entries.  if this would cause us to go over our limit, we fail.
                   2265:         *
                   2266:         * 2: still holding the write lock, we create any anonymous maps that
                   2267:         *    need to be created.  then we increment its wiring count.
                   2268:         *
                   2269:         * 3: we downgrade to a read lock, and call uvm_fault_wire to fault
1.56      thorpej  2270:         *    in the pages for any newly wired area (wired_count == 1).
1.54      thorpej  2271:         *
                   2272:         *    downgrading to a read lock for uvm_fault_wire avoids a possible
                   2273:         *    deadlock with another thread that may have faulted on one of
                   2274:         *    the pages to be wired (it would mark the page busy, blocking
                   2275:         *    us, then in turn block on the map lock that we hold).  because
                   2276:         *    of problems in the recursive lock package, we cannot upgrade
                   2277:         *    to a write lock in vm_map_lookup.  thus, any actions that
                   2278:         *    require the write lock must be done beforehand.  because we
                   2279:         *    keep the read lock on the map, the copy-on-write status of the
                   2280:         *    entries we modify here cannot change.
                   2281:         */
                   2282:
                   2283:        for (size = 0, entry = map->header.next; entry != &map->header;
                   2284:             entry = entry->next) {
                   2285:                if (entry->protection != VM_PROT_NONE &&
1.55      thorpej  2286:                    VM_MAPENT_ISWIRED(entry) == 0) { /* not already wired? */
1.54      thorpej  2287:                        size += entry->end - entry->start;
                   2288:                }
                   2289:        }
                   2290:
                   2291:        if (atop(size) + uvmexp.wired > uvmexp.wiredmax) {
                   2292:                vm_map_unlock(map);
                   2293:                return (KERN_NO_SPACE);         /* XXX overloaded */
                   2294:        }
                   2295:
                   2296:        /* XXX non-pmap_wired_count case must be handled by caller */
                   2297: #ifdef pmap_wired_count
                   2298:        if (limit != 0 &&
                   2299:            (size + ptoa(pmap_wired_count(vm_map_pmap(map))) > limit)) {
                   2300:                vm_map_unlock(map);
                   2301:                return (KERN_NO_SPACE);         /* XXX overloaded */
                   2302:        }
                   2303: #endif
                   2304:
                   2305:        /*
                   2306:         * Pass 2.
                   2307:         */
                   2308:
                   2309:        for (entry = map->header.next; entry != &map->header;
                   2310:             entry = entry->next) {
                   2311:                if (entry->protection == VM_PROT_NONE)
                   2312:                        continue;
1.55      thorpej  2313:                if (VM_MAPENT_ISWIRED(entry) == 0) { /* not already wired? */
1.54      thorpej  2314:                        /*
                   2315:                         * perform actions of vm_map_lookup that need the
                   2316:                         * write lock on the map: create an anonymous map
                   2317:                         * for a copy-on-write region, or an anonymous map
                   2318:                         * for a zero-fill region.  (XXXCDC: submap case
                   2319:                         * ok?)
                   2320:                         */
                   2321:                        if (!UVM_ET_ISSUBMAP(entry)) {  /* not submap */
                   2322:                                if (UVM_ET_ISNEEDSCOPY(entry) &&
                   2323:                                    ((entry->protection & VM_PROT_WRITE) ||
                   2324:                                     (entry->object.uvm_obj == NULL))) {
                   2325:                                        amap_copy(map, entry, M_WAITOK, TRUE,
                   2326:                                            entry->start, entry->end);
                   2327:                                        /* XXXCDC: wait OK? */
                   2328:                                }
                   2329:                        }
1.55      thorpej  2330:                }
1.54      thorpej  2331:                entry->wired_count++;
                   2332:        }
                   2333:
                   2334:        /*
                   2335:         * Pass 3.
                   2336:         */
                   2337:
1.60      thorpej  2338: #ifdef DIAGNOSTIC
                   2339:        timestamp_save = map->timestamp;
                   2340: #endif
                   2341:        vm_map_busy(map);
1.54      thorpej  2342:        vm_map_downgrade(map);
                   2343:
                   2344:        rv = KERN_SUCCESS;
                   2345:        for (entry = map->header.next; entry != &map->header;
                   2346:             entry = entry->next) {
                   2347:                if (entry->wired_count == 1) {
                   2348:                        rv = uvm_fault_wire(map, entry->start, entry->end,
                   2349:                             entry->protection);
                   2350:                        if (rv) {
                   2351:                                /*
                   2352:                                 * wiring failed.  break out of the loop.
                   2353:                                 * we'll clean up the map below, once we
                   2354:                                 * have a write lock again.
                   2355:                                 */
                   2356:                                break;
                   2357:                        }
                   2358:                }
                   2359:        }
                   2360:
                   2361:        if (rv) {       /* failed? */
                   2362:                /*
                   2363:                 * Get back an exclusive (write) lock.
                   2364:                 */
                   2365:                vm_map_upgrade(map);
1.60      thorpej  2366:                vm_map_unbusy(map);
                   2367:
                   2368: #ifdef DIAGNOSTIC
                   2369:                if (timestamp_save != map->timestamp)
                   2370:                        panic("uvm_map_pageable_all: stale map");
                   2371: #endif
1.54      thorpej  2372:
                   2373:                /*
                   2374:                 * first drop the wiring count on all the entries
                   2375:                 * which haven't actually been wired yet.
1.67      thorpej  2376:                 *
                   2377:                 * Skip VM_PROT_NONE entries like we did above.
1.54      thorpej  2378:                 */
                   2379:                failed_entry = entry;
                   2380:                for (/* nothing */; entry != &map->header;
1.67      thorpej  2381:                     entry = entry->next) {
                   2382:                        if (entry->protection == VM_PROT_NONE)
                   2383:                                continue;
1.54      thorpej  2384:                        entry->wired_count--;
1.67      thorpej  2385:                }
1.54      thorpej  2386:
                   2387:                /*
                   2388:                 * now, unwire all the entries that were successfully
                   2389:                 * wired above.
1.67      thorpej  2390:                 *
                   2391:                 * Skip VM_PROT_NONE entries like we did above.
1.54      thorpej  2392:                 */
                   2393:                for (entry = map->header.next; entry != failed_entry;
                   2394:                     entry = entry->next) {
1.67      thorpej  2395:                        if (entry->protection == VM_PROT_NONE)
                   2396:                                continue;
1.54      thorpej  2397:                        entry->wired_count--;
1.67      thorpej  2398:                        if (VM_MAPENT_ISWIRED(entry))
1.54      thorpej  2399:                                uvm_map_entry_unwire(map, entry);
                   2400:                }
                   2401:                vm_map_unlock(map);
                   2402:                UVMHIST_LOG(maphist,"<- done (RV=%d)", rv,0,0,0);
                   2403:                return (rv);
                   2404:        }
                   2405:
                   2406:        /* We are holding a read lock here. */
1.60      thorpej  2407:        vm_map_unbusy(map);
1.54      thorpej  2408:        vm_map_unlock_read(map);
                   2409:
                   2410:        UVMHIST_LOG(maphist,"<- done (OK WIRE)",0,0,0,0);
                   2411:        return (KERN_SUCCESS);
                   2412: }
                   2413:
                   2414: /*
1.61      thorpej  2415:  * uvm_map_clean: clean out a map range
1.1       mrg      2416:  *
                   2417:  * => valid flags:
1.61      thorpej  2418:  *   if (flags & PGO_CLEANIT): dirty pages are cleaned first
1.1       mrg      2419:  *   if (flags & PGO_SYNCIO): dirty pages are written synchronously
                   2420:  *   if (flags & PGO_DEACTIVATE): any cached pages are deactivated after clean
                   2421:  *   if (flags & PGO_FREE): any cached pages are freed after clean
                   2422:  * => returns an error if any part of the specified range isn't mapped
                   2423:  * => never a need to flush amap layer since the anonymous memory has
1.61      thorpej  2424:  *     no permanent home, but may deactivate pages there
                   2425:  * => called from sys_msync() and sys_madvise()
1.1       mrg      2426:  * => caller must not write-lock map (read OK).
                   2427:  * => we may sleep while cleaning if SYNCIO [with map read-locked]
                   2428:  */
                   2429:
1.61      thorpej  2430: int    amap_clean_works = 1;   /* XXX for now, just in case... */
                   2431:
1.10      mrg      2432: int
                   2433: uvm_map_clean(map, start, end, flags)
                   2434:        vm_map_t map;
1.24      eeh      2435:        vaddr_t start, end;
1.10      mrg      2436:        int flags;
                   2437: {
1.61      thorpej  2438:        vm_map_entry_t current, entry;
                   2439:        struct uvm_object *uobj;
                   2440:        struct vm_amap *amap;
                   2441:        struct vm_anon *anon;
                   2442:        struct vm_page *pg;
                   2443:        vaddr_t offset;
1.24      eeh      2444:        vsize_t size;
1.61      thorpej  2445:        int rv, error, refs;
1.10      mrg      2446:        UVMHIST_FUNC("uvm_map_clean"); UVMHIST_CALLED(maphist);
                   2447:        UVMHIST_LOG(maphist,"(map=0x%x,start=0x%x,end=0x%x,flags=0x%x)",
                   2448:        map, start, end, flags);
                   2449:
1.61      thorpej  2450: #ifdef DIAGNOSTIC
                   2451:        if ((flags & (PGO_FREE|PGO_DEACTIVATE)) == (PGO_FREE|PGO_DEACTIVATE))
                   2452:                panic("uvm_map_clean: FREE and DEACTIVATE");
                   2453: #endif
                   2454:
1.10      mrg      2455:        vm_map_lock_read(map);
                   2456:        VM_MAP_RANGE_CHECK(map, start, end);
1.61      thorpej  2457:        if (uvm_map_lookup_entry(map, start, &entry) == FALSE) {
1.10      mrg      2458:                vm_map_unlock_read(map);
                   2459:                return(KERN_INVALID_ADDRESS);
                   2460:        }
                   2461:
                   2462:        /*
                   2463:         * Make a first pass to check for holes.
                   2464:         */
                   2465:        for (current = entry; current->start < end; current = current->next) {
                   2466:                if (UVM_ET_ISSUBMAP(current)) {
                   2467:                        vm_map_unlock_read(map);
1.38      mrg      2468:                        return (KERN_INVALID_ARGUMENT);
1.10      mrg      2469:                }
                   2470:                if (end > current->end && (current->next == &map->header ||
                   2471:                    current->end != current->next->start)) {
                   2472:                        vm_map_unlock_read(map);
1.38      mrg      2473:                        return (KERN_INVALID_ADDRESS);
1.10      mrg      2474:                }
                   2475:        }
                   2476:
1.61      thorpej  2477:        error = KERN_SUCCESS;
                   2478:
1.10      mrg      2479:        for (current = entry; current->start < end; current = current->next) {
1.61      thorpej  2480:                amap = current->aref.ar_amap;   /* top layer */
                   2481:                uobj = current->object.uvm_obj; /* bottom layer */
                   2482:
                   2483: #ifdef DIAGNOSTIC
                   2484:                if (start < current->start)
                   2485:                        panic("uvm_map_clean: hole");
                   2486: #endif
1.1       mrg      2487:
1.10      mrg      2488:                /*
1.61      thorpej  2489:                 * No amap cleaning necessary if:
                   2490:                 *
                   2491:                 *      (1) There's no amap.
                   2492:                 *
                   2493:                 *      (2) We're not deactivating or freeing pages.
1.10      mrg      2494:                 */
1.61      thorpej  2495:                if (amap == NULL ||
                   2496:                    (flags & (PGO_DEACTIVATE|PGO_FREE)) == 0)
                   2497:                        goto flush_object;
                   2498:
                   2499:                /* XXX for now, just in case... */
                   2500:                if (amap_clean_works == 0)
                   2501:                        goto flush_object;
                   2502:
                   2503:                amap_lock(amap);
                   2504:
                   2505:                offset = start - current->start;
                   2506:                size = (end <= current->end ? end : current->end) -
                   2507:                    start;
                   2508:
                   2509:                for (/* nothing */; size != 0; size -= PAGE_SIZE,
                   2510:                     offset += PAGE_SIZE) {
                   2511:                        anon = amap_lookup(&current->aref, offset);
                   2512:                        if (anon == NULL)
                   2513:                                continue;
                   2514:
                   2515:                        simple_lock(&anon->an_lock);
                   2516:
1.63      thorpej  2517:                        pg = anon->u.an_page;
                   2518:                        if (pg == NULL) {
                   2519:                                simple_unlock(&anon->an_lock);
                   2520:                                continue;
                   2521:                        }
                   2522:
1.61      thorpej  2523:                        switch (flags & (PGO_CLEANIT|PGO_FREE|PGO_DEACTIVATE)) {
                   2524:                        /*
                   2525:                         * XXX In these first 3 cases, we always just
                   2526:                         * XXX deactivate the page.  We may want to
                   2527:                         * XXX handle the different cases more
                   2528:                         * XXX specifically, in the future.
                   2529:                         */
                   2530:                        case PGO_CLEANIT|PGO_FREE:
                   2531:                        case PGO_CLEANIT|PGO_DEACTIVATE:
                   2532:                        case PGO_DEACTIVATE:
1.68      thorpej  2533:  deactivate_it:
1.61      thorpej  2534:                                /* skip the page if it's loaned or wired */
                   2535:                                if (pg->loan_count != 0 ||
                   2536:                                    pg->wire_count != 0) {
                   2537:                                        simple_unlock(&anon->an_lock);
                   2538:                                        continue;
                   2539:                                }
                   2540:
                   2541:                                uvm_lock_pageq();
                   2542:
                   2543:                                /*
                   2544:                                 * skip the page if it's not actually owned
                   2545:                                 * by the anon (may simply be loaned to the
                   2546:                                 * anon).
                   2547:                                 */
                   2548:                                if ((pg->pqflags & PQ_ANON) == 0) {
                   2549: #ifdef DIAGNOSTIC
                   2550:                                        if (pg->uobject != NULL)
                   2551:                                                panic("uvm_map_clean: "
                   2552:                                                    "page anon vs. object "
                   2553:                                                    "inconsistency");
                   2554: #endif
                   2555:                                        uvm_unlock_pageq();
                   2556:                                        simple_unlock(&anon->an_lock);
                   2557:                                        continue;
                   2558:                                }
                   2559:
                   2560: #ifdef DIAGNOSTIC
                   2561:                                if (pg->uanon != anon)
                   2562:                                        panic("uvm_map_clean: anon "
                   2563:                                            "inconsistency");
                   2564: #endif
                   2565:
                   2566:                                /* zap all mappings for the page. */
1.69      chs      2567:                                pmap_page_protect(pg, VM_PROT_NONE);
1.61      thorpej  2568:
                   2569:                                /* ...and deactivate the page. */
                   2570:                                uvm_pagedeactivate(pg);
                   2571:
                   2572:                                uvm_unlock_pageq();
                   2573:                                simple_unlock(&anon->an_lock);
                   2574:                                continue;
                   2575:
                   2576:                        case PGO_FREE:
1.68      thorpej  2577:                                /*
                   2578:                                 * If there are multiple references to
                   2579:                                 * the amap, just deactivate the page.
                   2580:                                 */
                   2581:                                if (amap_refs(amap) > 1)
                   2582:                                        goto deactivate_it;
                   2583:
1.62      thorpej  2584:                                /* XXX skip the page if it's wired */
                   2585:                                if (pg->wire_count != 0) {
                   2586:                                        simple_unlock(&anon->an_lock);
                   2587:                                        continue;
                   2588:                                }
1.66      thorpej  2589:                                amap_unadd(&current->aref, offset);
1.61      thorpej  2590:                                refs = --anon->an_ref;
                   2591:                                simple_unlock(&anon->an_lock);
                   2592:                                if (refs == 0)
                   2593:                                        uvm_anfree(anon);
                   2594:                                continue;
                   2595:
                   2596:                        default:
                   2597:                                panic("uvm_map_clean: wierd flags");
                   2598:                        }
                   2599: #ifdef DIAGNOSTIC
                   2600:                        panic("uvm_map_clean: unreachable code");
                   2601: #endif
                   2602:                }
                   2603:
                   2604:                amap_unlock(amap);
1.1       mrg      2605:
1.61      thorpej  2606:  flush_object:
1.10      mrg      2607:                /*
1.33      chuck    2608:                 * flush pages if we've got a valid backing object.
1.10      mrg      2609:                 */
1.1       mrg      2610:
1.61      thorpej  2611:                offset = current->offset + (start - current->start);
                   2612:                size = (end <= current->end ? end : current->end) - start;
                   2613:
                   2614:                if (uobj != NULL) {
                   2615:                        simple_lock(&uobj->vmobjlock);
                   2616:                        rv = uobj->pgops->pgo_flush(uobj, offset,
                   2617:                            offset + size, flags);
                   2618:                        simple_unlock(&uobj->vmobjlock);
                   2619:
                   2620:                        if (rv == FALSE)
                   2621:                                error = KERN_FAILURE;
1.10      mrg      2622:                }
                   2623:                start += size;
                   2624:        }
1.61      thorpej  2625:
1.1       mrg      2626:        vm_map_unlock_read(map);
1.61      thorpej  2627:        return (error);
1.1       mrg      2628: }
                   2629:
                   2630:
                   2631: /*
                   2632:  * uvm_map_checkprot: check protection in map
                   2633:  *
                   2634:  * => must allow specified protection in a fully allocated region.
                   2635:  * => map must be read or write locked by caller.
                   2636:  */
                   2637:
1.10      mrg      2638: boolean_t
                   2639: uvm_map_checkprot(map, start, end, protection)
                   2640:        vm_map_t       map;
1.24      eeh      2641:        vaddr_t    start, end;
1.10      mrg      2642:        vm_prot_t      protection;
                   2643: {
                   2644:         vm_map_entry_t entry;
                   2645:         vm_map_entry_t tmp_entry;
                   2646:
                   2647:         if (!uvm_map_lookup_entry(map, start, &tmp_entry)) {
                   2648:                 return(FALSE);
                   2649:         }
                   2650:
                   2651:         entry = tmp_entry;
                   2652:
                   2653:         while (start < end) {
                   2654:                 if (entry == &map->header) {
                   2655:                         return(FALSE);
                   2656:                 }
                   2657:
                   2658:                /*
                   2659:                 * no holes allowed
                   2660:                 */
                   2661:
                   2662:                 if (start < entry->start) {
                   2663:                         return(FALSE);
                   2664:                 }
                   2665:
                   2666:                /*
                   2667:                 * check protection associated with entry
                   2668:                 */
1.1       mrg      2669:
1.10      mrg      2670:                 if ((entry->protection & protection) != protection) {
                   2671:                         return(FALSE);
                   2672:                 }
                   2673:
                   2674:                 /* go to next entry */
                   2675:
                   2676:                 start = entry->end;
                   2677:                 entry = entry->next;
                   2678:         }
                   2679:         return(TRUE);
1.1       mrg      2680: }
                   2681:
                   2682: /*
                   2683:  * uvmspace_alloc: allocate a vmspace structure.
                   2684:  *
                   2685:  * - structure includes vm_map and pmap
                   2686:  * - XXX: no locking on this structure
                   2687:  * - refcnt set to 1, rest must be init'd by caller
                   2688:  */
1.10      mrg      2689: struct vmspace *
                   2690: uvmspace_alloc(min, max, pageable)
1.24      eeh      2691:        vaddr_t min, max;
1.10      mrg      2692:        int pageable;
                   2693: {
                   2694:        struct vmspace *vm;
                   2695:        UVMHIST_FUNC("uvmspace_alloc"); UVMHIST_CALLED(maphist);
                   2696:
1.25      thorpej  2697:        vm = pool_get(&uvm_vmspace_pool, PR_WAITOK);
1.15      thorpej  2698:        uvmspace_init(vm, NULL, min, max, pageable);
                   2699:        UVMHIST_LOG(maphist,"<- done (vm=0x%x)", vm,0,0,0);
                   2700:        return (vm);
                   2701: }
                   2702:
                   2703: /*
                   2704:  * uvmspace_init: initialize a vmspace structure.
                   2705:  *
                   2706:  * - XXX: no locking on this structure
                   2707:  * - refcnt set to 1, rest must me init'd by caller
                   2708:  */
                   2709: void
                   2710: uvmspace_init(vm, pmap, min, max, pageable)
                   2711:        struct vmspace *vm;
                   2712:        struct pmap *pmap;
1.24      eeh      2713:        vaddr_t min, max;
1.15      thorpej  2714:        boolean_t pageable;
                   2715: {
                   2716:        UVMHIST_FUNC("uvmspace_init"); UVMHIST_CALLED(maphist);
                   2717:
1.23      perry    2718:        memset(vm, 0, sizeof(*vm));
1.15      thorpej  2719:
1.45      thorpej  2720:        uvm_map_setup(&vm->vm_map, min, max, pageable ? VM_MAP_PAGEABLE : 0);
1.15      thorpej  2721:
                   2722:        if (pmap)
                   2723:                pmap_reference(pmap);
                   2724:        else
                   2725:                pmap = pmap_create();
                   2726:        vm->vm_map.pmap = pmap;
                   2727:
1.10      mrg      2728:        vm->vm_refcnt = 1;
1.15      thorpej  2729:        UVMHIST_LOG(maphist,"<- done",0,0,0,0);
1.1       mrg      2730: }
                   2731:
                   2732: /*
                   2733:  * uvmspace_share: share a vmspace between two proceses
                   2734:  *
                   2735:  * - XXX: no locking on vmspace
                   2736:  * - used for vfork, threads(?)
                   2737:  */
                   2738:
1.10      mrg      2739: void
                   2740: uvmspace_share(p1, p2)
                   2741:        struct proc *p1, *p2;
1.1       mrg      2742: {
1.10      mrg      2743:        p2->p_vmspace = p1->p_vmspace;
                   2744:        p1->p_vmspace->vm_refcnt++;
1.1       mrg      2745: }
                   2746:
                   2747: /*
                   2748:  * uvmspace_unshare: ensure that process "p" has its own, unshared, vmspace
                   2749:  *
                   2750:  * - XXX: no locking on vmspace
                   2751:  */
                   2752:
1.10      mrg      2753: void
                   2754: uvmspace_unshare(p)
                   2755:        struct proc *p;
                   2756: {
                   2757:        struct vmspace *nvm, *ovm = p->p_vmspace;
1.1       mrg      2758:
1.10      mrg      2759:        if (ovm->vm_refcnt == 1)
                   2760:                /* nothing to do: vmspace isn't shared in the first place */
                   2761:                return;
1.1       mrg      2762:
1.10      mrg      2763:        /* make a new vmspace, still holding old one */
                   2764:        nvm = uvmspace_fork(ovm);
                   2765:
1.12      thorpej  2766:        pmap_deactivate(p);             /* unbind old vmspace */
1.10      mrg      2767:        p->p_vmspace = nvm;
                   2768:        pmap_activate(p);               /* switch to new vmspace */
1.13      thorpej  2769:
1.10      mrg      2770:        uvmspace_free(ovm);             /* drop reference to old vmspace */
1.1       mrg      2771: }
                   2772:
                   2773: /*
                   2774:  * uvmspace_exec: the process wants to exec a new program
                   2775:  *
                   2776:  * - XXX: no locking on vmspace
                   2777:  */
                   2778:
1.10      mrg      2779: void
                   2780: uvmspace_exec(p)
                   2781:        struct proc *p;
1.1       mrg      2782: {
1.10      mrg      2783:        struct vmspace *nvm, *ovm = p->p_vmspace;
                   2784:        vm_map_t map = &ovm->vm_map;
1.1       mrg      2785:
1.71      chs      2786: #ifdef __sparc__
1.10      mrg      2787:        /* XXX cgd 960926: the sparc #ifdef should be a MD hook */
                   2788:        kill_user_windows(p);   /* before stack addresses go away */
1.1       mrg      2789: #endif
                   2790:
1.10      mrg      2791:        /*
                   2792:         * see if more than one process is using this vmspace...
                   2793:         */
1.1       mrg      2794:
1.10      mrg      2795:        if (ovm->vm_refcnt == 1) {
1.1       mrg      2796:
1.10      mrg      2797:                /*
                   2798:                 * if p is the only process using its vmspace then we can safely
                   2799:                 * recycle that vmspace for the program that is being exec'd.
                   2800:                 */
1.1       mrg      2801:
                   2802: #ifdef SYSVSHM
1.10      mrg      2803:                /*
                   2804:                 * SYSV SHM semantics require us to kill all segments on an exec
                   2805:                 */
                   2806:                if (ovm->vm_shm)
                   2807:                        shmexit(ovm);
                   2808: #endif
1.54      thorpej  2809:
                   2810:                /*
                   2811:                 * POSIX 1003.1b -- "lock future mappings" is revoked
                   2812:                 * when a process execs another program image.
                   2813:                 */
                   2814:                vm_map_lock(map);
1.61      thorpej  2815:                vm_map_modflags(map, 0, VM_MAP_WIREFUTURE);
1.54      thorpej  2816:                vm_map_unlock(map);
1.10      mrg      2817:
                   2818:                /*
                   2819:                 * now unmap the old program
                   2820:                 */
1.29      chuck    2821:                uvm_unmap(map, VM_MIN_ADDRESS, VM_MAXUSER_ADDRESS);
1.10      mrg      2822:
                   2823:        } else {
                   2824:
                   2825:                /*
                   2826:                 * p's vmspace is being shared, so we can't reuse it for p since
                   2827:                 * it is still being used for others.   allocate a new vmspace
                   2828:                 * for p
                   2829:                 */
                   2830:                nvm = uvmspace_alloc(map->min_offset, map->max_offset,
1.45      thorpej  2831:                         (map->flags & VM_MAP_PAGEABLE) ? TRUE : FALSE);
1.1       mrg      2832:
1.10      mrg      2833:                /*
                   2834:                 * install new vmspace and drop our ref to the old one.
                   2835:                 */
                   2836:
1.12      thorpej  2837:                pmap_deactivate(p);
1.10      mrg      2838:                p->p_vmspace = nvm;
                   2839:                pmap_activate(p);
1.13      thorpej  2840:
1.10      mrg      2841:                uvmspace_free(ovm);
                   2842:        }
1.1       mrg      2843: }
                   2844:
                   2845: /*
                   2846:  * uvmspace_free: free a vmspace data structure
                   2847:  *
                   2848:  * - XXX: no locking on vmspace
                   2849:  */
                   2850:
1.10      mrg      2851: void
                   2852: uvmspace_free(vm)
                   2853:        struct vmspace *vm;
1.1       mrg      2854: {
1.10      mrg      2855:        vm_map_entry_t dead_entries;
                   2856:        UVMHIST_FUNC("uvmspace_free"); UVMHIST_CALLED(maphist);
1.1       mrg      2857:
1.10      mrg      2858:        UVMHIST_LOG(maphist,"(vm=0x%x) ref=%d", vm, vm->vm_refcnt,0,0);
                   2859:        if (--vm->vm_refcnt == 0) {
                   2860:                /*
                   2861:                 * lock the map, to wait out all other references to it.  delete
                   2862:                 * all of the mappings and pages they hold, then call the pmap
                   2863:                 * module to reclaim anything left.
                   2864:                 */
                   2865:                vm_map_lock(&vm->vm_map);
                   2866:                if (vm->vm_map.nentries) {
                   2867:                        (void)uvm_unmap_remove(&vm->vm_map,
                   2868:                            vm->vm_map.min_offset, vm->vm_map.max_offset,
1.29      chuck    2869:                            &dead_entries);
1.10      mrg      2870:                        if (dead_entries != NULL)
                   2871:                                uvm_unmap_detach(dead_entries, 0);
                   2872:                }
                   2873:                pmap_destroy(vm->vm_map.pmap);
                   2874:                vm->vm_map.pmap = NULL;
1.25      thorpej  2875:                pool_put(&uvm_vmspace_pool, vm);
1.10      mrg      2876:        }
                   2877:        UVMHIST_LOG(maphist,"<- done", 0,0,0,0);
1.1       mrg      2878: }
                   2879:
                   2880: /*
                   2881:  *   F O R K   -   m a i n   e n t r y   p o i n t
                   2882:  */
                   2883: /*
                   2884:  * uvmspace_fork: fork a process' main map
                   2885:  *
                   2886:  * => create a new vmspace for child process from parent.
                   2887:  * => parent's map must not be locked.
                   2888:  */
                   2889:
1.10      mrg      2890: struct vmspace *
                   2891: uvmspace_fork(vm1)
                   2892:        struct vmspace *vm1;
                   2893: {
                   2894:        struct vmspace *vm2;
                   2895:        vm_map_t        old_map = &vm1->vm_map;
                   2896:        vm_map_t        new_map;
                   2897:        vm_map_entry_t  old_entry;
                   2898:        vm_map_entry_t  new_entry;
                   2899:        pmap_t          new_pmap;
1.14      chuck    2900:        boolean_t       protect_child;
1.10      mrg      2901:        UVMHIST_FUNC("uvmspace_fork"); UVMHIST_CALLED(maphist);
1.1       mrg      2902:
1.10      mrg      2903:        vm_map_lock(old_map);
1.1       mrg      2904:
1.10      mrg      2905:        vm2 = uvmspace_alloc(old_map->min_offset, old_map->max_offset,
1.45      thorpej  2906:                      (old_map->flags & VM_MAP_PAGEABLE) ? TRUE : FALSE);
1.23      perry    2907:        memcpy(&vm2->vm_startcopy, &vm1->vm_startcopy,
1.1       mrg      2908:        (caddr_t) (vm1 + 1) - (caddr_t) &vm1->vm_startcopy);
1.10      mrg      2909:        new_map = &vm2->vm_map;           /* XXX */
                   2910:        new_pmap = new_map->pmap;
                   2911:
                   2912:        old_entry = old_map->header.next;
                   2913:
                   2914:        /*
                   2915:         * go entry-by-entry
                   2916:         */
1.1       mrg      2917:
1.10      mrg      2918:        while (old_entry != &old_map->header) {
1.1       mrg      2919:
1.10      mrg      2920:                /*
                   2921:                 * first, some sanity checks on the old entry
                   2922:                 */
                   2923:                if (UVM_ET_ISSUBMAP(old_entry))
1.29      chuck    2924:                    panic("fork: encountered a submap during fork (illegal)");
                   2925:
                   2926:                if (!UVM_ET_ISCOPYONWRITE(old_entry) &&
1.10      mrg      2927:                            UVM_ET_ISNEEDSCOPY(old_entry))
1.1       mrg      2928:        panic("fork: non-copy_on_write map entry marked needs_copy (illegal)");
                   2929:
                   2930:
1.10      mrg      2931:                switch (old_entry->inheritance) {
1.77.2.1! fvdl     2932:                case MAP_INHERIT_NONE:
1.10      mrg      2933:                        /*
                   2934:                         * drop the mapping
                   2935:                         */
                   2936:                        break;
                   2937:
1.77.2.1! fvdl     2938:                case MAP_INHERIT_SHARE:
1.10      mrg      2939:                        /*
                   2940:                         * share the mapping: this means we want the old and
                   2941:                         * new entries to share amaps and backing objects.
                   2942:                         */
                   2943:
                   2944:                        /*
                   2945:                         * if the old_entry needs a new amap (due to prev fork)
                   2946:                         * then we need to allocate it now so that we have
                   2947:                         * something we own to share with the new_entry.   [in
                   2948:                         * other words, we need to clear needs_copy]
                   2949:                         */
                   2950:
                   2951:                        if (UVM_ET_ISNEEDSCOPY(old_entry)) {
                   2952:                                /* get our own amap, clears needs_copy */
                   2953:                                amap_copy(old_map, old_entry, M_WAITOK, FALSE,
                   2954:                                    0, 0);
                   2955:                                /* XXXCDC: WAITOK??? */
                   2956:                        }
                   2957:
                   2958:                        new_entry = uvm_mapent_alloc(new_map);
                   2959:                        /* old_entry -> new_entry */
                   2960:                        uvm_mapent_copy(old_entry, new_entry);
                   2961:
                   2962:                        /* new pmap has nothing wired in it */
                   2963:                        new_entry->wired_count = 0;
                   2964:
                   2965:                        /*
1.29      chuck    2966:                         * gain reference to object backing the map (can't
                   2967:                         * be a submap, already checked this case).
1.10      mrg      2968:                         */
                   2969:                        if (new_entry->aref.ar_amap)
                   2970:                                /* share reference */
                   2971:                                amap_ref(new_entry, AMAP_SHARED);
                   2972:
                   2973:                        if (new_entry->object.uvm_obj &&
                   2974:                            new_entry->object.uvm_obj->pgops->pgo_reference)
                   2975:                                new_entry->object.uvm_obj->
                   2976:                                    pgops->pgo_reference(
                   2977:                                        new_entry->object.uvm_obj);
                   2978:
                   2979:                        /* insert entry at end of new_map's entry list */
                   2980:                        uvm_map_entry_link(new_map, new_map->header.prev,
                   2981:                            new_entry);
                   2982:
                   2983:                        /*
                   2984:                         * pmap_copy the mappings: this routine is optional
                   2985:                         * but if it is there it will reduce the number of
                   2986:                         * page faults in the new proc.
                   2987:                         */
                   2988:
                   2989:                        pmap_copy(new_pmap, old_map->pmap, new_entry->start,
                   2990:                            (old_entry->end - old_entry->start),
                   2991:                            old_entry->start);
                   2992:
                   2993:                        break;
                   2994:
1.77.2.1! fvdl     2995:                case MAP_INHERIT_COPY:
1.10      mrg      2996:
                   2997:                        /*
                   2998:                         * copy-on-write the mapping (using mmap's
                   2999:                         * MAP_PRIVATE semantics)
1.29      chuck    3000:                         *
                   3001:                         * allocate new_entry, adjust reference counts.
                   3002:                         * (note that new references are read-only).
1.10      mrg      3003:                         */
                   3004:
                   3005:                        new_entry = uvm_mapent_alloc(new_map);
                   3006:                        /* old_entry -> new_entry */
                   3007:                        uvm_mapent_copy(old_entry, new_entry);
                   3008:
                   3009:                        if (new_entry->aref.ar_amap)
                   3010:                                amap_ref(new_entry, 0);
                   3011:
                   3012:                        if (new_entry->object.uvm_obj &&
                   3013:                            new_entry->object.uvm_obj->pgops->pgo_reference)
                   3014:                                new_entry->object.uvm_obj->pgops->pgo_reference
                   3015:                                    (new_entry->object.uvm_obj);
                   3016:
                   3017:                        /* new pmap has nothing wired in it */
                   3018:                        new_entry->wired_count = 0;
                   3019:
                   3020:                        new_entry->etype |=
                   3021:                            (UVM_ET_COPYONWRITE|UVM_ET_NEEDSCOPY);
                   3022:                        uvm_map_entry_link(new_map, new_map->header.prev,
                   3023:                            new_entry);
                   3024:
1.14      chuck    3025:                        /*
1.10      mrg      3026:                         * the new entry will need an amap.  it will either
                   3027:                         * need to be copied from the old entry or created
1.14      chuck    3028:                         * from scratch (if the old entry does not have an
                   3029:                         * amap).  can we defer this process until later
                   3030:                         * (by setting "needs_copy") or do we need to copy
                   3031:                         * the amap now?
1.10      mrg      3032:                         *
1.14      chuck    3033:                         * we must copy the amap now if any of the following
1.10      mrg      3034:                         * conditions hold:
1.14      chuck    3035:                         * 1. the old entry has an amap and that amap is
                   3036:                         *    being shared.  this means that the old (parent)
                   3037:                         *    process is sharing the amap with another
                   3038:                         *    process.  if we do not clear needs_copy here
                   3039:                         *    we will end up in a situation where both the
                   3040:                         *    parent and child process are refering to the
                   3041:                         *    same amap with "needs_copy" set.  if the
                   3042:                         *    parent write-faults, the fault routine will
                   3043:                         *    clear "needs_copy" in the parent by allocating
                   3044:                         *    a new amap.   this is wrong because the
                   3045:                         *    parent is supposed to be sharing the old amap
                   3046:                         *    and the new amap will break that.
1.10      mrg      3047:                         *
1.14      chuck    3048:                         * 2. if the old entry has an amap and a non-zero
                   3049:                         *    wire count then we are going to have to call
                   3050:                         *    amap_cow_now to avoid page faults in the
                   3051:                         *    parent process.   since amap_cow_now requires
                   3052:                         *    "needs_copy" to be clear we might as well
                   3053:                         *    clear it here as well.
1.10      mrg      3054:                         *
                   3055:                         */
                   3056:
1.14      chuck    3057:                        if (old_entry->aref.ar_amap != NULL) {
                   3058:
1.34      chuck    3059:                          if ((amap_flags(old_entry->aref.ar_amap) &
1.14      chuck    3060:                               AMAP_SHARED) != 0 ||
1.55      thorpej  3061:                              VM_MAPENT_ISWIRED(old_entry)) {
1.14      chuck    3062:
                   3063:                            amap_copy(new_map, new_entry, M_WAITOK, FALSE,
                   3064:                                      0, 0);
                   3065:                            /* XXXCDC: M_WAITOK ... ok? */
                   3066:                          }
1.10      mrg      3067:                        }
1.14      chuck    3068:
1.10      mrg      3069:                        /*
1.14      chuck    3070:                         * if the parent's entry is wired down, then the
                   3071:                         * parent process does not want page faults on
                   3072:                         * access to that memory.  this means that we
                   3073:                         * cannot do copy-on-write because we can't write
                   3074:                         * protect the old entry.   in this case we
                   3075:                         * resolve all copy-on-write faults now, using
                   3076:                         * amap_cow_now.   note that we have already
                   3077:                         * allocated any needed amap (above).
1.10      mrg      3078:                         */
                   3079:
1.55      thorpej  3080:                        if (VM_MAPENT_ISWIRED(old_entry)) {
1.1       mrg      3081:
1.14      chuck    3082:                          /*
                   3083:                           * resolve all copy-on-write faults now
                   3084:                           * (note that there is nothing to do if
                   3085:                           * the old mapping does not have an amap).
                   3086:                           * XXX: is it worthwhile to bother with pmap_copy
                   3087:                           * in this case?
                   3088:                           */
                   3089:                          if (old_entry->aref.ar_amap)
                   3090:                            amap_cow_now(new_map, new_entry);
                   3091:
                   3092:                        } else {
                   3093:
                   3094:                          /*
                   3095:                           * setup mappings to trigger copy-on-write faults
                   3096:                           * we must write-protect the parent if it has
                   3097:                           * an amap and it is not already "needs_copy"...
                   3098:                           * if it is already "needs_copy" then the parent
                   3099:                           * has already been write-protected by a previous
                   3100:                           * fork operation.
                   3101:                           *
                   3102:                           * if we do not write-protect the parent, then
                   3103:                           * we must be sure to write-protect the child
                   3104:                           * after the pmap_copy() operation.
                   3105:                           *
                   3106:                           * XXX: pmap_copy should have some way of telling
                   3107:                           * us that it didn't do anything so we can avoid
                   3108:                           * calling pmap_protect needlessly.
                   3109:                           */
                   3110:
                   3111:                          if (old_entry->aref.ar_amap) {
                   3112:
                   3113:                            if (!UVM_ET_ISNEEDSCOPY(old_entry)) {
                   3114:                              if (old_entry->max_protection & VM_PROT_WRITE) {
                   3115:                                pmap_protect(old_map->pmap,
                   3116:                                             old_entry->start,
                   3117:                                             old_entry->end,
                   3118:                                             old_entry->protection &
                   3119:                                             ~VM_PROT_WRITE);
                   3120:                              }
                   3121:                              old_entry->etype |= UVM_ET_NEEDSCOPY;
                   3122:                            }
                   3123:
                   3124:                            /*
                   3125:                             * parent must now be write-protected
                   3126:                             */
                   3127:                            protect_child = FALSE;
                   3128:                          } else {
                   3129:
                   3130:                            /*
                   3131:                             * we only need to protect the child if the
                   3132:                             * parent has write access.
                   3133:                             */
                   3134:                            if (old_entry->max_protection & VM_PROT_WRITE)
                   3135:                              protect_child = TRUE;
                   3136:                            else
                   3137:                              protect_child = FALSE;
                   3138:
                   3139:                          }
                   3140:
                   3141:                          /*
                   3142:                           * copy the mappings
                   3143:                           * XXX: need a way to tell if this does anything
                   3144:                           */
1.1       mrg      3145:
1.14      chuck    3146:                          pmap_copy(new_pmap, old_map->pmap,
1.10      mrg      3147:                                    new_entry->start,
1.14      chuck    3148:                                    (old_entry->end - old_entry->start),
1.10      mrg      3149:                                    old_entry->start);
1.14      chuck    3150:
                   3151:                          /*
                   3152:                           * protect the child's mappings if necessary
                   3153:                           */
                   3154:                          if (protect_child) {
                   3155:                            pmap_protect(new_pmap, new_entry->start,
                   3156:                                         new_entry->end,
                   3157:                                         new_entry->protection &
                   3158:                                                  ~VM_PROT_WRITE);
                   3159:                          }
1.10      mrg      3160:
                   3161:                        }
                   3162:                        break;
1.14      chuck    3163:                }  /* end of switch statement */
1.10      mrg      3164:                old_entry = old_entry->next;
1.1       mrg      3165:        }
                   3166:
1.10      mrg      3167:        new_map->size = old_map->size;
                   3168:        vm_map_unlock(old_map);
1.1       mrg      3169:
                   3170: #ifdef SYSVSHM
1.10      mrg      3171:        if (vm1->vm_shm)
                   3172:                shmfork(vm1, vm2);
1.39      thorpej  3173: #endif
                   3174:
                   3175: #ifdef PMAP_FORK
                   3176:        pmap_fork(vm1->vm_map.pmap, vm2->vm_map.pmap);
1.1       mrg      3177: #endif
                   3178:
1.10      mrg      3179:        UVMHIST_LOG(maphist,"<- done",0,0,0,0);
                   3180:        return(vm2);
1.1       mrg      3181: }
                   3182:
                   3183:
                   3184: #if defined(DDB)
                   3185:
                   3186: /*
                   3187:  * DDB hooks
                   3188:  */
                   3189:
                   3190: /*
                   3191:  * uvm_map_print: print out a map
                   3192:  */
                   3193:
1.10      mrg      3194: void
                   3195: uvm_map_print(map, full)
                   3196:        vm_map_t map;
                   3197:        boolean_t full;
                   3198: {
1.1       mrg      3199:
1.10      mrg      3200:        uvm_map_printit(map, full, printf);
1.1       mrg      3201: }
                   3202:
                   3203: /*
                   3204:  * uvm_map_printit: actually prints the map
                   3205:  */
                   3206:
1.10      mrg      3207: void
                   3208: uvm_map_printit(map, full, pr)
                   3209:        vm_map_t map;
                   3210:        boolean_t full;
                   3211:        void (*pr) __P((const char *, ...));
                   3212: {
                   3213:        vm_map_entry_t entry;
                   3214:
                   3215:        (*pr)("MAP %p: [0x%lx->0x%lx]\n", map, map->min_offset,map->max_offset);
1.53      thorpej  3216:        (*pr)("\t#ent=%d, sz=%d, ref=%d, version=%d, flags=0x%x\n",
                   3217:            map->nentries, map->size, map->ref_count, map->timestamp,
                   3218:            map->flags);
1.16      chuck    3219: #ifdef pmap_resident_count
                   3220:        (*pr)("\tpmap=%p(resident=%d)\n", map->pmap,
                   3221:            pmap_resident_count(map->pmap));
                   3222: #else
                   3223:        /* XXXCDC: this should be required ... */
                   3224:        (*pr)("\tpmap=%p(resident=<<NOT SUPPORTED!!!>>)\n", map->pmap);
                   3225: #endif
1.10      mrg      3226:        if (!full)
                   3227:                return;
                   3228:        for (entry = map->header.next; entry != &map->header;
                   3229:            entry = entry->next) {
1.70      kleink   3230:                (*pr)(" - %p: 0x%lx->0x%lx: obj=%p/0x%llx, amap=%p/%d\n",
1.10      mrg      3231:                    entry, entry->start, entry->end, entry->object.uvm_obj,
1.70      kleink   3232:                    (long long)entry->offset, entry->aref.ar_amap, entry->aref.ar_pageoff);
1.10      mrg      3233:                (*pr)(
1.29      chuck    3234: "\tsubmap=%c, cow=%c, nc=%c, prot(max)=%d/%d, inh=%d, wc=%d, adv=%d\n",
1.10      mrg      3235:                    (entry->etype & UVM_ET_SUBMAP) ? 'T' : 'F',
                   3236:                    (entry->etype & UVM_ET_COPYONWRITE) ? 'T' : 'F',
                   3237:                    (entry->etype & UVM_ET_NEEDSCOPY) ? 'T' : 'F',
                   3238:                    entry->protection, entry->max_protection,
                   3239:                    entry->inheritance, entry->wired_count, entry->advice);
                   3240:        }
1.1       mrg      3241: }
                   3242:
                   3243: /*
                   3244:  * uvm_object_print: print out an object
                   3245:  */
                   3246:
1.10      mrg      3247: void
                   3248: uvm_object_print(uobj, full)
                   3249:        struct uvm_object *uobj;
                   3250:        boolean_t full;
                   3251: {
1.1       mrg      3252:
1.10      mrg      3253:        uvm_object_printit(uobj, full, printf);
1.1       mrg      3254: }
                   3255:
                   3256: /*
                   3257:  * uvm_object_printit: actually prints the object
                   3258:  */
                   3259:
1.10      mrg      3260: void
                   3261: uvm_object_printit(uobj, full, pr)
                   3262:        struct uvm_object *uobj;
                   3263:        boolean_t full;
                   3264:        void (*pr) __P((const char *, ...));
                   3265: {
                   3266:        struct vm_page *pg;
                   3267:        int cnt = 0;
                   3268:
1.71      chs      3269:        (*pr)("OBJECT %p: locked=%d, pgops=%p, npages=%d, ",
                   3270:            uobj, uobj->vmobjlock.lock_data, uobj->pgops, uobj->uo_npages);
1.42      thorpej  3271:        if (UVM_OBJ_IS_KERN_OBJECT(uobj))
1.10      mrg      3272:                (*pr)("refs=<SYSTEM>\n");
                   3273:        else
                   3274:                (*pr)("refs=%d\n", uobj->uo_refs);
                   3275:
1.71      chs      3276:        if (!full) {
                   3277:                return;
                   3278:        }
1.10      mrg      3279:        (*pr)("  PAGES <pg,offset>:\n  ");
1.71      chs      3280:        for (pg = TAILQ_FIRST(&uobj->memq);
                   3281:             pg != NULL;
                   3282:             pg = TAILQ_NEXT(pg, listq), cnt++) {
1.10      mrg      3283:                (*pr)("<%p,0x%lx> ", pg, pg->offset);
1.71      chs      3284:                if ((cnt % 3) == 2) {
                   3285:                        (*pr)("\n  ");
                   3286:                }
                   3287:        }
                   3288:        if ((cnt % 3) != 2) {
                   3289:                (*pr)("\n");
1.10      mrg      3290:        }
1.1       mrg      3291: }
                   3292:
1.71      chs      3293: const char page_flagbits[] =
                   3294:        "\20\4CLEAN\5BUSY\6WANTED\7TABLED\12FAKE\13FILLED\14DIRTY\15RELEASED"
                   3295:        "\16FAULTING\17CLEANCHK";
                   3296: const char page_pqflagbits[] =
                   3297:        "\20\1FREE\2INACTIVE\3ACTIVE\4LAUNDRY\5ANON\6AOBJ";
                   3298:
1.1       mrg      3299: /*
                   3300:  * uvm_page_print: print out a page
                   3301:  */
                   3302:
1.10      mrg      3303: void
                   3304: uvm_page_print(pg, full)
                   3305:        struct vm_page *pg;
                   3306:        boolean_t full;
                   3307: {
1.1       mrg      3308:
1.10      mrg      3309:        uvm_page_printit(pg, full, printf);
1.1       mrg      3310: }
                   3311:
                   3312: /*
                   3313:  * uvm_page_printit: actually print the page
                   3314:  */
                   3315:
1.10      mrg      3316: void
                   3317: uvm_page_printit(pg, full, pr)
                   3318:        struct vm_page *pg;
                   3319:        boolean_t full;
                   3320:        void (*pr) __P((const char *, ...));
                   3321: {
                   3322:        struct vm_page *lcv;
                   3323:        struct uvm_object *uobj;
                   3324:        struct pglist *pgl;
1.71      chs      3325:        char pgbuf[128];
                   3326:        char pqbuf[128];
1.1       mrg      3327:
1.10      mrg      3328:        (*pr)("PAGE %p:\n", pg);
1.71      chs      3329:        bitmask_snprintf(pg->flags, page_flagbits, pgbuf, sizeof(pgbuf));
                   3330:        bitmask_snprintf(pg->pqflags, page_pqflagbits, pqbuf, sizeof(pqbuf));
                   3331:        (*pr)("  flags=%s, pqflags=%s, vers=%d, wire_count=%d, pa=0x%lx\n",
                   3332:            pgbuf, pqbuf, pg->version, pg->wire_count, (long)pg->phys_addr);
1.10      mrg      3333:        (*pr)("  uobject=%p, uanon=%p, offset=0x%lx loan_count=%d\n",
1.71      chs      3334:            pg->uobject, pg->uanon, pg->offset, pg->loan_count);
1.1       mrg      3335: #if defined(UVM_PAGE_TRKOWN)
1.10      mrg      3336:        if (pg->flags & PG_BUSY)
                   3337:                (*pr)("  owning process = %d, tag=%s\n",
                   3338:                    pg->owner, pg->owner_tag);
                   3339:        else
                   3340:                (*pr)("  page not busy, no owner\n");
1.1       mrg      3341: #else
1.10      mrg      3342:        (*pr)("  [page ownership tracking disabled]\n");
1.1       mrg      3343: #endif
                   3344:
1.10      mrg      3345:        if (!full)
                   3346:                return;
                   3347:
                   3348:        /* cross-verify object/anon */
                   3349:        if ((pg->pqflags & PQ_FREE) == 0) {
                   3350:                if (pg->pqflags & PQ_ANON) {
                   3351:                        if (pg->uanon == NULL || pg->uanon->u.an_page != pg)
                   3352:                            (*pr)("  >>> ANON DOES NOT POINT HERE <<< (%p)\n",
                   3353:                                (pg->uanon) ? pg->uanon->u.an_page : NULL);
                   3354:                        else
                   3355:                                (*pr)("  anon backpointer is OK\n");
                   3356:                } else {
                   3357:                        uobj = pg->uobject;
                   3358:                        if (uobj) {
                   3359:                                (*pr)("  checking object list\n");
                   3360:                                for (lcv = uobj->memq.tqh_first ; lcv ;
                   3361:                                    lcv = lcv->listq.tqe_next) {
                   3362:                                        if (lcv == pg) break;
                   3363:                                }
                   3364:                                if (lcv)
                   3365:                                        (*pr)("  page found on object list\n");
                   3366:                                else
                   3367:                        (*pr)("  >>> PAGE NOT FOUND ON OBJECT LIST! <<<\n");
                   3368:                        }
                   3369:                }
                   3370:        }
1.1       mrg      3371:
1.10      mrg      3372:        /* cross-verify page queue */
1.73      thorpej  3373:        if (pg->pqflags & PQ_FREE) {
                   3374:                int fl = uvm_page_lookup_freelist(pg);
                   3375:                pgl = &uvm.page_free[fl].pgfl_queues[((pg)->flags & PG_ZERO) ?
                   3376:                    PGFL_ZEROS : PGFL_UNKNOWN];
                   3377:        }
1.10      mrg      3378:        else if (pg->pqflags & PQ_INACTIVE)
                   3379:                pgl = (pg->pqflags & PQ_SWAPBACKED) ?
                   3380:                    &uvm.page_inactive_swp : &uvm.page_inactive_obj;
                   3381:        else if (pg->pqflags & PQ_ACTIVE)
                   3382:                pgl = &uvm.page_active;
                   3383:        else
                   3384:                pgl = NULL;
                   3385:
                   3386:        if (pgl) {
                   3387:                (*pr)("  checking pageq list\n");
                   3388:                for (lcv = pgl->tqh_first ; lcv ; lcv = lcv->pageq.tqe_next) {
                   3389:                        if (lcv == pg) break;
                   3390:                }
                   3391:                if (lcv)
                   3392:                        (*pr)("  page found on pageq list\n");
                   3393:                else
                   3394:                        (*pr)("  >>> PAGE NOT FOUND ON PAGEQ LIST! <<<\n");
                   3395:        }
1.1       mrg      3396: }
                   3397: #endif

CVSweb <webmaster@jp.NetBSD.org>