[BACK]Return to uvm_map.c CVS log [TXT][DIR] Up to [cvs.NetBSD.org] / src / sys / uvm

Annotation of src/sys/uvm/uvm_map.c, Revision 1.330.2.1

1.330.2.1! martin      1: /*     $NetBSD: uvm_map.c,v 1.330 2014/07/18 12:36:57 christos Exp $   */
1.1       mrg         2:
1.98      chs         3: /*
1.1       mrg         4:  * Copyright (c) 1997 Charles D. Cranor and Washington University.
1.98      chs         5:  * Copyright (c) 1991, 1993, The Regents of the University of California.
1.1       mrg         6:  *
                      7:  * All rights reserved.
                      8:  *
                      9:  * This code is derived from software contributed to Berkeley by
                     10:  * The Mach Operating System project at Carnegie-Mellon University.
                     11:  *
                     12:  * Redistribution and use in source and binary forms, with or without
                     13:  * modification, are permitted provided that the following conditions
                     14:  * are met:
                     15:  * 1. Redistributions of source code must retain the above copyright
                     16:  *    notice, this list of conditions and the following disclaimer.
                     17:  * 2. Redistributions in binary form must reproduce the above copyright
                     18:  *    notice, this list of conditions and the following disclaimer in the
                     19:  *    documentation and/or other materials provided with the distribution.
1.295     chuck      20:  * 3. Neither the name of the University nor the names of its contributors
1.1       mrg        21:  *    may be used to endorse or promote products derived from this software
                     22:  *    without specific prior written permission.
                     23:  *
                     24:  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
                     25:  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
                     26:  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
                     27:  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
                     28:  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
                     29:  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
                     30:  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
                     31:  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
                     32:  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
                     33:  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
                     34:  * SUCH DAMAGE.
                     35:  *
                     36:  *     @(#)vm_map.c    8.3 (Berkeley) 1/12/94
1.3       mrg        37:  * from: Id: uvm_map.c,v 1.1.2.27 1998/02/07 01:16:54 chs Exp
1.1       mrg        38:  *
                     39:  *
                     40:  * Copyright (c) 1987, 1990 Carnegie-Mellon University.
                     41:  * All rights reserved.
1.98      chs        42:  *
1.1       mrg        43:  * Permission to use, copy, modify and distribute this software and
                     44:  * its documentation is hereby granted, provided that both the copyright
                     45:  * notice and this permission notice appear in all copies of the
                     46:  * software, derivative works or modified versions, and any portions
                     47:  * thereof, and that both notices appear in supporting documentation.
1.98      chs        48:  *
                     49:  * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
                     50:  * CONDITION.  CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND
1.1       mrg        51:  * FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
1.98      chs        52:  *
1.1       mrg        53:  * Carnegie Mellon requests users of this software to return to
                     54:  *
                     55:  *  Software Distribution Coordinator  or  Software.Distribution@CS.CMU.EDU
                     56:  *  School of Computer Science
                     57:  *  Carnegie Mellon University
                     58:  *  Pittsburgh PA 15213-3890
                     59:  *
                     60:  * any improvements or extensions that they make and grant Carnegie the
                     61:  * rights to redistribute these changes.
                     62:  */
                     63:
1.114     lukem      64: /*
                     65:  * uvm_map.c: uvm map operations
                     66:  */
                     67:
                     68: #include <sys/cdefs.h>
1.330.2.1! martin     69: __KERNEL_RCSID(0, "$NetBSD: uvm_map.c,v 1.330 2014/07/18 12:36:57 christos Exp $");
1.114     lukem      70:
1.21      jonathan   71: #include "opt_ddb.h"
1.6       mrg        72: #include "opt_uvmhist.h"
1.169     petrov     73: #include "opt_uvm.h"
1.31      tron       74: #include "opt_sysv.h"
1.1       mrg        75:
                     76: #include <sys/param.h>
                     77: #include <sys/systm.h>
                     78: #include <sys/mman.h>
                     79: #include <sys/proc.h>
1.25      thorpej    80: #include <sys/pool.h>
1.104     chs        81: #include <sys/kernel.h>
1.112     thorpej    82: #include <sys/mount.h>
1.109     thorpej    83: #include <sys/vnode.h>
1.244     yamt       84: #include <sys/lockdebug.h>
1.248     ad         85: #include <sys/atomic.h>
1.288     drochner   86: #ifndef __USER_VA0_IS_SAFE
                     87: #include <sys/sysctl.h>
                     88: #include <sys/kauth.h>
1.290     drochner   89: #include "opt_user_va0_disable_default.h"
1.288     drochner   90: #endif
1.1       mrg        91:
                     92: #ifdef SYSVSHM
                     93: #include <sys/shm.h>
                     94: #endif
                     95:
                     96: #include <uvm/uvm.h>
1.271     yamt       97: #include <uvm/uvm_readahead.h>
1.21      jonathan   98:
1.270     pooka      99: #if defined(DDB) || defined(DEBUGPRINT)
1.21      jonathan  100: #include <uvm/uvm_ddb.h>
                    101: #endif
                    102:
1.318     matt      103: #ifdef UVMHIST
1.328     matt      104: static struct kern_history_ent maphistbuf[100];
                    105: UVMHIST_DEFINE(maphist) = UVMHIST_INITIALIZER(maphist, maphistbuf);
1.318     matt      106: #endif
                    107:
1.258     ad        108: #if !defined(UVMMAP_COUNTERS)
1.207     yamt      109:
                    110: #define        UVMMAP_EVCNT_DEFINE(name)       /* nothing */
                    111: #define UVMMAP_EVCNT_INCR(ev)          /* nothing */
                    112: #define UVMMAP_EVCNT_DECR(ev)          /* nothing */
                    113:
                    114: #else /* defined(UVMMAP_NOCOUNTERS) */
                    115:
1.228     yamt      116: #include <sys/evcnt.h>
1.207     yamt      117: #define        UVMMAP_EVCNT_DEFINE(name) \
                    118: struct evcnt uvmmap_evcnt_##name = EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, \
                    119:     "uvmmap", #name); \
                    120: EVCNT_ATTACH_STATIC(uvmmap_evcnt_##name);
                    121: #define        UVMMAP_EVCNT_INCR(ev)           uvmmap_evcnt_##ev.ev_count++
                    122: #define        UVMMAP_EVCNT_DECR(ev)           uvmmap_evcnt_##ev.ev_count--
                    123:
                    124: #endif /* defined(UVMMAP_NOCOUNTERS) */
                    125:
                    126: UVMMAP_EVCNT_DEFINE(ubackmerge)
                    127: UVMMAP_EVCNT_DEFINE(uforwmerge)
                    128: UVMMAP_EVCNT_DEFINE(ubimerge)
                    129: UVMMAP_EVCNT_DEFINE(unomerge)
                    130: UVMMAP_EVCNT_DEFINE(kbackmerge)
                    131: UVMMAP_EVCNT_DEFINE(kforwmerge)
                    132: UVMMAP_EVCNT_DEFINE(kbimerge)
                    133: UVMMAP_EVCNT_DEFINE(knomerge)
                    134: UVMMAP_EVCNT_DEFINE(map_call)
                    135: UVMMAP_EVCNT_DEFINE(mlk_call)
                    136: UVMMAP_EVCNT_DEFINE(mlk_hint)
1.263     matt      137: UVMMAP_EVCNT_DEFINE(mlk_list)
                    138: UVMMAP_EVCNT_DEFINE(mlk_tree)
                    139: UVMMAP_EVCNT_DEFINE(mlk_treeloop)
                    140: UVMMAP_EVCNT_DEFINE(mlk_listloop)
1.169     petrov    141:
1.87      enami     142: const char vmmapbsy[] = "vmmapbsy";
1.1       mrg       143:
                    144: /*
1.248     ad        145:  * cache for vmspace structures.
1.25      thorpej   146:  */
                    147:
1.248     ad        148: static struct pool_cache uvm_vmspace_cache;
1.25      thorpej   149:
1.26      thorpej   150: /*
1.248     ad        151:  * cache for dynamically-allocated map entries.
1.26      thorpej   152:  */
                    153:
1.248     ad        154: static struct pool_cache uvm_map_entry_cache;
1.130     thorpej   155:
1.40      thorpej   156: #ifdef PMAP_GROWKERNEL
                    157: /*
                    158:  * This global represents the end of the kernel virtual address
                    159:  * space.  If we want to exceed this, we must grow the kernel
                    160:  * virtual address space dynamically.
                    161:  *
                    162:  * Note, this variable is locked by kernel_map's lock.
                    163:  */
                    164: vaddr_t uvm_maxkaddr;
                    165: #endif
                    166:
1.288     drochner  167: #ifndef __USER_VA0_IS_SAFE
1.290     drochner  168: #ifndef __USER_VA0_DISABLE_DEFAULT
                    169: #define __USER_VA0_DISABLE_DEFAULT 1
1.288     drochner  170: #endif
1.290     drochner  171: #ifdef USER_VA0_DISABLE_DEFAULT /* kernel config option overrides */
                    172: #undef __USER_VA0_DISABLE_DEFAULT
                    173: #define __USER_VA0_DISABLE_DEFAULT USER_VA0_DISABLE_DEFAULT
1.288     drochner  174: #endif
1.290     drochner  175: static int user_va0_disable = __USER_VA0_DISABLE_DEFAULT;
1.288     drochner  176: #endif
                    177:
1.25      thorpej   178: /*
1.1       mrg       179:  * macros
                    180:  */
                    181:
                    182: /*
1.194     yamt      183:  * UVM_ET_ISCOMPATIBLE: check some requirements for map entry merging
                    184:  */
1.311     para      185: extern struct vm_map *pager_map;
1.194     yamt      186:
                    187: #define        UVM_ET_ISCOMPATIBLE(ent, type, uobj, meflags, \
                    188:     prot, maxprot, inh, adv, wire) \
                    189:        ((ent)->etype == (type) && \
1.311     para      190:        (((ent)->flags ^ (meflags)) & (UVM_MAP_NOMERGE)) == 0 && \
1.194     yamt      191:        (ent)->object.uvm_obj == (uobj) && \
                    192:        (ent)->protection == (prot) && \
                    193:        (ent)->max_protection == (maxprot) && \
                    194:        (ent)->inheritance == (inh) && \
                    195:        (ent)->advice == (adv) && \
                    196:        (ent)->wired_count == (wire))
                    197:
                    198: /*
1.1       mrg       199:  * uvm_map_entry_link: insert entry into a map
                    200:  *
                    201:  * => map must be locked
                    202:  */
1.10      mrg       203: #define uvm_map_entry_link(map, after_where, entry) do { \
1.218     yamt      204:        uvm_mapent_check(entry); \
1.10      mrg       205:        (map)->nentries++; \
                    206:        (entry)->prev = (after_where); \
                    207:        (entry)->next = (after_where)->next; \
                    208:        (entry)->prev->next = (entry); \
                    209:        (entry)->next->prev = (entry); \
1.144     yamt      210:        uvm_rb_insert((map), (entry)); \
1.124     perry     211: } while (/*CONSTCOND*/ 0)
1.10      mrg       212:
1.1       mrg       213: /*
                    214:  * uvm_map_entry_unlink: remove entry from a map
                    215:  *
                    216:  * => map must be locked
                    217:  */
1.10      mrg       218: #define uvm_map_entry_unlink(map, entry) do { \
1.221     yamt      219:        KASSERT((entry) != (map)->first_free); \
                    220:        KASSERT((entry) != (map)->hint); \
1.218     yamt      221:        uvm_mapent_check(entry); \
1.10      mrg       222:        (map)->nentries--; \
                    223:        (entry)->next->prev = (entry)->prev; \
                    224:        (entry)->prev->next = (entry)->next; \
1.144     yamt      225:        uvm_rb_remove((map), (entry)); \
1.124     perry     226: } while (/*CONSTCOND*/ 0)
1.1       mrg       227:
                    228: /*
                    229:  * SAVE_HINT: saves the specified entry as the hint for future lookups.
                    230:  *
1.248     ad        231:  * => map need not be locked.
1.1       mrg       232:  */
1.248     ad        233: #define SAVE_HINT(map, check, value) do { \
1.258     ad        234:        if ((map)->hint == (check)) \
                    235:                (map)->hint = (value); \
1.124     perry     236: } while (/*CONSTCOND*/ 0)
1.1       mrg       237:
                    238: /*
1.221     yamt      239:  * clear_hints: ensure that hints don't point to the entry.
                    240:  *
                    241:  * => map must be write-locked.
                    242:  */
                    243: static void
                    244: clear_hints(struct vm_map *map, struct vm_map_entry *ent)
                    245: {
                    246:
                    247:        SAVE_HINT(map, ent, ent->prev);
                    248:        if (map->first_free == ent) {
                    249:                map->first_free = ent->prev;
                    250:        }
                    251: }
                    252:
                    253: /*
1.1       mrg       254:  * VM_MAP_RANGE_CHECK: check and correct range
                    255:  *
                    256:  * => map must at least be read locked
                    257:  */
                    258:
1.10      mrg       259: #define VM_MAP_RANGE_CHECK(map, start, end) do { \
1.139     enami     260:        if (start < vm_map_min(map))            \
                    261:                start = vm_map_min(map);        \
                    262:        if (end > vm_map_max(map))              \
                    263:                end = vm_map_max(map);          \
                    264:        if (start > end)                        \
                    265:                start = end;                    \
1.124     perry     266: } while (/*CONSTCOND*/ 0)
1.1       mrg       267:
                    268: /*
                    269:  * local prototypes
                    270:  */
                    271:
1.138     enami     272: static struct vm_map_entry *
                    273:                uvm_mapent_alloc(struct vm_map *, int);
                    274: static void    uvm_mapent_copy(struct vm_map_entry *, struct vm_map_entry *);
                    275: static void    uvm_mapent_free(struct vm_map_entry *);
1.218     yamt      276: #if defined(DEBUG)
                    277: static void    _uvm_mapent_check(const struct vm_map_entry *, const char *,
                    278:                    int);
                    279: #define        uvm_mapent_check(map)   _uvm_mapent_check(map, __FILE__, __LINE__)
                    280: #else /* defined(DEBUG) */
                    281: #define        uvm_mapent_check(e)     /* nothing */
                    282: #endif /* defined(DEBUG) */
1.219     yamt      283:
1.138     enami     284: static void    uvm_map_entry_unwire(struct vm_map *, struct vm_map_entry *);
                    285: static void    uvm_map_reference_amap(struct vm_map_entry *, int);
1.140     enami     286: static int     uvm_map_space_avail(vaddr_t *, vsize_t, voff_t, vsize_t, int,
1.304     matt      287:                    int, struct vm_map_entry *);
1.138     enami     288: static void    uvm_map_unreference_amap(struct vm_map_entry *, int);
1.1       mrg       289:
1.222     yamt      290: int _uvm_map_sanity(struct vm_map *);
                    291: int _uvm_tree_sanity(struct vm_map *);
1.263     matt      292: static vsize_t uvm_rb_maxgap(const struct vm_map_entry *);
1.144     yamt      293:
1.263     matt      294: #define        ROOT_ENTRY(map)         ((struct vm_map_entry *)(map)->rb_tree.rbt_root)
                    295: #define        LEFT_ENTRY(entry)       ((struct vm_map_entry *)(entry)->rb_node.rb_left)
                    296: #define        RIGHT_ENTRY(entry)      ((struct vm_map_entry *)(entry)->rb_node.rb_right)
                    297: #define        PARENT_ENTRY(map, entry) \
                    298:        (ROOT_ENTRY(map) == (entry) \
1.293     rmind     299:            ? NULL : (struct vm_map_entry *)RB_FATHER(&(entry)->rb_node))
1.263     matt      300:
                    301: static int
1.293     rmind     302: uvm_map_compare_nodes(void *ctx, const void *nparent, const void *nkey)
1.144     yamt      303: {
1.293     rmind     304:        const struct vm_map_entry *eparent = nparent;
                    305:        const struct vm_map_entry *ekey = nkey;
1.144     yamt      306:
1.263     matt      307:        KASSERT(eparent->start < ekey->start || eparent->start >= ekey->end);
                    308:        KASSERT(ekey->start < eparent->start || ekey->start >= eparent->end);
1.164     junyoung  309:
1.293     rmind     310:        if (eparent->start < ekey->start)
1.263     matt      311:                return -1;
1.293     rmind     312:        if (eparent->end >= ekey->start)
1.263     matt      313:                return 1;
                    314:        return 0;
1.144     yamt      315: }
                    316:
1.263     matt      317: static int
1.293     rmind     318: uvm_map_compare_key(void *ctx, const void *nparent, const void *vkey)
1.144     yamt      319: {
1.293     rmind     320:        const struct vm_map_entry *eparent = nparent;
1.263     matt      321:        const vaddr_t va = *(const vaddr_t *) vkey;
1.144     yamt      322:
1.293     rmind     323:        if (eparent->start < va)
1.263     matt      324:                return -1;
1.293     rmind     325:        if (eparent->end >= va)
1.263     matt      326:                return 1;
                    327:        return 0;
1.144     yamt      328: }
                    329:
1.293     rmind     330: static const rb_tree_ops_t uvm_map_tree_ops = {
1.263     matt      331:        .rbto_compare_nodes = uvm_map_compare_nodes,
                    332:        .rbto_compare_key = uvm_map_compare_key,
1.293     rmind     333:        .rbto_node_offset = offsetof(struct vm_map_entry, rb_node),
                    334:        .rbto_context = NULL
1.263     matt      335: };
1.144     yamt      336:
1.293     rmind     337: /*
                    338:  * uvm_rb_gap: return the gap size between our entry and next entry.
                    339:  */
1.206     perry     340: static inline vsize_t
1.263     matt      341: uvm_rb_gap(const struct vm_map_entry *entry)
1.144     yamt      342: {
1.293     rmind     343:
1.144     yamt      344:        KASSERT(entry->next != NULL);
                    345:        return entry->next->start - entry->end;
                    346: }
                    347:
                    348: static vsize_t
1.263     matt      349: uvm_rb_maxgap(const struct vm_map_entry *entry)
1.144     yamt      350: {
1.263     matt      351:        struct vm_map_entry *child;
                    352:        vsize_t maxgap = entry->gap;
1.144     yamt      353:
1.263     matt      354:        /*
                    355:         * We need maxgap to be the largest gap of us or any of our
                    356:         * descendents.  Since each of our children's maxgap is the
                    357:         * cached value of their largest gap of themselves or their
                    358:         * descendents, we can just use that value and avoid recursing
                    359:         * down the tree to calculate it.
                    360:         */
                    361:        if ((child = LEFT_ENTRY(entry)) != NULL && maxgap < child->maxgap)
                    362:                maxgap = child->maxgap;
1.144     yamt      363:
1.263     matt      364:        if ((child = RIGHT_ENTRY(entry)) != NULL && maxgap < child->maxgap)
                    365:                maxgap = child->maxgap;
1.144     yamt      366:
1.263     matt      367:        return maxgap;
1.144     yamt      368: }
                    369:
1.263     matt      370: static void
1.144     yamt      371: uvm_rb_fixup(struct vm_map *map, struct vm_map_entry *entry)
                    372: {
1.263     matt      373:        struct vm_map_entry *parent;
                    374:
                    375:        KASSERT(entry->gap == uvm_rb_gap(entry));
                    376:        entry->maxgap = uvm_rb_maxgap(entry);
                    377:
                    378:        while ((parent = PARENT_ENTRY(map, entry)) != NULL) {
                    379:                struct vm_map_entry *brother;
                    380:                vsize_t maxgap = parent->gap;
1.293     rmind     381:                unsigned int which;
1.263     matt      382:
                    383:                KDASSERT(parent->gap == uvm_rb_gap(parent));
                    384:                if (maxgap < entry->maxgap)
                    385:                        maxgap = entry->maxgap;
                    386:                /*
1.293     rmind     387:                 * Since we work towards the root, we know entry's maxgap
                    388:                 * value is OK, but its brothers may now be out-of-date due
                    389:                 * to rebalancing.  So refresh it.
1.263     matt      390:                 */
1.293     rmind     391:                which = RB_POSITION(&entry->rb_node) ^ RB_DIR_OTHER;
                    392:                brother = (struct vm_map_entry *)parent->rb_node.rb_nodes[which];
1.263     matt      393:                if (brother != NULL) {
                    394:                        KDASSERT(brother->gap == uvm_rb_gap(brother));
                    395:                        brother->maxgap = uvm_rb_maxgap(brother);
                    396:                        if (maxgap < brother->maxgap)
                    397:                                maxgap = brother->maxgap;
                    398:                }
                    399:
                    400:                parent->maxgap = maxgap;
                    401:                entry = parent;
                    402:        }
1.144     yamt      403: }
                    404:
1.203     thorpej   405: static void
1.144     yamt      406: uvm_rb_insert(struct vm_map *map, struct vm_map_entry *entry)
                    407: {
1.326     martin    408:        struct vm_map_entry *ret __diagused;
1.293     rmind     409:
1.263     matt      410:        entry->gap = entry->maxgap = uvm_rb_gap(entry);
                    411:        if (entry->prev != &map->header)
                    412:                entry->prev->gap = uvm_rb_gap(entry->prev);
1.144     yamt      413:
1.293     rmind     414:        ret = rb_tree_insert_node(&map->rb_tree, entry);
                    415:        KASSERTMSG(ret == entry,
1.305     jym       416:            "uvm_rb_insert: map %p: duplicate entry %p", map, ret);
1.263     matt      417:
                    418:        /*
                    419:         * If the previous entry is not our immediate left child, then it's an
                    420:         * ancestor and will be fixed up on the way to the root.  We don't
                    421:         * have to check entry->prev against &map->header since &map->header
                    422:         * will never be in the tree.
                    423:         */
                    424:        uvm_rb_fixup(map,
                    425:            LEFT_ENTRY(entry) == entry->prev ? entry->prev : entry);
1.144     yamt      426: }
                    427:
1.203     thorpej   428: static void
1.144     yamt      429: uvm_rb_remove(struct vm_map *map, struct vm_map_entry *entry)
                    430: {
1.263     matt      431:        struct vm_map_entry *prev_parent = NULL, *next_parent = NULL;
1.144     yamt      432:
1.263     matt      433:        /*
                    434:         * If we are removing an interior node, then an adjacent node will
                    435:         * be used to replace its position in the tree.  Therefore we will
                    436:         * need to fixup the tree starting at the parent of the replacement
                    437:         * node.  So record their parents for later use.
                    438:         */
1.144     yamt      439:        if (entry->prev != &map->header)
1.263     matt      440:                prev_parent = PARENT_ENTRY(map, entry->prev);
                    441:        if (entry->next != &map->header)
                    442:                next_parent = PARENT_ENTRY(map, entry->next);
                    443:
1.293     rmind     444:        rb_tree_remove_node(&map->rb_tree, entry);
1.263     matt      445:
                    446:        /*
                    447:         * If the previous node has a new parent, fixup the tree starting
                    448:         * at the previous node's old parent.
                    449:         */
                    450:        if (entry->prev != &map->header) {
                    451:                /*
                    452:                 * Update the previous entry's gap due to our absence.
                    453:                 */
                    454:                entry->prev->gap = uvm_rb_gap(entry->prev);
1.144     yamt      455:                uvm_rb_fixup(map, entry->prev);
1.263     matt      456:                if (prev_parent != NULL
                    457:                    && prev_parent != entry
                    458:                    && prev_parent != PARENT_ENTRY(map, entry->prev))
                    459:                        uvm_rb_fixup(map, prev_parent);
                    460:        }
                    461:
                    462:        /*
                    463:         * If the next node has a new parent, fixup the tree starting
                    464:         * at the next node's old parent.
                    465:         */
                    466:        if (entry->next != &map->header) {
                    467:                uvm_rb_fixup(map, entry->next);
                    468:                if (next_parent != NULL
                    469:                    && next_parent != entry
                    470:                    && next_parent != PARENT_ENTRY(map, entry->next))
                    471:                        uvm_rb_fixup(map, next_parent);
                    472:        }
1.144     yamt      473: }
                    474:
1.222     yamt      475: #if defined(DEBUG)
                    476: int uvm_debug_check_map = 0;
1.159     yamt      477: int uvm_debug_check_rbtree = 0;
1.222     yamt      478: #define uvm_map_check(map, name) \
                    479:        _uvm_map_check((map), (name), __FILE__, __LINE__)
                    480: static void
                    481: _uvm_map_check(struct vm_map *map, const char *name,
                    482:     const char *file, int line)
                    483: {
                    484:
                    485:        if ((uvm_debug_check_map && _uvm_map_sanity(map)) ||
                    486:            (uvm_debug_check_rbtree && _uvm_tree_sanity(map))) {
                    487:                panic("uvm_map_check failed: \"%s\" map=%p (%s:%d)",
                    488:                    name, map, file, line);
                    489:        }
                    490: }
                    491: #else /* defined(DEBUG) */
                    492: #define uvm_map_check(map, name)       /* nothing */
                    493: #endif /* defined(DEBUG) */
                    494:
                    495: #if defined(DEBUG) || defined(DDB)
                    496: int
                    497: _uvm_map_sanity(struct vm_map *map)
                    498: {
1.234     thorpej   499:        bool first_free_found = false;
                    500:        bool hint_found = false;
1.222     yamt      501:        const struct vm_map_entry *e;
1.272     yamt      502:        struct vm_map_entry *hint = map->hint;
1.222     yamt      503:
                    504:        e = &map->header;
                    505:        for (;;) {
                    506:                if (map->first_free == e) {
1.234     thorpej   507:                        first_free_found = true;
1.222     yamt      508:                } else if (!first_free_found && e->next->start > e->end) {
                    509:                        printf("first_free %p should be %p\n",
                    510:                            map->first_free, e);
                    511:                        return -1;
                    512:                }
1.272     yamt      513:                if (hint == e) {
1.234     thorpej   514:                        hint_found = true;
1.222     yamt      515:                }
                    516:
                    517:                e = e->next;
                    518:                if (e == &map->header) {
                    519:                        break;
                    520:                }
                    521:        }
                    522:        if (!first_free_found) {
                    523:                printf("stale first_free\n");
                    524:                return -1;
                    525:        }
                    526:        if (!hint_found) {
                    527:                printf("stale hint\n");
                    528:                return -1;
                    529:        }
                    530:        return 0;
                    531: }
1.144     yamt      532:
                    533: int
1.222     yamt      534: _uvm_tree_sanity(struct vm_map *map)
1.144     yamt      535: {
                    536:        struct vm_map_entry *tmp, *trtmp;
                    537:        int n = 0, i = 1;
                    538:
1.263     matt      539:        for (tmp = map->header.next; tmp != &map->header; tmp = tmp->next) {
                    540:                if (tmp->gap != uvm_rb_gap(tmp)) {
                    541:                        printf("%d/%d gap %lx != %lx %s\n",
1.222     yamt      542:                            n + 1, map->nentries,
1.263     matt      543:                            (ulong)tmp->gap, (ulong)uvm_rb_gap(tmp),
1.144     yamt      544:                            tmp->next == &map->header ? "(last)" : "");
                    545:                        goto error;
                    546:                }
1.263     matt      547:                /*
                    548:                 * If any entries are out of order, tmp->gap will be unsigned
                    549:                 * and will likely exceed the size of the map.
                    550:                 */
1.273     yamt      551:                if (tmp->gap >= vm_map_max(map) - vm_map_min(map)) {
                    552:                        printf("too large gap %zu\n", (size_t)tmp->gap);
                    553:                        goto error;
                    554:                }
1.263     matt      555:                n++;
                    556:        }
                    557:
                    558:        if (n != map->nentries) {
                    559:                printf("nentries: %d vs %d\n", n, map->nentries);
                    560:                goto error;
1.144     yamt      561:        }
1.263     matt      562:
1.144     yamt      563:        trtmp = NULL;
1.263     matt      564:        for (tmp = map->header.next; tmp != &map->header; tmp = tmp->next) {
                    565:                if (tmp->maxgap != uvm_rb_maxgap(tmp)) {
                    566:                        printf("maxgap %lx != %lx\n",
                    567:                            (ulong)tmp->maxgap,
                    568:                            (ulong)uvm_rb_maxgap(tmp));
1.144     yamt      569:                        goto error;
                    570:                }
                    571:                if (trtmp != NULL && trtmp->start >= tmp->start) {
1.285     matt      572:                        printf("corrupt: 0x%"PRIxVADDR"x >= 0x%"PRIxVADDR"x\n",
1.222     yamt      573:                            trtmp->start, tmp->start);
1.144     yamt      574:                        goto error;
                    575:                }
                    576:
                    577:                trtmp = tmp;
                    578:        }
                    579:
1.263     matt      580:        for (tmp = map->header.next; tmp != &map->header;
1.144     yamt      581:            tmp = tmp->next, i++) {
1.293     rmind     582:                trtmp = rb_tree_iterate(&map->rb_tree, tmp, RB_DIR_LEFT);
1.263     matt      583:                if (trtmp == NULL)
                    584:                        trtmp = &map->header;
                    585:                if (tmp->prev != trtmp) {
                    586:                        printf("lookup: %d: %p->prev=%p: %p\n",
                    587:                            i, tmp, tmp->prev, trtmp);
                    588:                        goto error;
                    589:                }
1.293     rmind     590:                trtmp = rb_tree_iterate(&map->rb_tree, tmp, RB_DIR_RIGHT);
1.263     matt      591:                if (trtmp == NULL)
                    592:                        trtmp = &map->header;
                    593:                if (tmp->next != trtmp) {
                    594:                        printf("lookup: %d: %p->next=%p: %p\n",
                    595:                            i, tmp, tmp->next, trtmp);
                    596:                        goto error;
                    597:                }
1.293     rmind     598:                trtmp = rb_tree_find_node(&map->rb_tree, &tmp->start);
1.144     yamt      599:                if (trtmp != tmp) {
1.222     yamt      600:                        printf("lookup: %d: %p - %p: %p\n", i, tmp, trtmp,
1.263     matt      601:                            PARENT_ENTRY(map, tmp));
1.144     yamt      602:                        goto error;
                    603:                }
                    604:        }
                    605:
                    606:        return (0);
                    607:  error:
                    608:        return (-1);
                    609: }
1.222     yamt      610: #endif /* defined(DEBUG) || defined(DDB) */
1.144     yamt      611:
1.1       mrg       612: /*
1.238     ad        613:  * vm_map_lock: acquire an exclusive (write) lock on a map.
                    614:  *
                    615:  * => The locking protocol provides for guaranteed upgrade from shared ->
                    616:  *    exclusive by whichever thread currently has the map marked busy.
                    617:  *    See "LOCKING PROTOCOL NOTES" in uvm_map.h.  This is horrible; among
                    618:  *    other problems, it defeats any fairness guarantees provided by RW
                    619:  *    locks.
                    620:  */
                    621:
                    622: void
                    623: vm_map_lock(struct vm_map *map)
                    624: {
                    625:
                    626:        for (;;) {
                    627:                rw_enter(&map->lock, RW_WRITER);
1.314     rmind     628:                if (map->busy == NULL || map->busy == curlwp) {
1.249     yamt      629:                        break;
1.314     rmind     630:                }
1.238     ad        631:                mutex_enter(&map->misc_lock);
                    632:                rw_exit(&map->lock);
1.314     rmind     633:                if (map->busy != NULL) {
1.248     ad        634:                        cv_wait(&map->cv, &map->misc_lock);
1.314     rmind     635:                }
1.238     ad        636:                mutex_exit(&map->misc_lock);
                    637:        }
                    638:        map->timestamp++;
                    639: }
                    640:
                    641: /*
                    642:  * vm_map_lock_try: try to lock a map, failing if it is already locked.
                    643:  */
                    644:
                    645: bool
                    646: vm_map_lock_try(struct vm_map *map)
                    647: {
                    648:
1.314     rmind     649:        if (!rw_tryenter(&map->lock, RW_WRITER)) {
1.238     ad        650:                return false;
1.314     rmind     651:        }
1.238     ad        652:        if (map->busy != NULL) {
                    653:                rw_exit(&map->lock);
                    654:                return false;
                    655:        }
                    656:        map->timestamp++;
                    657:        return true;
                    658: }
                    659:
                    660: /*
                    661:  * vm_map_unlock: release an exclusive lock on a map.
                    662:  */
                    663:
                    664: void
                    665: vm_map_unlock(struct vm_map *map)
                    666: {
                    667:
1.314     rmind     668:        KASSERT(rw_write_held(&map->lock));
                    669:        KASSERT(map->busy == NULL || map->busy == curlwp);
                    670:        rw_exit(&map->lock);
1.238     ad        671: }
                    672:
                    673: /*
                    674:  * vm_map_unbusy: mark the map as unbusy, and wake any waiters that
                    675:  *     want an exclusive lock.
                    676:  */
                    677:
                    678: void
                    679: vm_map_unbusy(struct vm_map *map)
                    680: {
                    681:
                    682:        KASSERT(map->busy == curlwp);
                    683:
                    684:        /*
                    685:         * Safe to clear 'busy' and 'waiters' with only a read lock held:
                    686:         *
                    687:         * o they can only be set with a write lock held
                    688:         * o writers are blocked out with a read or write hold
                    689:         * o at any time, only one thread owns the set of values
                    690:         */
1.248     ad        691:        mutex_enter(&map->misc_lock);
1.238     ad        692:        map->busy = NULL;
                    693:        cv_broadcast(&map->cv);
                    694:        mutex_exit(&map->misc_lock);
                    695: }
                    696:
                    697: /*
1.248     ad        698:  * vm_map_lock_read: acquire a shared (read) lock on a map.
                    699:  */
                    700:
                    701: void
                    702: vm_map_lock_read(struct vm_map *map)
                    703: {
                    704:
                    705:        rw_enter(&map->lock, RW_READER);
                    706: }
                    707:
                    708: /*
                    709:  * vm_map_unlock_read: release a shared lock on a map.
                    710:  */
1.314     rmind     711:
1.248     ad        712: void
                    713: vm_map_unlock_read(struct vm_map *map)
                    714: {
                    715:
                    716:        rw_exit(&map->lock);
                    717: }
                    718:
                    719: /*
                    720:  * vm_map_busy: mark a map as busy.
                    721:  *
                    722:  * => the caller must hold the map write locked
                    723:  */
                    724:
                    725: void
                    726: vm_map_busy(struct vm_map *map)
                    727: {
                    728:
                    729:        KASSERT(rw_write_held(&map->lock));
                    730:        KASSERT(map->busy == NULL);
                    731:
                    732:        map->busy = curlwp;
                    733: }
                    734:
                    735: /*
                    736:  * vm_map_locked_p: return true if the map is write locked.
1.269     yamt      737:  *
                    738:  * => only for debug purposes like KASSERTs.
                    739:  * => should not be used to verify that a map is not locked.
1.248     ad        740:  */
                    741:
                    742: bool
                    743: vm_map_locked_p(struct vm_map *map)
                    744: {
                    745:
1.314     rmind     746:        return rw_write_held(&map->lock);
1.248     ad        747: }
                    748:
                    749: /*
1.1       mrg       750:  * uvm_mapent_alloc: allocate a map entry
                    751:  */
                    752:
1.203     thorpej   753: static struct vm_map_entry *
1.138     enami     754: uvm_mapent_alloc(struct vm_map *map, int flags)
1.10      mrg       755: {
1.99      chs       756:        struct vm_map_entry *me;
1.127     thorpej   757:        int pflags = (flags & UVM_FLAG_NOWAIT) ? PR_NOWAIT : PR_WAITOK;
1.104     chs       758:        UVMHIST_FUNC("uvm_mapent_alloc"); UVMHIST_CALLED(maphist);
1.1       mrg       759:
1.311     para      760:        me = pool_cache_get(&uvm_map_entry_cache, pflags);
1.314     rmind     761:        if (__predict_false(me == NULL)) {
1.311     para      762:                return NULL;
1.314     rmind     763:        }
1.311     para      764:        me->flags = 0;
                    765:
1.104     chs       766:        UVMHIST_LOG(maphist, "<- new entry=0x%x [kentry=%d]", me,
1.314     rmind     767:            (map == kernel_map), 0, 0);
                    768:        return me;
1.1       mrg       769: }
                    770:
                    771: /*
                    772:  * uvm_mapent_free: free map entry
                    773:  */
                    774:
1.203     thorpej   775: static void
1.138     enami     776: uvm_mapent_free(struct vm_map_entry *me)
1.1       mrg       777: {
1.104     chs       778:        UVMHIST_FUNC("uvm_mapent_free"); UVMHIST_CALLED(maphist);
                    779:
1.98      chs       780:        UVMHIST_LOG(maphist,"<- freeing map entry=0x%x [flags=%d]",
1.1       mrg       781:                me, me->flags, 0, 0);
1.311     para      782:        pool_cache_put(&uvm_map_entry_cache, me);
1.1       mrg       783: }
                    784:
                    785: /*
                    786:  * uvm_mapent_copy: copy a map entry, preserving flags
                    787:  */
                    788:
1.206     perry     789: static inline void
1.138     enami     790: uvm_mapent_copy(struct vm_map_entry *src, struct vm_map_entry *dst)
1.10      mrg       791: {
1.139     enami     792:
1.106     chs       793:        memcpy(dst, src, ((char *)&src->uvm_map_entry_stop_copy) -
1.139     enami     794:            ((char *)src));
1.1       mrg       795: }
                    796:
1.218     yamt      797: #if defined(DEBUG)
                    798: static void
                    799: _uvm_mapent_check(const struct vm_map_entry *entry, const char *file, int line)
                    800: {
                    801:
                    802:        if (entry->start >= entry->end) {
                    803:                goto bad;
                    804:        }
                    805:        if (UVM_ET_ISOBJ(entry)) {
                    806:                if (entry->object.uvm_obj == NULL) {
                    807:                        goto bad;
                    808:                }
                    809:        } else if (UVM_ET_ISSUBMAP(entry)) {
                    810:                if (entry->object.sub_map == NULL) {
                    811:                        goto bad;
                    812:                }
                    813:        } else {
                    814:                if (entry->object.uvm_obj != NULL ||
                    815:                    entry->object.sub_map != NULL) {
                    816:                        goto bad;
                    817:                }
                    818:        }
                    819:        if (!UVM_ET_ISOBJ(entry)) {
                    820:                if (entry->offset != 0) {
                    821:                        goto bad;
                    822:                }
                    823:        }
                    824:
                    825:        return;
                    826:
                    827: bad:
                    828:        panic("%s: bad entry %p (%s:%d)", __func__, entry, file, line);
                    829: }
                    830: #endif /* defined(DEBUG) */
                    831:
1.1       mrg       832: /*
                    833:  * uvm_map_entry_unwire: unwire a map entry
                    834:  *
                    835:  * => map should be locked by caller
                    836:  */
                    837:
1.206     perry     838: static inline void
1.138     enami     839: uvm_map_entry_unwire(struct vm_map *map, struct vm_map_entry *entry)
1.10      mrg       840: {
1.139     enami     841:
1.10      mrg       842:        entry->wired_count = 0;
1.57      thorpej   843:        uvm_fault_unwire_locked(map, entry->start, entry->end);
1.1       mrg       844: }
                    845:
1.85      chs       846:
                    847: /*
                    848:  * wrapper for calling amap_ref()
                    849:  */
1.206     perry     850: static inline void
1.138     enami     851: uvm_map_reference_amap(struct vm_map_entry *entry, int flags)
1.85      chs       852: {
1.139     enami     853:
1.99      chs       854:        amap_ref(entry->aref.ar_amap, entry->aref.ar_pageoff,
1.139     enami     855:            (entry->end - entry->start) >> PAGE_SHIFT, flags);
1.85      chs       856: }
                    857:
                    858:
                    859: /*
1.98      chs       860:  * wrapper for calling amap_unref()
1.85      chs       861:  */
1.206     perry     862: static inline void
1.138     enami     863: uvm_map_unreference_amap(struct vm_map_entry *entry, int flags)
1.85      chs       864: {
1.139     enami     865:
1.99      chs       866:        amap_unref(entry->aref.ar_amap, entry->aref.ar_pageoff,
1.139     enami     867:            (entry->end - entry->start) >> PAGE_SHIFT, flags);
1.85      chs       868: }
                    869:
                    870:
1.1       mrg       871: /*
1.248     ad        872:  * uvm_map_init: init mapping system at boot time.
1.1       mrg       873:  */
                    874:
1.10      mrg       875: void
1.138     enami     876: uvm_map_init(void)
1.1       mrg       877: {
                    878: #if defined(UVMHIST)
1.297     mrg       879:        static struct kern_history_ent pdhistbuf[100];
1.1       mrg       880: #endif
1.10      mrg       881:
                    882:        /*
                    883:         * first, init logging system.
                    884:         */
1.1       mrg       885:
1.10      mrg       886:        UVMHIST_FUNC("uvm_map_init");
1.328     matt      887:        UVMHIST_LINK_STATIC(maphist);
1.10      mrg       888:        UVMHIST_INIT_STATIC(pdhist, pdhistbuf);
                    889:        UVMHIST_CALLED(maphist);
                    890:        UVMHIST_LOG(maphist,"<starting uvm map system>", 0, 0, 0, 0);
                    891:
                    892:        /*
1.174     yamt      893:         * initialize the global lock for kernel map entry.
1.10      mrg       894:         */
                    895:
1.238     ad        896:        mutex_init(&uvm_kentry_lock, MUTEX_DRIVER, IPL_VM);
1.311     para      897: }
1.248     ad        898:
1.311     para      899: /*
                    900:  * uvm_map_init_caches: init mapping system caches.
                    901:  */
                    902: void
                    903: uvm_map_init_caches(void)
1.312     rmind     904: {
1.248     ad        905:        /*
                    906:         * initialize caches.
                    907:         */
                    908:
                    909:        pool_cache_bootstrap(&uvm_map_entry_cache, sizeof(struct vm_map_entry),
                    910:            0, 0, 0, "vmmpepl", NULL, IPL_NONE, NULL, NULL, NULL);
                    911:        pool_cache_bootstrap(&uvm_vmspace_cache, sizeof(struct vmspace),
                    912:            0, 0, 0, "vmsppl", NULL, IPL_NONE, NULL, NULL, NULL);
1.1       mrg       913: }
                    914:
                    915: /*
                    916:  * clippers
                    917:  */
                    918:
                    919: /*
1.218     yamt      920:  * uvm_mapent_splitadj: adjust map entries for splitting, after uvm_mapent_copy.
                    921:  */
                    922:
                    923: static void
                    924: uvm_mapent_splitadj(struct vm_map_entry *entry1, struct vm_map_entry *entry2,
                    925:     vaddr_t splitat)
                    926: {
                    927:        vaddr_t adj;
                    928:
                    929:        KASSERT(entry1->start < splitat);
                    930:        KASSERT(splitat < entry1->end);
                    931:
                    932:        adj = splitat - entry1->start;
                    933:        entry1->end = entry2->start = splitat;
                    934:
                    935:        if (entry1->aref.ar_amap) {
                    936:                amap_splitref(&entry1->aref, &entry2->aref, adj);
                    937:        }
                    938:        if (UVM_ET_ISSUBMAP(entry1)) {
                    939:                /* ... unlikely to happen, but play it safe */
                    940:                 uvm_map_reference(entry1->object.sub_map);
                    941:        } else if (UVM_ET_ISOBJ(entry1)) {
                    942:                KASSERT(entry1->object.uvm_obj != NULL); /* suppress coverity */
                    943:                entry2->offset += adj;
                    944:                if (entry1->object.uvm_obj->pgops &&
                    945:                    entry1->object.uvm_obj->pgops->pgo_reference)
                    946:                        entry1->object.uvm_obj->pgops->pgo_reference(
                    947:                            entry1->object.uvm_obj);
                    948:        }
                    949: }
                    950:
                    951: /*
1.1       mrg       952:  * uvm_map_clip_start: ensure that the entry begins at or after
                    953:  *     the starting address, if it doesn't we split the entry.
1.98      chs       954:  *
1.1       mrg       955:  * => caller should use UVM_MAP_CLIP_START macro rather than calling
                    956:  *    this directly
                    957:  * => map must be locked by caller
                    958:  */
                    959:
1.99      chs       960: void
1.138     enami     961: uvm_map_clip_start(struct vm_map *map, struct vm_map_entry *entry,
1.311     para      962:     vaddr_t start)
1.1       mrg       963: {
1.99      chs       964:        struct vm_map_entry *new_entry;
1.1       mrg       965:
                    966:        /* uvm_map_simplify_entry(map, entry); */ /* XXX */
                    967:
1.222     yamt      968:        uvm_map_check(map, "clip_start entry");
1.218     yamt      969:        uvm_mapent_check(entry);
1.144     yamt      970:
1.10      mrg       971:        /*
                    972:         * Split off the front portion.  note that we must insert the new
                    973:         * entry BEFORE this one, so that this entry has the specified
1.1       mrg       974:         * starting address.
1.10      mrg       975:         */
1.311     para      976:        new_entry = uvm_mapent_alloc(map, 0);
1.1       mrg       977:        uvm_mapent_copy(entry, new_entry); /* entry -> new_entry */
1.218     yamt      978:        uvm_mapent_splitadj(new_entry, entry, start);
1.10      mrg       979:        uvm_map_entry_link(map, entry->prev, new_entry);
1.85      chs       980:
1.222     yamt      981:        uvm_map_check(map, "clip_start leave");
1.1       mrg       982: }
                    983:
                    984: /*
                    985:  * uvm_map_clip_end: ensure that the entry ends at or before
                    986:  *     the ending address, if it does't we split the reference
1.98      chs       987:  *
1.1       mrg       988:  * => caller should use UVM_MAP_CLIP_END macro rather than calling
                    989:  *    this directly
                    990:  * => map must be locked by caller
                    991:  */
                    992:
1.10      mrg       993: void
1.311     para      994: uvm_map_clip_end(struct vm_map *map, struct vm_map_entry *entry, vaddr_t end)
1.1       mrg       995: {
1.218     yamt      996:        struct vm_map_entry *new_entry;
1.1       mrg       997:
1.222     yamt      998:        uvm_map_check(map, "clip_end entry");
1.218     yamt      999:        uvm_mapent_check(entry);
1.174     yamt     1000:
1.1       mrg      1001:        /*
                   1002:         *      Create a new entry and insert it
                   1003:         *      AFTER the specified entry
                   1004:         */
1.311     para     1005:        new_entry = uvm_mapent_alloc(map, 0);
1.1       mrg      1006:        uvm_mapent_copy(entry, new_entry); /* entry -> new_entry */
1.218     yamt     1007:        uvm_mapent_splitadj(entry, new_entry, end);
1.1       mrg      1008:        uvm_map_entry_link(map, entry, new_entry);
                   1009:
1.222     yamt     1010:        uvm_map_check(map, "clip_end leave");
1.1       mrg      1011: }
                   1012:
                   1013: /*
                   1014:  *   M A P   -   m a i n   e n t r y   p o i n t
                   1015:  */
                   1016: /*
                   1017:  * uvm_map: establish a valid mapping in a map
                   1018:  *
                   1019:  * => assume startp is page aligned.
                   1020:  * => assume size is a multiple of PAGE_SIZE.
                   1021:  * => assume sys_mmap provides enough of a "hint" to have us skip
                   1022:  *     over text/data/bss area.
                   1023:  * => map must be unlocked (we will lock it)
                   1024:  * => <uobj,uoffset> value meanings (4 cases):
1.139     enami    1025:  *      [1] <NULL,uoffset>             == uoffset is a hint for PMAP_PREFER
1.1       mrg      1026:  *      [2] <NULL,UVM_UNKNOWN_OFFSET>  == don't PMAP_PREFER
                   1027:  *      [3] <uobj,uoffset>             == normal mapping
                   1028:  *      [4] <uobj,UVM_UNKNOWN_OFFSET>  == uvm_map finds offset based on VA
1.98      chs      1029:  *
1.1       mrg      1030:  *    case [4] is for kernel mappings where we don't know the offset until
1.8       chuck    1031:  *    we've found a virtual address.   note that kernel object offsets are
                   1032:  *    always relative to vm_map_min(kernel_map).
1.81      thorpej  1033:  *
1.165     yamt     1034:  * => if `align' is non-zero, we align the virtual address to the specified
                   1035:  *     alignment.
                   1036:  *     this is provided as a mechanism for large pages.
1.81      thorpej  1037:  *
1.1       mrg      1038:  * => XXXCDC: need way to map in external amap?
                   1039:  */
                   1040:
1.10      mrg      1041: int
1.138     enami    1042: uvm_map(struct vm_map *map, vaddr_t *startp /* IN/OUT */, vsize_t size,
                   1043:     struct uvm_object *uobj, voff_t uoffset, vsize_t align, uvm_flag_t flags)
1.10      mrg      1044: {
1.174     yamt     1045:        struct uvm_map_args args;
                   1046:        struct vm_map_entry *new_entry;
                   1047:        int error;
                   1048:
1.187     yamt     1049:        KASSERT((size & PAGE_MASK) == 0);
1.174     yamt     1050:
1.288     drochner 1051: #ifndef __USER_VA0_IS_SAFE
                   1052:        if ((flags & UVM_FLAG_FIXED) && *startp == 0 &&
1.290     drochner 1053:            !VM_MAP_IS_KERNEL(map) && user_va0_disable)
1.288     drochner 1054:                return EACCES;
                   1055: #endif
                   1056:
1.174     yamt     1057:        /*
                   1058:         * for pager_map, allocate the new entry first to avoid sleeping
                   1059:         * for memory while we have the map locked.
                   1060:         */
                   1061:
                   1062:        new_entry = NULL;
1.311     para     1063:        if (map == pager_map) {
1.174     yamt     1064:                new_entry = uvm_mapent_alloc(map, (flags & UVM_FLAG_NOWAIT));
                   1065:                if (__predict_false(new_entry == NULL))
                   1066:                        return ENOMEM;
                   1067:        }
                   1068:        if (map == pager_map)
                   1069:                flags |= UVM_FLAG_NOMERGE;
                   1070:
                   1071:        error = uvm_map_prepare(map, *startp, size, uobj, uoffset, align,
                   1072:            flags, &args);
                   1073:        if (!error) {
                   1074:                error = uvm_map_enter(map, &args, new_entry);
                   1075:                *startp = args.uma_start;
1.189     yamt     1076:        } else if (new_entry) {
                   1077:                uvm_mapent_free(new_entry);
1.174     yamt     1078:        }
                   1079:
1.187     yamt     1080: #if defined(DEBUG)
                   1081:        if (!error && VM_MAP_IS_KERNEL(map)) {
1.264     ad       1082:                uvm_km_check_empty(map, *startp, *startp + size);
1.187     yamt     1083:        }
                   1084: #endif /* defined(DEBUG) */
                   1085:
1.174     yamt     1086:        return error;
                   1087: }
                   1088:
1.307     yamt     1089: /*
                   1090:  * uvm_map_prepare:
                   1091:  *
                   1092:  * called with map unlocked.
                   1093:  * on success, returns the map locked.
                   1094:  */
                   1095:
1.174     yamt     1096: int
                   1097: uvm_map_prepare(struct vm_map *map, vaddr_t start, vsize_t size,
                   1098:     struct uvm_object *uobj, voff_t uoffset, vsize_t align, uvm_flag_t flags,
                   1099:     struct uvm_map_args *args)
                   1100: {
                   1101:        struct vm_map_entry *prev_entry;
                   1102:        vm_prot_t prot = UVM_PROTECTION(flags);
                   1103:        vm_prot_t maxprot = UVM_MAXPROTECTION(flags);
                   1104:
                   1105:        UVMHIST_FUNC("uvm_map_prepare");
1.10      mrg      1106:        UVMHIST_CALLED(maphist);
1.1       mrg      1107:
1.174     yamt     1108:        UVMHIST_LOG(maphist, "(map=0x%x, start=0x%x, size=%d, flags=0x%x)",
                   1109:            map, start, size, flags);
1.10      mrg      1110:        UVMHIST_LOG(maphist, "  uobj/offset 0x%x/%d", uobj, uoffset,0,0);
1.107     chs      1111:
                   1112:        /*
                   1113:         * detect a popular device driver bug.
                   1114:         */
                   1115:
1.314     rmind    1116:        KASSERT(doing_shutdown || curlwp != NULL);
1.1       mrg      1117:
1.10      mrg      1118:        /*
1.144     yamt     1119:         * zero-sized mapping doesn't make any sense.
                   1120:         */
                   1121:        KASSERT(size > 0);
                   1122:
1.180     yamt     1123:        KASSERT((~flags & (UVM_FLAG_NOWAIT | UVM_FLAG_WAITVA)) != 0);
                   1124:
1.222     yamt     1125:        uvm_map_check(map, "map entry");
1.144     yamt     1126:
                   1127:        /*
1.106     chs      1128:         * check sanity of protection code
1.10      mrg      1129:         */
1.1       mrg      1130:
1.10      mrg      1131:        if ((prot & maxprot) != prot) {
1.98      chs      1132:                UVMHIST_LOG(maphist, "<- prot. failure:  prot=0x%x, max=0x%x",
1.10      mrg      1133:                prot, maxprot,0,0);
1.94      chs      1134:                return EACCES;
1.10      mrg      1135:        }
1.1       mrg      1136:
1.10      mrg      1137:        /*
1.106     chs      1138:         * figure out where to put new VM range
1.10      mrg      1139:         */
1.180     yamt     1140: retry:
1.234     thorpej  1141:        if (vm_map_lock_try(map) == false) {
1.314     rmind    1142:                if ((flags & UVM_FLAG_TRYLOCK) != 0) {
1.94      chs      1143:                        return EAGAIN;
1.106     chs      1144:                }
1.10      mrg      1145:                vm_map_lock(map); /* could sleep here */
                   1146:        }
1.226     yamt     1147:        prev_entry = uvm_map_findspace(map, start, size, &start,
                   1148:            uobj, uoffset, align, flags);
                   1149:        if (prev_entry == NULL) {
1.180     yamt     1150:                unsigned int timestamp;
                   1151:
                   1152:                timestamp = map->timestamp;
                   1153:                UVMHIST_LOG(maphist,"waiting va timestamp=0x%x",
                   1154:                            timestamp,0,0,0);
                   1155:                map->flags |= VM_MAP_WANTVA;
1.10      mrg      1156:                vm_map_unlock(map);
1.180     yamt     1157:
                   1158:                /*
1.226     yamt     1159:                 * try to reclaim kva and wait until someone does unmap.
1.238     ad       1160:                 * fragile locking here, so we awaken every second to
                   1161:                 * recheck the condition.
1.180     yamt     1162:                 */
                   1163:
1.238     ad       1164:                mutex_enter(&map->misc_lock);
1.180     yamt     1165:                while ((map->flags & VM_MAP_WANTVA) != 0 &&
                   1166:                   map->timestamp == timestamp) {
1.226     yamt     1167:                        if ((flags & UVM_FLAG_WAITVA) == 0) {
1.238     ad       1168:                                mutex_exit(&map->misc_lock);
1.226     yamt     1169:                                UVMHIST_LOG(maphist,
                   1170:                                    "<- uvm_map_findspace failed!", 0,0,0,0);
                   1171:                                return ENOMEM;
                   1172:                        } else {
1.238     ad       1173:                                cv_timedwait(&map->cv, &map->misc_lock, hz);
1.226     yamt     1174:                        }
1.180     yamt     1175:                }
1.238     ad       1176:                mutex_exit(&map->misc_lock);
1.180     yamt     1177:                goto retry;
1.10      mrg      1178:        }
1.1       mrg      1179:
1.40      thorpej  1180: #ifdef PMAP_GROWKERNEL
1.152     simonb   1181:        /*
                   1182:         * If the kernel pmap can't map the requested space,
                   1183:         * then allocate more resources for it.
                   1184:         */
1.229     yamt     1185:        if (map == kernel_map && uvm_maxkaddr < (start + size))
                   1186:                uvm_maxkaddr = pmap_growkernel(start + size);
1.10      mrg      1187: #endif
                   1188:
1.207     yamt     1189:        UVMMAP_EVCNT_INCR(map_call);
1.10      mrg      1190:
                   1191:        /*
                   1192:         * if uobj is null, then uoffset is either a VAC hint for PMAP_PREFER
1.98      chs      1193:         * [typically from uvm_map_reserve] or it is UVM_UNKNOWN_OFFSET.   in
                   1194:         * either case we want to zero it  before storing it in the map entry
1.10      mrg      1195:         * (because it looks strange and confusing when debugging...)
1.98      chs      1196:         *
                   1197:         * if uobj is not null
1.10      mrg      1198:         *   if uoffset is not UVM_UNKNOWN_OFFSET then we have a normal mapping
                   1199:         *      and we do not need to change uoffset.
                   1200:         *   if uoffset is UVM_UNKNOWN_OFFSET then we need to find the offset
                   1201:         *      now (based on the starting address of the map).   this case is
                   1202:         *      for kernel object mappings where we don't know the offset until
                   1203:         *      the virtual address is found (with uvm_map_findspace).   the
                   1204:         *      offset is the distance we are from the start of the map.
                   1205:         */
                   1206:
                   1207:        if (uobj == NULL) {
                   1208:                uoffset = 0;
                   1209:        } else {
                   1210:                if (uoffset == UVM_UNKNOWN_OFFSET) {
1.85      chs      1211:                        KASSERT(UVM_OBJ_IS_KERN_OBJECT(uobj));
1.174     yamt     1212:                        uoffset = start - vm_map_min(kernel_map);
1.10      mrg      1213:                }
                   1214:        }
                   1215:
1.174     yamt     1216:        args->uma_flags = flags;
                   1217:        args->uma_prev = prev_entry;
                   1218:        args->uma_start = start;
                   1219:        args->uma_size = size;
                   1220:        args->uma_uobj = uobj;
                   1221:        args->uma_uoffset = uoffset;
                   1222:
1.276     matt     1223:        UVMHIST_LOG(maphist, "<- done!", 0,0,0,0);
1.174     yamt     1224:        return 0;
                   1225: }
                   1226:
1.307     yamt     1227: /*
                   1228:  * uvm_map_enter:
                   1229:  *
                   1230:  * called with map locked.
                   1231:  * unlock the map before returning.
                   1232:  */
                   1233:
1.174     yamt     1234: int
                   1235: uvm_map_enter(struct vm_map *map, const struct uvm_map_args *args,
                   1236:     struct vm_map_entry *new_entry)
                   1237: {
                   1238:        struct vm_map_entry *prev_entry = args->uma_prev;
                   1239:        struct vm_map_entry *dead = NULL;
                   1240:
                   1241:        const uvm_flag_t flags = args->uma_flags;
                   1242:        const vm_prot_t prot = UVM_PROTECTION(flags);
                   1243:        const vm_prot_t maxprot = UVM_MAXPROTECTION(flags);
                   1244:        const vm_inherit_t inherit = UVM_INHERIT(flags);
                   1245:        const int amapwaitflag = (flags & UVM_FLAG_NOWAIT) ?
                   1246:            AMAP_EXTEND_NOWAIT : 0;
                   1247:        const int advice = UVM_ADVICE(flags);
                   1248:
                   1249:        vaddr_t start = args->uma_start;
                   1250:        vsize_t size = args->uma_size;
                   1251:        struct uvm_object *uobj = args->uma_uobj;
                   1252:        voff_t uoffset = args->uma_uoffset;
                   1253:
                   1254:        const int kmap = (vm_map_pmap(map) == pmap_kernel());
                   1255:        int merged = 0;
                   1256:        int error;
1.176     yamt     1257:        int newetype;
1.174     yamt     1258:
                   1259:        UVMHIST_FUNC("uvm_map_enter");
                   1260:        UVMHIST_CALLED(maphist);
                   1261:
                   1262:        UVMHIST_LOG(maphist, "(map=0x%x, start=0x%x, size=%d, flags=0x%x)",
                   1263:            map, start, size, flags);
                   1264:        UVMHIST_LOG(maphist, "  uobj/offset 0x%x/%d", uobj, uoffset,0,0);
                   1265:
1.221     yamt     1266:        KASSERT(map->hint == prev_entry); /* bimerge case assumes this */
1.307     yamt     1267:        KASSERT(vm_map_locked_p(map));
1.221     yamt     1268:
1.176     yamt     1269:        if (uobj)
                   1270:                newetype = UVM_ET_OBJ;
                   1271:        else
                   1272:                newetype = 0;
                   1273:
                   1274:        if (flags & UVM_FLAG_COPYONW) {
                   1275:                newetype |= UVM_ET_COPYONWRITE;
                   1276:                if ((flags & UVM_FLAG_OVERLAY) == 0)
                   1277:                        newetype |= UVM_ET_NEEDSCOPY;
                   1278:        }
                   1279:
1.10      mrg      1280:        /*
1.106     chs      1281:         * try and insert in map by extending previous entry, if possible.
1.10      mrg      1282:         * XXX: we don't try and pull back the next entry.   might be useful
                   1283:         * for a stack, but we are currently allocating our stack in advance.
                   1284:         */
                   1285:
1.121     atatat   1286:        if (flags & UVM_FLAG_NOMERGE)
                   1287:                goto nomerge;
                   1288:
1.194     yamt     1289:        if (prev_entry->end == start &&
1.121     atatat   1290:            prev_entry != &map->header &&
1.312     rmind    1291:            UVM_ET_ISCOMPATIBLE(prev_entry, newetype, uobj, 0,
1.194     yamt     1292:            prot, maxprot, inherit, advice, 0)) {
1.161     matt     1293:
1.10      mrg      1294:                if (uobj && prev_entry->offset +
                   1295:                    (prev_entry->end - prev_entry->start) != uoffset)
1.121     atatat   1296:                        goto forwardmerge;
1.10      mrg      1297:
                   1298:                /*
1.98      chs      1299:                 * can't extend a shared amap.  note: no need to lock amap to
1.34      chuck    1300:                 * look at refs since we don't care about its exact value.
1.10      mrg      1301:                 * if it is one (i.e. we have only reference) it will stay there
                   1302:                 */
1.85      chs      1303:
1.10      mrg      1304:                if (prev_entry->aref.ar_amap &&
1.34      chuck    1305:                    amap_refs(prev_entry->aref.ar_amap) != 1) {
1.121     atatat   1306:                        goto forwardmerge;
1.10      mrg      1307:                }
1.85      chs      1308:
1.119     chs      1309:                if (prev_entry->aref.ar_amap) {
1.139     enami    1310:                        error = amap_extend(prev_entry, size,
1.126     bouyer   1311:                            amapwaitflag | AMAP_EXTEND_FORWARDS);
1.174     yamt     1312:                        if (error)
1.191     yamt     1313:                                goto nomerge;
1.119     chs      1314:                }
1.10      mrg      1315:
1.258     ad       1316:                if (kmap) {
1.207     yamt     1317:                        UVMMAP_EVCNT_INCR(kbackmerge);
1.258     ad       1318:                } else {
1.207     yamt     1319:                        UVMMAP_EVCNT_INCR(ubackmerge);
1.258     ad       1320:                }
1.10      mrg      1321:                UVMHIST_LOG(maphist,"  starting back merge", 0, 0, 0, 0);
                   1322:
                   1323:                /*
                   1324:                 * drop our reference to uobj since we are extending a reference
                   1325:                 * that we already have (the ref count can not drop to zero).
                   1326:                 */
1.119     chs      1327:
1.10      mrg      1328:                if (uobj && uobj->pgops->pgo_detach)
                   1329:                        uobj->pgops->pgo_detach(uobj);
                   1330:
1.263     matt     1331:                /*
                   1332:                 * Now that we've merged the entries, note that we've grown
                   1333:                 * and our gap has shrunk.  Then fix the tree.
                   1334:                 */
1.10      mrg      1335:                prev_entry->end += size;
1.263     matt     1336:                prev_entry->gap -= size;
1.145     yamt     1337:                uvm_rb_fixup(map, prev_entry);
                   1338:
1.222     yamt     1339:                uvm_map_check(map, "map backmerged");
1.10      mrg      1340:
                   1341:                UVMHIST_LOG(maphist,"<- done (via backmerge)!", 0, 0, 0, 0);
1.121     atatat   1342:                merged++;
1.106     chs      1343:        }
1.10      mrg      1344:
1.121     atatat   1345: forwardmerge:
1.194     yamt     1346:        if (prev_entry->next->start == (start + size) &&
1.121     atatat   1347:            prev_entry->next != &map->header &&
1.312     rmind    1348:            UVM_ET_ISCOMPATIBLE(prev_entry->next, newetype, uobj, 0,
1.194     yamt     1349:            prot, maxprot, inherit, advice, 0)) {
1.161     matt     1350:
1.121     atatat   1351:                if (uobj && prev_entry->next->offset != uoffset + size)
                   1352:                        goto nomerge;
                   1353:
                   1354:                /*
                   1355:                 * can't extend a shared amap.  note: no need to lock amap to
                   1356:                 * look at refs since we don't care about its exact value.
1.122     atatat   1357:                 * if it is one (i.e. we have only reference) it will stay there.
                   1358:                 *
                   1359:                 * note that we also can't merge two amaps, so if we
                   1360:                 * merged with the previous entry which has an amap,
                   1361:                 * and the next entry also has an amap, we give up.
                   1362:                 *
1.125     atatat   1363:                 * Interesting cases:
                   1364:                 * amap, new, amap -> give up second merge (single fwd extend)
                   1365:                 * amap, new, none -> double forward extend (extend again here)
                   1366:                 * none, new, amap -> double backward extend (done here)
                   1367:                 * uobj, new, amap -> single backward extend (done here)
                   1368:                 *
1.122     atatat   1369:                 * XXX should we attempt to deal with someone refilling
                   1370:                 * the deallocated region between two entries that are
                   1371:                 * backed by the same amap (ie, arefs is 2, "prev" and
                   1372:                 * "next" refer to it, and adding this allocation will
                   1373:                 * close the hole, thus restoring arefs to 1 and
                   1374:                 * deallocating the "next" vm_map_entry)?  -- @@@
1.121     atatat   1375:                 */
                   1376:
                   1377:                if (prev_entry->next->aref.ar_amap &&
1.122     atatat   1378:                    (amap_refs(prev_entry->next->aref.ar_amap) != 1 ||
                   1379:                     (merged && prev_entry->aref.ar_amap))) {
1.121     atatat   1380:                        goto nomerge;
                   1381:                }
                   1382:
1.122     atatat   1383:                if (merged) {
1.123     atatat   1384:                        /*
                   1385:                         * Try to extend the amap of the previous entry to
                   1386:                         * cover the next entry as well.  If it doesn't work
                   1387:                         * just skip on, don't actually give up, since we've
                   1388:                         * already completed the back merge.
                   1389:                         */
1.125     atatat   1390:                        if (prev_entry->aref.ar_amap) {
                   1391:                                if (amap_extend(prev_entry,
                   1392:                                    prev_entry->next->end -
                   1393:                                    prev_entry->next->start,
1.126     bouyer   1394:                                    amapwaitflag | AMAP_EXTEND_FORWARDS))
1.142     enami    1395:                                        goto nomerge;
1.125     atatat   1396:                        }
                   1397:
                   1398:                        /*
                   1399:                         * Try to extend the amap of the *next* entry
                   1400:                         * back to cover the new allocation *and* the
                   1401:                         * previous entry as well (the previous merge
                   1402:                         * didn't have an amap already otherwise we
                   1403:                         * wouldn't be checking here for an amap).  If
                   1404:                         * it doesn't work just skip on, again, don't
                   1405:                         * actually give up, since we've already
                   1406:                         * completed the back merge.
                   1407:                         */
                   1408:                        else if (prev_entry->next->aref.ar_amap) {
                   1409:                                if (amap_extend(prev_entry->next,
                   1410:                                    prev_entry->end -
1.141     atatat   1411:                                    prev_entry->start,
1.126     bouyer   1412:                                    amapwaitflag | AMAP_EXTEND_BACKWARDS))
1.142     enami    1413:                                        goto nomerge;
1.125     atatat   1414:                        }
                   1415:                } else {
                   1416:                        /*
                   1417:                         * Pull the next entry's amap backwards to cover this
                   1418:                         * new allocation.
                   1419:                         */
                   1420:                        if (prev_entry->next->aref.ar_amap) {
                   1421:                                error = amap_extend(prev_entry->next, size,
1.126     bouyer   1422:                                    amapwaitflag | AMAP_EXTEND_BACKWARDS);
1.174     yamt     1423:                                if (error)
1.191     yamt     1424:                                        goto nomerge;
1.125     atatat   1425:                        }
1.122     atatat   1426:                }
                   1427:
1.121     atatat   1428:                if (merged) {
                   1429:                        if (kmap) {
1.207     yamt     1430:                                UVMMAP_EVCNT_DECR(kbackmerge);
                   1431:                                UVMMAP_EVCNT_INCR(kbimerge);
1.121     atatat   1432:                        } else {
1.207     yamt     1433:                                UVMMAP_EVCNT_DECR(ubackmerge);
                   1434:                                UVMMAP_EVCNT_INCR(ubimerge);
1.121     atatat   1435:                        }
1.122     atatat   1436:                } else {
1.258     ad       1437:                        if (kmap) {
1.207     yamt     1438:                                UVMMAP_EVCNT_INCR(kforwmerge);
1.258     ad       1439:                        } else {
1.207     yamt     1440:                                UVMMAP_EVCNT_INCR(uforwmerge);
1.258     ad       1441:                        }
1.121     atatat   1442:                }
                   1443:                UVMHIST_LOG(maphist,"  starting forward merge", 0, 0, 0, 0);
1.10      mrg      1444:
1.121     atatat   1445:                /*
                   1446:                 * drop our reference to uobj since we are extending a reference
                   1447:                 * that we already have (the ref count can not drop to zero).
                   1448:                 */
1.319     chs      1449:                if (uobj && uobj->pgops->pgo_detach)
1.121     atatat   1450:                        uobj->pgops->pgo_detach(uobj);
1.1       mrg      1451:
1.121     atatat   1452:                if (merged) {
1.174     yamt     1453:                        dead = prev_entry->next;
1.121     atatat   1454:                        prev_entry->end = dead->end;
                   1455:                        uvm_map_entry_unlink(map, dead);
1.125     atatat   1456:                        if (dead->aref.ar_amap != NULL) {
                   1457:                                prev_entry->aref = dead->aref;
                   1458:                                dead->aref.ar_amap = NULL;
                   1459:                        }
1.121     atatat   1460:                } else {
                   1461:                        prev_entry->next->start -= size;
1.263     matt     1462:                        if (prev_entry != &map->header) {
                   1463:                                prev_entry->gap -= size;
                   1464:                                KASSERT(prev_entry->gap == uvm_rb_gap(prev_entry));
1.145     yamt     1465:                                uvm_rb_fixup(map, prev_entry);
1.263     matt     1466:                        }
1.121     atatat   1467:                        if (uobj)
                   1468:                                prev_entry->next->offset = uoffset;
                   1469:                }
1.145     yamt     1470:
1.222     yamt     1471:                uvm_map_check(map, "map forwardmerged");
1.1       mrg      1472:
1.121     atatat   1473:                UVMHIST_LOG(maphist,"<- done forwardmerge", 0, 0, 0, 0);
                   1474:                merged++;
1.106     chs      1475:        }
1.121     atatat   1476:
                   1477: nomerge:
                   1478:        if (!merged) {
                   1479:                UVMHIST_LOG(maphist,"  allocating new map entry", 0, 0, 0, 0);
1.258     ad       1480:                if (kmap) {
1.207     yamt     1481:                        UVMMAP_EVCNT_INCR(knomerge);
1.258     ad       1482:                } else {
1.207     yamt     1483:                        UVMMAP_EVCNT_INCR(unomerge);
1.258     ad       1484:                }
1.106     chs      1485:
1.10      mrg      1486:                /*
1.121     atatat   1487:                 * allocate new entry and link it in.
1.10      mrg      1488:                 */
1.106     chs      1489:
1.121     atatat   1490:                if (new_entry == NULL) {
1.126     bouyer   1491:                        new_entry = uvm_mapent_alloc(map,
1.127     thorpej  1492:                                (flags & UVM_FLAG_NOWAIT));
1.126     bouyer   1493:                        if (__predict_false(new_entry == NULL)) {
1.174     yamt     1494:                                error = ENOMEM;
                   1495:                                goto done;
1.126     bouyer   1496:                        }
1.121     atatat   1497:                }
1.174     yamt     1498:                new_entry->start = start;
1.121     atatat   1499:                new_entry->end = new_entry->start + size;
                   1500:                new_entry->object.uvm_obj = uobj;
                   1501:                new_entry->offset = uoffset;
                   1502:
1.176     yamt     1503:                new_entry->etype = newetype;
1.121     atatat   1504:
1.161     matt     1505:                if (flags & UVM_FLAG_NOMERGE) {
                   1506:                        new_entry->flags |= UVM_MAP_NOMERGE;
                   1507:                }
1.121     atatat   1508:
                   1509:                new_entry->protection = prot;
                   1510:                new_entry->max_protection = maxprot;
                   1511:                new_entry->inheritance = inherit;
                   1512:                new_entry->wired_count = 0;
                   1513:                new_entry->advice = advice;
                   1514:                if (flags & UVM_FLAG_OVERLAY) {
                   1515:
                   1516:                        /*
                   1517:                         * to_add: for BSS we overallocate a little since we
                   1518:                         * are likely to extend
                   1519:                         */
                   1520:
                   1521:                        vaddr_t to_add = (flags & UVM_FLAG_AMAPPAD) ?
                   1522:                                UVM_AMAP_CHUNK << PAGE_SHIFT : 0;
1.126     bouyer   1523:                        struct vm_amap *amap = amap_alloc(size, to_add,
1.227     yamt     1524:                            (flags & UVM_FLAG_NOWAIT));
1.126     bouyer   1525:                        if (__predict_false(amap == NULL)) {
1.174     yamt     1526:                                error = ENOMEM;
                   1527:                                goto done;
1.126     bouyer   1528:                        }
1.121     atatat   1529:                        new_entry->aref.ar_pageoff = 0;
                   1530:                        new_entry->aref.ar_amap = amap;
                   1531:                } else {
                   1532:                        new_entry->aref.ar_pageoff = 0;
                   1533:                        new_entry->aref.ar_amap = NULL;
                   1534:                }
                   1535:                uvm_map_entry_link(map, prev_entry, new_entry);
1.1       mrg      1536:
1.121     atatat   1537:                /*
                   1538:                 * Update the free space hint
                   1539:                 */
1.10      mrg      1540:
1.121     atatat   1541:                if ((map->first_free == prev_entry) &&
                   1542:                    (prev_entry->end >= new_entry->start))
                   1543:                        map->first_free = new_entry;
1.174     yamt     1544:
                   1545:                new_entry = NULL;
1.121     atatat   1546:        }
1.10      mrg      1547:
1.146     yamt     1548:        map->size += size;
                   1549:
1.10      mrg      1550:        UVMHIST_LOG(maphist,"<- done!", 0, 0, 0, 0);
1.174     yamt     1551:
                   1552:        error = 0;
                   1553: done:
1.311     para     1554:        vm_map_unlock(map);
                   1555:
                   1556:        if (new_entry) {
                   1557:                uvm_mapent_free(new_entry);
1.174     yamt     1558:        }
1.311     para     1559:
1.174     yamt     1560:        if (dead) {
                   1561:                KDASSERT(merged);
1.311     para     1562:                uvm_mapent_free(dead);
1.248     ad       1563:        }
1.311     para     1564:
1.174     yamt     1565:        return error;
1.1       mrg      1566: }
                   1567:
                   1568: /*
1.247     yamt     1569:  * uvm_map_lookup_entry_bytree: lookup an entry in tree
                   1570:  */
                   1571:
1.263     matt     1572: static inline bool
1.247     yamt     1573: uvm_map_lookup_entry_bytree(struct vm_map *map, vaddr_t address,
                   1574:     struct vm_map_entry **entry        /* OUT */)
                   1575: {
                   1576:        struct vm_map_entry *prev = &map->header;
1.263     matt     1577:        struct vm_map_entry *cur = ROOT_ENTRY(map);
1.247     yamt     1578:
                   1579:        while (cur) {
1.263     matt     1580:                UVMMAP_EVCNT_INCR(mlk_treeloop);
1.247     yamt     1581:                if (address >= cur->start) {
                   1582:                        if (address < cur->end) {
                   1583:                                *entry = cur;
                   1584:                                return true;
                   1585:                        }
                   1586:                        prev = cur;
1.263     matt     1587:                        cur = RIGHT_ENTRY(cur);
1.247     yamt     1588:                } else
1.263     matt     1589:                        cur = LEFT_ENTRY(cur);
1.247     yamt     1590:        }
                   1591:        *entry = prev;
                   1592:        return false;
                   1593: }
                   1594:
                   1595: /*
1.1       mrg      1596:  * uvm_map_lookup_entry: find map entry at or before an address
                   1597:  *
                   1598:  * => map must at least be read-locked by caller
                   1599:  * => entry is returned in "entry"
                   1600:  * => return value is true if address is in the returned entry
                   1601:  */
                   1602:
1.233     thorpej  1603: bool
1.138     enami    1604: uvm_map_lookup_entry(struct vm_map *map, vaddr_t address,
                   1605:     struct vm_map_entry **entry        /* OUT */)
1.1       mrg      1606: {
1.99      chs      1607:        struct vm_map_entry *cur;
1.234     thorpej  1608:        bool use_tree = false;
1.1       mrg      1609:        UVMHIST_FUNC("uvm_map_lookup_entry");
                   1610:        UVMHIST_CALLED(maphist);
                   1611:
                   1612:        UVMHIST_LOG(maphist,"(map=0x%x,addr=0x%x,ent=0x%x)",
1.10      mrg      1613:            map, address, entry, 0);
1.1       mrg      1614:
                   1615:        /*
1.10      mrg      1616:         * start looking either from the head of the
                   1617:         * list, or from the hint.
1.1       mrg      1618:         */
                   1619:
                   1620:        cur = map->hint;
                   1621:
                   1622:        if (cur == &map->header)
                   1623:                cur = cur->next;
                   1624:
1.207     yamt     1625:        UVMMAP_EVCNT_INCR(mlk_call);
1.1       mrg      1626:        if (address >= cur->start) {
1.99      chs      1627:
1.139     enami    1628:                /*
1.10      mrg      1629:                 * go from hint to end of list.
1.1       mrg      1630:                 *
1.10      mrg      1631:                 * but first, make a quick check to see if
                   1632:                 * we are already looking at the entry we
                   1633:                 * want (which is usually the case).
                   1634:                 * note also that we don't need to save the hint
                   1635:                 * here... it is the same hint (unless we are
                   1636:                 * at the header, in which case the hint didn't
                   1637:                 * buy us anything anyway).
1.1       mrg      1638:                 */
1.99      chs      1639:
1.144     yamt     1640:                if (cur != &map->header && cur->end > address) {
1.207     yamt     1641:                        UVMMAP_EVCNT_INCR(mlk_hint);
1.1       mrg      1642:                        *entry = cur;
                   1643:                        UVMHIST_LOG(maphist,"<- got it via hint (0x%x)",
1.10      mrg      1644:                            cur, 0, 0, 0);
1.218     yamt     1645:                        uvm_mapent_check(*entry);
1.234     thorpej  1646:                        return (true);
1.1       mrg      1647:                }
1.144     yamt     1648:
1.263     matt     1649:                if (map->nentries > 15)
1.234     thorpej  1650:                        use_tree = true;
1.10      mrg      1651:        } else {
1.99      chs      1652:
1.139     enami    1653:                /*
1.144     yamt     1654:                 * invalid hint.  use tree.
1.1       mrg      1655:                 */
1.234     thorpej  1656:                use_tree = true;
1.144     yamt     1657:        }
                   1658:
1.222     yamt     1659:        uvm_map_check(map, __func__);
1.144     yamt     1660:
                   1661:        if (use_tree) {
                   1662:                /*
                   1663:                 * Simple lookup in the tree.  Happens when the hint is
                   1664:                 * invalid, or nentries reach a threshold.
                   1665:                 */
1.263     matt     1666:                UVMMAP_EVCNT_INCR(mlk_tree);
1.247     yamt     1667:                if (uvm_map_lookup_entry_bytree(map, address, entry)) {
                   1668:                        goto got;
                   1669:                } else {
                   1670:                        goto failed;
1.144     yamt     1671:                }
1.1       mrg      1672:        }
                   1673:
                   1674:        /*
1.10      mrg      1675:         * search linearly
1.1       mrg      1676:         */
                   1677:
1.263     matt     1678:        UVMMAP_EVCNT_INCR(mlk_list);
1.144     yamt     1679:        while (cur != &map->header) {
1.263     matt     1680:                UVMMAP_EVCNT_INCR(mlk_listloop);
1.1       mrg      1681:                if (cur->end > address) {
                   1682:                        if (address >= cur->start) {
1.139     enami    1683:                                /*
1.10      mrg      1684:                                 * save this lookup for future
                   1685:                                 * hints, and return
1.1       mrg      1686:                                 */
                   1687:
                   1688:                                *entry = cur;
1.144     yamt     1689: got:
                   1690:                                SAVE_HINT(map, map->hint, *entry);
1.1       mrg      1691:                                UVMHIST_LOG(maphist,"<- search got it (0x%x)",
1.10      mrg      1692:                                        cur, 0, 0, 0);
1.144     yamt     1693:                                KDASSERT((*entry)->start <= address);
                   1694:                                KDASSERT(address < (*entry)->end);
1.218     yamt     1695:                                uvm_mapent_check(*entry);
1.234     thorpej  1696:                                return (true);
1.1       mrg      1697:                        }
                   1698:                        break;
                   1699:                }
                   1700:                cur = cur->next;
                   1701:        }
                   1702:        *entry = cur->prev;
1.144     yamt     1703: failed:
1.82      thorpej  1704:        SAVE_HINT(map, map->hint, *entry);
1.1       mrg      1705:        UVMHIST_LOG(maphist,"<- failed!",0,0,0,0);
1.147     yamt     1706:        KDASSERT((*entry) == &map->header || (*entry)->end <= address);
1.144     yamt     1707:        KDASSERT((*entry)->next == &map->header ||
                   1708:            address < (*entry)->next->start);
1.234     thorpej  1709:        return (false);
1.1       mrg      1710: }
                   1711:
                   1712: /*
1.140     enami    1713:  * See if the range between start and start + length fits in the gap
                   1714:  * entry->next->start and entry->end.  Returns 1 if fits, 0 if doesn't
                   1715:  * fit, and -1 address wraps around.
                   1716:  */
1.203     thorpej  1717: static int
1.232     yamt     1718: uvm_map_space_avail(vaddr_t *start, vsize_t length, voff_t uoffset,
1.304     matt     1719:     vsize_t align, int flags, int topdown, struct vm_map_entry *entry)
1.140     enami    1720: {
                   1721:        vaddr_t end;
                   1722:
                   1723: #ifdef PMAP_PREFER
                   1724:        /*
                   1725:         * push start address forward as needed to avoid VAC alias problems.
                   1726:         * we only do this if a valid offset is specified.
                   1727:         */
                   1728:
                   1729:        if (uoffset != UVM_UNKNOWN_OFFSET)
1.182     atatat   1730:                PMAP_PREFER(uoffset, start, length, topdown);
1.140     enami    1731: #endif
1.304     matt     1732:        if ((flags & UVM_FLAG_COLORMATCH) != 0) {
                   1733:                KASSERT(align < uvmexp.ncolors);
                   1734:                if (uvmexp.ncolors > 1) {
                   1735:                        const u_int colormask = uvmexp.colormask;
                   1736:                        const u_int colorsize = colormask + 1;
                   1737:                        vaddr_t hint = atop(*start);
                   1738:                        const u_int color = hint & colormask;
                   1739:                        if (color != align) {
                   1740:                                hint -= color;  /* adjust to color boundary */
                   1741:                                KASSERT((hint & colormask) == 0);
                   1742:                                if (topdown) {
                   1743:                                        if (align > color)
                   1744:                                                hint -= colorsize;
                   1745:                                } else {
                   1746:                                        if (align < color)
                   1747:                                                hint += colorsize;
                   1748:                                }
                   1749:                                *start = ptoa(hint + align); /* adjust to color */
                   1750:                        }
                   1751:                }
                   1752:        } else if (align != 0) {
1.140     enami    1753:                if ((*start & (align - 1)) != 0) {
                   1754:                        if (topdown)
                   1755:                                *start &= ~(align - 1);
                   1756:                        else
                   1757:                                *start = roundup(*start, align);
                   1758:                }
                   1759:                /*
                   1760:                 * XXX Should we PMAP_PREFER() here again?
1.182     atatat   1761:                 * eh...i think we're okay
1.140     enami    1762:                 */
                   1763:        }
                   1764:
                   1765:        /*
                   1766:         * Find the end of the proposed new region.  Be sure we didn't
                   1767:         * wrap around the address; if so, we lose.  Otherwise, if the
                   1768:         * proposed new region fits before the next entry, we win.
                   1769:         */
                   1770:
                   1771:        end = *start + length;
                   1772:        if (end < *start)
                   1773:                return (-1);
                   1774:
                   1775:        if (entry->next->start >= end && *start >= entry->end)
                   1776:                return (1);
                   1777:
                   1778:        return (0);
                   1779: }
                   1780:
                   1781: /*
1.1       mrg      1782:  * uvm_map_findspace: find "length" sized space in "map".
                   1783:  *
1.167     junyoung 1784:  * => "hint" is a hint about where we want it, unless UVM_FLAG_FIXED is
                   1785:  *     set in "flags" (in which case we insist on using "hint").
1.1       mrg      1786:  * => "result" is VA returned
                   1787:  * => uobj/uoffset are to be used to handle VAC alignment, if required
1.167     junyoung 1788:  * => if "align" is non-zero, we attempt to align to that value.
1.1       mrg      1789:  * => caller must at least have read-locked map
                   1790:  * => returns NULL on failure, or pointer to prev. map entry if success
                   1791:  * => note this is a cross between the old vm_map_findspace and vm_map_find
                   1792:  */
                   1793:
1.99      chs      1794: struct vm_map_entry *
1.138     enami    1795: uvm_map_findspace(struct vm_map *map, vaddr_t hint, vsize_t length,
1.232     yamt     1796:     vaddr_t *result /* OUT */, struct uvm_object *uobj, voff_t uoffset,
1.138     enami    1797:     vsize_t align, int flags)
1.1       mrg      1798: {
1.140     enami    1799:        struct vm_map_entry *entry;
1.144     yamt     1800:        struct vm_map_entry *child, *prev, *tmp;
1.326     martin   1801:        vaddr_t orig_hint __diagused;
1.131     atatat   1802:        const int topdown = map->flags & VM_MAP_TOPDOWN;
1.1       mrg      1803:        UVMHIST_FUNC("uvm_map_findspace");
                   1804:        UVMHIST_CALLED(maphist);
                   1805:
1.98      chs      1806:        UVMHIST_LOG(maphist, "(map=0x%x, hint=0x%x, len=%d, flags=0x%x)",
1.140     enami    1807:            map, hint, length, flags);
1.304     matt     1808:        KASSERT((flags & UVM_FLAG_COLORMATCH) != 0 || (align & (align - 1)) == 0);
                   1809:        KASSERT((flags & UVM_FLAG_COLORMATCH) == 0 || align < uvmexp.ncolors);
1.85      chs      1810:        KASSERT((flags & UVM_FLAG_FIXED) == 0 || align == 0);
1.81      thorpej  1811:
1.222     yamt     1812:        uvm_map_check(map, "map_findspace entry");
1.144     yamt     1813:
1.81      thorpej  1814:        /*
                   1815:         * remember the original hint.  if we are aligning, then we
                   1816:         * may have to try again with no alignment constraint if
                   1817:         * we fail the first time.
                   1818:         */
1.85      chs      1819:
1.81      thorpej  1820:        orig_hint = hint;
1.184     chs      1821:        if (hint < vm_map_min(map)) {   /* check ranges ... */
1.81      thorpej  1822:                if (flags & UVM_FLAG_FIXED) {
1.1       mrg      1823:                        UVMHIST_LOG(maphist,"<- VA below map range",0,0,0,0);
1.139     enami    1824:                        return (NULL);
1.1       mrg      1825:                }
1.184     chs      1826:                hint = vm_map_min(map);
1.1       mrg      1827:        }
1.184     chs      1828:        if (hint > vm_map_max(map)) {
1.1       mrg      1829:                UVMHIST_LOG(maphist,"<- VA 0x%x > range [0x%x->0x%x]",
1.184     chs      1830:                    hint, vm_map_min(map), vm_map_max(map), 0);
1.139     enami    1831:                return (NULL);
1.1       mrg      1832:        }
                   1833:
                   1834:        /*
                   1835:         * Look for the first possible address; if there's already
                   1836:         * something at this address, we have to start after it.
                   1837:         */
                   1838:
1.131     atatat   1839:        /*
                   1840:         * @@@: there are four, no, eight cases to consider.
                   1841:         *
                   1842:         * 0: found,     fixed,     bottom up -> fail
                   1843:         * 1: found,     fixed,     top down  -> fail
1.140     enami    1844:         * 2: found,     not fixed, bottom up -> start after entry->end,
                   1845:         *                                       loop up
                   1846:         * 3: found,     not fixed, top down  -> start before entry->start,
                   1847:         *                                       loop down
                   1848:         * 4: not found, fixed,     bottom up -> check entry->next->start, fail
                   1849:         * 5: not found, fixed,     top down  -> check entry->next->start, fail
                   1850:         * 6: not found, not fixed, bottom up -> check entry->next->start,
                   1851:         *                                       loop up
                   1852:         * 7: not found, not fixed, top down  -> check entry->next->start,
                   1853:         *                                       loop down
1.131     atatat   1854:         *
                   1855:         * as you can see, it reduces to roughly five cases, and that
                   1856:         * adding top down mapping only adds one unique case (without
                   1857:         * it, there would be four cases).
                   1858:         */
                   1859:
1.184     chs      1860:        if ((flags & UVM_FLAG_FIXED) == 0 && hint == vm_map_min(map)) {
1.140     enami    1861:                entry = map->first_free;
1.1       mrg      1862:        } else {
1.140     enami    1863:                if (uvm_map_lookup_entry(map, hint, &entry)) {
1.1       mrg      1864:                        /* "hint" address already in use ... */
1.81      thorpej  1865:                        if (flags & UVM_FLAG_FIXED) {
1.140     enami    1866:                                UVMHIST_LOG(maphist, "<- fixed & VA in use",
1.10      mrg      1867:                                    0, 0, 0, 0);
1.139     enami    1868:                                return (NULL);
1.1       mrg      1869:                        }
1.140     enami    1870:                        if (topdown)
                   1871:                                /* Start from lower gap. */
                   1872:                                entry = entry->prev;
                   1873:                } else if (flags & UVM_FLAG_FIXED) {
                   1874:                        if (entry->next->start >= hint + length &&
                   1875:                            hint + length > hint)
                   1876:                                goto found;
                   1877:
                   1878:                        /* "hint" address is gap but too small */
                   1879:                        UVMHIST_LOG(maphist, "<- fixed mapping failed",
                   1880:                            0, 0, 0, 0);
                   1881:                        return (NULL); /* only one shot at it ... */
                   1882:                } else {
                   1883:                        /*
                   1884:                         * See if given hint fits in this gap.
                   1885:                         */
                   1886:                        switch (uvm_map_space_avail(&hint, length,
1.304     matt     1887:                            uoffset, align, flags, topdown, entry)) {
1.140     enami    1888:                        case 1:
                   1889:                                goto found;
                   1890:                        case -1:
                   1891:                                goto wraparound;
                   1892:                        }
                   1893:
1.148     yamt     1894:                        if (topdown) {
1.140     enami    1895:                                /*
                   1896:                                 * Still there is a chance to fit
                   1897:                                 * if hint > entry->end.
                   1898:                                 */
1.148     yamt     1899:                        } else {
1.168     junyoung 1900:                                /* Start from higher gap. */
1.148     yamt     1901:                                entry = entry->next;
                   1902:                                if (entry == &map->header)
                   1903:                                        goto notfound;
1.140     enami    1904:                                goto nextgap;
1.148     yamt     1905:                        }
1.1       mrg      1906:                }
                   1907:        }
                   1908:
                   1909:        /*
1.144     yamt     1910:         * Note that all UVM_FLAGS_FIXED case is already handled.
                   1911:         */
                   1912:        KDASSERT((flags & UVM_FLAG_FIXED) == 0);
                   1913:
                   1914:        /* Try to find the space in the red-black tree */
                   1915:
                   1916:        /* Check slot before any entry */
                   1917:        hint = topdown ? entry->next->start - length : entry->end;
1.304     matt     1918:        switch (uvm_map_space_avail(&hint, length, uoffset, align, flags,
1.144     yamt     1919:            topdown, entry)) {
                   1920:        case 1:
                   1921:                goto found;
                   1922:        case -1:
                   1923:                goto wraparound;
                   1924:        }
                   1925:
                   1926: nextgap:
1.148     yamt     1927:        KDASSERT((flags & UVM_FLAG_FIXED) == 0);
1.144     yamt     1928:        /* If there is not enough space in the whole tree, we fail */
1.263     matt     1929:        tmp = ROOT_ENTRY(map);
                   1930:        if (tmp == NULL || tmp->maxgap < length)
1.144     yamt     1931:                goto notfound;
                   1932:
                   1933:        prev = NULL; /* previous candidate */
                   1934:
                   1935:        /* Find an entry close to hint that has enough space */
                   1936:        for (; tmp;) {
1.263     matt     1937:                KASSERT(tmp->next->start == tmp->end + tmp->gap);
1.144     yamt     1938:                if (topdown) {
                   1939:                        if (tmp->next->start < hint + length &&
                   1940:                            (prev == NULL || tmp->end > prev->end)) {
1.263     matt     1941:                                if (tmp->gap >= length)
1.144     yamt     1942:                                        prev = tmp;
1.263     matt     1943:                                else if ((child = LEFT_ENTRY(tmp)) != NULL
                   1944:                                    && child->maxgap >= length)
1.144     yamt     1945:                                        prev = tmp;
                   1946:                        }
                   1947:                } else {
                   1948:                        if (tmp->end >= hint &&
                   1949:                            (prev == NULL || tmp->end < prev->end)) {
1.263     matt     1950:                                if (tmp->gap >= length)
1.144     yamt     1951:                                        prev = tmp;
1.263     matt     1952:                                else if ((child = RIGHT_ENTRY(tmp)) != NULL
                   1953:                                    && child->maxgap >= length)
1.144     yamt     1954:                                        prev = tmp;
                   1955:                        }
                   1956:                }
                   1957:                if (tmp->next->start < hint + length)
1.263     matt     1958:                        child = RIGHT_ENTRY(tmp);
1.144     yamt     1959:                else if (tmp->end > hint)
1.263     matt     1960:                        child = LEFT_ENTRY(tmp);
1.144     yamt     1961:                else {
1.263     matt     1962:                        if (tmp->gap >= length)
1.144     yamt     1963:                                break;
                   1964:                        if (topdown)
1.263     matt     1965:                                child = LEFT_ENTRY(tmp);
1.144     yamt     1966:                        else
1.263     matt     1967:                                child = RIGHT_ENTRY(tmp);
1.144     yamt     1968:                }
1.263     matt     1969:                if (child == NULL || child->maxgap < length)
1.144     yamt     1970:                        break;
                   1971:                tmp = child;
                   1972:        }
                   1973:
1.148     yamt     1974:        if (tmp != NULL && tmp->start < hint && hint < tmp->next->start) {
1.164     junyoung 1975:                /*
1.144     yamt     1976:                 * Check if the entry that we found satifies the
                   1977:                 * space requirement
                   1978:                 */
1.148     yamt     1979:                if (topdown) {
1.149     yamt     1980:                        if (hint > tmp->next->start - length)
                   1981:                                hint = tmp->next->start - length;
1.148     yamt     1982:                } else {
1.149     yamt     1983:                        if (hint < tmp->end)
                   1984:                                hint = tmp->end;
1.148     yamt     1985:                }
                   1986:                switch (uvm_map_space_avail(&hint, length, uoffset, align,
1.304     matt     1987:                    flags, topdown, tmp)) {
1.148     yamt     1988:                case 1:
1.144     yamt     1989:                        entry = tmp;
                   1990:                        goto found;
1.148     yamt     1991:                case -1:
                   1992:                        goto wraparound;
1.144     yamt     1993:                }
1.263     matt     1994:                if (tmp->gap >= length)
1.144     yamt     1995:                        goto listsearch;
                   1996:        }
                   1997:        if (prev == NULL)
                   1998:                goto notfound;
                   1999:
1.148     yamt     2000:        if (topdown) {
1.150     yamt     2001:                KASSERT(orig_hint >= prev->next->start - length ||
1.148     yamt     2002:                    prev->next->start - length > prev->next->start);
                   2003:                hint = prev->next->start - length;
                   2004:        } else {
1.150     yamt     2005:                KASSERT(orig_hint <= prev->end);
1.148     yamt     2006:                hint = prev->end;
                   2007:        }
                   2008:        switch (uvm_map_space_avail(&hint, length, uoffset, align,
1.304     matt     2009:            flags, topdown, prev)) {
1.148     yamt     2010:        case 1:
1.144     yamt     2011:                entry = prev;
                   2012:                goto found;
1.148     yamt     2013:        case -1:
                   2014:                goto wraparound;
1.144     yamt     2015:        }
1.263     matt     2016:        if (prev->gap >= length)
1.144     yamt     2017:                goto listsearch;
1.164     junyoung 2018:
1.144     yamt     2019:        if (topdown)
1.263     matt     2020:                tmp = LEFT_ENTRY(prev);
1.144     yamt     2021:        else
1.263     matt     2022:                tmp = RIGHT_ENTRY(prev);
1.144     yamt     2023:        for (;;) {
1.263     matt     2024:                KASSERT(tmp && tmp->maxgap >= length);
1.144     yamt     2025:                if (topdown)
1.263     matt     2026:                        child = RIGHT_ENTRY(tmp);
1.144     yamt     2027:                else
1.263     matt     2028:                        child = LEFT_ENTRY(tmp);
                   2029:                if (child && child->maxgap >= length) {
1.144     yamt     2030:                        tmp = child;
                   2031:                        continue;
                   2032:                }
1.263     matt     2033:                if (tmp->gap >= length)
1.144     yamt     2034:                        break;
                   2035:                if (topdown)
1.263     matt     2036:                        tmp = LEFT_ENTRY(tmp);
1.144     yamt     2037:                else
1.263     matt     2038:                        tmp = RIGHT_ENTRY(tmp);
1.144     yamt     2039:        }
1.164     junyoung 2040:
1.148     yamt     2041:        if (topdown) {
1.150     yamt     2042:                KASSERT(orig_hint >= tmp->next->start - length ||
1.148     yamt     2043:                    tmp->next->start - length > tmp->next->start);
                   2044:                hint = tmp->next->start - length;
                   2045:        } else {
1.150     yamt     2046:                KASSERT(orig_hint <= tmp->end);
1.148     yamt     2047:                hint = tmp->end;
                   2048:        }
1.144     yamt     2049:        switch (uvm_map_space_avail(&hint, length, uoffset, align,
1.304     matt     2050:            flags, topdown, tmp)) {
1.144     yamt     2051:        case 1:
                   2052:                entry = tmp;
                   2053:                goto found;
1.148     yamt     2054:        case -1:
                   2055:                goto wraparound;
1.144     yamt     2056:        }
                   2057:
1.164     junyoung 2058:        /*
1.144     yamt     2059:         * The tree fails to find an entry because of offset or alignment
                   2060:         * restrictions.  Search the list instead.
                   2061:         */
                   2062:  listsearch:
                   2063:        /*
1.1       mrg      2064:         * Look through the rest of the map, trying to fit a new region in
                   2065:         * the gap between existing regions, or after the very last region.
1.140     enami    2066:         * note: entry->end = base VA of current gap,
                   2067:         *       entry->next->start = VA of end of current gap
1.1       mrg      2068:         */
1.99      chs      2069:
1.140     enami    2070:        for (;;) {
                   2071:                /* Update hint for current gap. */
                   2072:                hint = topdown ? entry->next->start - length : entry->end;
                   2073:
                   2074:                /* See if it fits. */
                   2075:                switch (uvm_map_space_avail(&hint, length, uoffset, align,
1.304     matt     2076:                    flags, topdown, entry)) {
1.140     enami    2077:                case 1:
                   2078:                        goto found;
                   2079:                case -1:
                   2080:                        goto wraparound;
                   2081:                }
                   2082:
                   2083:                /* Advance to next/previous gap */
                   2084:                if (topdown) {
                   2085:                        if (entry == &map->header) {
                   2086:                                UVMHIST_LOG(maphist, "<- failed (off start)",
                   2087:                                    0,0,0,0);
                   2088:                                goto notfound;
1.134     matt     2089:                        }
1.140     enami    2090:                        entry = entry->prev;
                   2091:                } else {
                   2092:                        entry = entry->next;
                   2093:                        if (entry == &map->header) {
                   2094:                                UVMHIST_LOG(maphist, "<- failed (off end)",
1.81      thorpej  2095:                                    0,0,0,0);
1.140     enami    2096:                                goto notfound;
1.81      thorpej  2097:                        }
1.1       mrg      2098:                }
                   2099:        }
1.140     enami    2100:
                   2101:  found:
1.82      thorpej  2102:        SAVE_HINT(map, map->hint, entry);
1.1       mrg      2103:        *result = hint;
                   2104:        UVMHIST_LOG(maphist,"<- got it!  (result=0x%x)", hint, 0,0,0);
1.148     yamt     2105:        KASSERT( topdown || hint >= orig_hint);
                   2106:        KASSERT(!topdown || hint <= orig_hint);
1.144     yamt     2107:        KASSERT(entry->end <= hint);
                   2108:        KASSERT(hint + length <= entry->next->start);
1.1       mrg      2109:        return (entry);
1.140     enami    2110:
                   2111:  wraparound:
                   2112:        UVMHIST_LOG(maphist, "<- failed (wrap around)", 0,0,0,0);
                   2113:
1.165     yamt     2114:        return (NULL);
                   2115:
1.140     enami    2116:  notfound:
1.165     yamt     2117:        UVMHIST_LOG(maphist, "<- failed (notfound)", 0,0,0,0);
                   2118:
1.140     enami    2119:        return (NULL);
1.1       mrg      2120: }
                   2121:
                   2122: /*
                   2123:  *   U N M A P   -   m a i n   h e l p e r   f u n c t i o n s
                   2124:  */
                   2125:
                   2126: /*
                   2127:  * uvm_unmap_remove: remove mappings from a vm_map (from "start" up to "stop")
                   2128:  *
1.98      chs      2129:  * => caller must check alignment and size
1.1       mrg      2130:  * => map must be locked by caller
                   2131:  * => we return a list of map entries that we've remove from the map
                   2132:  *    in "entry_list"
                   2133:  */
                   2134:
1.94      chs      2135: void
1.138     enami    2136: uvm_unmap_remove(struct vm_map *map, vaddr_t start, vaddr_t end,
1.311     para     2137:     struct vm_map_entry **entry_list /* OUT */, int flags)
1.10      mrg      2138: {
1.99      chs      2139:        struct vm_map_entry *entry, *first_entry, *next;
1.24      eeh      2140:        vaddr_t len;
1.99      chs      2141:        UVMHIST_FUNC("uvm_unmap_remove"); UVMHIST_CALLED(maphist);
1.10      mrg      2142:
                   2143:        UVMHIST_LOG(maphist,"(map=0x%x, start=0x%x, end=0x%x)",
                   2144:            map, start, end, 0);
                   2145:        VM_MAP_RANGE_CHECK(map, start, end);
                   2146:
1.222     yamt     2147:        uvm_map_check(map, "unmap_remove entry");
1.144     yamt     2148:
1.10      mrg      2149:        /*
                   2150:         * find first entry
                   2151:         */
1.99      chs      2152:
1.234     thorpej  2153:        if (uvm_map_lookup_entry(map, start, &first_entry) == true) {
1.29      chuck    2154:                /* clip and go... */
1.10      mrg      2155:                entry = first_entry;
1.311     para     2156:                UVM_MAP_CLIP_START(map, entry, start);
1.10      mrg      2157:                /* critical!  prevents stale hint */
1.82      thorpej  2158:                SAVE_HINT(map, entry, entry->prev);
1.10      mrg      2159:        } else {
                   2160:                entry = first_entry->next;
                   2161:        }
                   2162:
                   2163:        /*
                   2164:         * Save the free space hint
                   2165:         */
                   2166:
1.220     yamt     2167:        if (map->first_free != &map->header && map->first_free->start >= start)
1.10      mrg      2168:                map->first_free = entry->prev;
                   2169:
                   2170:        /*
                   2171:         * note: we now re-use first_entry for a different task.  we remove
                   2172:         * a number of map entries from the map and save them in a linked
                   2173:         * list headed by "first_entry".  once we remove them from the map
                   2174:         * the caller should unlock the map and drop the references to the
                   2175:         * backing objects [c.f. uvm_unmap_detach].  the object is to
1.100     wiz      2176:         * separate unmapping from reference dropping.  why?
1.10      mrg      2177:         *   [1] the map has to be locked for unmapping
                   2178:         *   [2] the map need not be locked for reference dropping
                   2179:         *   [3] dropping references may trigger pager I/O, and if we hit
                   2180:         *       a pager that does synchronous I/O we may have to wait for it.
                   2181:         *   [4] we would like all waiting for I/O to occur with maps unlocked
1.98      chs      2182:         *       so that we don't block other threads.
1.10      mrg      2183:         */
1.99      chs      2184:
1.10      mrg      2185:        first_entry = NULL;
1.106     chs      2186:        *entry_list = NULL;
1.10      mrg      2187:
                   2188:        /*
1.98      chs      2189:         * break up the area into map entry sized regions and unmap.  note
1.10      mrg      2190:         * that all mappings have to be removed before we can even consider
                   2191:         * dropping references to amaps or VM objects (otherwise we could end
                   2192:         * up with a mapping to a page on the free list which would be very bad)
                   2193:         */
                   2194:
                   2195:        while ((entry != &map->header) && (entry->start < end)) {
1.311     para     2196:                KASSERT((entry->flags & UVM_MAP_STATIC) == 0);
1.174     yamt     2197:
1.311     para     2198:                UVM_MAP_CLIP_END(map, entry, end);
1.10      mrg      2199:                next = entry->next;
                   2200:                len = entry->end - entry->start;
1.81      thorpej  2201:
1.10      mrg      2202:                /*
                   2203:                 * unwire before removing addresses from the pmap; otherwise
                   2204:                 * unwiring will put the entries back into the pmap (XXX).
                   2205:                 */
1.1       mrg      2206:
1.106     chs      2207:                if (VM_MAPENT_ISWIRED(entry)) {
1.10      mrg      2208:                        uvm_map_entry_unwire(map, entry);
1.106     chs      2209:                }
1.187     yamt     2210:                if (flags & UVM_FLAG_VAONLY) {
                   2211:
                   2212:                        /* nothing */
                   2213:
                   2214:                } else if ((map->flags & VM_MAP_PAGEABLE) == 0) {
1.10      mrg      2215:
1.106     chs      2216:                        /*
                   2217:                         * if the map is non-pageable, any pages mapped there
                   2218:                         * must be wired and entered with pmap_kenter_pa(),
                   2219:                         * and we should free any such pages immediately.
1.287     joerg    2220:                         * this is mostly used for kmem_map.
1.106     chs      2221:                         */
1.292     rmind    2222:                        KASSERT(vm_map_pmap(map) == pmap_kernel());
1.99      chs      2223:
1.323     para     2224:                        uvm_km_pgremove_intrsafe(map, entry->start, entry->end);
1.106     chs      2225:                } else if (UVM_ET_ISOBJ(entry) &&
                   2226:                           UVM_OBJ_IS_KERN_OBJECT(entry->object.uvm_obj)) {
1.300     yamt     2227:                        panic("%s: kernel object %p %p\n",
                   2228:                            __func__, map, entry);
1.106     chs      2229:                } else if (UVM_ET_ISOBJ(entry) || entry->aref.ar_amap) {
1.29      chuck    2230:                        /*
1.298     rmind    2231:                         * remove mappings the standard way.  lock object
                   2232:                         * and/or amap to ensure vm_page state does not
                   2233:                         * change while in pmap_remove().
1.139     enami    2234:                         */
1.99      chs      2235:
1.298     rmind    2236:                        uvm_map_lock_entry(entry);
1.29      chuck    2237:                        pmap_remove(map->pmap, entry->start, entry->end);
1.298     rmind    2238:                        uvm_map_unlock_entry(entry);
1.10      mrg      2239:                }
                   2240:
1.177     yamt     2241: #if defined(DEBUG)
1.323     para     2242:                /*
                   2243:                 * check if there's remaining mapping,
                   2244:                 * which is a bug in caller.
                   2245:                 */
1.177     yamt     2246:
1.323     para     2247:                vaddr_t va;
                   2248:                for (va = entry->start; va < entry->end;
                   2249:                    va += PAGE_SIZE) {
                   2250:                        if (pmap_extract(vm_map_pmap(map), va, NULL)) {
                   2251:                                panic("%s: %#"PRIxVADDR" has mapping",
                   2252:                                    __func__, va);
1.177     yamt     2253:                        }
1.323     para     2254:                }
1.187     yamt     2255:
1.323     para     2256:                if (VM_MAP_IS_KERNEL(map)) {
                   2257:                        uvm_km_check_empty(map, entry->start,
                   2258:                            entry->end);
1.177     yamt     2259:                }
                   2260: #endif /* defined(DEBUG) */
                   2261:
1.10      mrg      2262:                /*
1.98      chs      2263:                 * remove entry from map and put it on our list of entries
1.106     chs      2264:                 * that we've nuked.  then go to next entry.
1.10      mrg      2265:                 */
1.99      chs      2266:
1.10      mrg      2267:                UVMHIST_LOG(maphist, "  removed map entry 0x%x", entry, 0, 0,0);
1.82      thorpej  2268:
                   2269:                /* critical!  prevents stale hint */
                   2270:                SAVE_HINT(map, entry, entry->prev);
                   2271:
1.10      mrg      2272:                uvm_map_entry_unlink(map, entry);
1.146     yamt     2273:                KASSERT(map->size >= len);
1.10      mrg      2274:                map->size -= len;
1.131     atatat   2275:                entry->prev = NULL;
1.10      mrg      2276:                entry->next = first_entry;
                   2277:                first_entry = entry;
1.106     chs      2278:                entry = next;
1.10      mrg      2279:        }
1.292     rmind    2280:
                   2281:        /*
                   2282:         * Note: if map is dying, leave pmap_update() for pmap_destroy(),
                   2283:         * which will be called later.
                   2284:         */
1.120     chs      2285:        if ((map->flags & VM_MAP_DYING) == 0) {
                   2286:                pmap_update(vm_map_pmap(map));
1.292     rmind    2287:        } else {
                   2288:                KASSERT(vm_map_pmap(map) != pmap_kernel());
1.120     chs      2289:        }
1.10      mrg      2290:
1.222     yamt     2291:        uvm_map_check(map, "unmap_remove leave");
1.144     yamt     2292:
1.10      mrg      2293:        /*
                   2294:         * now we've cleaned up the map and are ready for the caller to drop
1.98      chs      2295:         * references to the mapped objects.
1.10      mrg      2296:         */
                   2297:
                   2298:        *entry_list = first_entry;
                   2299:        UVMHIST_LOG(maphist,"<- done!", 0, 0, 0, 0);
1.180     yamt     2300:
                   2301:        if (map->flags & VM_MAP_WANTVA) {
1.238     ad       2302:                mutex_enter(&map->misc_lock);
1.180     yamt     2303:                map->flags &= ~VM_MAP_WANTVA;
1.238     ad       2304:                cv_broadcast(&map->cv);
                   2305:                mutex_exit(&map->misc_lock);
1.180     yamt     2306:        }
1.1       mrg      2307: }
                   2308:
                   2309: /*
                   2310:  * uvm_unmap_detach: drop references in a chain of map entries
                   2311:  *
                   2312:  * => we will free the map entries as we traverse the list.
                   2313:  */
                   2314:
1.10      mrg      2315: void
1.138     enami    2316: uvm_unmap_detach(struct vm_map_entry *first_entry, int flags)
1.1       mrg      2317: {
1.99      chs      2318:        struct vm_map_entry *next_entry;
1.10      mrg      2319:        UVMHIST_FUNC("uvm_unmap_detach"); UVMHIST_CALLED(maphist);
1.1       mrg      2320:
1.10      mrg      2321:        while (first_entry) {
1.85      chs      2322:                KASSERT(!VM_MAPENT_ISWIRED(first_entry));
1.10      mrg      2323:                UVMHIST_LOG(maphist,
1.98      chs      2324:                    "  detach 0x%x: amap=0x%x, obj=0x%x, submap?=%d",
                   2325:                    first_entry, first_entry->aref.ar_amap,
1.29      chuck    2326:                    first_entry->object.uvm_obj,
                   2327:                    UVM_ET_ISSUBMAP(first_entry));
1.1       mrg      2328:
1.10      mrg      2329:                /*
                   2330:                 * drop reference to amap, if we've got one
                   2331:                 */
                   2332:
                   2333:                if (first_entry->aref.ar_amap)
1.85      chs      2334:                        uvm_map_unreference_amap(first_entry, flags);
1.10      mrg      2335:
                   2336:                /*
                   2337:                 * drop reference to our backing object, if we've got one
                   2338:                 */
1.85      chs      2339:
1.120     chs      2340:                KASSERT(!UVM_ET_ISSUBMAP(first_entry));
                   2341:                if (UVM_ET_ISOBJ(first_entry) &&
                   2342:                    first_entry->object.uvm_obj->pgops->pgo_detach) {
                   2343:                        (*first_entry->object.uvm_obj->pgops->pgo_detach)
                   2344:                                (first_entry->object.uvm_obj);
1.10      mrg      2345:                }
                   2346:                next_entry = first_entry->next;
                   2347:                uvm_mapent_free(first_entry);
                   2348:                first_entry = next_entry;
                   2349:        }
                   2350:        UVMHIST_LOG(maphist, "<- done", 0,0,0,0);
1.1       mrg      2351: }
                   2352:
                   2353: /*
                   2354:  *   E X T R A C T I O N   F U N C T I O N S
                   2355:  */
                   2356:
1.98      chs      2357: /*
1.1       mrg      2358:  * uvm_map_reserve: reserve space in a vm_map for future use.
                   2359:  *
1.98      chs      2360:  * => we reserve space in a map by putting a dummy map entry in the
1.1       mrg      2361:  *    map (dummy means obj=NULL, amap=NULL, prot=VM_PROT_NONE)
                   2362:  * => map should be unlocked (we will write lock it)
                   2363:  * => we return true if we were able to reserve space
                   2364:  * => XXXCDC: should be inline?
                   2365:  */
                   2366:
1.10      mrg      2367: int
1.138     enami    2368: uvm_map_reserve(struct vm_map *map, vsize_t size,
                   2369:     vaddr_t offset     /* hint for pmap_prefer */,
1.243     yamt     2370:     vsize_t align      /* alignment */,
1.210     yamt     2371:     vaddr_t *raddr     /* IN:hint, OUT: reserved VA */,
1.324     matt     2372:     uvm_flag_t flags   /* UVM_FLAG_FIXED or UVM_FLAG_COLORMATCH or 0 */)
1.1       mrg      2373: {
1.98      chs      2374:        UVMHIST_FUNC("uvm_map_reserve"); UVMHIST_CALLED(maphist);
1.85      chs      2375:
1.10      mrg      2376:        UVMHIST_LOG(maphist, "(map=0x%x, size=0x%x, offset=0x%x,addr=0x%x)",
1.139     enami    2377:            map,size,offset,raddr);
1.85      chs      2378:
1.10      mrg      2379:        size = round_page(size);
1.85      chs      2380:
1.10      mrg      2381:        /*
                   2382:         * reserve some virtual space.
                   2383:         */
1.85      chs      2384:
1.243     yamt     2385:        if (uvm_map(map, raddr, size, NULL, offset, align,
1.10      mrg      2386:            UVM_MAPFLAG(UVM_PROT_NONE, UVM_PROT_NONE, UVM_INH_NONE,
1.210     yamt     2387:            UVM_ADV_RANDOM, UVM_FLAG_NOMERGE|flags)) != 0) {
1.10      mrg      2388:            UVMHIST_LOG(maphist, "<- done (no VM)", 0,0,0,0);
1.234     thorpej  2389:                return (false);
1.98      chs      2390:        }
1.85      chs      2391:
1.10      mrg      2392:        UVMHIST_LOG(maphist, "<- done (*raddr=0x%x)", *raddr,0,0,0);
1.234     thorpej  2393:        return (true);
1.1       mrg      2394: }
                   2395:
                   2396: /*
1.98      chs      2397:  * uvm_map_replace: replace a reserved (blank) area of memory with
1.1       mrg      2398:  * real mappings.
                   2399:  *
1.98      chs      2400:  * => caller must WRITE-LOCK the map
1.234     thorpej  2401:  * => we return true if replacement was a success
1.1       mrg      2402:  * => we expect the newents chain to have nnewents entrys on it and
                   2403:  *    we expect newents->prev to point to the last entry on the list
                   2404:  * => note newents is allowed to be NULL
                   2405:  */
                   2406:
1.275     yamt     2407: static int
1.138     enami    2408: uvm_map_replace(struct vm_map *map, vaddr_t start, vaddr_t end,
1.275     yamt     2409:     struct vm_map_entry *newents, int nnewents, vsize_t nsize,
                   2410:     struct vm_map_entry **oldentryp)
1.10      mrg      2411: {
1.99      chs      2412:        struct vm_map_entry *oldent, *last;
1.1       mrg      2413:
1.222     yamt     2414:        uvm_map_check(map, "map_replace entry");
1.144     yamt     2415:
1.10      mrg      2416:        /*
                   2417:         * first find the blank map entry at the specified address
                   2418:         */
1.85      chs      2419:
1.10      mrg      2420:        if (!uvm_map_lookup_entry(map, start, &oldent)) {
1.234     thorpej  2421:                return (false);
1.10      mrg      2422:        }
1.85      chs      2423:
1.10      mrg      2424:        /*
                   2425:         * check to make sure we have a proper blank entry
                   2426:         */
1.1       mrg      2427:
1.311     para     2428:        if (end < oldent->end) {
                   2429:                UVM_MAP_CLIP_END(map, oldent, end);
1.210     yamt     2430:        }
1.98      chs      2431:        if (oldent->start != start || oldent->end != end ||
1.10      mrg      2432:            oldent->object.uvm_obj != NULL || oldent->aref.ar_amap != NULL) {
1.234     thorpej  2433:                return (false);
1.10      mrg      2434:        }
1.1       mrg      2435:
                   2436: #ifdef DIAGNOSTIC
1.99      chs      2437:
1.10      mrg      2438:        /*
                   2439:         * sanity check the newents chain
                   2440:         */
1.99      chs      2441:
1.10      mrg      2442:        {
1.99      chs      2443:                struct vm_map_entry *tmpent = newents;
1.10      mrg      2444:                int nent = 0;
1.275     yamt     2445:                vsize_t sz = 0;
1.24      eeh      2446:                vaddr_t cur = start;
1.10      mrg      2447:
                   2448:                while (tmpent) {
                   2449:                        nent++;
1.275     yamt     2450:                        sz += tmpent->end - tmpent->start;
1.10      mrg      2451:                        if (tmpent->start < cur)
                   2452:                                panic("uvm_map_replace1");
1.275     yamt     2453:                        if (tmpent->start >= tmpent->end || tmpent->end > end) {
1.286     matt     2454:                                panic("uvm_map_replace2: "
                   2455:                                    "tmpent->start=0x%"PRIxVADDR
                   2456:                                    ", tmpent->end=0x%"PRIxVADDR
                   2457:                                    ", end=0x%"PRIxVADDR,
                   2458:                                    tmpent->start, tmpent->end, end);
1.10      mrg      2459:                        }
                   2460:                        cur = tmpent->end;
                   2461:                        if (tmpent->next) {
                   2462:                                if (tmpent->next->prev != tmpent)
                   2463:                                        panic("uvm_map_replace3");
                   2464:                        } else {
                   2465:                                if (newents->prev != tmpent)
                   2466:                                        panic("uvm_map_replace4");
                   2467:                        }
                   2468:                        tmpent = tmpent->next;
                   2469:                }
                   2470:                if (nent != nnewents)
                   2471:                        panic("uvm_map_replace5");
1.275     yamt     2472:                if (sz != nsize)
                   2473:                        panic("uvm_map_replace6");
1.10      mrg      2474:        }
                   2475: #endif
                   2476:
                   2477:        /*
                   2478:         * map entry is a valid blank!   replace it.   (this does all the
                   2479:         * work of map entry link/unlink...).
                   2480:         */
                   2481:
                   2482:        if (newents) {
1.99      chs      2483:                last = newents->prev;
1.10      mrg      2484:
                   2485:                /* critical: flush stale hints out of map */
1.82      thorpej  2486:                SAVE_HINT(map, map->hint, newents);
1.10      mrg      2487:                if (map->first_free == oldent)
                   2488:                        map->first_free = last;
                   2489:
                   2490:                last->next = oldent->next;
                   2491:                last->next->prev = last;
1.144     yamt     2492:
                   2493:                /* Fix RB tree */
                   2494:                uvm_rb_remove(map, oldent);
                   2495:
1.10      mrg      2496:                newents->prev = oldent->prev;
                   2497:                newents->prev->next = newents;
                   2498:                map->nentries = map->nentries + (nnewents - 1);
                   2499:
1.144     yamt     2500:                /* Fixup the RB tree */
                   2501:                {
                   2502:                        int i;
                   2503:                        struct vm_map_entry *tmp;
                   2504:
                   2505:                        tmp = newents;
                   2506:                        for (i = 0; i < nnewents && tmp; i++) {
                   2507:                                uvm_rb_insert(map, tmp);
                   2508:                                tmp = tmp->next;
                   2509:                        }
                   2510:                }
1.10      mrg      2511:        } else {
                   2512:                /* NULL list of new entries: just remove the old one */
1.221     yamt     2513:                clear_hints(map, oldent);
1.10      mrg      2514:                uvm_map_entry_unlink(map, oldent);
                   2515:        }
1.275     yamt     2516:        map->size -= end - start - nsize;
1.10      mrg      2517:
1.222     yamt     2518:        uvm_map_check(map, "map_replace leave");
1.10      mrg      2519:
                   2520:        /*
1.209     yamt     2521:         * now we can free the old blank entry and return.
1.10      mrg      2522:         */
1.1       mrg      2523:
1.253     yamt     2524:        *oldentryp = oldent;
1.234     thorpej  2525:        return (true);
1.1       mrg      2526: }
                   2527:
                   2528: /*
                   2529:  * uvm_map_extract: extract a mapping from a map and put it somewhere
                   2530:  *     (maybe removing the old mapping)
                   2531:  *
                   2532:  * => maps should be unlocked (we will write lock them)
                   2533:  * => returns 0 on success, error code otherwise
                   2534:  * => start must be page aligned
                   2535:  * => len must be page sized
                   2536:  * => flags:
                   2537:  *      UVM_EXTRACT_REMOVE: remove mappings from srcmap
                   2538:  *      UVM_EXTRACT_CONTIG: abort if unmapped area (advisory only)
                   2539:  *      UVM_EXTRACT_QREF: for a temporary extraction do quick obj refs
                   2540:  *      UVM_EXTRACT_FIXPROT: set prot to maxprot as we go
                   2541:  *    >>>NOTE: if you set REMOVE, you are not allowed to use CONTIG or QREF!<<<
                   2542:  *    >>>NOTE: QREF's must be unmapped via the QREF path, thus should only
                   2543:  *             be used from within the kernel in a kernel level map <<<
                   2544:  */
                   2545:
1.10      mrg      2546: int
1.138     enami    2547: uvm_map_extract(struct vm_map *srcmap, vaddr_t start, vsize_t len,
                   2548:     struct vm_map *dstmap, vaddr_t *dstaddrp, int flags)
1.10      mrg      2549: {
1.163     mycroft  2550:        vaddr_t dstaddr, end, newend, oldoffset, fudge, orig_fudge;
1.99      chs      2551:        struct vm_map_entry *chain, *endchain, *entry, *orig_entry, *newentry,
                   2552:            *deadentry, *oldentry;
1.253     yamt     2553:        struct vm_map_entry *resentry = NULL; /* a dummy reservation entry */
1.325     martin   2554:        vsize_t elen __unused;
1.10      mrg      2555:        int nchain, error, copy_ok;
1.275     yamt     2556:        vsize_t nsize;
1.10      mrg      2557:        UVMHIST_FUNC("uvm_map_extract"); UVMHIST_CALLED(maphist);
1.85      chs      2558:
1.10      mrg      2559:        UVMHIST_LOG(maphist,"(srcmap=0x%x,start=0x%x, len=0x%x", srcmap, start,
                   2560:            len,0);
                   2561:        UVMHIST_LOG(maphist," ...,dstmap=0x%x, flags=0x%x)", dstmap,flags,0,0);
                   2562:
                   2563:        /*
                   2564:         * step 0: sanity check: start must be on a page boundary, length
                   2565:         * must be page sized.  can't ask for CONTIG/QREF if you asked for
                   2566:         * REMOVE.
                   2567:         */
                   2568:
1.85      chs      2569:        KASSERT((start & PAGE_MASK) == 0 && (len & PAGE_MASK) == 0);
                   2570:        KASSERT((flags & UVM_EXTRACT_REMOVE) == 0 ||
                   2571:                (flags & (UVM_EXTRACT_CONTIG|UVM_EXTRACT_QREF)) == 0);
1.10      mrg      2572:
                   2573:        /*
                   2574:         * step 1: reserve space in the target map for the extracted area
                   2575:         */
                   2576:
1.210     yamt     2577:        if ((flags & UVM_EXTRACT_RESERVED) == 0) {
                   2578:                dstaddr = vm_map_min(dstmap);
1.324     matt     2579:                if (!uvm_map_reserve(dstmap, len, start,
                   2580:                    atop(start) & uvmexp.colormask, &dstaddr,
                   2581:                    UVM_FLAG_COLORMATCH))
1.210     yamt     2582:                        return (ENOMEM);
1.324     matt     2583:                KASSERT((atop(start ^ dstaddr) & uvmexp.colormask) == 0);
1.210     yamt     2584:                *dstaddrp = dstaddr;    /* pass address back to caller */
                   2585:                UVMHIST_LOG(maphist, "  dstaddr=0x%x", dstaddr,0,0,0);
                   2586:        } else {
                   2587:                dstaddr = *dstaddrp;
                   2588:        }
1.10      mrg      2589:
                   2590:        /*
1.98      chs      2591:         * step 2: setup for the extraction process loop by init'ing the
1.10      mrg      2592:         * map entry chain, locking src map, and looking up the first useful
                   2593:         * entry in the map.
                   2594:         */
1.1       mrg      2595:
1.10      mrg      2596:        end = start + len;
                   2597:        newend = dstaddr + len;
                   2598:        chain = endchain = NULL;
                   2599:        nchain = 0;
1.275     yamt     2600:        nsize = 0;
1.10      mrg      2601:        vm_map_lock(srcmap);
                   2602:
                   2603:        if (uvm_map_lookup_entry(srcmap, start, &entry)) {
                   2604:
                   2605:                /* "start" is within an entry */
                   2606:                if (flags & UVM_EXTRACT_QREF) {
1.85      chs      2607:
1.10      mrg      2608:                        /*
                   2609:                         * for quick references we don't clip the entry, so
                   2610:                         * the entry may map space "before" the starting
                   2611:                         * virtual address... this is the "fudge" factor
                   2612:                         * (which can be non-zero only the first time
                   2613:                         * through the "while" loop in step 3).
                   2614:                         */
1.85      chs      2615:
1.10      mrg      2616:                        fudge = start - entry->start;
                   2617:                } else {
1.85      chs      2618:
1.10      mrg      2619:                        /*
                   2620:                         * normal reference: we clip the map to fit (thus
                   2621:                         * fudge is zero)
                   2622:                         */
1.85      chs      2623:
1.311     para     2624:                        UVM_MAP_CLIP_START(srcmap, entry, start);
1.82      thorpej  2625:                        SAVE_HINT(srcmap, srcmap->hint, entry->prev);
1.10      mrg      2626:                        fudge = 0;
                   2627:                }
1.85      chs      2628:        } else {
1.1       mrg      2629:
1.10      mrg      2630:                /* "start" is not within an entry ... skip to next entry */
                   2631:                if (flags & UVM_EXTRACT_CONTIG) {
                   2632:                        error = EINVAL;
                   2633:                        goto bad;    /* definite hole here ... */
                   2634:                }
1.1       mrg      2635:
1.10      mrg      2636:                entry = entry->next;
                   2637:                fudge = 0;
                   2638:        }
1.85      chs      2639:
1.10      mrg      2640:        /* save values from srcmap for step 6 */
                   2641:        orig_entry = entry;
                   2642:        orig_fudge = fudge;
1.1       mrg      2643:
1.10      mrg      2644:        /*
                   2645:         * step 3: now start looping through the map entries, extracting
                   2646:         * as we go.
                   2647:         */
1.1       mrg      2648:
1.10      mrg      2649:        while (entry->start < end && entry != &srcmap->header) {
1.85      chs      2650:
1.10      mrg      2651:                /* if we are not doing a quick reference, clip it */
                   2652:                if ((flags & UVM_EXTRACT_QREF) == 0)
1.311     para     2653:                        UVM_MAP_CLIP_END(srcmap, entry, end);
1.10      mrg      2654:
                   2655:                /* clear needs_copy (allow chunking) */
                   2656:                if (UVM_ET_ISNEEDSCOPY(entry)) {
1.212     yamt     2657:                        amap_copy(srcmap, entry,
                   2658:                            AMAP_COPY_NOWAIT|AMAP_COPY_NOMERGE, start, end);
1.10      mrg      2659:                        if (UVM_ET_ISNEEDSCOPY(entry)) {  /* failed? */
                   2660:                                error = ENOMEM;
                   2661:                                goto bad;
                   2662:                        }
1.85      chs      2663:
1.10      mrg      2664:                        /* amap_copy could clip (during chunk)!  update fudge */
                   2665:                        if (fudge) {
1.163     mycroft  2666:                                fudge = start - entry->start;
1.10      mrg      2667:                                orig_fudge = fudge;
                   2668:                        }
                   2669:                }
1.1       mrg      2670:
1.10      mrg      2671:                /* calculate the offset of this from "start" */
                   2672:                oldoffset = (entry->start + fudge) - start;
1.1       mrg      2673:
1.10      mrg      2674:                /* allocate a new map entry */
1.126     bouyer   2675:                newentry = uvm_mapent_alloc(dstmap, 0);
1.10      mrg      2676:                if (newentry == NULL) {
                   2677:                        error = ENOMEM;
                   2678:                        goto bad;
                   2679:                }
                   2680:
                   2681:                /* set up new map entry */
                   2682:                newentry->next = NULL;
                   2683:                newentry->prev = endchain;
                   2684:                newentry->start = dstaddr + oldoffset;
                   2685:                newentry->end =
                   2686:                    newentry->start + (entry->end - (entry->start + fudge));
1.37      chs      2687:                if (newentry->end > newend || newentry->end < newentry->start)
1.10      mrg      2688:                        newentry->end = newend;
                   2689:                newentry->object.uvm_obj = entry->object.uvm_obj;
                   2690:                if (newentry->object.uvm_obj) {
                   2691:                        if (newentry->object.uvm_obj->pgops->pgo_reference)
                   2692:                                newentry->object.uvm_obj->pgops->
                   2693:                                    pgo_reference(newentry->object.uvm_obj);
                   2694:                                newentry->offset = entry->offset + fudge;
                   2695:                } else {
                   2696:                        newentry->offset = 0;
                   2697:                }
                   2698:                newentry->etype = entry->etype;
1.98      chs      2699:                newentry->protection = (flags & UVM_EXTRACT_FIXPROT) ?
                   2700:                        entry->max_protection : entry->protection;
1.10      mrg      2701:                newentry->max_protection = entry->max_protection;
                   2702:                newentry->inheritance = entry->inheritance;
                   2703:                newentry->wired_count = 0;
                   2704:                newentry->aref.ar_amap = entry->aref.ar_amap;
                   2705:                if (newentry->aref.ar_amap) {
1.34      chuck    2706:                        newentry->aref.ar_pageoff =
                   2707:                            entry->aref.ar_pageoff + (fudge >> PAGE_SHIFT);
1.85      chs      2708:                        uvm_map_reference_amap(newentry, AMAP_SHARED |
1.10      mrg      2709:                            ((flags & UVM_EXTRACT_QREF) ? AMAP_REFALL : 0));
                   2710:                } else {
1.34      chuck    2711:                        newentry->aref.ar_pageoff = 0;
1.10      mrg      2712:                }
                   2713:                newentry->advice = entry->advice;
1.245     yamt     2714:                if ((flags & UVM_EXTRACT_QREF) != 0) {
                   2715:                        newentry->flags |= UVM_MAP_NOMERGE;
                   2716:                }
1.10      mrg      2717:
                   2718:                /* now link it on the chain */
                   2719:                nchain++;
1.275     yamt     2720:                nsize += newentry->end - newentry->start;
1.10      mrg      2721:                if (endchain == NULL) {
                   2722:                        chain = endchain = newentry;
                   2723:                } else {
                   2724:                        endchain->next = newentry;
                   2725:                        endchain = newentry;
                   2726:                }
                   2727:
                   2728:                /* end of 'while' loop! */
1.98      chs      2729:                if ((flags & UVM_EXTRACT_CONTIG) && entry->end < end &&
1.10      mrg      2730:                    (entry->next == &srcmap->header ||
                   2731:                    entry->next->start != entry->end)) {
                   2732:                        error = EINVAL;
                   2733:                        goto bad;
                   2734:                }
                   2735:                entry = entry->next;
                   2736:                fudge = 0;
                   2737:        }
                   2738:
                   2739:        /*
                   2740:         * step 4: close off chain (in format expected by uvm_map_replace)
                   2741:         */
                   2742:
                   2743:        if (chain)
                   2744:                chain->prev = endchain;
                   2745:
                   2746:        /*
                   2747:         * step 5: attempt to lock the dest map so we can pmap_copy.
1.98      chs      2748:         * note usage of copy_ok:
1.10      mrg      2749:         *   1 => dstmap locked, pmap_copy ok, and we "replace" here (step 5)
                   2750:         *   0 => dstmap unlocked, NO pmap_copy, and we will "replace" in step 7
                   2751:         */
1.85      chs      2752:
1.234     thorpej  2753:        if (srcmap == dstmap || vm_map_lock_try(dstmap) == true) {
1.10      mrg      2754:                copy_ok = 1;
                   2755:                if (!uvm_map_replace(dstmap, dstaddr, dstaddr+len, chain,
1.275     yamt     2756:                    nchain, nsize, &resentry)) {
1.10      mrg      2757:                        if (srcmap != dstmap)
                   2758:                                vm_map_unlock(dstmap);
                   2759:                        error = EIO;
                   2760:                        goto bad;
                   2761:                }
                   2762:        } else {
                   2763:                copy_ok = 0;
                   2764:                /* replace defered until step 7 */
                   2765:        }
                   2766:
                   2767:        /*
                   2768:         * step 6: traverse the srcmap a second time to do the following:
                   2769:         *  - if we got a lock on the dstmap do pmap_copy
                   2770:         *  - if UVM_EXTRACT_REMOVE remove the entries
                   2771:         * we make use of orig_entry and orig_fudge (saved in step 2)
                   2772:         */
                   2773:
                   2774:        if (copy_ok || (flags & UVM_EXTRACT_REMOVE)) {
                   2775:
                   2776:                /* purge possible stale hints from srcmap */
                   2777:                if (flags & UVM_EXTRACT_REMOVE) {
1.82      thorpej  2778:                        SAVE_HINT(srcmap, srcmap->hint, orig_entry->prev);
1.220     yamt     2779:                        if (srcmap->first_free != &srcmap->header &&
                   2780:                            srcmap->first_free->start >= start)
1.10      mrg      2781:                                srcmap->first_free = orig_entry->prev;
                   2782:                }
                   2783:
                   2784:                entry = orig_entry;
                   2785:                fudge = orig_fudge;
                   2786:                deadentry = NULL;       /* for UVM_EXTRACT_REMOVE */
                   2787:
                   2788:                while (entry->start < end && entry != &srcmap->header) {
                   2789:                        if (copy_ok) {
1.74      thorpej  2790:                                oldoffset = (entry->start + fudge) - start;
1.90      chs      2791:                                elen = MIN(end, entry->end) -
1.74      thorpej  2792:                                    (entry->start + fudge);
                   2793:                                pmap_copy(dstmap->pmap, srcmap->pmap,
                   2794:                                    dstaddr + oldoffset, elen,
                   2795:                                    entry->start + fudge);
1.10      mrg      2796:                        }
                   2797:
1.74      thorpej  2798:                        /* we advance "entry" in the following if statement */
1.10      mrg      2799:                        if (flags & UVM_EXTRACT_REMOVE) {
1.298     rmind    2800:                                uvm_map_lock_entry(entry);
1.98      chs      2801:                                pmap_remove(srcmap->pmap, entry->start,
1.20      chuck    2802:                                                entry->end);
1.298     rmind    2803:                                uvm_map_unlock_entry(entry);
1.139     enami    2804:                                oldentry = entry;       /* save entry */
                   2805:                                entry = entry->next;    /* advance */
1.20      chuck    2806:                                uvm_map_entry_unlink(srcmap, oldentry);
                   2807:                                                        /* add to dead list */
                   2808:                                oldentry->next = deadentry;
                   2809:                                deadentry = oldentry;
1.139     enami    2810:                        } else {
                   2811:                                entry = entry->next;            /* advance */
1.10      mrg      2812:                        }
                   2813:
                   2814:                        /* end of 'while' loop */
                   2815:                        fudge = 0;
                   2816:                }
1.105     chris    2817:                pmap_update(srcmap->pmap);
1.10      mrg      2818:
                   2819:                /*
                   2820:                 * unlock dstmap.  we will dispose of deadentry in
                   2821:                 * step 7 if needed
                   2822:                 */
1.85      chs      2823:
1.10      mrg      2824:                if (copy_ok && srcmap != dstmap)
                   2825:                        vm_map_unlock(dstmap);
                   2826:
1.99      chs      2827:        } else {
                   2828:                deadentry = NULL;
1.10      mrg      2829:        }
                   2830:
                   2831:        /*
                   2832:         * step 7: we are done with the source map, unlock.   if copy_ok
                   2833:         * is 0 then we have not replaced the dummy mapping in dstmap yet
                   2834:         * and we need to do so now.
                   2835:         */
                   2836:
                   2837:        vm_map_unlock(srcmap);
                   2838:        if ((flags & UVM_EXTRACT_REMOVE) && deadentry)
                   2839:                uvm_unmap_detach(deadentry, 0);   /* dispose of old entries */
                   2840:
                   2841:        /* now do the replacement if we didn't do it in step 5 */
                   2842:        if (copy_ok == 0) {
                   2843:                vm_map_lock(dstmap);
                   2844:                error = uvm_map_replace(dstmap, dstaddr, dstaddr+len, chain,
1.275     yamt     2845:                    nchain, nsize, &resentry);
1.10      mrg      2846:                vm_map_unlock(dstmap);
                   2847:
1.234     thorpej  2848:                if (error == false) {
1.10      mrg      2849:                        error = EIO;
                   2850:                        goto bad2;
                   2851:                }
                   2852:        }
1.144     yamt     2853:
1.253     yamt     2854:        if (resentry != NULL)
                   2855:                uvm_mapent_free(resentry);
                   2856:
1.139     enami    2857:        return (0);
1.10      mrg      2858:
                   2859:        /*
                   2860:         * bad: failure recovery
                   2861:         */
                   2862: bad:
                   2863:        vm_map_unlock(srcmap);
                   2864: bad2:                  /* src already unlocked */
                   2865:        if (chain)
                   2866:                uvm_unmap_detach(chain,
                   2867:                    (flags & UVM_EXTRACT_QREF) ? AMAP_REFALL : 0);
1.144     yamt     2868:
1.253     yamt     2869:        if (resentry != NULL)
                   2870:                uvm_mapent_free(resentry);
                   2871:
1.210     yamt     2872:        if ((flags & UVM_EXTRACT_RESERVED) == 0) {
                   2873:                uvm_unmap(dstmap, dstaddr, dstaddr+len);   /* ??? */
                   2874:        }
1.139     enami    2875:        return (error);
1.10      mrg      2876: }
                   2877:
                   2878: /* end of extraction functions */
1.1       mrg      2879:
                   2880: /*
                   2881:  * uvm_map_submap: punch down part of a map into a submap
                   2882:  *
                   2883:  * => only the kernel_map is allowed to be submapped
                   2884:  * => the purpose of submapping is to break up the locking granularity
                   2885:  *     of a larger map
                   2886:  * => the range specified must have been mapped previously with a uvm_map()
                   2887:  *     call [with uobj==NULL] to create a blank map entry in the main map.
                   2888:  *     [And it had better still be blank!]
                   2889:  * => maps which contain submaps should never be copied or forked.
1.98      chs      2890:  * => to remove a submap, use uvm_unmap() on the main map
1.1       mrg      2891:  *     and then uvm_map_deallocate() the submap.
                   2892:  * => main map must be unlocked.
                   2893:  * => submap must have been init'd and have a zero reference count.
                   2894:  *     [need not be locked as we don't actually reference it]
                   2895:  */
1.85      chs      2896:
1.10      mrg      2897: int
1.138     enami    2898: uvm_map_submap(struct vm_map *map, vaddr_t start, vaddr_t end,
                   2899:     struct vm_map *submap)
1.10      mrg      2900: {
1.99      chs      2901:        struct vm_map_entry *entry;
1.94      chs      2902:        int error;
1.1       mrg      2903:
1.10      mrg      2904:        vm_map_lock(map);
1.85      chs      2905:        VM_MAP_RANGE_CHECK(map, start, end);
1.1       mrg      2906:
1.10      mrg      2907:        if (uvm_map_lookup_entry(map, start, &entry)) {
1.311     para     2908:                UVM_MAP_CLIP_START(map, entry, start);
                   2909:                UVM_MAP_CLIP_END(map, entry, end);      /* to be safe */
1.94      chs      2910:        } else {
1.10      mrg      2911:                entry = NULL;
                   2912:        }
1.1       mrg      2913:
1.98      chs      2914:        if (entry != NULL &&
1.10      mrg      2915:            entry->start == start && entry->end == end &&
                   2916:            entry->object.uvm_obj == NULL && entry->aref.ar_amap == NULL &&
                   2917:            !UVM_ET_ISCOPYONWRITE(entry) && !UVM_ET_ISNEEDSCOPY(entry)) {
1.29      chuck    2918:                entry->etype |= UVM_ET_SUBMAP;
1.10      mrg      2919:                entry->object.sub_map = submap;
                   2920:                entry->offset = 0;
                   2921:                uvm_map_reference(submap);
1.94      chs      2922:                error = 0;
1.10      mrg      2923:        } else {
1.94      chs      2924:                error = EINVAL;
1.10      mrg      2925:        }
                   2926:        vm_map_unlock(map);
1.174     yamt     2927:
1.94      chs      2928:        return error;
1.1       mrg      2929: }
                   2930:
1.175     yamt     2931: /*
1.1       mrg      2932:  * uvm_map_protect: change map protection
                   2933:  *
                   2934:  * => set_max means set max_protection.
                   2935:  * => map must be unlocked.
                   2936:  */
                   2937:
1.139     enami    2938: #define MASK(entry)    (UVM_ET_ISCOPYONWRITE(entry) ? \
1.36      mycroft  2939:                         ~VM_PROT_WRITE : VM_PROT_ALL)
1.1       mrg      2940:
1.10      mrg      2941: int
1.138     enami    2942: uvm_map_protect(struct vm_map *map, vaddr_t start, vaddr_t end,
1.233     thorpej  2943:     vm_prot_t new_prot, bool set_max)
1.10      mrg      2944: {
1.99      chs      2945:        struct vm_map_entry *current, *entry;
1.94      chs      2946:        int error = 0;
1.10      mrg      2947:        UVMHIST_FUNC("uvm_map_protect"); UVMHIST_CALLED(maphist);
                   2948:        UVMHIST_LOG(maphist,"(map=0x%x,start=0x%x,end=0x%x,new_prot=0x%x)",
1.85      chs      2949:                    map, start, end, new_prot);
                   2950:
1.10      mrg      2951:        vm_map_lock(map);
                   2952:        VM_MAP_RANGE_CHECK(map, start, end);
                   2953:        if (uvm_map_lookup_entry(map, start, &entry)) {
1.311     para     2954:                UVM_MAP_CLIP_START(map, entry, start);
1.10      mrg      2955:        } else {
                   2956:                entry = entry->next;
                   2957:        }
                   2958:
1.1       mrg      2959:        /*
1.10      mrg      2960:         * make a first pass to check for protection violations.
1.1       mrg      2961:         */
                   2962:
1.10      mrg      2963:        current = entry;
                   2964:        while ((current != &map->header) && (current->start < end)) {
1.65      thorpej  2965:                if (UVM_ET_ISSUBMAP(current)) {
1.94      chs      2966:                        error = EINVAL;
1.65      thorpej  2967:                        goto out;
                   2968:                }
1.10      mrg      2969:                if ((new_prot & current->max_protection) != new_prot) {
1.94      chs      2970:                        error = EACCES;
1.65      thorpej  2971:                        goto out;
1.112     thorpej  2972:                }
                   2973:                /*
                   2974:                 * Don't allow VM_PROT_EXECUTE to be set on entries that
                   2975:                 * point to vnodes that are associated with a NOEXEC file
                   2976:                 * system.
                   2977:                 */
                   2978:                if (UVM_ET_ISOBJ(current) &&
                   2979:                    UVM_OBJ_IS_VNODE(current->object.uvm_obj)) {
                   2980:                        struct vnode *vp =
                   2981:                            (struct vnode *) current->object.uvm_obj;
                   2982:
                   2983:                        if ((new_prot & VM_PROT_EXECUTE) != 0 &&
                   2984:                            (vp->v_mount->mnt_flag & MNT_NOEXEC) != 0) {
                   2985:                                error = EACCES;
                   2986:                                goto out;
                   2987:                        }
1.10      mrg      2988:                }
1.224     elad     2989:
1.65      thorpej  2990:                current = current->next;
1.10      mrg      2991:        }
                   2992:
                   2993:        /* go back and fix up protections (no need to clip this time). */
                   2994:
                   2995:        current = entry;
                   2996:        while ((current != &map->header) && (current->start < end)) {
                   2997:                vm_prot_t old_prot;
1.85      chs      2998:
1.311     para     2999:                UVM_MAP_CLIP_END(map, current, end);
1.10      mrg      3000:                old_prot = current->protection;
                   3001:                if (set_max)
                   3002:                        current->protection =
                   3003:                            (current->max_protection = new_prot) & old_prot;
                   3004:                else
                   3005:                        current->protection = new_prot;
                   3006:
                   3007:                /*
1.98      chs      3008:                 * update physical map if necessary.  worry about copy-on-write
1.10      mrg      3009:                 * here -- CHECK THIS XXX
                   3010:                 */
                   3011:
                   3012:                if (current->protection != old_prot) {
1.29      chuck    3013:                        /* update pmap! */
1.298     rmind    3014:                        uvm_map_lock_entry(current);
1.29      chuck    3015:                        pmap_protect(map->pmap, current->start, current->end,
                   3016:                            current->protection & MASK(entry));
1.298     rmind    3017:                        uvm_map_unlock_entry(current);
1.109     thorpej  3018:
                   3019:                        /*
                   3020:                         * If this entry points at a vnode, and the
                   3021:                         * protection includes VM_PROT_EXECUTE, mark
1.111     thorpej  3022:                         * the vnode as VEXECMAP.
1.109     thorpej  3023:                         */
                   3024:                        if (UVM_ET_ISOBJ(current)) {
                   3025:                                struct uvm_object *uobj =
                   3026:                                    current->object.uvm_obj;
                   3027:
                   3028:                                if (UVM_OBJ_IS_VNODE(uobj) &&
1.241     ad       3029:                                    (current->protection & VM_PROT_EXECUTE)) {
1.110     thorpej  3030:                                        vn_markexec((struct vnode *) uobj);
1.241     ad       3031:                                }
1.109     thorpej  3032:                        }
1.65      thorpej  3033:                }
1.10      mrg      3034:
1.65      thorpej  3035:                /*
                   3036:                 * If the map is configured to lock any future mappings,
                   3037:                 * wire this entry now if the old protection was VM_PROT_NONE
                   3038:                 * and the new protection is not VM_PROT_NONE.
                   3039:                 */
                   3040:
                   3041:                if ((map->flags & VM_MAP_WIREFUTURE) != 0 &&
                   3042:                    VM_MAPENT_ISWIRED(entry) == 0 &&
                   3043:                    old_prot == VM_PROT_NONE &&
                   3044:                    new_prot != VM_PROT_NONE) {
                   3045:                        if (uvm_map_pageable(map, entry->start,
1.234     thorpej  3046:                            entry->end, false,
1.94      chs      3047:                            UVM_LK_ENTER|UVM_LK_EXIT) != 0) {
1.99      chs      3048:
1.65      thorpej  3049:                                /*
                   3050:                                 * If locking the entry fails, remember the
                   3051:                                 * error if it's the first one.  Note we
                   3052:                                 * still continue setting the protection in
1.94      chs      3053:                                 * the map, but will return the error
                   3054:                                 * condition regardless.
1.65      thorpej  3055:                                 *
                   3056:                                 * XXX Ignore what the actual error is,
                   3057:                                 * XXX just call it a resource shortage
                   3058:                                 * XXX so that it doesn't get confused
                   3059:                                 * XXX what uvm_map_protect() itself would
                   3060:                                 * XXX normally return.
                   3061:                                 */
1.99      chs      3062:
1.94      chs      3063:                                error = ENOMEM;
1.65      thorpej  3064:                        }
1.10      mrg      3065:                }
                   3066:                current = current->next;
                   3067:        }
1.105     chris    3068:        pmap_update(map->pmap);
1.85      chs      3069:
1.65      thorpej  3070:  out:
1.10      mrg      3071:        vm_map_unlock(map);
1.174     yamt     3072:
1.94      chs      3073:        UVMHIST_LOG(maphist, "<- done, error=%d",error,0,0,0);
                   3074:        return error;
1.1       mrg      3075: }
                   3076:
                   3077: #undef  MASK
                   3078:
1.98      chs      3079: /*
1.1       mrg      3080:  * uvm_map_inherit: set inheritance code for range of addrs in map.
                   3081:  *
                   3082:  * => map must be unlocked
                   3083:  * => note that the inherit code is used during a "fork".  see fork
                   3084:  *     code for details.
                   3085:  */
                   3086:
1.10      mrg      3087: int
1.138     enami    3088: uvm_map_inherit(struct vm_map *map, vaddr_t start, vaddr_t end,
                   3089:     vm_inherit_t new_inheritance)
1.10      mrg      3090: {
1.99      chs      3091:        struct vm_map_entry *entry, *temp_entry;
1.10      mrg      3092:        UVMHIST_FUNC("uvm_map_inherit"); UVMHIST_CALLED(maphist);
                   3093:        UVMHIST_LOG(maphist,"(map=0x%x,start=0x%x,end=0x%x,new_inh=0x%x)",
                   3094:            map, start, end, new_inheritance);
                   3095:
                   3096:        switch (new_inheritance) {
1.80      wiz      3097:        case MAP_INHERIT_NONE:
                   3098:        case MAP_INHERIT_COPY:
                   3099:        case MAP_INHERIT_SHARE:
1.330     christos 3100:        case MAP_INHERIT_ZERO:
1.10      mrg      3101:                break;
                   3102:        default:
                   3103:                UVMHIST_LOG(maphist,"<- done (INVALID ARG)",0,0,0,0);
1.94      chs      3104:                return EINVAL;
1.10      mrg      3105:        }
1.1       mrg      3106:
1.10      mrg      3107:        vm_map_lock(map);
                   3108:        VM_MAP_RANGE_CHECK(map, start, end);
                   3109:        if (uvm_map_lookup_entry(map, start, &temp_entry)) {
                   3110:                entry = temp_entry;
1.311     para     3111:                UVM_MAP_CLIP_START(map, entry, start);
1.10      mrg      3112:        }  else {
                   3113:                entry = temp_entry->next;
                   3114:        }
                   3115:        while ((entry != &map->header) && (entry->start < end)) {
1.311     para     3116:                UVM_MAP_CLIP_END(map, entry, end);
1.10      mrg      3117:                entry->inheritance = new_inheritance;
                   3118:                entry = entry->next;
                   3119:        }
                   3120:        vm_map_unlock(map);
                   3121:        UVMHIST_LOG(maphist,"<- done (OK)",0,0,0,0);
1.94      chs      3122:        return 0;
1.41      mrg      3123: }
                   3124:
1.98      chs      3125: /*
1.41      mrg      3126:  * uvm_map_advice: set advice code for range of addrs in map.
                   3127:  *
                   3128:  * => map must be unlocked
                   3129:  */
                   3130:
                   3131: int
1.138     enami    3132: uvm_map_advice(struct vm_map *map, vaddr_t start, vaddr_t end, int new_advice)
1.41      mrg      3133: {
1.99      chs      3134:        struct vm_map_entry *entry, *temp_entry;
1.41      mrg      3135:        UVMHIST_FUNC("uvm_map_advice"); UVMHIST_CALLED(maphist);
                   3136:        UVMHIST_LOG(maphist,"(map=0x%x,start=0x%x,end=0x%x,new_adv=0x%x)",
                   3137:            map, start, end, new_advice);
                   3138:
                   3139:        vm_map_lock(map);
                   3140:        VM_MAP_RANGE_CHECK(map, start, end);
                   3141:        if (uvm_map_lookup_entry(map, start, &temp_entry)) {
                   3142:                entry = temp_entry;
1.311     para     3143:                UVM_MAP_CLIP_START(map, entry, start);
1.41      mrg      3144:        } else {
                   3145:                entry = temp_entry->next;
                   3146:        }
1.61      thorpej  3147:
                   3148:        /*
                   3149:         * XXXJRT: disallow holes?
                   3150:         */
                   3151:
1.41      mrg      3152:        while ((entry != &map->header) && (entry->start < end)) {
1.311     para     3153:                UVM_MAP_CLIP_END(map, entry, end);
1.41      mrg      3154:
                   3155:                switch (new_advice) {
                   3156:                case MADV_NORMAL:
                   3157:                case MADV_RANDOM:
                   3158:                case MADV_SEQUENTIAL:
                   3159:                        /* nothing special here */
                   3160:                        break;
                   3161:
                   3162:                default:
1.50      mrg      3163:                        vm_map_unlock(map);
1.41      mrg      3164:                        UVMHIST_LOG(maphist,"<- done (INVALID ARG)",0,0,0,0);
1.94      chs      3165:                        return EINVAL;
1.41      mrg      3166:                }
                   3167:                entry->advice = new_advice;
                   3168:                entry = entry->next;
                   3169:        }
                   3170:
                   3171:        vm_map_unlock(map);
                   3172:        UVMHIST_LOG(maphist,"<- done (OK)",0,0,0,0);
1.94      chs      3173:        return 0;
1.1       mrg      3174: }
                   3175:
                   3176: /*
1.271     yamt     3177:  * uvm_map_willneed: apply MADV_WILLNEED
                   3178:  */
                   3179:
                   3180: int
                   3181: uvm_map_willneed(struct vm_map *map, vaddr_t start, vaddr_t end)
                   3182: {
                   3183:        struct vm_map_entry *entry;
                   3184:        UVMHIST_FUNC("uvm_map_willneed"); UVMHIST_CALLED(maphist);
                   3185:        UVMHIST_LOG(maphist,"(map=0x%lx,start=0x%lx,end=0x%lx)",
                   3186:            map, start, end, 0);
                   3187:
                   3188:        vm_map_lock_read(map);
                   3189:        VM_MAP_RANGE_CHECK(map, start, end);
                   3190:        if (!uvm_map_lookup_entry(map, start, &entry)) {
                   3191:                entry = entry->next;
                   3192:        }
                   3193:        while (entry->start < end) {
                   3194:                struct vm_amap * const amap = entry->aref.ar_amap;
                   3195:                struct uvm_object * const uobj = entry->object.uvm_obj;
                   3196:
                   3197:                KASSERT(entry != &map->header);
                   3198:                KASSERT(start < entry->end);
                   3199:                /*
1.296     yamt     3200:                 * For now, we handle only the easy but commonly-requested case.
                   3201:                 * ie. start prefetching of backing uobj pages.
1.271     yamt     3202:                 *
1.296     yamt     3203:                 * XXX It might be useful to pmap_enter() the already-in-core
                   3204:                 * pages by inventing a "weak" mode for uvm_fault() which would
                   3205:                 * only do the PGO_LOCKED pgo_get().
1.271     yamt     3206:                 */
                   3207:                if (UVM_ET_ISOBJ(entry) && amap == NULL && uobj != NULL) {
                   3208:                        off_t offset;
                   3209:                        off_t size;
                   3210:
                   3211:                        offset = entry->offset;
                   3212:                        if (start < entry->start) {
                   3213:                                offset += entry->start - start;
                   3214:                        }
                   3215:                        size = entry->offset + (entry->end - entry->start);
                   3216:                        if (entry->end < end) {
                   3217:                                size -= end - entry->end;
                   3218:                        }
                   3219:                        uvm_readahead(uobj, offset, size);
                   3220:                }
                   3221:                entry = entry->next;
                   3222:        }
                   3223:        vm_map_unlock_read(map);
                   3224:        UVMHIST_LOG(maphist,"<- done (OK)",0,0,0,0);
                   3225:        return 0;
                   3226: }
                   3227:
                   3228: /*
1.1       mrg      3229:  * uvm_map_pageable: sets the pageability of a range in a map.
                   3230:  *
1.56      thorpej  3231:  * => wires map entries.  should not be used for transient page locking.
                   3232:  *     for that, use uvm_fault_wire()/uvm_fault_unwire() (see uvm_vslock()).
1.216     drochner 3233:  * => regions specified as not pageable require lock-down (wired) memory
1.1       mrg      3234:  *     and page tables.
1.59      thorpej  3235:  * => map must never be read-locked
1.234     thorpej  3236:  * => if islocked is true, map is already write-locked
1.59      thorpej  3237:  * => we always unlock the map, since we must downgrade to a read-lock
                   3238:  *     to call uvm_fault_wire()
1.1       mrg      3239:  * => XXXCDC: check this and try and clean it up.
                   3240:  */
                   3241:
1.19      kleink   3242: int
1.138     enami    3243: uvm_map_pageable(struct vm_map *map, vaddr_t start, vaddr_t end,
1.233     thorpej  3244:     bool new_pageable, int lockflags)
1.1       mrg      3245: {
1.99      chs      3246:        struct vm_map_entry *entry, *start_entry, *failed_entry;
1.10      mrg      3247:        int rv;
1.60      thorpej  3248: #ifdef DIAGNOSTIC
                   3249:        u_int timestamp_save;
                   3250: #endif
1.10      mrg      3251:        UVMHIST_FUNC("uvm_map_pageable"); UVMHIST_CALLED(maphist);
                   3252:        UVMHIST_LOG(maphist,"(map=0x%x,start=0x%x,end=0x%x,new_pageable=0x%x)",
1.85      chs      3253:                    map, start, end, new_pageable);
                   3254:        KASSERT(map->flags & VM_MAP_PAGEABLE);
1.45      thorpej  3255:
1.64      thorpej  3256:        if ((lockflags & UVM_LK_ENTER) == 0)
1.59      thorpej  3257:                vm_map_lock(map);
1.10      mrg      3258:        VM_MAP_RANGE_CHECK(map, start, end);
                   3259:
1.98      chs      3260:        /*
1.10      mrg      3261:         * only one pageability change may take place at one time, since
                   3262:         * uvm_fault_wire assumes it will be called only once for each
                   3263:         * wiring/unwiring.  therefore, we have to make sure we're actually
                   3264:         * changing the pageability for the entire region.  we do so before
1.98      chs      3265:         * making any changes.
1.10      mrg      3266:         */
                   3267:
1.234     thorpej  3268:        if (uvm_map_lookup_entry(map, start, &start_entry) == false) {
1.64      thorpej  3269:                if ((lockflags & UVM_LK_EXIT) == 0)
                   3270:                        vm_map_unlock(map);
1.85      chs      3271:
1.94      chs      3272:                UVMHIST_LOG(maphist,"<- done (fault)",0,0,0,0);
                   3273:                return EFAULT;
1.10      mrg      3274:        }
                   3275:        entry = start_entry;
                   3276:
1.98      chs      3277:        /*
1.100     wiz      3278:         * handle wiring and unwiring separately.
1.10      mrg      3279:         */
1.1       mrg      3280:
1.56      thorpej  3281:        if (new_pageable) {             /* unwire */
1.311     para     3282:                UVM_MAP_CLIP_START(map, entry, start);
1.85      chs      3283:
1.10      mrg      3284:                /*
                   3285:                 * unwiring.  first ensure that the range to be unwired is
1.98      chs      3286:                 * really wired down and that there are no holes.
1.10      mrg      3287:                 */
1.85      chs      3288:
1.10      mrg      3289:                while ((entry != &map->header) && (entry->start < end)) {
                   3290:                        if (entry->wired_count == 0 ||
                   3291:                            (entry->end < end &&
1.55      thorpej  3292:                             (entry->next == &map->header ||
                   3293:                              entry->next->start > entry->end))) {
1.64      thorpej  3294:                                if ((lockflags & UVM_LK_EXIT) == 0)
                   3295:                                        vm_map_unlock(map);
1.94      chs      3296:                                UVMHIST_LOG(maphist, "<- done (INVAL)",0,0,0,0);
                   3297:                                return EINVAL;
1.10      mrg      3298:                        }
                   3299:                        entry = entry->next;
                   3300:                }
                   3301:
1.98      chs      3302:                /*
1.56      thorpej  3303:                 * POSIX 1003.1b - a single munlock call unlocks a region,
                   3304:                 * regardless of the number of mlock calls made on that
                   3305:                 * region.
1.10      mrg      3306:                 */
1.85      chs      3307:
1.10      mrg      3308:                entry = start_entry;
                   3309:                while ((entry != &map->header) && (entry->start < end)) {
1.311     para     3310:                        UVM_MAP_CLIP_END(map, entry, end);
1.56      thorpej  3311:                        if (VM_MAPENT_ISWIRED(entry))
1.10      mrg      3312:                                uvm_map_entry_unwire(map, entry);
                   3313:                        entry = entry->next;
                   3314:                }
1.64      thorpej  3315:                if ((lockflags & UVM_LK_EXIT) == 0)
                   3316:                        vm_map_unlock(map);
1.10      mrg      3317:                UVMHIST_LOG(maphist,"<- done (OK UNWIRE)",0,0,0,0);
1.94      chs      3318:                return 0;
1.10      mrg      3319:        }
                   3320:
                   3321:        /*
                   3322:         * wire case: in two passes [XXXCDC: ugly block of code here]
                   3323:         *
                   3324:         * 1: holding the write lock, we create any anonymous maps that need
                   3325:         *    to be created.  then we clip each map entry to the region to
1.98      chs      3326:         *    be wired and increment its wiring count.
1.10      mrg      3327:         *
                   3328:         * 2: we downgrade to a read lock, and call uvm_fault_wire to fault
1.56      thorpej  3329:         *    in the pages for any newly wired area (wired_count == 1).
1.10      mrg      3330:         *
                   3331:         *    downgrading to a read lock for uvm_fault_wire avoids a possible
                   3332:         *    deadlock with another thread that may have faulted on one of
                   3333:         *    the pages to be wired (it would mark the page busy, blocking
                   3334:         *    us, then in turn block on the map lock that we hold).  because
                   3335:         *    of problems in the recursive lock package, we cannot upgrade
                   3336:         *    to a write lock in vm_map_lookup.  thus, any actions that
                   3337:         *    require the write lock must be done beforehand.  because we
                   3338:         *    keep the read lock on the map, the copy-on-write status of the
                   3339:         *    entries we modify here cannot change.
                   3340:         */
                   3341:
                   3342:        while ((entry != &map->header) && (entry->start < end)) {
1.55      thorpej  3343:                if (VM_MAPENT_ISWIRED(entry) == 0) { /* not already wired? */
1.85      chs      3344:
                   3345:                        /*
1.10      mrg      3346:                         * perform actions of vm_map_lookup that need the
                   3347:                         * write lock on the map: create an anonymous map
                   3348:                         * for a copy-on-write region, or an anonymous map
1.29      chuck    3349:                         * for a zero-fill region.  (XXXCDC: submap case
                   3350:                         * ok?)
1.10      mrg      3351:                         */
1.85      chs      3352:
1.29      chuck    3353:                        if (!UVM_ET_ISSUBMAP(entry)) {  /* not submap */
1.98      chs      3354:                                if (UVM_ET_ISNEEDSCOPY(entry) &&
1.117     chs      3355:                                    ((entry->max_protection & VM_PROT_WRITE) ||
1.54      thorpej  3356:                                     (entry->object.uvm_obj == NULL))) {
1.212     yamt     3357:                                        amap_copy(map, entry, 0, start, end);
1.10      mrg      3358:                                        /* XXXCDC: wait OK? */
                   3359:                                }
                   3360:                        }
1.55      thorpej  3361:                }
1.311     para     3362:                UVM_MAP_CLIP_START(map, entry, start);
                   3363:                UVM_MAP_CLIP_END(map, entry, end);
1.10      mrg      3364:                entry->wired_count++;
                   3365:
                   3366:                /*
1.98      chs      3367:                 * Check for holes
1.10      mrg      3368:                 */
1.85      chs      3369:
1.54      thorpej  3370:                if (entry->protection == VM_PROT_NONE ||
                   3371:                    (entry->end < end &&
                   3372:                     (entry->next == &map->header ||
                   3373:                      entry->next->start > entry->end))) {
1.85      chs      3374:
1.10      mrg      3375:                        /*
                   3376:                         * found one.  amap creation actions do not need to
1.98      chs      3377:                         * be undone, but the wired counts need to be restored.
1.10      mrg      3378:                         */
1.85      chs      3379:
1.10      mrg      3380:                        while (entry != &map->header && entry->end > start) {
                   3381:                                entry->wired_count--;
                   3382:                                entry = entry->prev;
                   3383:                        }
1.64      thorpej  3384:                        if ((lockflags & UVM_LK_EXIT) == 0)
                   3385:                                vm_map_unlock(map);
1.10      mrg      3386:                        UVMHIST_LOG(maphist,"<- done (INVALID WIRE)",0,0,0,0);
1.94      chs      3387:                        return EINVAL;
1.10      mrg      3388:                }
                   3389:                entry = entry->next;
                   3390:        }
                   3391:
                   3392:        /*
                   3393:         * Pass 2.
                   3394:         */
1.51      thorpej  3395:
1.60      thorpej  3396: #ifdef DIAGNOSTIC
                   3397:        timestamp_save = map->timestamp;
                   3398: #endif
                   3399:        vm_map_busy(map);
1.249     yamt     3400:        vm_map_unlock(map);
1.10      mrg      3401:
                   3402:        rv = 0;
                   3403:        entry = start_entry;
                   3404:        while (entry != &map->header && entry->start < end) {
1.51      thorpej  3405:                if (entry->wired_count == 1) {
1.44      thorpej  3406:                        rv = uvm_fault_wire(map, entry->start, entry->end,
1.216     drochner 3407:                            entry->max_protection, 1);
1.10      mrg      3408:                        if (rv) {
1.94      chs      3409:
1.51      thorpej  3410:                                /*
                   3411:                                 * wiring failed.  break out of the loop.
                   3412:                                 * we'll clean up the map below, once we
                   3413:                                 * have a write lock again.
                   3414:                                 */
1.94      chs      3415:
1.51      thorpej  3416:                                break;
1.10      mrg      3417:                        }
                   3418:                }
                   3419:                entry = entry->next;
                   3420:        }
                   3421:
1.139     enami    3422:        if (rv) {       /* failed? */
1.85      chs      3423:
1.52      thorpej  3424:                /*
                   3425:                 * Get back to an exclusive (write) lock.
                   3426:                 */
1.85      chs      3427:
1.249     yamt     3428:                vm_map_lock(map);
1.60      thorpej  3429:                vm_map_unbusy(map);
                   3430:
                   3431: #ifdef DIAGNOSTIC
1.252     yamt     3432:                if (timestamp_save + 1 != map->timestamp)
1.60      thorpej  3433:                        panic("uvm_map_pageable: stale map");
                   3434: #endif
1.10      mrg      3435:
1.51      thorpej  3436:                /*
                   3437:                 * first drop the wiring count on all the entries
                   3438:                 * which haven't actually been wired yet.
                   3439:                 */
1.85      chs      3440:
1.54      thorpej  3441:                failed_entry = entry;
                   3442:                while (entry != &map->header && entry->start < end) {
1.51      thorpej  3443:                        entry->wired_count--;
1.54      thorpej  3444:                        entry = entry->next;
                   3445:                }
1.51      thorpej  3446:
                   3447:                /*
1.54      thorpej  3448:                 * now, unwire all the entries that were successfully
                   3449:                 * wired above.
1.51      thorpej  3450:                 */
1.85      chs      3451:
1.54      thorpej  3452:                entry = start_entry;
                   3453:                while (entry != failed_entry) {
                   3454:                        entry->wired_count--;
1.55      thorpej  3455:                        if (VM_MAPENT_ISWIRED(entry) == 0)
1.54      thorpej  3456:                                uvm_map_entry_unwire(map, entry);
                   3457:                        entry = entry->next;
                   3458:                }
1.64      thorpej  3459:                if ((lockflags & UVM_LK_EXIT) == 0)
                   3460:                        vm_map_unlock(map);
1.10      mrg      3461:                UVMHIST_LOG(maphist, "<- done (RV=%d)", rv,0,0,0);
1.139     enami    3462:                return (rv);
1.10      mrg      3463:        }
1.51      thorpej  3464:
1.64      thorpej  3465:        if ((lockflags & UVM_LK_EXIT) == 0) {
                   3466:                vm_map_unbusy(map);
                   3467:        } else {
1.85      chs      3468:
1.64      thorpej  3469:                /*
                   3470:                 * Get back to an exclusive (write) lock.
                   3471:                 */
1.85      chs      3472:
1.249     yamt     3473:                vm_map_lock(map);
1.64      thorpej  3474:                vm_map_unbusy(map);
                   3475:        }
                   3476:
1.10      mrg      3477:        UVMHIST_LOG(maphist,"<- done (OK WIRE)",0,0,0,0);
1.94      chs      3478:        return 0;
1.1       mrg      3479: }
                   3480:
                   3481: /*
1.54      thorpej  3482:  * uvm_map_pageable_all: special case of uvm_map_pageable - affects
                   3483:  * all mapped regions.
                   3484:  *
                   3485:  * => map must not be locked.
                   3486:  * => if no flags are specified, all regions are unwired.
                   3487:  * => XXXJRT: has some of the same problems as uvm_map_pageable() above.
                   3488:  */
                   3489:
                   3490: int
1.138     enami    3491: uvm_map_pageable_all(struct vm_map *map, int flags, vsize_t limit)
1.54      thorpej  3492: {
1.99      chs      3493:        struct vm_map_entry *entry, *failed_entry;
1.54      thorpej  3494:        vsize_t size;
                   3495:        int rv;
1.60      thorpej  3496: #ifdef DIAGNOSTIC
                   3497:        u_int timestamp_save;
                   3498: #endif
1.54      thorpej  3499:        UVMHIST_FUNC("uvm_map_pageable_all"); UVMHIST_CALLED(maphist);
                   3500:        UVMHIST_LOG(maphist,"(map=0x%x,flags=0x%x)", map, flags, 0, 0);
                   3501:
1.85      chs      3502:        KASSERT(map->flags & VM_MAP_PAGEABLE);
1.54      thorpej  3503:
                   3504:        vm_map_lock(map);
                   3505:
                   3506:        /*
                   3507:         * handle wiring and unwiring separately.
                   3508:         */
                   3509:
                   3510:        if (flags == 0) {                       /* unwire */
1.99      chs      3511:
1.54      thorpej  3512:                /*
1.56      thorpej  3513:                 * POSIX 1003.1b -- munlockall unlocks all regions,
                   3514:                 * regardless of how many times mlockall has been called.
1.54      thorpej  3515:                 */
1.99      chs      3516:
1.54      thorpej  3517:                for (entry = map->header.next; entry != &map->header;
                   3518:                     entry = entry->next) {
1.56      thorpej  3519:                        if (VM_MAPENT_ISWIRED(entry))
                   3520:                                uvm_map_entry_unwire(map, entry);
1.54      thorpej  3521:                }
1.238     ad       3522:                map->flags &= ~VM_MAP_WIREFUTURE;
1.54      thorpej  3523:                vm_map_unlock(map);
                   3524:                UVMHIST_LOG(maphist,"<- done (OK UNWIRE)",0,0,0,0);
1.94      chs      3525:                return 0;
1.54      thorpej  3526:        }
                   3527:
                   3528:        if (flags & MCL_FUTURE) {
1.99      chs      3529:
1.54      thorpej  3530:                /*
                   3531:                 * must wire all future mappings; remember this.
                   3532:                 */
1.99      chs      3533:
1.238     ad       3534:                map->flags |= VM_MAP_WIREFUTURE;
1.54      thorpej  3535:        }
                   3536:
                   3537:        if ((flags & MCL_CURRENT) == 0) {
1.99      chs      3538:
1.54      thorpej  3539:                /*
                   3540:                 * no more work to do!
                   3541:                 */
1.99      chs      3542:
1.54      thorpej  3543:                UVMHIST_LOG(maphist,"<- done (OK no wire)",0,0,0,0);
                   3544:                vm_map_unlock(map);
1.94      chs      3545:                return 0;
1.54      thorpej  3546:        }
                   3547:
                   3548:        /*
                   3549:         * wire case: in three passes [XXXCDC: ugly block of code here]
                   3550:         *
                   3551:         * 1: holding the write lock, count all pages mapped by non-wired
                   3552:         *    entries.  if this would cause us to go over our limit, we fail.
                   3553:         *
                   3554:         * 2: still holding the write lock, we create any anonymous maps that
                   3555:         *    need to be created.  then we increment its wiring count.
                   3556:         *
                   3557:         * 3: we downgrade to a read lock, and call uvm_fault_wire to fault
1.56      thorpej  3558:         *    in the pages for any newly wired area (wired_count == 1).
1.54      thorpej  3559:         *
                   3560:         *    downgrading to a read lock for uvm_fault_wire avoids a possible
                   3561:         *    deadlock with another thread that may have faulted on one of
                   3562:         *    the pages to be wired (it would mark the page busy, blocking
                   3563:         *    us, then in turn block on the map lock that we hold).  because
                   3564:         *    of problems in the recursive lock package, we cannot upgrade
                   3565:         *    to a write lock in vm_map_lookup.  thus, any actions that
                   3566:         *    require the write lock must be done beforehand.  because we
                   3567:         *    keep the read lock on the map, the copy-on-write status of the
                   3568:         *    entries we modify here cannot change.
                   3569:         */
                   3570:
                   3571:        for (size = 0, entry = map->header.next; entry != &map->header;
                   3572:             entry = entry->next) {
                   3573:                if (entry->protection != VM_PROT_NONE &&
1.55      thorpej  3574:                    VM_MAPENT_ISWIRED(entry) == 0) { /* not already wired? */
1.54      thorpej  3575:                        size += entry->end - entry->start;
                   3576:                }
                   3577:        }
                   3578:
                   3579:        if (atop(size) + uvmexp.wired > uvmexp.wiredmax) {
                   3580:                vm_map_unlock(map);
1.94      chs      3581:                return ENOMEM;
1.54      thorpej  3582:        }
                   3583:
                   3584:        if (limit != 0 &&
                   3585:            (size + ptoa(pmap_wired_count(vm_map_pmap(map))) > limit)) {
                   3586:                vm_map_unlock(map);
1.94      chs      3587:                return ENOMEM;
1.54      thorpej  3588:        }
                   3589:
                   3590:        /*
                   3591:         * Pass 2.
                   3592:         */
                   3593:
                   3594:        for (entry = map->header.next; entry != &map->header;
                   3595:             entry = entry->next) {
                   3596:                if (entry->protection == VM_PROT_NONE)
                   3597:                        continue;
1.55      thorpej  3598:                if (VM_MAPENT_ISWIRED(entry) == 0) { /* not already wired? */
1.99      chs      3599:
1.54      thorpej  3600:                        /*
                   3601:                         * perform actions of vm_map_lookup that need the
                   3602:                         * write lock on the map: create an anonymous map
                   3603:                         * for a copy-on-write region, or an anonymous map
                   3604:                         * for a zero-fill region.  (XXXCDC: submap case
                   3605:                         * ok?)
                   3606:                         */
1.99      chs      3607:
1.54      thorpej  3608:                        if (!UVM_ET_ISSUBMAP(entry)) {  /* not submap */
1.98      chs      3609:                                if (UVM_ET_ISNEEDSCOPY(entry) &&
1.117     chs      3610:                                    ((entry->max_protection & VM_PROT_WRITE) ||
1.54      thorpej  3611:                                     (entry->object.uvm_obj == NULL))) {
1.212     yamt     3612:                                        amap_copy(map, entry, 0, entry->start,
                   3613:                                            entry->end);
1.54      thorpej  3614:                                        /* XXXCDC: wait OK? */
                   3615:                                }
                   3616:                        }
1.55      thorpej  3617:                }
1.54      thorpej  3618:                entry->wired_count++;
                   3619:        }
                   3620:
                   3621:        /*
                   3622:         * Pass 3.
                   3623:         */
                   3624:
1.60      thorpej  3625: #ifdef DIAGNOSTIC
                   3626:        timestamp_save = map->timestamp;
                   3627: #endif
                   3628:        vm_map_busy(map);
1.249     yamt     3629:        vm_map_unlock(map);
1.54      thorpej  3630:
1.94      chs      3631:        rv = 0;
1.54      thorpej  3632:        for (entry = map->header.next; entry != &map->header;
                   3633:             entry = entry->next) {
                   3634:                if (entry->wired_count == 1) {
                   3635:                        rv = uvm_fault_wire(map, entry->start, entry->end,
1.216     drochner 3636:                            entry->max_protection, 1);
1.54      thorpej  3637:                        if (rv) {
1.99      chs      3638:
1.54      thorpej  3639:                                /*
                   3640:                                 * wiring failed.  break out of the loop.
                   3641:                                 * we'll clean up the map below, once we
                   3642:                                 * have a write lock again.
                   3643:                                 */
1.99      chs      3644:
1.54      thorpej  3645:                                break;
                   3646:                        }
                   3647:                }
                   3648:        }
                   3649:
1.99      chs      3650:        if (rv) {
                   3651:
1.54      thorpej  3652:                /*
                   3653:                 * Get back an exclusive (write) lock.
                   3654:                 */
1.99      chs      3655:
1.249     yamt     3656:                vm_map_lock(map);
1.60      thorpej  3657:                vm_map_unbusy(map);
                   3658:
                   3659: #ifdef DIAGNOSTIC
1.252     yamt     3660:                if (timestamp_save + 1 != map->timestamp)
1.60      thorpej  3661:                        panic("uvm_map_pageable_all: stale map");
                   3662: #endif
1.54      thorpej  3663:
                   3664:                /*
                   3665:                 * first drop the wiring count on all the entries
                   3666:                 * which haven't actually been wired yet.
1.67      thorpej  3667:                 *
                   3668:                 * Skip VM_PROT_NONE entries like we did above.
1.54      thorpej  3669:                 */
1.99      chs      3670:
1.54      thorpej  3671:                failed_entry = entry;
                   3672:                for (/* nothing */; entry != &map->header;
1.67      thorpej  3673:                     entry = entry->next) {
                   3674:                        if (entry->protection == VM_PROT_NONE)
                   3675:                                continue;
1.54      thorpej  3676:                        entry->wired_count--;
1.67      thorpej  3677:                }
1.54      thorpej  3678:
                   3679:                /*
                   3680:                 * now, unwire all the entries that were successfully
                   3681:                 * wired above.
1.67      thorpej  3682:                 *
                   3683:                 * Skip VM_PROT_NONE entries like we did above.
1.54      thorpej  3684:                 */
1.99      chs      3685:
1.54      thorpej  3686:                for (entry = map->header.next; entry != failed_entry;
                   3687:                     entry = entry->next) {
1.67      thorpej  3688:                        if (entry->protection == VM_PROT_NONE)
                   3689:                                continue;
1.54      thorpej  3690:                        entry->wired_count--;
1.67      thorpej  3691:                        if (VM_MAPENT_ISWIRED(entry))
1.54      thorpej  3692:                                uvm_map_entry_unwire(map, entry);
                   3693:                }
                   3694:                vm_map_unlock(map);
                   3695:                UVMHIST_LOG(maphist,"<- done (RV=%d)", rv,0,0,0);
                   3696:                return (rv);
                   3697:        }
                   3698:
1.60      thorpej  3699:        vm_map_unbusy(map);
1.54      thorpej  3700:
                   3701:        UVMHIST_LOG(maphist,"<- done (OK WIRE)",0,0,0,0);
1.94      chs      3702:        return 0;
1.54      thorpej  3703: }
                   3704:
                   3705: /*
1.61      thorpej  3706:  * uvm_map_clean: clean out a map range
1.1       mrg      3707:  *
                   3708:  * => valid flags:
1.61      thorpej  3709:  *   if (flags & PGO_CLEANIT): dirty pages are cleaned first
1.1       mrg      3710:  *   if (flags & PGO_SYNCIO): dirty pages are written synchronously
                   3711:  *   if (flags & PGO_DEACTIVATE): any cached pages are deactivated after clean
                   3712:  *   if (flags & PGO_FREE): any cached pages are freed after clean
                   3713:  * => returns an error if any part of the specified range isn't mapped
1.98      chs      3714:  * => never a need to flush amap layer since the anonymous memory has
1.61      thorpej  3715:  *     no permanent home, but may deactivate pages there
                   3716:  * => called from sys_msync() and sys_madvise()
1.1       mrg      3717:  * => caller must not write-lock map (read OK).
                   3718:  * => we may sleep while cleaning if SYNCIO [with map read-locked]
                   3719:  */
                   3720:
1.10      mrg      3721: int
1.138     enami    3722: uvm_map_clean(struct vm_map *map, vaddr_t start, vaddr_t end, int flags)
1.10      mrg      3723: {
1.99      chs      3724:        struct vm_map_entry *current, *entry;
1.61      thorpej  3725:        struct uvm_object *uobj;
                   3726:        struct vm_amap *amap;
1.298     rmind    3727:        struct vm_anon *anon, *anon_tofree;
1.61      thorpej  3728:        struct vm_page *pg;
                   3729:        vaddr_t offset;
1.24      eeh      3730:        vsize_t size;
1.188     dbj      3731:        voff_t uoff;
1.106     chs      3732:        int error, refs;
1.10      mrg      3733:        UVMHIST_FUNC("uvm_map_clean"); UVMHIST_CALLED(maphist);
1.85      chs      3734:
1.10      mrg      3735:        UVMHIST_LOG(maphist,"(map=0x%x,start=0x%x,end=0x%x,flags=0x%x)",
1.85      chs      3736:                    map, start, end, flags);
                   3737:        KASSERT((flags & (PGO_FREE|PGO_DEACTIVATE)) !=
                   3738:                (PGO_FREE|PGO_DEACTIVATE));
1.61      thorpej  3739:
1.10      mrg      3740:        vm_map_lock_read(map);
                   3741:        VM_MAP_RANGE_CHECK(map, start, end);
1.234     thorpej  3742:        if (uvm_map_lookup_entry(map, start, &entry) == false) {
1.10      mrg      3743:                vm_map_unlock_read(map);
1.94      chs      3744:                return EFAULT;
1.10      mrg      3745:        }
                   3746:
                   3747:        /*
1.186     chs      3748:         * Make a first pass to check for holes and wiring problems.
1.10      mrg      3749:         */
1.85      chs      3750:
1.10      mrg      3751:        for (current = entry; current->start < end; current = current->next) {
                   3752:                if (UVM_ET_ISSUBMAP(current)) {
                   3753:                        vm_map_unlock_read(map);
1.94      chs      3754:                        return EINVAL;
1.10      mrg      3755:                }
1.186     chs      3756:                if ((flags & PGO_FREE) != 0 && VM_MAPENT_ISWIRED(entry)) {
                   3757:                        vm_map_unlock_read(map);
                   3758:                        return EBUSY;
                   3759:                }
1.90      chs      3760:                if (end <= current->end) {
                   3761:                        break;
                   3762:                }
                   3763:                if (current->end != current->next->start) {
1.10      mrg      3764:                        vm_map_unlock_read(map);
1.94      chs      3765:                        return EFAULT;
1.10      mrg      3766:                }
                   3767:        }
                   3768:
1.94      chs      3769:        error = 0;
1.90      chs      3770:        for (current = entry; start < end; current = current->next) {
1.283     uebayasi 3771:                amap = current->aref.ar_amap;   /* upper layer */
                   3772:                uobj = current->object.uvm_obj; /* lower layer */
1.85      chs      3773:                KASSERT(start >= current->start);
1.1       mrg      3774:
1.10      mrg      3775:                /*
1.61      thorpej  3776:                 * No amap cleaning necessary if:
                   3777:                 *
                   3778:                 *      (1) There's no amap.
                   3779:                 *
                   3780:                 *      (2) We're not deactivating or freeing pages.
1.10      mrg      3781:                 */
1.85      chs      3782:
1.90      chs      3783:                if (amap == NULL || (flags & (PGO_DEACTIVATE|PGO_FREE)) == 0)
1.61      thorpej  3784:                        goto flush_object;
                   3785:
                   3786:                offset = start - current->start;
1.90      chs      3787:                size = MIN(end, current->end) - start;
1.303     rmind    3788:                anon_tofree = NULL;
                   3789:
                   3790:                amap_lock(amap);
1.90      chs      3791:                for ( ; size != 0; size -= PAGE_SIZE, offset += PAGE_SIZE) {
1.61      thorpej  3792:                        anon = amap_lookup(&current->aref, offset);
                   3793:                        if (anon == NULL)
                   3794:                                continue;
                   3795:
1.298     rmind    3796:                        KASSERT(anon->an_lock == amap->am_lock);
1.192     yamt     3797:                        pg = anon->an_page;
1.63      thorpej  3798:                        if (pg == NULL) {
                   3799:                                continue;
                   3800:                        }
1.330.2.1! martin   3801:                        if (pg->flags & PG_BUSY) {
        !          3802:                                continue;
        !          3803:                        }
1.63      thorpej  3804:
1.61      thorpej  3805:                        switch (flags & (PGO_CLEANIT|PGO_FREE|PGO_DEACTIVATE)) {
1.85      chs      3806:
1.61      thorpej  3807:                        /*
1.115     chs      3808:                         * In these first 3 cases, we just deactivate the page.
1.61      thorpej  3809:                         */
1.85      chs      3810:
1.61      thorpej  3811:                        case PGO_CLEANIT|PGO_FREE:
                   3812:                        case PGO_CLEANIT|PGO_DEACTIVATE:
                   3813:                        case PGO_DEACTIVATE:
1.68      thorpej  3814:  deactivate_it:
1.61      thorpej  3815:                                /*
1.115     chs      3816:                                 * skip the page if it's loaned or wired,
                   3817:                                 * since it shouldn't be on a paging queue
                   3818:                                 * at all in these cases.
1.61      thorpej  3819:                                 */
1.85      chs      3820:
1.248     ad       3821:                                mutex_enter(&uvm_pageqlock);
1.115     chs      3822:                                if (pg->loan_count != 0 ||
                   3823:                                    pg->wire_count != 0) {
1.248     ad       3824:                                        mutex_exit(&uvm_pageqlock);
1.61      thorpej  3825:                                        continue;
                   3826:                                }
1.85      chs      3827:                                KASSERT(pg->uanon == anon);
1.61      thorpej  3828:                                uvm_pagedeactivate(pg);
1.248     ad       3829:                                mutex_exit(&uvm_pageqlock);
1.61      thorpej  3830:                                continue;
                   3831:
                   3832:                        case PGO_FREE:
1.85      chs      3833:
1.68      thorpej  3834:                                /*
                   3835:                                 * If there are multiple references to
                   3836:                                 * the amap, just deactivate the page.
                   3837:                                 */
1.85      chs      3838:
1.68      thorpej  3839:                                if (amap_refs(amap) > 1)
                   3840:                                        goto deactivate_it;
                   3841:
1.115     chs      3842:                                /* skip the page if it's wired */
1.62      thorpej  3843:                                if (pg->wire_count != 0) {
                   3844:                                        continue;
                   3845:                                }
1.66      thorpej  3846:                                amap_unadd(&current->aref, offset);
1.61      thorpej  3847:                                refs = --anon->an_ref;
1.298     rmind    3848:                                if (refs == 0) {
                   3849:                                        anon->an_link = anon_tofree;
                   3850:                                        anon_tofree = anon;
                   3851:                                }
1.61      thorpej  3852:                                continue;
                   3853:                        }
                   3854:                }
1.303     rmind    3855:                uvm_anon_freelst(amap, anon_tofree);
1.1       mrg      3856:
1.61      thorpej  3857:  flush_object:
1.10      mrg      3858:                /*
1.33      chuck    3859:                 * flush pages if we've got a valid backing object.
1.116     chs      3860:                 * note that we must always clean object pages before
                   3861:                 * freeing them since otherwise we could reveal stale
                   3862:                 * data from files.
1.10      mrg      3863:                 */
1.1       mrg      3864:
1.188     dbj      3865:                uoff = current->offset + (start - current->start);
1.90      chs      3866:                size = MIN(end, current->end) - start;
1.61      thorpej  3867:                if (uobj != NULL) {
1.298     rmind    3868:                        mutex_enter(uobj->vmobjlock);
1.136     thorpej  3869:                        if (uobj->pgops->pgo_put != NULL)
1.188     dbj      3870:                                error = (uobj->pgops->pgo_put)(uobj, uoff,
                   3871:                                    uoff + size, flags | PGO_CLEANIT);
1.136     thorpej  3872:                        else
                   3873:                                error = 0;
1.10      mrg      3874:                }
                   3875:                start += size;
                   3876:        }
1.1       mrg      3877:        vm_map_unlock_read(map);
1.98      chs      3878:        return (error);
1.1       mrg      3879: }
                   3880:
                   3881:
                   3882: /*
                   3883:  * uvm_map_checkprot: check protection in map
                   3884:  *
                   3885:  * => must allow specified protection in a fully allocated region.
                   3886:  * => map must be read or write locked by caller.
                   3887:  */
                   3888:
1.233     thorpej  3889: bool
1.138     enami    3890: uvm_map_checkprot(struct vm_map *map, vaddr_t start, vaddr_t end,
                   3891:     vm_prot_t protection)
1.10      mrg      3892: {
1.99      chs      3893:        struct vm_map_entry *entry;
                   3894:        struct vm_map_entry *tmp_entry;
1.10      mrg      3895:
1.94      chs      3896:        if (!uvm_map_lookup_entry(map, start, &tmp_entry)) {
1.234     thorpej  3897:                return (false);
1.94      chs      3898:        }
                   3899:        entry = tmp_entry;
                   3900:        while (start < end) {
                   3901:                if (entry == &map->header) {
1.234     thorpej  3902:                        return (false);
1.94      chs      3903:                }
1.85      chs      3904:
1.10      mrg      3905:                /*
                   3906:                 * no holes allowed
                   3907:                 */
                   3908:
1.94      chs      3909:                if (start < entry->start) {
1.234     thorpej  3910:                        return (false);
1.94      chs      3911:                }
1.10      mrg      3912:
                   3913:                /*
                   3914:                 * check protection associated with entry
                   3915:                 */
1.1       mrg      3916:
1.94      chs      3917:                if ((entry->protection & protection) != protection) {
1.234     thorpej  3918:                        return (false);
1.94      chs      3919:                }
                   3920:                start = entry->end;
                   3921:                entry = entry->next;
                   3922:        }
1.234     thorpej  3923:        return (true);
1.1       mrg      3924: }
                   3925:
                   3926: /*
                   3927:  * uvmspace_alloc: allocate a vmspace structure.
                   3928:  *
                   3929:  * - structure includes vm_map and pmap
                   3930:  * - XXX: no locking on this structure
                   3931:  * - refcnt set to 1, rest must be init'd by caller
                   3932:  */
1.10      mrg      3933: struct vmspace *
1.327     martin   3934: uvmspace_alloc(vaddr_t vmin, vaddr_t vmax, bool topdown)
1.10      mrg      3935: {
                   3936:        struct vmspace *vm;
                   3937:        UVMHIST_FUNC("uvmspace_alloc"); UVMHIST_CALLED(maphist);
                   3938:
1.248     ad       3939:        vm = pool_cache_get(&uvm_vmspace_cache, PR_WAITOK);
1.327     martin   3940:        uvmspace_init(vm, NULL, vmin, vmax, topdown);
1.15      thorpej  3941:        UVMHIST_LOG(maphist,"<- done (vm=0x%x)", vm,0,0,0);
                   3942:        return (vm);
                   3943: }
                   3944:
                   3945: /*
                   3946:  * uvmspace_init: initialize a vmspace structure.
                   3947:  *
                   3948:  * - XXX: no locking on this structure
1.132     matt     3949:  * - refcnt set to 1, rest must be init'd by caller
1.15      thorpej  3950:  */
                   3951: void
1.327     martin   3952: uvmspace_init(struct vmspace *vm, struct pmap *pmap, vaddr_t vmin,
                   3953:     vaddr_t vmax, bool topdown)
1.15      thorpej  3954: {
                   3955:        UVMHIST_FUNC("uvmspace_init"); UVMHIST_CALLED(maphist);
                   3956:
1.23      perry    3957:        memset(vm, 0, sizeof(*vm));
1.199     christos 3958:        uvm_map_setup(&vm->vm_map, vmin, vmax, VM_MAP_PAGEABLE
1.327     martin   3959:            | (topdown ? VM_MAP_TOPDOWN : 0)
1.131     atatat   3960:            );
1.15      thorpej  3961:        if (pmap)
                   3962:                pmap_reference(pmap);
                   3963:        else
                   3964:                pmap = pmap_create();
                   3965:        vm->vm_map.pmap = pmap;
1.10      mrg      3966:        vm->vm_refcnt = 1;
1.15      thorpej  3967:        UVMHIST_LOG(maphist,"<- done",0,0,0,0);
1.1       mrg      3968: }
                   3969:
                   3970: /*
1.168     junyoung 3971:  * uvmspace_share: share a vmspace between two processes
1.1       mrg      3972:  *
                   3973:  * - used for vfork, threads(?)
                   3974:  */
                   3975:
1.10      mrg      3976: void
1.138     enami    3977: uvmspace_share(struct proc *p1, struct proc *p2)
1.1       mrg      3978: {
1.139     enami    3979:
1.215     yamt     3980:        uvmspace_addref(p1->p_vmspace);
1.10      mrg      3981:        p2->p_vmspace = p1->p_vmspace;
1.1       mrg      3982: }
                   3983:
1.282     rmind    3984: #if 0
                   3985:
1.1       mrg      3986: /*
                   3987:  * uvmspace_unshare: ensure that process "p" has its own, unshared, vmspace
                   3988:  *
                   3989:  * - XXX: no locking on vmspace
                   3990:  */
                   3991:
1.10      mrg      3992: void
1.138     enami    3993: uvmspace_unshare(struct lwp *l)
1.10      mrg      3994: {
1.128     thorpej  3995:        struct proc *p = l->l_proc;
1.10      mrg      3996:        struct vmspace *nvm, *ovm = p->p_vmspace;
1.85      chs      3997:
1.10      mrg      3998:        if (ovm->vm_refcnt == 1)
                   3999:                /* nothing to do: vmspace isn't shared in the first place */
                   4000:                return;
1.85      chs      4001:
1.10      mrg      4002:        /* make a new vmspace, still holding old one */
                   4003:        nvm = uvmspace_fork(ovm);
                   4004:
1.254     ad       4005:        kpreempt_disable();
1.128     thorpej  4006:        pmap_deactivate(l);             /* unbind old vmspace */
1.98      chs      4007:        p->p_vmspace = nvm;
1.128     thorpej  4008:        pmap_activate(l);               /* switch to new vmspace */
1.254     ad       4009:        kpreempt_enable();
1.13      thorpej  4010:
1.10      mrg      4011:        uvmspace_free(ovm);             /* drop reference to old vmspace */
1.1       mrg      4012: }
                   4013:
1.282     rmind    4014: #endif
                   4015:
1.317     martin   4016:
                   4017: /*
                   4018:  * uvmspace_spawn: a new process has been spawned and needs a vmspace
                   4019:  */
                   4020:
                   4021: void
1.327     martin   4022: uvmspace_spawn(struct lwp *l, vaddr_t start, vaddr_t end, bool topdown)
1.317     martin   4023: {
                   4024:        struct proc *p = l->l_proc;
                   4025:        struct vmspace *nvm;
                   4026:
                   4027: #ifdef __HAVE_CPU_VMSPACE_EXEC
                   4028:        cpu_vmspace_exec(l, start, end);
                   4029: #endif
                   4030:
1.327     martin   4031:        nvm = uvmspace_alloc(start, end, topdown);
1.317     martin   4032:        kpreempt_disable();
                   4033:        p->p_vmspace = nvm;
                   4034:        pmap_activate(l);
                   4035:        kpreempt_enable();
                   4036: }
                   4037:
1.1       mrg      4038: /*
                   4039:  * uvmspace_exec: the process wants to exec a new program
                   4040:  */
                   4041:
1.10      mrg      4042: void
1.327     martin   4043: uvmspace_exec(struct lwp *l, vaddr_t start, vaddr_t end, bool topdown)
1.1       mrg      4044: {
1.128     thorpej  4045:        struct proc *p = l->l_proc;
1.10      mrg      4046:        struct vmspace *nvm, *ovm = p->p_vmspace;
1.302     martin   4047:        struct vm_map *map;
1.1       mrg      4048:
1.317     martin   4049:        KASSERT(ovm != NULL);
1.294     matt     4050: #ifdef __HAVE_CPU_VMSPACE_EXEC
                   4051:        cpu_vmspace_exec(l, start, end);
                   4052: #endif
1.1       mrg      4053:
1.302     martin   4054:        map = &ovm->vm_map;
                   4055:        /*
1.10      mrg      4056:         * see if more than one process is using this vmspace...
                   4057:         */
1.1       mrg      4058:
1.327     martin   4059:        if (ovm->vm_refcnt == 1
                   4060:            && topdown == ((ovm->vm_map.flags & VM_MAP_TOPDOWN) != 0)) {
1.1       mrg      4061:
1.10      mrg      4062:                /*
                   4063:                 * if p is the only process using its vmspace then we can safely
                   4064:                 * recycle that vmspace for the program that is being exec'd.
1.327     martin   4065:                 * But only if TOPDOWN matches the requested value for the new
                   4066:                 * vm space!
1.10      mrg      4067:                 */
1.1       mrg      4068:
                   4069: #ifdef SYSVSHM
1.10      mrg      4070:                /*
                   4071:                 * SYSV SHM semantics require us to kill all segments on an exec
                   4072:                 */
1.99      chs      4073:
1.10      mrg      4074:                if (ovm->vm_shm)
                   4075:                        shmexit(ovm);
                   4076: #endif
1.54      thorpej  4077:
                   4078:                /*
                   4079:                 * POSIX 1003.1b -- "lock future mappings" is revoked
                   4080:                 * when a process execs another program image.
                   4081:                 */
1.99      chs      4082:
1.238     ad       4083:                map->flags &= ~VM_MAP_WIREFUTURE;
1.10      mrg      4084:
                   4085:                /*
                   4086:                 * now unmap the old program
                   4087:                 */
1.99      chs      4088:
1.120     chs      4089:                pmap_remove_all(map->pmap);
1.184     chs      4090:                uvm_unmap(map, vm_map_min(map), vm_map_max(map));
1.144     yamt     4091:                KASSERT(map->header.prev == &map->header);
                   4092:                KASSERT(map->nentries == 0);
1.93      eeh      4093:
                   4094:                /*
                   4095:                 * resize the map
                   4096:                 */
1.99      chs      4097:
1.184     chs      4098:                vm_map_setmin(map, start);
                   4099:                vm_map_setmax(map, end);
1.10      mrg      4100:        } else {
                   4101:
                   4102:                /*
                   4103:                 * p's vmspace is being shared, so we can't reuse it for p since
                   4104:                 * it is still being used for others.   allocate a new vmspace
                   4105:                 * for p
                   4106:                 */
1.99      chs      4107:
1.327     martin   4108:                nvm = uvmspace_alloc(start, end, topdown);
1.1       mrg      4109:
1.10      mrg      4110:                /*
                   4111:                 * install new vmspace and drop our ref to the old one.
                   4112:                 */
                   4113:
1.254     ad       4114:                kpreempt_disable();
1.128     thorpej  4115:                pmap_deactivate(l);
1.10      mrg      4116:                p->p_vmspace = nvm;
1.128     thorpej  4117:                pmap_activate(l);
1.254     ad       4118:                kpreempt_enable();
1.13      thorpej  4119:
1.10      mrg      4120:                uvmspace_free(ovm);
                   4121:        }
1.1       mrg      4122: }
                   4123:
                   4124: /*
1.215     yamt     4125:  * uvmspace_addref: add a referece to a vmspace.
                   4126:  */
                   4127:
                   4128: void
                   4129: uvmspace_addref(struct vmspace *vm)
                   4130: {
                   4131:        struct vm_map *map = &vm->vm_map;
                   4132:
                   4133:        KASSERT((map->flags & VM_MAP_DYING) == 0);
                   4134:
1.238     ad       4135:        mutex_enter(&map->misc_lock);
1.215     yamt     4136:        KASSERT(vm->vm_refcnt > 0);
                   4137:        vm->vm_refcnt++;
1.238     ad       4138:        mutex_exit(&map->misc_lock);
1.215     yamt     4139: }
                   4140:
                   4141: /*
1.1       mrg      4142:  * uvmspace_free: free a vmspace data structure
                   4143:  */
                   4144:
1.10      mrg      4145: void
1.138     enami    4146: uvmspace_free(struct vmspace *vm)
1.1       mrg      4147: {
1.99      chs      4148:        struct vm_map_entry *dead_entries;
1.171     pk       4149:        struct vm_map *map = &vm->vm_map;
1.172     he       4150:        int n;
                   4151:
1.10      mrg      4152:        UVMHIST_FUNC("uvmspace_free"); UVMHIST_CALLED(maphist);
1.1       mrg      4153:
1.10      mrg      4154:        UVMHIST_LOG(maphist,"(vm=0x%x) ref=%d", vm, vm->vm_refcnt,0,0);
1.238     ad       4155:        mutex_enter(&map->misc_lock);
1.172     he       4156:        n = --vm->vm_refcnt;
1.238     ad       4157:        mutex_exit(&map->misc_lock);
1.171     pk       4158:        if (n > 0)
1.120     chs      4159:                return;
1.99      chs      4160:
1.120     chs      4161:        /*
                   4162:         * at this point, there should be no other references to the map.
                   4163:         * delete all of the mappings, then destroy the pmap.
                   4164:         */
1.99      chs      4165:
1.120     chs      4166:        map->flags |= VM_MAP_DYING;
                   4167:        pmap_remove_all(map->pmap);
1.92      thorpej  4168: #ifdef SYSVSHM
1.120     chs      4169:        /* Get rid of any SYSV shared memory segments. */
                   4170:        if (vm->vm_shm != NULL)
                   4171:                shmexit(vm);
1.92      thorpej  4172: #endif
1.314     rmind    4173:
1.120     chs      4174:        if (map->nentries) {
1.184     chs      4175:                uvm_unmap_remove(map, vm_map_min(map), vm_map_max(map),
1.311     para     4176:                    &dead_entries, 0);
1.120     chs      4177:                if (dead_entries != NULL)
                   4178:                        uvm_unmap_detach(dead_entries, 0);
1.10      mrg      4179:        }
1.146     yamt     4180:        KASSERT(map->nentries == 0);
                   4181:        KASSERT(map->size == 0);
1.314     rmind    4182:
1.239     ad       4183:        mutex_destroy(&map->misc_lock);
                   4184:        rw_destroy(&map->lock);
1.255     ad       4185:        cv_destroy(&map->cv);
1.120     chs      4186:        pmap_destroy(map->pmap);
1.248     ad       4187:        pool_cache_put(&uvm_vmspace_cache, vm);
1.1       mrg      4188: }
                   4189:
1.329     christos 4190: static struct vm_map_entry *
                   4191: uvm_mapent_clone(struct vm_map *new_map, struct vm_map_entry *old_entry,
                   4192:     int flags)
                   4193: {
                   4194:        struct vm_map_entry *new_entry;
                   4195:
                   4196:        new_entry = uvm_mapent_alloc(new_map, 0);
                   4197:        /* old_entry -> new_entry */
                   4198:        uvm_mapent_copy(old_entry, new_entry);
                   4199:
                   4200:        /* new pmap has nothing wired in it */
                   4201:        new_entry->wired_count = 0;
                   4202:
                   4203:        /*
                   4204:         * gain reference to object backing the map (can't
                   4205:         * be a submap, already checked this case).
                   4206:         */
                   4207:
                   4208:        if (new_entry->aref.ar_amap)
                   4209:                uvm_map_reference_amap(new_entry, flags);
                   4210:
                   4211:        if (new_entry->object.uvm_obj &&
                   4212:            new_entry->object.uvm_obj->pgops->pgo_reference)
                   4213:                new_entry->object.uvm_obj->pgops->pgo_reference(
                   4214:                        new_entry->object.uvm_obj);
                   4215:
                   4216:        /* insert entry at end of new_map's entry list */
                   4217:        uvm_map_entry_link(new_map, new_map->header.prev,
                   4218:            new_entry);
                   4219:
                   4220:        return new_entry;
                   4221: }
                   4222:
                   4223: /*
                   4224:  * share the mapping: this means we want the old and
                   4225:  * new entries to share amaps and backing objects.
                   4226:  */
                   4227: static void
                   4228: uvm_mapent_forkshared(struct vm_map *new_map, struct vm_map *old_map,
                   4229:     struct vm_map_entry *old_entry)
                   4230: {
                   4231:        /*
                   4232:         * if the old_entry needs a new amap (due to prev fork)
                   4233:         * then we need to allocate it now so that we have
                   4234:         * something we own to share with the new_entry.   [in
                   4235:         * other words, we need to clear needs_copy]
                   4236:         */
                   4237:
                   4238:        if (UVM_ET_ISNEEDSCOPY(old_entry)) {
                   4239:                /* get our own amap, clears needs_copy */
                   4240:                amap_copy(old_map, old_entry, AMAP_COPY_NOCHUNK,
                   4241:                    0, 0);
                   4242:                /* XXXCDC: WAITOK??? */
                   4243:        }
                   4244:
                   4245:        uvm_mapent_clone(new_map, old_entry, AMAP_SHARED);
                   4246: }
                   4247:
                   4248:
                   4249: static void
                   4250: uvm_mapent_forkcopy(struct vm_map *new_map, struct vm_map *old_map,
                   4251:     struct vm_map_entry *old_entry)
                   4252: {
                   4253:        struct vm_map_entry *new_entry;
                   4254:
                   4255:        /*
                   4256:         * copy-on-write the mapping (using mmap's
                   4257:         * MAP_PRIVATE semantics)
                   4258:         *
                   4259:         * allocate new_entry, adjust reference counts.
                   4260:         * (note that new references are read-only).
                   4261:         */
                   4262:
                   4263:        new_entry = uvm_mapent_clone(new_map, old_entry, 0);
                   4264:
                   4265:        new_entry->etype |=
                   4266:            (UVM_ET_COPYONWRITE|UVM_ET_NEEDSCOPY);
                   4267:
                   4268:        /*
                   4269:         * the new entry will need an amap.  it will either
                   4270:         * need to be copied from the old entry or created
                   4271:         * from scratch (if the old entry does not have an
                   4272:         * amap).  can we defer this process until later
                   4273:         * (by setting "needs_copy") or do we need to copy
                   4274:         * the amap now?
                   4275:         *
                   4276:         * we must copy the amap now if any of the following
                   4277:         * conditions hold:
                   4278:         * 1. the old entry has an amap and that amap is
                   4279:         *    being shared.  this means that the old (parent)
                   4280:         *    process is sharing the amap with another
                   4281:         *    process.  if we do not clear needs_copy here
                   4282:         *    we will end up in a situation where both the
                   4283:         *    parent and child process are refering to the
                   4284:         *    same amap with "needs_copy" set.  if the
                   4285:         *    parent write-faults, the fault routine will
                   4286:         *    clear "needs_copy" in the parent by allocating
                   4287:         *    a new amap.   this is wrong because the
                   4288:         *    parent is supposed to be sharing the old amap
                   4289:         *    and the new amap will break that.
                   4290:         *
                   4291:         * 2. if the old entry has an amap and a non-zero
                   4292:         *    wire count then we are going to have to call
                   4293:         *    amap_cow_now to avoid page faults in the
                   4294:         *    parent process.   since amap_cow_now requires
                   4295:         *    "needs_copy" to be clear we might as well
                   4296:         *    clear it here as well.
                   4297:         *
                   4298:         */
                   4299:
                   4300:        if (old_entry->aref.ar_amap != NULL) {
                   4301:                if ((amap_flags(old_entry->aref.ar_amap) & AMAP_SHARED) != 0 ||
                   4302:                    VM_MAPENT_ISWIRED(old_entry)) {
                   4303:
                   4304:                        amap_copy(new_map, new_entry,
                   4305:                            AMAP_COPY_NOCHUNK, 0, 0);
                   4306:                        /* XXXCDC: M_WAITOK ... ok? */
                   4307:                }
                   4308:        }
                   4309:
                   4310:        /*
                   4311:         * if the parent's entry is wired down, then the
                   4312:         * parent process does not want page faults on
                   4313:         * access to that memory.  this means that we
                   4314:         * cannot do copy-on-write because we can't write
                   4315:         * protect the old entry.   in this case we
                   4316:         * resolve all copy-on-write faults now, using
                   4317:         * amap_cow_now.   note that we have already
                   4318:         * allocated any needed amap (above).
                   4319:         */
                   4320:
                   4321:        if (VM_MAPENT_ISWIRED(old_entry)) {
                   4322:
                   4323:                /*
                   4324:                 * resolve all copy-on-write faults now
                   4325:                 * (note that there is nothing to do if
                   4326:                 * the old mapping does not have an amap).
                   4327:                 */
                   4328:                if (old_entry->aref.ar_amap)
                   4329:                        amap_cow_now(new_map, new_entry);
                   4330:
                   4331:        } else {
                   4332:                /*
                   4333:                 * setup mappings to trigger copy-on-write faults
                   4334:                 * we must write-protect the parent if it has
                   4335:                 * an amap and it is not already "needs_copy"...
                   4336:                 * if it is already "needs_copy" then the parent
                   4337:                 * has already been write-protected by a previous
                   4338:                 * fork operation.
                   4339:                 */
                   4340:                if (old_entry->aref.ar_amap &&
                   4341:                    !UVM_ET_ISNEEDSCOPY(old_entry)) {
                   4342:                        if (old_entry->max_protection & VM_PROT_WRITE) {
                   4343:                                pmap_protect(old_map->pmap,
                   4344:                                    old_entry->start, old_entry->end,
                   4345:                                    old_entry->protection & ~VM_PROT_WRITE);
                   4346:                        }
                   4347:                        old_entry->etype |= UVM_ET_NEEDSCOPY;
                   4348:                }
                   4349:        }
                   4350: }
                   4351:
1.1       mrg      4352: /*
1.330     christos 4353:  * zero the mapping: the new entry will be zero initialized
                   4354:  */
                   4355: static void
                   4356: uvm_mapent_forkzero(struct vm_map *new_map, struct vm_map *old_map,
                   4357:     struct vm_map_entry *old_entry)
                   4358: {
                   4359:        struct vm_map_entry *new_entry;
                   4360:
                   4361:        new_entry = uvm_mapent_clone(new_map, old_entry, 0);
                   4362:
                   4363:        new_entry->etype |=
                   4364:            (UVM_ET_COPYONWRITE|UVM_ET_NEEDSCOPY);
                   4365:
                   4366:        if (new_entry->aref.ar_amap) {
                   4367:                uvm_map_unreference_amap(new_entry, 0);
                   4368:                new_entry->aref.ar_pageoff = 0;
                   4369:                new_entry->aref.ar_amap = NULL;
                   4370:        }
                   4371:
                   4372:        if (UVM_ET_ISOBJ(new_entry)) {
                   4373:                if (new_entry->object.uvm_obj->pgops->pgo_detach)
                   4374:                        new_entry->object.uvm_obj->pgops->pgo_detach(
                   4375:                            new_entry->object.uvm_obj);
                   4376:                new_entry->object.uvm_obj = NULL;
                   4377:                new_entry->etype &= ~UVM_ET_OBJ;
                   4378:        }
                   4379: }
                   4380:
                   4381: /*
1.1       mrg      4382:  *   F O R K   -   m a i n   e n t r y   p o i n t
                   4383:  */
                   4384: /*
                   4385:  * uvmspace_fork: fork a process' main map
                   4386:  *
                   4387:  * => create a new vmspace for child process from parent.
                   4388:  * => parent's map must not be locked.
                   4389:  */
                   4390:
1.10      mrg      4391: struct vmspace *
1.138     enami    4392: uvmspace_fork(struct vmspace *vm1)
1.10      mrg      4393: {
                   4394:        struct vmspace *vm2;
1.99      chs      4395:        struct vm_map *old_map = &vm1->vm_map;
                   4396:        struct vm_map *new_map;
                   4397:        struct vm_map_entry *old_entry;
1.10      mrg      4398:        UVMHIST_FUNC("uvmspace_fork"); UVMHIST_CALLED(maphist);
1.1       mrg      4399:
1.10      mrg      4400:        vm_map_lock(old_map);
1.1       mrg      4401:
1.327     martin   4402:        vm2 = uvmspace_alloc(vm_map_min(old_map), vm_map_max(old_map),
                   4403:            vm1->vm_map.flags & VM_MAP_TOPDOWN);
1.23      perry    4404:        memcpy(&vm2->vm_startcopy, &vm1->vm_startcopy,
1.235     christos 4405:            (char *) (vm1 + 1) - (char *) &vm1->vm_startcopy);
1.10      mrg      4406:        new_map = &vm2->vm_map;           /* XXX */
                   4407:
                   4408:        old_entry = old_map->header.next;
1.162     pooka    4409:        new_map->size = old_map->size;
1.10      mrg      4410:
                   4411:        /*
                   4412:         * go entry-by-entry
                   4413:         */
1.1       mrg      4414:
1.10      mrg      4415:        while (old_entry != &old_map->header) {
1.1       mrg      4416:
1.10      mrg      4417:                /*
                   4418:                 * first, some sanity checks on the old entry
                   4419:                 */
1.99      chs      4420:
1.94      chs      4421:                KASSERT(!UVM_ET_ISSUBMAP(old_entry));
                   4422:                KASSERT(UVM_ET_ISCOPYONWRITE(old_entry) ||
                   4423:                        !UVM_ET_ISNEEDSCOPY(old_entry));
1.1       mrg      4424:
1.10      mrg      4425:                switch (old_entry->inheritance) {
1.80      wiz      4426:                case MAP_INHERIT_NONE:
1.10      mrg      4427:                        /*
1.162     pooka    4428:                         * drop the mapping, modify size
1.10      mrg      4429:                         */
1.162     pooka    4430:                        new_map->size -= old_entry->end - old_entry->start;
1.10      mrg      4431:                        break;
                   4432:
1.80      wiz      4433:                case MAP_INHERIT_SHARE:
1.329     christos 4434:                        uvm_mapent_forkshared(new_map, old_map, old_entry);
1.10      mrg      4435:                        break;
                   4436:
1.80      wiz      4437:                case MAP_INHERIT_COPY:
1.329     christos 4438:                        uvm_mapent_forkcopy(new_map, old_map, old_entry);
                   4439:                        break;
1.10      mrg      4440:
1.330     christos 4441:                case MAP_INHERIT_ZERO:
                   4442:                        uvm_mapent_forkzero(new_map, old_map, old_entry);
                   4443:                        break;
1.329     christos 4444:                default:
                   4445:                        KASSERT(0);
1.10      mrg      4446:                        break;
1.329     christos 4447:                }
1.10      mrg      4448:                old_entry = old_entry->next;
1.1       mrg      4449:        }
                   4450:
1.268     ad       4451:        pmap_update(old_map->pmap);
1.98      chs      4452:        vm_map_unlock(old_map);
1.1       mrg      4453:
                   4454: #ifdef SYSVSHM
1.10      mrg      4455:        if (vm1->vm_shm)
                   4456:                shmfork(vm1, vm2);
1.39      thorpej  4457: #endif
                   4458:
                   4459: #ifdef PMAP_FORK
                   4460:        pmap_fork(vm1->vm_map.pmap, vm2->vm_map.pmap);
1.1       mrg      4461: #endif
                   4462:
1.10      mrg      4463:        UVMHIST_LOG(maphist,"<- done",0,0,0,0);
1.139     enami    4464:        return (vm2);
1.1       mrg      4465: }
                   4466:
                   4467:
1.174     yamt     4468: /*
1.194     yamt     4469:  * uvm_mapent_trymerge: try to merge an entry with its neighbors.
                   4470:  *
                   4471:  * => called with map locked.
                   4472:  * => return non zero if successfully merged.
                   4473:  */
                   4474:
                   4475: int
                   4476: uvm_mapent_trymerge(struct vm_map *map, struct vm_map_entry *entry, int flags)
                   4477: {
                   4478:        struct uvm_object *uobj;
                   4479:        struct vm_map_entry *next;
                   4480:        struct vm_map_entry *prev;
1.195     yamt     4481:        vsize_t size;
1.194     yamt     4482:        int merged = 0;
1.233     thorpej  4483:        bool copying;
1.194     yamt     4484:        int newetype;
                   4485:
                   4486:        if (entry->aref.ar_amap != NULL) {
                   4487:                return 0;
                   4488:        }
                   4489:        if ((entry->flags & UVM_MAP_NOMERGE) != 0) {
                   4490:                return 0;
                   4491:        }
                   4492:
                   4493:        uobj = entry->object.uvm_obj;
1.195     yamt     4494:        size = entry->end - entry->start;
1.194     yamt     4495:        copying = (flags & UVM_MERGE_COPYING) != 0;
                   4496:        newetype = copying ? (entry->etype & ~UVM_ET_NEEDSCOPY) : entry->etype;
                   4497:
                   4498:        next = entry->next;
                   4499:        if (next != &map->header &&
                   4500:            next->start == entry->end &&
                   4501:            ((copying && next->aref.ar_amap != NULL &&
                   4502:            amap_refs(next->aref.ar_amap) == 1) ||
                   4503:            (!copying && next->aref.ar_amap == NULL)) &&
                   4504:            UVM_ET_ISCOMPATIBLE(next, newetype,
                   4505:            uobj, entry->flags, entry->protection,
                   4506:            entry->max_protection, entry->inheritance, entry->advice,
1.195     yamt     4507:            entry->wired_count) &&
                   4508:            (uobj == NULL || entry->offset + size == next->offset)) {
1.194     yamt     4509:                int error;
                   4510:
                   4511:                if (copying) {
1.195     yamt     4512:                        error = amap_extend(next, size,
1.194     yamt     4513:                            AMAP_EXTEND_NOWAIT|AMAP_EXTEND_BACKWARDS);
                   4514:                } else {
                   4515:                        error = 0;
                   4516:                }
                   4517:                if (error == 0) {
1.197     yamt     4518:                        if (uobj) {
                   4519:                                if (uobj->pgops->pgo_detach) {
                   4520:                                        uobj->pgops->pgo_detach(uobj);
                   4521:                                }
1.194     yamt     4522:                        }
                   4523:
                   4524:                        entry->end = next->end;
1.221     yamt     4525:                        clear_hints(map, next);
1.194     yamt     4526:                        uvm_map_entry_unlink(map, next);
                   4527:                        if (copying) {
                   4528:                                entry->aref = next->aref;
                   4529:                                entry->etype &= ~UVM_ET_NEEDSCOPY;
                   4530:                        }
1.222     yamt     4531:                        uvm_map_check(map, "trymerge forwardmerge");
1.311     para     4532:                        uvm_mapent_free(next);
1.194     yamt     4533:                        merged++;
                   4534:                }
                   4535:        }
                   4536:
                   4537:        prev = entry->prev;
                   4538:        if (prev != &map->header &&
                   4539:            prev->end == entry->start &&
                   4540:            ((copying && !merged && prev->aref.ar_amap != NULL &&
                   4541:            amap_refs(prev->aref.ar_amap) == 1) ||
                   4542:            (!copying && prev->aref.ar_amap == NULL)) &&
                   4543:            UVM_ET_ISCOMPATIBLE(prev, newetype,
                   4544:            uobj, entry->flags, entry->protection,
                   4545:            entry->max_protection, entry->inheritance, entry->advice,
1.195     yamt     4546:            entry->wired_count) &&
1.196     yamt     4547:            (uobj == NULL ||
                   4548:            prev->offset + prev->end - prev->start == entry->offset)) {
1.194     yamt     4549:                int error;
                   4550:
                   4551:                if (copying) {
1.195     yamt     4552:                        error = amap_extend(prev, size,
1.194     yamt     4553:                            AMAP_EXTEND_NOWAIT|AMAP_EXTEND_FORWARDS);
                   4554:                } else {
                   4555:                        error = 0;
                   4556:                }
                   4557:                if (error == 0) {
1.197     yamt     4558:                        if (uobj) {
                   4559:                                if (uobj->pgops->pgo_detach) {
                   4560:                                        uobj->pgops->pgo_detach(uobj);
                   4561:                                }
                   4562:                                entry->offset = prev->offset;
1.194     yamt     4563:                        }
                   4564:
                   4565:                        entry->start = prev->start;
1.221     yamt     4566:                        clear_hints(map, prev);
1.194     yamt     4567:                        uvm_map_entry_unlink(map, prev);
                   4568:                        if (copying) {
                   4569:                                entry->aref = prev->aref;
                   4570:                                entry->etype &= ~UVM_ET_NEEDSCOPY;
                   4571:                        }
1.222     yamt     4572:                        uvm_map_check(map, "trymerge backmerge");
1.311     para     4573:                        uvm_mapent_free(prev);
1.194     yamt     4574:                        merged++;
                   4575:                }
                   4576:        }
                   4577:
                   4578:        return merged;
                   4579: }
                   4580:
1.211     yamt     4581: /*
                   4582:  * uvm_map_setup: init map
                   4583:  *
                   4584:  * => map must not be in service yet.
                   4585:  */
                   4586:
                   4587: void
                   4588: uvm_map_setup(struct vm_map *map, vaddr_t vmin, vaddr_t vmax, int flags)
                   4589: {
                   4590:
1.263     matt     4591:        rb_tree_init(&map->rb_tree, &uvm_map_tree_ops);
1.211     yamt     4592:        map->header.next = map->header.prev = &map->header;
                   4593:        map->nentries = 0;
                   4594:        map->size = 0;
                   4595:        map->ref_count = 1;
                   4596:        vm_map_setmin(map, vmin);
                   4597:        vm_map_setmax(map, vmax);
                   4598:        map->flags = flags;
                   4599:        map->first_free = &map->header;
                   4600:        map->hint = &map->header;
                   4601:        map->timestamp = 0;
1.238     ad       4602:        map->busy = NULL;
                   4603:
1.240     ad       4604:        rw_init(&map->lock);
1.238     ad       4605:        cv_init(&map->cv, "vm_map");
1.314     rmind    4606:        mutex_init(&map->misc_lock, MUTEX_DRIVER, IPL_NONE);
1.211     yamt     4607: }
                   4608:
                   4609: /*
                   4610:  *   U N M A P   -   m a i n   e n t r y   p o i n t
                   4611:  */
                   4612:
                   4613: /*
                   4614:  * uvm_unmap1: remove mappings from a vm_map (from "start" up to "stop")
                   4615:  *
                   4616:  * => caller must check alignment and size
                   4617:  * => map must be unlocked (we will lock it)
                   4618:  * => flags is UVM_FLAG_QUANTUM or 0.
                   4619:  */
                   4620:
                   4621: void
                   4622: uvm_unmap1(struct vm_map *map, vaddr_t start, vaddr_t end, int flags)
                   4623: {
                   4624:        struct vm_map_entry *dead_entries;
                   4625:        UVMHIST_FUNC("uvm_unmap"); UVMHIST_CALLED(maphist);
                   4626:
                   4627:        UVMHIST_LOG(maphist, "  (map=0x%x, start=0x%x, end=0x%x)",
                   4628:            map, start, end, 0);
1.246     xtraeme  4629:        if (map == kernel_map) {
1.244     yamt     4630:                LOCKDEBUG_MEM_CHECK((void *)start, end - start);
1.246     xtraeme  4631:        }
1.211     yamt     4632:        /*
                   4633:         * work now done by helper functions.   wipe the pmap's and then
                   4634:         * detach from the dead entries...
                   4635:         */
                   4636:        vm_map_lock(map);
1.311     para     4637:        uvm_unmap_remove(map, start, end, &dead_entries, flags);
1.211     yamt     4638:        vm_map_unlock(map);
                   4639:
                   4640:        if (dead_entries != NULL)
                   4641:                uvm_unmap_detach(dead_entries, 0);
                   4642:
                   4643:        UVMHIST_LOG(maphist, "<- done", 0,0,0,0);
                   4644: }
                   4645:
                   4646:
                   4647: /*
                   4648:  * uvm_map_reference: add reference to a map
                   4649:  *
1.238     ad       4650:  * => map need not be locked (we use misc_lock).
1.211     yamt     4651:  */
                   4652:
                   4653: void
                   4654: uvm_map_reference(struct vm_map *map)
                   4655: {
1.238     ad       4656:        mutex_enter(&map->misc_lock);
1.211     yamt     4657:        map->ref_count++;
1.238     ad       4658:        mutex_exit(&map->misc_lock);
1.211     yamt     4659: }
                   4660:
1.233     thorpej  4661: bool
1.226     yamt     4662: vm_map_starved_p(struct vm_map *map)
                   4663: {
                   4664:
                   4665:        if ((map->flags & VM_MAP_WANTVA) != 0) {
1.234     thorpej  4666:                return true;
1.226     yamt     4667:        }
                   4668:        /* XXX */
                   4669:        if ((vm_map_max(map) - vm_map_min(map)) / 16 * 15 < map->size) {
1.234     thorpej  4670:                return true;
1.226     yamt     4671:        }
1.234     thorpej  4672:        return false;
1.226     yamt     4673: }
1.247     yamt     4674:
1.298     rmind    4675: void
                   4676: uvm_map_lock_entry(struct vm_map_entry *entry)
                   4677: {
                   4678:
1.299     rmind    4679:        if (entry->aref.ar_amap != NULL) {
                   4680:                amap_lock(entry->aref.ar_amap);
                   4681:        }
1.298     rmind    4682:        if (UVM_ET_ISOBJ(entry)) {
                   4683:                mutex_enter(entry->object.uvm_obj->vmobjlock);
                   4684:        }
                   4685: }
                   4686:
                   4687: void
                   4688: uvm_map_unlock_entry(struct vm_map_entry *entry)
                   4689: {
                   4690:
1.299     rmind    4691:        if (UVM_ET_ISOBJ(entry)) {
                   4692:                mutex_exit(entry->object.uvm_obj->vmobjlock);
                   4693:        }
1.298     rmind    4694:        if (entry->aref.ar_amap != NULL) {
                   4695:                amap_unlock(entry->aref.ar_amap);
                   4696:        }
                   4697: }
                   4698:
1.270     pooka    4699: #if defined(DDB) || defined(DEBUGPRINT)
1.280     thorpej  4700:
                   4701: /*
                   4702:  * uvm_map_printit: actually prints the map
                   4703:  */
                   4704:
                   4705: void
                   4706: uvm_map_printit(struct vm_map *map, bool full,
                   4707:     void (*pr)(const char *, ...))
                   4708: {
                   4709:        struct vm_map_entry *entry;
                   4710:
                   4711:        (*pr)("MAP %p: [0x%lx->0x%lx]\n", map, vm_map_min(map),
                   4712:            vm_map_max(map));
                   4713:        (*pr)("\t#ent=%d, sz=%d, ref=%d, version=%d, flags=0x%x\n",
                   4714:            map->nentries, map->size, map->ref_count, map->timestamp,
                   4715:            map->flags);
                   4716:        (*pr)("\tpmap=%p(resident=%ld, wired=%ld)\n", map->pmap,
                   4717:            pmap_resident_count(map->pmap), pmap_wired_count(map->pmap));
                   4718:        if (!full)
                   4719:                return;
                   4720:        for (entry = map->header.next; entry != &map->header;
                   4721:            entry = entry->next) {
                   4722:                (*pr)(" - %p: 0x%lx->0x%lx: obj=%p/0x%llx, amap=%p/%d\n",
                   4723:                    entry, entry->start, entry->end, entry->object.uvm_obj,
                   4724:                    (long long)entry->offset, entry->aref.ar_amap,
                   4725:                    entry->aref.ar_pageoff);
                   4726:                (*pr)(
                   4727:                    "\tsubmap=%c, cow=%c, nc=%c, prot(max)=%d/%d, inh=%d, "
                   4728:                    "wc=%d, adv=%d\n",
                   4729:                    (entry->etype & UVM_ET_SUBMAP) ? 'T' : 'F',
                   4730:                    (entry->etype & UVM_ET_COPYONWRITE) ? 'T' : 'F',
                   4731:                    (entry->etype & UVM_ET_NEEDSCOPY) ? 'T' : 'F',
                   4732:                    entry->protection, entry->max_protection,
                   4733:                    entry->inheritance, entry->wired_count, entry->advice);
                   4734:        }
                   4735: }
                   4736:
1.247     yamt     4737: void
                   4738: uvm_whatis(uintptr_t addr, void (*pr)(const char *, ...))
                   4739: {
                   4740:        struct vm_map *map;
                   4741:
                   4742:        for (map = kernel_map;;) {
                   4743:                struct vm_map_entry *entry;
                   4744:
                   4745:                if (!uvm_map_lookup_entry_bytree(map, (vaddr_t)addr, &entry)) {
                   4746:                        break;
                   4747:                }
                   4748:                (*pr)("%p is %p+%zu from VMMAP %p\n",
                   4749:                    (void *)addr, (void *)entry->start,
                   4750:                    (size_t)(addr - (uintptr_t)entry->start), map);
                   4751:                if (!UVM_ET_ISSUBMAP(entry)) {
                   4752:                        break;
                   4753:                }
                   4754:                map = entry->object.sub_map;
                   4755:        }
                   4756: }
1.280     thorpej  4757:
                   4758: #endif /* DDB || DEBUGPRINT */
1.288     drochner 4759:
                   4760: #ifndef __USER_VA0_IS_SAFE
                   4761: static int
1.290     drochner 4762: sysctl_user_va0_disable(SYSCTLFN_ARGS)
1.288     drochner 4763: {
                   4764:        struct sysctlnode node;
                   4765:        int t, error;
                   4766:
                   4767:        node = *rnode;
                   4768:        node.sysctl_data = &t;
1.290     drochner 4769:        t = user_va0_disable;
1.288     drochner 4770:        error = sysctl_lookup(SYSCTLFN_CALL(&node));
                   4771:        if (error || newp == NULL)
                   4772:                return (error);
                   4773:
1.290     drochner 4774:        if (!t && user_va0_disable &&
1.316     elad     4775:            kauth_authorize_system(l->l_cred, KAUTH_SYSTEM_MAP_VA_ZERO, 0,
                   4776:            NULL, NULL, NULL))
1.288     drochner 4777:                return EPERM;
                   4778:
1.290     drochner 4779:        user_va0_disable = !!t;
1.288     drochner 4780:        return 0;
                   4781: }
                   4782:
                   4783: SYSCTL_SETUP(sysctl_uvmmap_setup, "sysctl uvmmap setup")
                   4784: {
                   4785:
                   4786:         sysctl_createv(clog, 0, NULL, NULL,
                   4787:                        CTLFLAG_PERMANENT|CTLFLAG_READWRITE,
1.289     drochner 4788:                        CTLTYPE_INT, "user_va0_disable",
1.288     drochner 4789:                        SYSCTL_DESCR("Disable VA 0"),
1.290     drochner 4790:                        sysctl_user_va0_disable, 0, &user_va0_disable, 0,
1.288     drochner 4791:                        CTL_VM, CTL_CREATE, CTL_EOL);
                   4792: }
                   4793: #endif

CVSweb <webmaster@jp.NetBSD.org>