Annotation of src/sys/uvm/uvm_map.c, Revision 1.239
1.239 ! ad 1: /* $NetBSD: uvm_map.c,v 1.238 2007/07/21 19:21:54 ad Exp $ */
1.1 mrg 2:
1.98 chs 3: /*
1.1 mrg 4: * Copyright (c) 1997 Charles D. Cranor and Washington University.
1.98 chs 5: * Copyright (c) 1991, 1993, The Regents of the University of California.
1.1 mrg 6: *
7: * All rights reserved.
8: *
9: * This code is derived from software contributed to Berkeley by
10: * The Mach Operating System project at Carnegie-Mellon University.
11: *
12: * Redistribution and use in source and binary forms, with or without
13: * modification, are permitted provided that the following conditions
14: * are met:
15: * 1. Redistributions of source code must retain the above copyright
16: * notice, this list of conditions and the following disclaimer.
17: * 2. Redistributions in binary form must reproduce the above copyright
18: * notice, this list of conditions and the following disclaimer in the
19: * documentation and/or other materials provided with the distribution.
20: * 3. All advertising materials mentioning features or use of this software
21: * must display the following acknowledgement:
22: * This product includes software developed by Charles D. Cranor,
1.98 chs 23: * Washington University, the University of California, Berkeley and
1.1 mrg 24: * its contributors.
25: * 4. Neither the name of the University nor the names of its contributors
26: * may be used to endorse or promote products derived from this software
27: * without specific prior written permission.
28: *
29: * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
30: * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
31: * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
32: * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
33: * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
34: * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
35: * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
36: * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
37: * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
38: * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
39: * SUCH DAMAGE.
40: *
41: * @(#)vm_map.c 8.3 (Berkeley) 1/12/94
1.3 mrg 42: * from: Id: uvm_map.c,v 1.1.2.27 1998/02/07 01:16:54 chs Exp
1.1 mrg 43: *
44: *
45: * Copyright (c) 1987, 1990 Carnegie-Mellon University.
46: * All rights reserved.
1.98 chs 47: *
1.1 mrg 48: * Permission to use, copy, modify and distribute this software and
49: * its documentation is hereby granted, provided that both the copyright
50: * notice and this permission notice appear in all copies of the
51: * software, derivative works or modified versions, and any portions
52: * thereof, and that both notices appear in supporting documentation.
1.98 chs 53: *
54: * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
55: * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND
1.1 mrg 56: * FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
1.98 chs 57: *
1.1 mrg 58: * Carnegie Mellon requests users of this software to return to
59: *
60: * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU
61: * School of Computer Science
62: * Carnegie Mellon University
63: * Pittsburgh PA 15213-3890
64: *
65: * any improvements or extensions that they make and grant Carnegie the
66: * rights to redistribute these changes.
67: */
68:
1.114 lukem 69: /*
70: * uvm_map.c: uvm map operations
71: */
72:
73: #include <sys/cdefs.h>
1.239 ! ad 74: __KERNEL_RCSID(0, "$NetBSD: uvm_map.c,v 1.238 2007/07/21 19:21:54 ad Exp $");
1.114 lukem 75:
1.21 jonathan 76: #include "opt_ddb.h"
1.6 mrg 77: #include "opt_uvmhist.h"
1.169 petrov 78: #include "opt_uvm.h"
1.31 tron 79: #include "opt_sysv.h"
1.1 mrg 80:
81: #include <sys/param.h>
82: #include <sys/systm.h>
83: #include <sys/mman.h>
84: #include <sys/proc.h>
85: #include <sys/malloc.h>
1.25 thorpej 86: #include <sys/pool.h>
1.104 chs 87: #include <sys/kernel.h>
1.112 thorpej 88: #include <sys/mount.h>
1.109 thorpej 89: #include <sys/vnode.h>
1.1 mrg 90:
91: #ifdef SYSVSHM
92: #include <sys/shm.h>
93: #endif
94:
95: #include <uvm/uvm.h>
1.151 chs 96: #undef RB_AUGMENT
97: #define RB_AUGMENT(x) uvm_rb_augment(x)
1.21 jonathan 98:
99: #ifdef DDB
100: #include <uvm/uvm_ddb.h>
101: #endif
102:
1.207 yamt 103: #if defined(UVMMAP_NOCOUNTERS)
104:
105: #define UVMMAP_EVCNT_DEFINE(name) /* nothing */
106: #define UVMMAP_EVCNT_INCR(ev) /* nothing */
107: #define UVMMAP_EVCNT_DECR(ev) /* nothing */
108:
109: #else /* defined(UVMMAP_NOCOUNTERS) */
110:
1.228 yamt 111: #include <sys/evcnt.h>
1.207 yamt 112: #define UVMMAP_EVCNT_DEFINE(name) \
113: struct evcnt uvmmap_evcnt_##name = EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, \
114: "uvmmap", #name); \
115: EVCNT_ATTACH_STATIC(uvmmap_evcnt_##name);
116: #define UVMMAP_EVCNT_INCR(ev) uvmmap_evcnt_##ev.ev_count++
117: #define UVMMAP_EVCNT_DECR(ev) uvmmap_evcnt_##ev.ev_count--
118:
119: #endif /* defined(UVMMAP_NOCOUNTERS) */
120:
121: UVMMAP_EVCNT_DEFINE(ubackmerge)
122: UVMMAP_EVCNT_DEFINE(uforwmerge)
123: UVMMAP_EVCNT_DEFINE(ubimerge)
124: UVMMAP_EVCNT_DEFINE(unomerge)
125: UVMMAP_EVCNT_DEFINE(kbackmerge)
126: UVMMAP_EVCNT_DEFINE(kforwmerge)
127: UVMMAP_EVCNT_DEFINE(kbimerge)
128: UVMMAP_EVCNT_DEFINE(knomerge)
129: UVMMAP_EVCNT_DEFINE(map_call)
130: UVMMAP_EVCNT_DEFINE(mlk_call)
131: UVMMAP_EVCNT_DEFINE(mlk_hint)
1.169 petrov 132:
1.208 yamt 133: UVMMAP_EVCNT_DEFINE(uke_alloc)
134: UVMMAP_EVCNT_DEFINE(uke_free)
135: UVMMAP_EVCNT_DEFINE(ukh_alloc)
136: UVMMAP_EVCNT_DEFINE(ukh_free)
137:
1.87 enami 138: const char vmmapbsy[] = "vmmapbsy";
1.1 mrg 139:
140: /*
1.25 thorpej 141: * pool for vmspace structures.
142: */
143:
1.166 simonb 144: POOL_INIT(uvm_vmspace_pool, sizeof(struct vmspace), 0, 0, 0, "vmsppl",
1.236 ad 145: &pool_allocator_nointr, IPL_NONE);
1.25 thorpej 146:
1.26 thorpej 147: /*
148: * pool for dynamically-allocated map entries.
149: */
150:
1.166 simonb 151: POOL_INIT(uvm_map_entry_pool, sizeof(struct vm_map_entry), 0, 0, 0, "vmmpepl",
1.236 ad 152: &pool_allocator_nointr, IPL_NONE);
1.130 thorpej 153:
154: MALLOC_DEFINE(M_VMMAP, "VM map", "VM map structures");
155: MALLOC_DEFINE(M_VMPMAP, "VM pmap", "VM pmap");
1.25 thorpej 156:
1.40 thorpej 157: #ifdef PMAP_GROWKERNEL
158: /*
159: * This global represents the end of the kernel virtual address
160: * space. If we want to exceed this, we must grow the kernel
161: * virtual address space dynamically.
162: *
163: * Note, this variable is locked by kernel_map's lock.
164: */
165: vaddr_t uvm_maxkaddr;
166: #endif
167:
1.25 thorpej 168: /*
1.1 mrg 169: * macros
170: */
171:
172: /*
1.174 yamt 173: * VM_MAP_USE_KMAPENT: determine if uvm_kmapent_alloc/free is used
174: * for the vm_map.
175: */
176: extern struct vm_map *pager_map; /* XXX */
1.219 yamt 177: #define VM_MAP_USE_KMAPENT_FLAGS(flags) \
178: (((flags) & VM_MAP_INTRSAFE) != 0)
1.174 yamt 179: #define VM_MAP_USE_KMAPENT(map) \
1.219 yamt 180: (VM_MAP_USE_KMAPENT_FLAGS((map)->flags) || (map) == kernel_map)
1.174 yamt 181:
182: /*
1.194 yamt 183: * UVM_ET_ISCOMPATIBLE: check some requirements for map entry merging
184: */
185:
186: #define UVM_ET_ISCOMPATIBLE(ent, type, uobj, meflags, \
187: prot, maxprot, inh, adv, wire) \
188: ((ent)->etype == (type) && \
189: (((ent)->flags ^ (meflags)) & (UVM_MAP_NOMERGE | UVM_MAP_QUANTUM)) \
190: == 0 && \
191: (ent)->object.uvm_obj == (uobj) && \
192: (ent)->protection == (prot) && \
193: (ent)->max_protection == (maxprot) && \
194: (ent)->inheritance == (inh) && \
195: (ent)->advice == (adv) && \
196: (ent)->wired_count == (wire))
197:
198: /*
1.1 mrg 199: * uvm_map_entry_link: insert entry into a map
200: *
201: * => map must be locked
202: */
1.10 mrg 203: #define uvm_map_entry_link(map, after_where, entry) do { \
1.218 yamt 204: uvm_mapent_check(entry); \
1.10 mrg 205: (map)->nentries++; \
206: (entry)->prev = (after_where); \
207: (entry)->next = (after_where)->next; \
208: (entry)->prev->next = (entry); \
209: (entry)->next->prev = (entry); \
1.144 yamt 210: uvm_rb_insert((map), (entry)); \
1.124 perry 211: } while (/*CONSTCOND*/ 0)
1.10 mrg 212:
1.1 mrg 213: /*
214: * uvm_map_entry_unlink: remove entry from a map
215: *
216: * => map must be locked
217: */
1.10 mrg 218: #define uvm_map_entry_unlink(map, entry) do { \
1.221 yamt 219: KASSERT((entry) != (map)->first_free); \
220: KASSERT((entry) != (map)->hint); \
1.218 yamt 221: uvm_mapent_check(entry); \
1.10 mrg 222: (map)->nentries--; \
223: (entry)->next->prev = (entry)->prev; \
224: (entry)->prev->next = (entry)->next; \
1.144 yamt 225: uvm_rb_remove((map), (entry)); \
1.124 perry 226: } while (/*CONSTCOND*/ 0)
1.1 mrg 227:
228: /*
229: * SAVE_HINT: saves the specified entry as the hint for future lookups.
230: *
231: * => map need not be locked (protected by hint_lock).
232: */
1.82 thorpej 233: #define SAVE_HINT(map,check,value) do { \
1.238 ad 234: mutex_enter(&(map)->hint_lock); \
1.82 thorpej 235: if ((map)->hint == (check)) \
236: (map)->hint = (value); \
1.238 ad 237: mutex_exit(&(map)->hint_lock); \
1.124 perry 238: } while (/*CONSTCOND*/ 0)
1.1 mrg 239:
240: /*
1.221 yamt 241: * clear_hints: ensure that hints don't point to the entry.
242: *
243: * => map must be write-locked.
244: */
245: static void
246: clear_hints(struct vm_map *map, struct vm_map_entry *ent)
247: {
248:
249: SAVE_HINT(map, ent, ent->prev);
250: if (map->first_free == ent) {
251: map->first_free = ent->prev;
252: }
253: }
254:
255: /*
1.1 mrg 256: * VM_MAP_RANGE_CHECK: check and correct range
257: *
258: * => map must at least be read locked
259: */
260:
1.10 mrg 261: #define VM_MAP_RANGE_CHECK(map, start, end) do { \
1.139 enami 262: if (start < vm_map_min(map)) \
263: start = vm_map_min(map); \
264: if (end > vm_map_max(map)) \
265: end = vm_map_max(map); \
266: if (start > end) \
267: start = end; \
1.124 perry 268: } while (/*CONSTCOND*/ 0)
1.1 mrg 269:
270: /*
271: * local prototypes
272: */
273:
1.138 enami 274: static struct vm_map_entry *
275: uvm_mapent_alloc(struct vm_map *, int);
1.174 yamt 276: static struct vm_map_entry *
277: uvm_mapent_alloc_split(struct vm_map *,
278: const struct vm_map_entry *, int,
279: struct uvm_mapent_reservation *);
1.138 enami 280: static void uvm_mapent_copy(struct vm_map_entry *, struct vm_map_entry *);
281: static void uvm_mapent_free(struct vm_map_entry *);
1.218 yamt 282: #if defined(DEBUG)
283: static void _uvm_mapent_check(const struct vm_map_entry *, const char *,
284: int);
285: #define uvm_mapent_check(map) _uvm_mapent_check(map, __FILE__, __LINE__)
286: #else /* defined(DEBUG) */
287: #define uvm_mapent_check(e) /* nothing */
288: #endif /* defined(DEBUG) */
1.174 yamt 289: static struct vm_map_entry *
290: uvm_kmapent_alloc(struct vm_map *, int);
291: static void uvm_kmapent_free(struct vm_map_entry *);
1.219 yamt 292: static vsize_t uvm_kmapent_overhead(vsize_t);
293:
1.138 enami 294: static void uvm_map_entry_unwire(struct vm_map *, struct vm_map_entry *);
295: static void uvm_map_reference_amap(struct vm_map_entry *, int);
1.140 enami 296: static int uvm_map_space_avail(vaddr_t *, vsize_t, voff_t, vsize_t, int,
297: struct vm_map_entry *);
1.138 enami 298: static void uvm_map_unreference_amap(struct vm_map_entry *, int);
1.1 mrg 299:
1.222 yamt 300: int _uvm_map_sanity(struct vm_map *);
301: int _uvm_tree_sanity(struct vm_map *);
1.144 yamt 302: static vsize_t uvm_rb_subtree_space(const struct vm_map_entry *);
303:
1.206 perry 304: static inline int
1.144 yamt 305: uvm_compare(const struct vm_map_entry *a, const struct vm_map_entry *b)
306: {
307:
308: if (a->start < b->start)
309: return (-1);
310: else if (a->start > b->start)
311: return (1);
1.164 junyoung 312:
1.144 yamt 313: return (0);
314: }
315:
1.206 perry 316: static inline void
1.144 yamt 317: uvm_rb_augment(struct vm_map_entry *entry)
318: {
319:
320: entry->space = uvm_rb_subtree_space(entry);
321: }
322:
323: RB_PROTOTYPE(uvm_tree, vm_map_entry, rb_entry, uvm_compare);
324:
325: RB_GENERATE(uvm_tree, vm_map_entry, rb_entry, uvm_compare);
326:
1.206 perry 327: static inline vsize_t
1.232 yamt 328: uvm_rb_space(const struct vm_map *map, const struct vm_map_entry *entry)
1.144 yamt 329: {
330: /* XXX map is not used */
331:
332: KASSERT(entry->next != NULL);
333: return entry->next->start - entry->end;
334: }
335:
336: static vsize_t
337: uvm_rb_subtree_space(const struct vm_map_entry *entry)
338: {
339: vaddr_t space, tmp;
340:
341: space = entry->ownspace;
342: if (RB_LEFT(entry, rb_entry)) {
343: tmp = RB_LEFT(entry, rb_entry)->space;
344: if (tmp > space)
345: space = tmp;
346: }
347:
348: if (RB_RIGHT(entry, rb_entry)) {
349: tmp = RB_RIGHT(entry, rb_entry)->space;
350: if (tmp > space)
351: space = tmp;
352: }
353:
354: return (space);
355: }
356:
1.206 perry 357: static inline void
1.144 yamt 358: uvm_rb_fixup(struct vm_map *map, struct vm_map_entry *entry)
359: {
360: /* We need to traverse to the very top */
361: do {
362: entry->ownspace = uvm_rb_space(map, entry);
363: entry->space = uvm_rb_subtree_space(entry);
364: } while ((entry = RB_PARENT(entry, rb_entry)) != NULL);
365: }
366:
1.203 thorpej 367: static void
1.144 yamt 368: uvm_rb_insert(struct vm_map *map, struct vm_map_entry *entry)
369: {
370: vaddr_t space = uvm_rb_space(map, entry);
371: struct vm_map_entry *tmp;
372:
373: entry->ownspace = entry->space = space;
374: tmp = RB_INSERT(uvm_tree, &(map)->rbhead, entry);
375: #ifdef DIAGNOSTIC
376: if (tmp != NULL)
377: panic("uvm_rb_insert: duplicate entry?");
378: #endif
379: uvm_rb_fixup(map, entry);
380: if (entry->prev != &map->header)
381: uvm_rb_fixup(map, entry->prev);
382: }
383:
1.203 thorpej 384: static void
1.144 yamt 385: uvm_rb_remove(struct vm_map *map, struct vm_map_entry *entry)
386: {
387: struct vm_map_entry *parent;
388:
389: parent = RB_PARENT(entry, rb_entry);
390: RB_REMOVE(uvm_tree, &(map)->rbhead, entry);
391: if (entry->prev != &map->header)
392: uvm_rb_fixup(map, entry->prev);
393: if (parent)
394: uvm_rb_fixup(map, parent);
395: }
396:
1.222 yamt 397: #if defined(DEBUG)
398: int uvm_debug_check_map = 0;
1.159 yamt 399: int uvm_debug_check_rbtree = 0;
1.222 yamt 400: #define uvm_map_check(map, name) \
401: _uvm_map_check((map), (name), __FILE__, __LINE__)
402: static void
403: _uvm_map_check(struct vm_map *map, const char *name,
404: const char *file, int line)
405: {
406:
407: if ((uvm_debug_check_map && _uvm_map_sanity(map)) ||
408: (uvm_debug_check_rbtree && _uvm_tree_sanity(map))) {
409: panic("uvm_map_check failed: \"%s\" map=%p (%s:%d)",
410: name, map, file, line);
411: }
412: }
413: #else /* defined(DEBUG) */
414: #define uvm_map_check(map, name) /* nothing */
415: #endif /* defined(DEBUG) */
416:
417: #if defined(DEBUG) || defined(DDB)
418: int
419: _uvm_map_sanity(struct vm_map *map)
420: {
1.234 thorpej 421: bool first_free_found = false;
422: bool hint_found = false;
1.222 yamt 423: const struct vm_map_entry *e;
424:
425: e = &map->header;
426: for (;;) {
427: if (map->first_free == e) {
1.234 thorpej 428: first_free_found = true;
1.222 yamt 429: } else if (!first_free_found && e->next->start > e->end) {
430: printf("first_free %p should be %p\n",
431: map->first_free, e);
432: return -1;
433: }
434: if (map->hint == e) {
1.234 thorpej 435: hint_found = true;
1.222 yamt 436: }
437:
438: e = e->next;
439: if (e == &map->header) {
440: break;
441: }
442: }
443: if (!first_free_found) {
444: printf("stale first_free\n");
445: return -1;
446: }
447: if (!hint_found) {
448: printf("stale hint\n");
449: return -1;
450: }
451: return 0;
452: }
1.144 yamt 453:
454: int
1.222 yamt 455: _uvm_tree_sanity(struct vm_map *map)
1.144 yamt 456: {
457: struct vm_map_entry *tmp, *trtmp;
458: int n = 0, i = 1;
459:
460: RB_FOREACH(tmp, uvm_tree, &map->rbhead) {
461: if (tmp->ownspace != uvm_rb_space(map, tmp)) {
1.222 yamt 462: printf("%d/%d ownspace %lx != %lx %s\n",
463: n + 1, map->nentries,
1.144 yamt 464: (ulong)tmp->ownspace, (ulong)uvm_rb_space(map, tmp),
465: tmp->next == &map->header ? "(last)" : "");
466: goto error;
467: }
468: }
469: trtmp = NULL;
470: RB_FOREACH(tmp, uvm_tree, &map->rbhead) {
471: if (tmp->space != uvm_rb_subtree_space(tmp)) {
1.222 yamt 472: printf("space %lx != %lx\n",
473: (ulong)tmp->space,
1.144 yamt 474: (ulong)uvm_rb_subtree_space(tmp));
475: goto error;
476: }
477: if (trtmp != NULL && trtmp->start >= tmp->start) {
1.222 yamt 478: printf("corrupt: 0x%lx >= 0x%lx\n",
479: trtmp->start, tmp->start);
1.144 yamt 480: goto error;
481: }
482: n++;
483:
484: trtmp = tmp;
485: }
486:
487: if (n != map->nentries) {
1.222 yamt 488: printf("nentries: %d vs %d\n", n, map->nentries);
1.144 yamt 489: goto error;
490: }
491:
492: for (tmp = map->header.next; tmp && tmp != &map->header;
493: tmp = tmp->next, i++) {
494: trtmp = RB_FIND(uvm_tree, &map->rbhead, tmp);
495: if (trtmp != tmp) {
1.222 yamt 496: printf("lookup: %d: %p - %p: %p\n", i, tmp, trtmp,
1.144 yamt 497: RB_PARENT(tmp, rb_entry));
498: goto error;
499: }
500: }
501:
502: return (0);
503: error:
504: return (-1);
505: }
1.222 yamt 506: #endif /* defined(DEBUG) || defined(DDB) */
1.144 yamt 507:
1.201 dsl 508: #ifdef DIAGNOSTIC
1.203 thorpej 509: static struct vm_map *uvm_kmapent_map(struct vm_map_entry *);
1.201 dsl 510: #endif
1.174 yamt 511:
1.1 mrg 512: /*
1.238 ad 513: * vm_map_lock: acquire an exclusive (write) lock on a map.
514: *
515: * => Note that "intrsafe" maps use only exclusive, spin locks.
516: *
517: * => The locking protocol provides for guaranteed upgrade from shared ->
518: * exclusive by whichever thread currently has the map marked busy.
519: * See "LOCKING PROTOCOL NOTES" in uvm_map.h. This is horrible; among
520: * other problems, it defeats any fairness guarantees provided by RW
521: * locks.
522: */
523:
524: void
525: vm_map_lock(struct vm_map *map)
526: {
527:
528: if ((map->flags & VM_MAP_INTRSAFE) != 0) {
529: mutex_spin_enter(&map->mutex);
530: return;
531: }
532:
533: for (;;) {
534: rw_enter(&map->lock, RW_WRITER);
535: if (map->busy == NULL)
536: break;
537: KASSERT(map->busy != curlwp);
538: mutex_enter(&map->misc_lock);
539: rw_exit(&map->lock);
540: cv_wait(&map->cv, &map->misc_lock);
541: mutex_exit(&map->misc_lock);
542: }
543:
544: map->timestamp++;
545: }
546:
547: /*
548: * vm_map_lock_try: try to lock a map, failing if it is already locked.
549: */
550:
551: bool
552: vm_map_lock_try(struct vm_map *map)
553: {
554:
555: if ((map->flags & VM_MAP_INTRSAFE) != 0)
556: return mutex_tryenter(&map->mutex);
557: if (!rw_tryenter(&map->lock, RW_WRITER))
558: return false;
559: if (map->busy != NULL) {
560: rw_exit(&map->lock);
561: return false;
562: }
563:
564: map->timestamp++;
565: return true;
566: }
567:
568: /*
569: * vm_map_unlock: release an exclusive lock on a map.
570: */
571:
572: void
573: vm_map_unlock(struct vm_map *map)
574: {
575:
576: if ((map->flags & VM_MAP_INTRSAFE) != 0)
577: mutex_spin_exit(&map->mutex);
578: else {
579: KASSERT(rw_write_held(&map->lock));
580: rw_exit(&map->lock);
581: }
582: }
583:
584: /*
585: * vm_map_upgrade: upgrade a shared lock to an exclusive lock.
586: *
587: * => the caller must hold the map busy
588: */
589:
590: void
591: vm_map_upgrade(struct vm_map *map)
592: {
593:
594: KASSERT(rw_read_held(&map->lock));
595: KASSERT(map->busy == curlwp);
596:
597: if (rw_tryupgrade(&map->lock))
598: return;
599:
600: rw_exit(&map->lock);
601: rw_enter(&map->lock, RW_WRITER);
602: }
603:
604: /*
605: * vm_map_unbusy: mark the map as unbusy, and wake any waiters that
606: * want an exclusive lock.
607: */
608:
609: void
610: vm_map_unbusy(struct vm_map *map)
611: {
612:
613: KASSERT(rw_lock_held(&map->lock));
614: KASSERT(map->busy == curlwp);
615:
616: /*
617: * Safe to clear 'busy' and 'waiters' with only a read lock held:
618: *
619: * o they can only be set with a write lock held
620: * o writers are blocked out with a read or write hold
621: * o at any time, only one thread owns the set of values
622: */
623: map->busy = NULL;
624: mutex_enter(&map->misc_lock);
625: cv_broadcast(&map->cv);
626: mutex_exit(&map->misc_lock);
627: }
628:
629: /*
1.1 mrg 630: * uvm_mapent_alloc: allocate a map entry
631: */
632:
1.203 thorpej 633: static struct vm_map_entry *
1.138 enami 634: uvm_mapent_alloc(struct vm_map *map, int flags)
1.10 mrg 635: {
1.99 chs 636: struct vm_map_entry *me;
1.127 thorpej 637: int pflags = (flags & UVM_FLAG_NOWAIT) ? PR_NOWAIT : PR_WAITOK;
1.104 chs 638: UVMHIST_FUNC("uvm_mapent_alloc"); UVMHIST_CALLED(maphist);
1.1 mrg 639:
1.174 yamt 640: if (VM_MAP_USE_KMAPENT(map)) {
641: me = uvm_kmapent_alloc(map, flags);
1.104 chs 642: } else {
1.126 bouyer 643: me = pool_get(&uvm_map_entry_pool, pflags);
644: if (__predict_false(me == NULL))
645: return NULL;
1.104 chs 646: me->flags = 0;
1.10 mrg 647: }
1.1 mrg 648:
1.104 chs 649: UVMHIST_LOG(maphist, "<- new entry=0x%x [kentry=%d]", me,
650: ((map->flags & VM_MAP_INTRSAFE) != 0 || map == kernel_map), 0, 0);
1.139 enami 651: return (me);
1.1 mrg 652: }
653:
654: /*
1.174 yamt 655: * uvm_mapent_alloc_split: allocate a map entry for clipping.
656: */
657:
1.203 thorpej 658: static struct vm_map_entry *
1.174 yamt 659: uvm_mapent_alloc_split(struct vm_map *map,
660: const struct vm_map_entry *old_entry, int flags,
1.232 yamt 661: struct uvm_mapent_reservation *umr)
1.174 yamt 662: {
663: struct vm_map_entry *me;
664:
665: KASSERT(!VM_MAP_USE_KMAPENT(map) ||
666: (old_entry->flags & UVM_MAP_QUANTUM) || !UMR_EMPTY(umr));
667:
668: if (old_entry->flags & UVM_MAP_QUANTUM) {
1.175 yamt 669: struct vm_map_kernel *vmk = vm_map_to_kernel(map);
1.174 yamt 670:
1.238 ad 671: mutex_spin_enter(&uvm_kentry_lock);
1.175 yamt 672: me = vmk->vmk_merged_entries;
1.174 yamt 673: KASSERT(me);
1.175 yamt 674: vmk->vmk_merged_entries = me->next;
1.238 ad 675: mutex_spin_exit(&uvm_kentry_lock);
1.174 yamt 676: KASSERT(me->flags & UVM_MAP_QUANTUM);
677: } else {
678: me = uvm_mapent_alloc(map, flags);
679: }
680:
681: return me;
682: }
683:
684: /*
1.1 mrg 685: * uvm_mapent_free: free map entry
686: */
687:
1.203 thorpej 688: static void
1.138 enami 689: uvm_mapent_free(struct vm_map_entry *me)
1.1 mrg 690: {
1.104 chs 691: UVMHIST_FUNC("uvm_mapent_free"); UVMHIST_CALLED(maphist);
692:
1.98 chs 693: UVMHIST_LOG(maphist,"<- freeing map entry=0x%x [flags=%d]",
1.1 mrg 694: me, me->flags, 0, 0);
1.174 yamt 695: if (me->flags & UVM_MAP_KERNEL) {
696: uvm_kmapent_free(me);
697: } else {
698: pool_put(&uvm_map_entry_pool, me);
699: }
700: }
701:
702: /*
1.194 yamt 703: * uvm_mapent_free_merged: free merged map entry
1.174 yamt 704: *
705: * => keep the entry if needed.
1.194 yamt 706: * => caller shouldn't hold map locked if VM_MAP_USE_KMAPENT(map) is true.
1.174 yamt 707: */
708:
1.203 thorpej 709: static void
1.181 yamt 710: uvm_mapent_free_merged(struct vm_map *map, struct vm_map_entry *me)
1.174 yamt 711: {
712:
1.181 yamt 713: KASSERT(!(me->flags & UVM_MAP_KERNEL) || uvm_kmapent_map(me) == map);
714:
1.174 yamt 715: if (me->flags & UVM_MAP_QUANTUM) {
716: /*
717: * keep this entry for later splitting.
718: */
1.175 yamt 719: struct vm_map_kernel *vmk;
1.174 yamt 720:
1.181 yamt 721: KASSERT(VM_MAP_IS_KERNEL(map));
722: KASSERT(!VM_MAP_USE_KMAPENT(map) ||
723: (me->flags & UVM_MAP_KERNEL));
1.174 yamt 724:
1.175 yamt 725: vmk = vm_map_to_kernel(map);
1.238 ad 726: mutex_spin_enter(&uvm_kentry_lock);
1.175 yamt 727: me->next = vmk->vmk_merged_entries;
728: vmk->vmk_merged_entries = me;
1.238 ad 729: mutex_spin_exit(&uvm_kentry_lock);
1.104 chs 730: } else {
1.174 yamt 731: uvm_mapent_free(me);
1.10 mrg 732: }
1.1 mrg 733: }
734:
735: /*
736: * uvm_mapent_copy: copy a map entry, preserving flags
737: */
738:
1.206 perry 739: static inline void
1.138 enami 740: uvm_mapent_copy(struct vm_map_entry *src, struct vm_map_entry *dst)
1.10 mrg 741: {
1.139 enami 742:
1.106 chs 743: memcpy(dst, src, ((char *)&src->uvm_map_entry_stop_copy) -
1.139 enami 744: ((char *)src));
1.1 mrg 745: }
746:
1.219 yamt 747: /*
748: * uvm_mapent_overhead: calculate maximum kva overhead necessary for
749: * map entries.
750: *
751: * => size and flags are the same as uvm_km_suballoc's ones.
752: */
753:
754: vsize_t
755: uvm_mapent_overhead(vsize_t size, int flags)
756: {
757:
758: if (VM_MAP_USE_KMAPENT_FLAGS(flags)) {
759: return uvm_kmapent_overhead(size);
760: }
761: return 0;
762: }
763:
1.218 yamt 764: #if defined(DEBUG)
765: static void
766: _uvm_mapent_check(const struct vm_map_entry *entry, const char *file, int line)
767: {
768:
769: if (entry->start >= entry->end) {
770: goto bad;
771: }
772: if (UVM_ET_ISOBJ(entry)) {
773: if (entry->object.uvm_obj == NULL) {
774: goto bad;
775: }
776: } else if (UVM_ET_ISSUBMAP(entry)) {
777: if (entry->object.sub_map == NULL) {
778: goto bad;
779: }
780: } else {
781: if (entry->object.uvm_obj != NULL ||
782: entry->object.sub_map != NULL) {
783: goto bad;
784: }
785: }
786: if (!UVM_ET_ISOBJ(entry)) {
787: if (entry->offset != 0) {
788: goto bad;
789: }
790: }
791:
792: return;
793:
794: bad:
795: panic("%s: bad entry %p (%s:%d)", __func__, entry, file, line);
796: }
797: #endif /* defined(DEBUG) */
798:
1.1 mrg 799: /*
800: * uvm_map_entry_unwire: unwire a map entry
801: *
802: * => map should be locked by caller
803: */
804:
1.206 perry 805: static inline void
1.138 enami 806: uvm_map_entry_unwire(struct vm_map *map, struct vm_map_entry *entry)
1.10 mrg 807: {
1.139 enami 808:
1.10 mrg 809: entry->wired_count = 0;
1.57 thorpej 810: uvm_fault_unwire_locked(map, entry->start, entry->end);
1.1 mrg 811: }
812:
1.85 chs 813:
814: /*
815: * wrapper for calling amap_ref()
816: */
1.206 perry 817: static inline void
1.138 enami 818: uvm_map_reference_amap(struct vm_map_entry *entry, int flags)
1.85 chs 819: {
1.139 enami 820:
1.99 chs 821: amap_ref(entry->aref.ar_amap, entry->aref.ar_pageoff,
1.139 enami 822: (entry->end - entry->start) >> PAGE_SHIFT, flags);
1.85 chs 823: }
824:
825:
826: /*
1.98 chs 827: * wrapper for calling amap_unref()
1.85 chs 828: */
1.206 perry 829: static inline void
1.138 enami 830: uvm_map_unreference_amap(struct vm_map_entry *entry, int flags)
1.85 chs 831: {
1.139 enami 832:
1.99 chs 833: amap_unref(entry->aref.ar_amap, entry->aref.ar_pageoff,
1.139 enami 834: (entry->end - entry->start) >> PAGE_SHIFT, flags);
1.85 chs 835: }
836:
837:
1.1 mrg 838: /*
839: * uvm_map_init: init mapping system at boot time. note that we allocate
1.99 chs 840: * and init the static pool of struct vm_map_entry *'s for the kernel here.
1.1 mrg 841: */
842:
1.10 mrg 843: void
1.138 enami 844: uvm_map_init(void)
1.1 mrg 845: {
846: #if defined(UVMHIST)
1.10 mrg 847: static struct uvm_history_ent maphistbuf[100];
848: static struct uvm_history_ent pdhistbuf[100];
1.1 mrg 849: #endif
1.10 mrg 850:
851: /*
852: * first, init logging system.
853: */
1.1 mrg 854:
1.10 mrg 855: UVMHIST_FUNC("uvm_map_init");
856: UVMHIST_INIT_STATIC(maphist, maphistbuf);
857: UVMHIST_INIT_STATIC(pdhist, pdhistbuf);
858: UVMHIST_CALLED(maphist);
859: UVMHIST_LOG(maphist,"<starting uvm map system>", 0, 0, 0, 0);
860:
861: /*
1.174 yamt 862: * initialize the global lock for kernel map entry.
1.10 mrg 863: */
864:
1.238 ad 865: mutex_init(&uvm_kentry_lock, MUTEX_DRIVER, IPL_VM);
1.1 mrg 866: }
867:
868: /*
869: * clippers
870: */
871:
872: /*
1.218 yamt 873: * uvm_mapent_splitadj: adjust map entries for splitting, after uvm_mapent_copy.
874: */
875:
876: static void
877: uvm_mapent_splitadj(struct vm_map_entry *entry1, struct vm_map_entry *entry2,
878: vaddr_t splitat)
879: {
880: vaddr_t adj;
881:
882: KASSERT(entry1->start < splitat);
883: KASSERT(splitat < entry1->end);
884:
885: adj = splitat - entry1->start;
886: entry1->end = entry2->start = splitat;
887:
888: if (entry1->aref.ar_amap) {
889: amap_splitref(&entry1->aref, &entry2->aref, adj);
890: }
891: if (UVM_ET_ISSUBMAP(entry1)) {
892: /* ... unlikely to happen, but play it safe */
893: uvm_map_reference(entry1->object.sub_map);
894: } else if (UVM_ET_ISOBJ(entry1)) {
895: KASSERT(entry1->object.uvm_obj != NULL); /* suppress coverity */
896: entry2->offset += adj;
897: if (entry1->object.uvm_obj->pgops &&
898: entry1->object.uvm_obj->pgops->pgo_reference)
899: entry1->object.uvm_obj->pgops->pgo_reference(
900: entry1->object.uvm_obj);
901: }
902: }
903:
904: /*
1.1 mrg 905: * uvm_map_clip_start: ensure that the entry begins at or after
906: * the starting address, if it doesn't we split the entry.
1.98 chs 907: *
1.1 mrg 908: * => caller should use UVM_MAP_CLIP_START macro rather than calling
909: * this directly
910: * => map must be locked by caller
911: */
912:
1.99 chs 913: void
1.138 enami 914: uvm_map_clip_start(struct vm_map *map, struct vm_map_entry *entry,
1.174 yamt 915: vaddr_t start, struct uvm_mapent_reservation *umr)
1.1 mrg 916: {
1.99 chs 917: struct vm_map_entry *new_entry;
1.1 mrg 918:
919: /* uvm_map_simplify_entry(map, entry); */ /* XXX */
920:
1.222 yamt 921: uvm_map_check(map, "clip_start entry");
1.218 yamt 922: uvm_mapent_check(entry);
1.144 yamt 923:
1.10 mrg 924: /*
925: * Split off the front portion. note that we must insert the new
926: * entry BEFORE this one, so that this entry has the specified
1.1 mrg 927: * starting address.
1.10 mrg 928: */
1.174 yamt 929: new_entry = uvm_mapent_alloc_split(map, entry, 0, umr);
1.1 mrg 930: uvm_mapent_copy(entry, new_entry); /* entry -> new_entry */
1.218 yamt 931: uvm_mapent_splitadj(new_entry, entry, start);
1.10 mrg 932: uvm_map_entry_link(map, entry->prev, new_entry);
1.85 chs 933:
1.222 yamt 934: uvm_map_check(map, "clip_start leave");
1.1 mrg 935: }
936:
937: /*
938: * uvm_map_clip_end: ensure that the entry ends at or before
939: * the ending address, if it does't we split the reference
1.98 chs 940: *
1.1 mrg 941: * => caller should use UVM_MAP_CLIP_END macro rather than calling
942: * this directly
943: * => map must be locked by caller
944: */
945:
1.10 mrg 946: void
1.174 yamt 947: uvm_map_clip_end(struct vm_map *map, struct vm_map_entry *entry, vaddr_t end,
948: struct uvm_mapent_reservation *umr)
1.1 mrg 949: {
1.218 yamt 950: struct vm_map_entry *new_entry;
1.1 mrg 951:
1.222 yamt 952: uvm_map_check(map, "clip_end entry");
1.218 yamt 953: uvm_mapent_check(entry);
1.174 yamt 954:
1.1 mrg 955: /*
956: * Create a new entry and insert it
957: * AFTER the specified entry
958: */
1.174 yamt 959: new_entry = uvm_mapent_alloc_split(map, entry, 0, umr);
1.1 mrg 960: uvm_mapent_copy(entry, new_entry); /* entry -> new_entry */
1.218 yamt 961: uvm_mapent_splitadj(entry, new_entry, end);
1.1 mrg 962: uvm_map_entry_link(map, entry, new_entry);
963:
1.222 yamt 964: uvm_map_check(map, "clip_end leave");
1.1 mrg 965: }
966:
1.226 yamt 967: static void
968: vm_map_drain(struct vm_map *map, uvm_flag_t flags)
969: {
970:
971: if (!VM_MAP_IS_KERNEL(map)) {
972: return;
973: }
974:
975: uvm_km_va_drain(map, flags);
976: }
1.1 mrg 977:
978: /*
979: * M A P - m a i n e n t r y p o i n t
980: */
981: /*
982: * uvm_map: establish a valid mapping in a map
983: *
984: * => assume startp is page aligned.
985: * => assume size is a multiple of PAGE_SIZE.
986: * => assume sys_mmap provides enough of a "hint" to have us skip
987: * over text/data/bss area.
988: * => map must be unlocked (we will lock it)
989: * => <uobj,uoffset> value meanings (4 cases):
1.139 enami 990: * [1] <NULL,uoffset> == uoffset is a hint for PMAP_PREFER
1.1 mrg 991: * [2] <NULL,UVM_UNKNOWN_OFFSET> == don't PMAP_PREFER
992: * [3] <uobj,uoffset> == normal mapping
993: * [4] <uobj,UVM_UNKNOWN_OFFSET> == uvm_map finds offset based on VA
1.98 chs 994: *
1.1 mrg 995: * case [4] is for kernel mappings where we don't know the offset until
1.8 chuck 996: * we've found a virtual address. note that kernel object offsets are
997: * always relative to vm_map_min(kernel_map).
1.81 thorpej 998: *
1.165 yamt 999: * => if `align' is non-zero, we align the virtual address to the specified
1000: * alignment.
1001: * this is provided as a mechanism for large pages.
1.81 thorpej 1002: *
1.1 mrg 1003: * => XXXCDC: need way to map in external amap?
1004: */
1005:
1.10 mrg 1006: int
1.138 enami 1007: uvm_map(struct vm_map *map, vaddr_t *startp /* IN/OUT */, vsize_t size,
1008: struct uvm_object *uobj, voff_t uoffset, vsize_t align, uvm_flag_t flags)
1.10 mrg 1009: {
1.174 yamt 1010: struct uvm_map_args args;
1011: struct vm_map_entry *new_entry;
1012: int error;
1013:
1.181 yamt 1014: KASSERT((flags & UVM_FLAG_QUANTUM) == 0 || VM_MAP_IS_KERNEL(map));
1.187 yamt 1015: KASSERT((size & PAGE_MASK) == 0);
1.174 yamt 1016:
1017: /*
1018: * for pager_map, allocate the new entry first to avoid sleeping
1019: * for memory while we have the map locked.
1020: *
1021: * besides, because we allocates entries for in-kernel maps
1022: * a bit differently (cf. uvm_kmapent_alloc/free), we need to
1023: * allocate them before locking the map.
1024: */
1025:
1026: new_entry = NULL;
1.181 yamt 1027: if (VM_MAP_USE_KMAPENT(map) || (flags & UVM_FLAG_QUANTUM) ||
1028: map == pager_map) {
1.174 yamt 1029: new_entry = uvm_mapent_alloc(map, (flags & UVM_FLAG_NOWAIT));
1030: if (__predict_false(new_entry == NULL))
1031: return ENOMEM;
1032: if (flags & UVM_FLAG_QUANTUM)
1033: new_entry->flags |= UVM_MAP_QUANTUM;
1034: }
1035: if (map == pager_map)
1036: flags |= UVM_FLAG_NOMERGE;
1037:
1038: error = uvm_map_prepare(map, *startp, size, uobj, uoffset, align,
1039: flags, &args);
1040: if (!error) {
1041: error = uvm_map_enter(map, &args, new_entry);
1042: *startp = args.uma_start;
1.189 yamt 1043: } else if (new_entry) {
1044: uvm_mapent_free(new_entry);
1.174 yamt 1045: }
1046:
1.187 yamt 1047: #if defined(DEBUG)
1048: if (!error && VM_MAP_IS_KERNEL(map)) {
1049: uvm_km_check_empty(*startp, *startp + size,
1050: (map->flags & VM_MAP_INTRSAFE) != 0);
1051: }
1052: #endif /* defined(DEBUG) */
1053:
1.174 yamt 1054: return error;
1055: }
1056:
1057: int
1058: uvm_map_prepare(struct vm_map *map, vaddr_t start, vsize_t size,
1059: struct uvm_object *uobj, voff_t uoffset, vsize_t align, uvm_flag_t flags,
1060: struct uvm_map_args *args)
1061: {
1062: struct vm_map_entry *prev_entry;
1063: vm_prot_t prot = UVM_PROTECTION(flags);
1064: vm_prot_t maxprot = UVM_MAXPROTECTION(flags);
1065:
1066: UVMHIST_FUNC("uvm_map_prepare");
1.10 mrg 1067: UVMHIST_CALLED(maphist);
1.1 mrg 1068:
1.174 yamt 1069: UVMHIST_LOG(maphist, "(map=0x%x, start=0x%x, size=%d, flags=0x%x)",
1070: map, start, size, flags);
1.10 mrg 1071: UVMHIST_LOG(maphist, " uobj/offset 0x%x/%d", uobj, uoffset,0,0);
1.107 chs 1072:
1073: /*
1074: * detect a popular device driver bug.
1075: */
1076:
1.139 enami 1077: KASSERT(doing_shutdown || curlwp != NULL ||
1.129 christos 1078: (map->flags & VM_MAP_INTRSAFE));
1.1 mrg 1079:
1.10 mrg 1080: /*
1.144 yamt 1081: * zero-sized mapping doesn't make any sense.
1082: */
1083: KASSERT(size > 0);
1084:
1.180 yamt 1085: KASSERT((~flags & (UVM_FLAG_NOWAIT | UVM_FLAG_WAITVA)) != 0);
1086:
1.222 yamt 1087: uvm_map_check(map, "map entry");
1.144 yamt 1088:
1089: /*
1.106 chs 1090: * check sanity of protection code
1.10 mrg 1091: */
1.1 mrg 1092:
1.10 mrg 1093: if ((prot & maxprot) != prot) {
1.98 chs 1094: UVMHIST_LOG(maphist, "<- prot. failure: prot=0x%x, max=0x%x",
1.10 mrg 1095: prot, maxprot,0,0);
1.94 chs 1096: return EACCES;
1.10 mrg 1097: }
1.1 mrg 1098:
1.10 mrg 1099: /*
1.106 chs 1100: * figure out where to put new VM range
1.10 mrg 1101: */
1.1 mrg 1102:
1.180 yamt 1103: retry:
1.234 thorpej 1104: if (vm_map_lock_try(map) == false) {
1.106 chs 1105: if (flags & UVM_FLAG_TRYLOCK) {
1.94 chs 1106: return EAGAIN;
1.106 chs 1107: }
1.10 mrg 1108: vm_map_lock(map); /* could sleep here */
1109: }
1.226 yamt 1110: prev_entry = uvm_map_findspace(map, start, size, &start,
1111: uobj, uoffset, align, flags);
1112: if (prev_entry == NULL) {
1.180 yamt 1113: unsigned int timestamp;
1114:
1115: timestamp = map->timestamp;
1116: UVMHIST_LOG(maphist,"waiting va timestamp=0x%x",
1117: timestamp,0,0,0);
1118: map->flags |= VM_MAP_WANTVA;
1.10 mrg 1119: vm_map_unlock(map);
1.180 yamt 1120:
1121: /*
1.226 yamt 1122: * try to reclaim kva and wait until someone does unmap.
1.238 ad 1123: * fragile locking here, so we awaken every second to
1124: * recheck the condition.
1.180 yamt 1125: */
1126:
1.226 yamt 1127: vm_map_drain(map, flags);
1128:
1.238 ad 1129: mutex_enter(&map->misc_lock);
1.180 yamt 1130: while ((map->flags & VM_MAP_WANTVA) != 0 &&
1131: map->timestamp == timestamp) {
1.226 yamt 1132: if ((flags & UVM_FLAG_WAITVA) == 0) {
1.238 ad 1133: mutex_exit(&map->misc_lock);
1.226 yamt 1134: UVMHIST_LOG(maphist,
1135: "<- uvm_map_findspace failed!", 0,0,0,0);
1136: return ENOMEM;
1137: } else {
1.238 ad 1138: cv_timedwait(&map->cv, &map->misc_lock, hz);
1.226 yamt 1139: }
1.180 yamt 1140: }
1.238 ad 1141: mutex_exit(&map->misc_lock);
1.180 yamt 1142: goto retry;
1.10 mrg 1143: }
1.1 mrg 1144:
1.40 thorpej 1145: #ifdef PMAP_GROWKERNEL
1.152 simonb 1146: /*
1147: * If the kernel pmap can't map the requested space,
1148: * then allocate more resources for it.
1149: */
1.229 yamt 1150: if (map == kernel_map && uvm_maxkaddr < (start + size))
1151: uvm_maxkaddr = pmap_growkernel(start + size);
1.10 mrg 1152: #endif
1153:
1.207 yamt 1154: UVMMAP_EVCNT_INCR(map_call);
1.10 mrg 1155:
1156: /*
1157: * if uobj is null, then uoffset is either a VAC hint for PMAP_PREFER
1.98 chs 1158: * [typically from uvm_map_reserve] or it is UVM_UNKNOWN_OFFSET. in
1159: * either case we want to zero it before storing it in the map entry
1.10 mrg 1160: * (because it looks strange and confusing when debugging...)
1.98 chs 1161: *
1162: * if uobj is not null
1.10 mrg 1163: * if uoffset is not UVM_UNKNOWN_OFFSET then we have a normal mapping
1164: * and we do not need to change uoffset.
1165: * if uoffset is UVM_UNKNOWN_OFFSET then we need to find the offset
1166: * now (based on the starting address of the map). this case is
1167: * for kernel object mappings where we don't know the offset until
1168: * the virtual address is found (with uvm_map_findspace). the
1169: * offset is the distance we are from the start of the map.
1170: */
1171:
1172: if (uobj == NULL) {
1173: uoffset = 0;
1174: } else {
1175: if (uoffset == UVM_UNKNOWN_OFFSET) {
1.85 chs 1176: KASSERT(UVM_OBJ_IS_KERN_OBJECT(uobj));
1.174 yamt 1177: uoffset = start - vm_map_min(kernel_map);
1.10 mrg 1178: }
1179: }
1180:
1.174 yamt 1181: args->uma_flags = flags;
1182: args->uma_prev = prev_entry;
1183: args->uma_start = start;
1184: args->uma_size = size;
1185: args->uma_uobj = uobj;
1186: args->uma_uoffset = uoffset;
1187:
1188: return 0;
1189: }
1190:
1191: int
1192: uvm_map_enter(struct vm_map *map, const struct uvm_map_args *args,
1193: struct vm_map_entry *new_entry)
1194: {
1195: struct vm_map_entry *prev_entry = args->uma_prev;
1196: struct vm_map_entry *dead = NULL;
1197:
1198: const uvm_flag_t flags = args->uma_flags;
1199: const vm_prot_t prot = UVM_PROTECTION(flags);
1200: const vm_prot_t maxprot = UVM_MAXPROTECTION(flags);
1201: const vm_inherit_t inherit = UVM_INHERIT(flags);
1202: const int amapwaitflag = (flags & UVM_FLAG_NOWAIT) ?
1203: AMAP_EXTEND_NOWAIT : 0;
1204: const int advice = UVM_ADVICE(flags);
1205: const int meflagval = (flags & UVM_FLAG_QUANTUM) ?
1206: UVM_MAP_QUANTUM : 0;
1207:
1208: vaddr_t start = args->uma_start;
1209: vsize_t size = args->uma_size;
1210: struct uvm_object *uobj = args->uma_uobj;
1211: voff_t uoffset = args->uma_uoffset;
1212:
1213: const int kmap = (vm_map_pmap(map) == pmap_kernel());
1214: int merged = 0;
1215: int error;
1.176 yamt 1216: int newetype;
1.174 yamt 1217:
1218: UVMHIST_FUNC("uvm_map_enter");
1219: UVMHIST_CALLED(maphist);
1220:
1221: UVMHIST_LOG(maphist, "(map=0x%x, start=0x%x, size=%d, flags=0x%x)",
1222: map, start, size, flags);
1223: UVMHIST_LOG(maphist, " uobj/offset 0x%x/%d", uobj, uoffset,0,0);
1224:
1.221 yamt 1225: KASSERT(map->hint == prev_entry); /* bimerge case assumes this */
1226:
1.174 yamt 1227: if (flags & UVM_FLAG_QUANTUM) {
1228: KASSERT(new_entry);
1229: KASSERT(new_entry->flags & UVM_MAP_QUANTUM);
1230: }
1231:
1.176 yamt 1232: if (uobj)
1233: newetype = UVM_ET_OBJ;
1234: else
1235: newetype = 0;
1236:
1237: if (flags & UVM_FLAG_COPYONW) {
1238: newetype |= UVM_ET_COPYONWRITE;
1239: if ((flags & UVM_FLAG_OVERLAY) == 0)
1240: newetype |= UVM_ET_NEEDSCOPY;
1241: }
1242:
1.10 mrg 1243: /*
1.106 chs 1244: * try and insert in map by extending previous entry, if possible.
1.10 mrg 1245: * XXX: we don't try and pull back the next entry. might be useful
1246: * for a stack, but we are currently allocating our stack in advance.
1247: */
1248:
1.121 atatat 1249: if (flags & UVM_FLAG_NOMERGE)
1250: goto nomerge;
1251:
1.194 yamt 1252: if (prev_entry->end == start &&
1.121 atatat 1253: prev_entry != &map->header &&
1.194 yamt 1254: UVM_ET_ISCOMPATIBLE(prev_entry, newetype, uobj, meflagval,
1255: prot, maxprot, inherit, advice, 0)) {
1.161 matt 1256:
1.10 mrg 1257: if (uobj && prev_entry->offset +
1258: (prev_entry->end - prev_entry->start) != uoffset)
1.121 atatat 1259: goto forwardmerge;
1.10 mrg 1260:
1261: /*
1.98 chs 1262: * can't extend a shared amap. note: no need to lock amap to
1.34 chuck 1263: * look at refs since we don't care about its exact value.
1.10 mrg 1264: * if it is one (i.e. we have only reference) it will stay there
1265: */
1.85 chs 1266:
1.10 mrg 1267: if (prev_entry->aref.ar_amap &&
1.34 chuck 1268: amap_refs(prev_entry->aref.ar_amap) != 1) {
1.121 atatat 1269: goto forwardmerge;
1.10 mrg 1270: }
1.85 chs 1271:
1.119 chs 1272: if (prev_entry->aref.ar_amap) {
1.139 enami 1273: error = amap_extend(prev_entry, size,
1.126 bouyer 1274: amapwaitflag | AMAP_EXTEND_FORWARDS);
1.174 yamt 1275: if (error)
1.191 yamt 1276: goto nomerge;
1.119 chs 1277: }
1.10 mrg 1278:
1.121 atatat 1279: if (kmap)
1.207 yamt 1280: UVMMAP_EVCNT_INCR(kbackmerge);
1.121 atatat 1281: else
1.207 yamt 1282: UVMMAP_EVCNT_INCR(ubackmerge);
1.10 mrg 1283: UVMHIST_LOG(maphist," starting back merge", 0, 0, 0, 0);
1284:
1285: /*
1286: * drop our reference to uobj since we are extending a reference
1287: * that we already have (the ref count can not drop to zero).
1288: */
1.119 chs 1289:
1.10 mrg 1290: if (uobj && uobj->pgops->pgo_detach)
1291: uobj->pgops->pgo_detach(uobj);
1292:
1293: prev_entry->end += size;
1.145 yamt 1294: uvm_rb_fixup(map, prev_entry);
1295:
1.222 yamt 1296: uvm_map_check(map, "map backmerged");
1.10 mrg 1297:
1298: UVMHIST_LOG(maphist,"<- done (via backmerge)!", 0, 0, 0, 0);
1.121 atatat 1299: merged++;
1.106 chs 1300: }
1.10 mrg 1301:
1.121 atatat 1302: forwardmerge:
1.194 yamt 1303: if (prev_entry->next->start == (start + size) &&
1.121 atatat 1304: prev_entry->next != &map->header &&
1.194 yamt 1305: UVM_ET_ISCOMPATIBLE(prev_entry->next, newetype, uobj, meflagval,
1306: prot, maxprot, inherit, advice, 0)) {
1.161 matt 1307:
1.121 atatat 1308: if (uobj && prev_entry->next->offset != uoffset + size)
1309: goto nomerge;
1310:
1311: /*
1312: * can't extend a shared amap. note: no need to lock amap to
1313: * look at refs since we don't care about its exact value.
1.122 atatat 1314: * if it is one (i.e. we have only reference) it will stay there.
1315: *
1316: * note that we also can't merge two amaps, so if we
1317: * merged with the previous entry which has an amap,
1318: * and the next entry also has an amap, we give up.
1319: *
1.125 atatat 1320: * Interesting cases:
1321: * amap, new, amap -> give up second merge (single fwd extend)
1322: * amap, new, none -> double forward extend (extend again here)
1323: * none, new, amap -> double backward extend (done here)
1324: * uobj, new, amap -> single backward extend (done here)
1325: *
1.122 atatat 1326: * XXX should we attempt to deal with someone refilling
1327: * the deallocated region between two entries that are
1328: * backed by the same amap (ie, arefs is 2, "prev" and
1329: * "next" refer to it, and adding this allocation will
1330: * close the hole, thus restoring arefs to 1 and
1331: * deallocating the "next" vm_map_entry)? -- @@@
1.121 atatat 1332: */
1333:
1334: if (prev_entry->next->aref.ar_amap &&
1.122 atatat 1335: (amap_refs(prev_entry->next->aref.ar_amap) != 1 ||
1336: (merged && prev_entry->aref.ar_amap))) {
1.121 atatat 1337: goto nomerge;
1338: }
1339:
1.122 atatat 1340: if (merged) {
1.123 atatat 1341: /*
1342: * Try to extend the amap of the previous entry to
1343: * cover the next entry as well. If it doesn't work
1344: * just skip on, don't actually give up, since we've
1345: * already completed the back merge.
1346: */
1.125 atatat 1347: if (prev_entry->aref.ar_amap) {
1348: if (amap_extend(prev_entry,
1349: prev_entry->next->end -
1350: prev_entry->next->start,
1.126 bouyer 1351: amapwaitflag | AMAP_EXTEND_FORWARDS))
1.142 enami 1352: goto nomerge;
1.125 atatat 1353: }
1354:
1355: /*
1356: * Try to extend the amap of the *next* entry
1357: * back to cover the new allocation *and* the
1358: * previous entry as well (the previous merge
1359: * didn't have an amap already otherwise we
1360: * wouldn't be checking here for an amap). If
1361: * it doesn't work just skip on, again, don't
1362: * actually give up, since we've already
1363: * completed the back merge.
1364: */
1365: else if (prev_entry->next->aref.ar_amap) {
1366: if (amap_extend(prev_entry->next,
1367: prev_entry->end -
1.141 atatat 1368: prev_entry->start,
1.126 bouyer 1369: amapwaitflag | AMAP_EXTEND_BACKWARDS))
1.142 enami 1370: goto nomerge;
1.125 atatat 1371: }
1372: } else {
1373: /*
1374: * Pull the next entry's amap backwards to cover this
1375: * new allocation.
1376: */
1377: if (prev_entry->next->aref.ar_amap) {
1378: error = amap_extend(prev_entry->next, size,
1.126 bouyer 1379: amapwaitflag | AMAP_EXTEND_BACKWARDS);
1.174 yamt 1380: if (error)
1.191 yamt 1381: goto nomerge;
1.125 atatat 1382: }
1.122 atatat 1383: }
1384:
1.121 atatat 1385: if (merged) {
1386: if (kmap) {
1.207 yamt 1387: UVMMAP_EVCNT_DECR(kbackmerge);
1388: UVMMAP_EVCNT_INCR(kbimerge);
1.121 atatat 1389: } else {
1.207 yamt 1390: UVMMAP_EVCNT_DECR(ubackmerge);
1391: UVMMAP_EVCNT_INCR(ubimerge);
1.121 atatat 1392: }
1.122 atatat 1393: } else {
1.121 atatat 1394: if (kmap)
1.207 yamt 1395: UVMMAP_EVCNT_INCR(kforwmerge);
1.121 atatat 1396: else
1.207 yamt 1397: UVMMAP_EVCNT_INCR(uforwmerge);
1.121 atatat 1398: }
1399: UVMHIST_LOG(maphist," starting forward merge", 0, 0, 0, 0);
1.10 mrg 1400:
1.121 atatat 1401: /*
1402: * drop our reference to uobj since we are extending a reference
1403: * that we already have (the ref count can not drop to zero).
1404: * (if merged, we've already detached)
1405: */
1406: if (uobj && uobj->pgops->pgo_detach && !merged)
1407: uobj->pgops->pgo_detach(uobj);
1.1 mrg 1408:
1.121 atatat 1409: if (merged) {
1.174 yamt 1410: dead = prev_entry->next;
1.121 atatat 1411: prev_entry->end = dead->end;
1412: uvm_map_entry_unlink(map, dead);
1.125 atatat 1413: if (dead->aref.ar_amap != NULL) {
1414: prev_entry->aref = dead->aref;
1415: dead->aref.ar_amap = NULL;
1416: }
1.121 atatat 1417: } else {
1418: prev_entry->next->start -= size;
1.145 yamt 1419: if (prev_entry != &map->header)
1420: uvm_rb_fixup(map, prev_entry);
1.121 atatat 1421: if (uobj)
1422: prev_entry->next->offset = uoffset;
1423: }
1.145 yamt 1424:
1.222 yamt 1425: uvm_map_check(map, "map forwardmerged");
1.1 mrg 1426:
1.121 atatat 1427: UVMHIST_LOG(maphist,"<- done forwardmerge", 0, 0, 0, 0);
1428: merged++;
1.106 chs 1429: }
1.121 atatat 1430:
1431: nomerge:
1432: if (!merged) {
1433: UVMHIST_LOG(maphist," allocating new map entry", 0, 0, 0, 0);
1434: if (kmap)
1.207 yamt 1435: UVMMAP_EVCNT_INCR(knomerge);
1.121 atatat 1436: else
1.207 yamt 1437: UVMMAP_EVCNT_INCR(unomerge);
1.106 chs 1438:
1.10 mrg 1439: /*
1.121 atatat 1440: * allocate new entry and link it in.
1.10 mrg 1441: */
1.106 chs 1442:
1.121 atatat 1443: if (new_entry == NULL) {
1.126 bouyer 1444: new_entry = uvm_mapent_alloc(map,
1.127 thorpej 1445: (flags & UVM_FLAG_NOWAIT));
1.126 bouyer 1446: if (__predict_false(new_entry == NULL)) {
1.174 yamt 1447: error = ENOMEM;
1448: goto done;
1.126 bouyer 1449: }
1.121 atatat 1450: }
1.174 yamt 1451: new_entry->start = start;
1.121 atatat 1452: new_entry->end = new_entry->start + size;
1453: new_entry->object.uvm_obj = uobj;
1454: new_entry->offset = uoffset;
1455:
1.176 yamt 1456: new_entry->etype = newetype;
1.121 atatat 1457:
1.161 matt 1458: if (flags & UVM_FLAG_NOMERGE) {
1459: new_entry->flags |= UVM_MAP_NOMERGE;
1460: }
1.121 atatat 1461:
1462: new_entry->protection = prot;
1463: new_entry->max_protection = maxprot;
1464: new_entry->inheritance = inherit;
1465: new_entry->wired_count = 0;
1466: new_entry->advice = advice;
1467: if (flags & UVM_FLAG_OVERLAY) {
1468:
1469: /*
1470: * to_add: for BSS we overallocate a little since we
1471: * are likely to extend
1472: */
1473:
1474: vaddr_t to_add = (flags & UVM_FLAG_AMAPPAD) ?
1475: UVM_AMAP_CHUNK << PAGE_SHIFT : 0;
1.126 bouyer 1476: struct vm_amap *amap = amap_alloc(size, to_add,
1.227 yamt 1477: (flags & UVM_FLAG_NOWAIT));
1.126 bouyer 1478: if (__predict_false(amap == NULL)) {
1.174 yamt 1479: error = ENOMEM;
1480: goto done;
1.126 bouyer 1481: }
1.121 atatat 1482: new_entry->aref.ar_pageoff = 0;
1483: new_entry->aref.ar_amap = amap;
1484: } else {
1485: new_entry->aref.ar_pageoff = 0;
1486: new_entry->aref.ar_amap = NULL;
1487: }
1488: uvm_map_entry_link(map, prev_entry, new_entry);
1.1 mrg 1489:
1.121 atatat 1490: /*
1491: * Update the free space hint
1492: */
1.10 mrg 1493:
1.121 atatat 1494: if ((map->first_free == prev_entry) &&
1495: (prev_entry->end >= new_entry->start))
1496: map->first_free = new_entry;
1.174 yamt 1497:
1498: new_entry = NULL;
1.121 atatat 1499: }
1.10 mrg 1500:
1.146 yamt 1501: map->size += size;
1502:
1.10 mrg 1503: UVMHIST_LOG(maphist,"<- done!", 0, 0, 0, 0);
1.174 yamt 1504:
1505: error = 0;
1506: done:
1.1 mrg 1507: vm_map_unlock(map);
1.174 yamt 1508: if (new_entry) {
1509: if (error == 0) {
1510: KDASSERT(merged);
1.181 yamt 1511: uvm_mapent_free_merged(map, new_entry);
1.174 yamt 1512: } else {
1513: uvm_mapent_free(new_entry);
1514: }
1515: }
1516: if (dead) {
1517: KDASSERT(merged);
1.181 yamt 1518: uvm_mapent_free_merged(map, dead);
1.174 yamt 1519: }
1520: return error;
1.1 mrg 1521: }
1522:
1523: /*
1524: * uvm_map_lookup_entry: find map entry at or before an address
1525: *
1526: * => map must at least be read-locked by caller
1527: * => entry is returned in "entry"
1528: * => return value is true if address is in the returned entry
1529: */
1530:
1.233 thorpej 1531: bool
1.138 enami 1532: uvm_map_lookup_entry(struct vm_map *map, vaddr_t address,
1533: struct vm_map_entry **entry /* OUT */)
1.1 mrg 1534: {
1.99 chs 1535: struct vm_map_entry *cur;
1.234 thorpej 1536: bool use_tree = false;
1.1 mrg 1537: UVMHIST_FUNC("uvm_map_lookup_entry");
1538: UVMHIST_CALLED(maphist);
1539:
1540: UVMHIST_LOG(maphist,"(map=0x%x,addr=0x%x,ent=0x%x)",
1.10 mrg 1541: map, address, entry, 0);
1.1 mrg 1542:
1543: /*
1.10 mrg 1544: * start looking either from the head of the
1545: * list, or from the hint.
1.1 mrg 1546: */
1547:
1.238 ad 1548: mutex_enter(&map->hint_lock);
1.1 mrg 1549: cur = map->hint;
1.238 ad 1550: mutex_exit(&map->hint_lock);
1.1 mrg 1551:
1552: if (cur == &map->header)
1553: cur = cur->next;
1554:
1.207 yamt 1555: UVMMAP_EVCNT_INCR(mlk_call);
1.1 mrg 1556: if (address >= cur->start) {
1.99 chs 1557:
1.139 enami 1558: /*
1.10 mrg 1559: * go from hint to end of list.
1.1 mrg 1560: *
1.10 mrg 1561: * but first, make a quick check to see if
1562: * we are already looking at the entry we
1563: * want (which is usually the case).
1564: * note also that we don't need to save the hint
1565: * here... it is the same hint (unless we are
1566: * at the header, in which case the hint didn't
1567: * buy us anything anyway).
1.1 mrg 1568: */
1.99 chs 1569:
1.144 yamt 1570: if (cur != &map->header && cur->end > address) {
1.207 yamt 1571: UVMMAP_EVCNT_INCR(mlk_hint);
1.1 mrg 1572: *entry = cur;
1573: UVMHIST_LOG(maphist,"<- got it via hint (0x%x)",
1.10 mrg 1574: cur, 0, 0, 0);
1.218 yamt 1575: uvm_mapent_check(*entry);
1.234 thorpej 1576: return (true);
1.1 mrg 1577: }
1.144 yamt 1578:
1579: if (map->nentries > 30)
1.234 thorpej 1580: use_tree = true;
1.10 mrg 1581: } else {
1.99 chs 1582:
1.139 enami 1583: /*
1.144 yamt 1584: * invalid hint. use tree.
1.1 mrg 1585: */
1.234 thorpej 1586: use_tree = true;
1.144 yamt 1587: }
1588:
1.222 yamt 1589: uvm_map_check(map, __func__);
1.144 yamt 1590:
1591: if (use_tree) {
1592: struct vm_map_entry *prev = &map->header;
1593: cur = RB_ROOT(&map->rbhead);
1.99 chs 1594:
1.144 yamt 1595: /*
1596: * Simple lookup in the tree. Happens when the hint is
1597: * invalid, or nentries reach a threshold.
1598: */
1599: while (cur) {
1600: if (address >= cur->start) {
1601: if (address < cur->end) {
1602: *entry = cur;
1603: goto got;
1604: }
1605: prev = cur;
1606: cur = RB_RIGHT(cur, rb_entry);
1607: } else
1608: cur = RB_LEFT(cur, rb_entry);
1609: }
1610: *entry = prev;
1611: goto failed;
1.1 mrg 1612: }
1613:
1614: /*
1.10 mrg 1615: * search linearly
1.1 mrg 1616: */
1617:
1.144 yamt 1618: while (cur != &map->header) {
1.1 mrg 1619: if (cur->end > address) {
1620: if (address >= cur->start) {
1.139 enami 1621: /*
1.10 mrg 1622: * save this lookup for future
1623: * hints, and return
1.1 mrg 1624: */
1625:
1626: *entry = cur;
1.144 yamt 1627: got:
1628: SAVE_HINT(map, map->hint, *entry);
1.1 mrg 1629: UVMHIST_LOG(maphist,"<- search got it (0x%x)",
1.10 mrg 1630: cur, 0, 0, 0);
1.144 yamt 1631: KDASSERT((*entry)->start <= address);
1632: KDASSERT(address < (*entry)->end);
1.218 yamt 1633: uvm_mapent_check(*entry);
1.234 thorpej 1634: return (true);
1.1 mrg 1635: }
1636: break;
1637: }
1638: cur = cur->next;
1639: }
1640: *entry = cur->prev;
1.144 yamt 1641: failed:
1.82 thorpej 1642: SAVE_HINT(map, map->hint, *entry);
1.1 mrg 1643: UVMHIST_LOG(maphist,"<- failed!",0,0,0,0);
1.147 yamt 1644: KDASSERT((*entry) == &map->header || (*entry)->end <= address);
1.144 yamt 1645: KDASSERT((*entry)->next == &map->header ||
1646: address < (*entry)->next->start);
1.234 thorpej 1647: return (false);
1.1 mrg 1648: }
1649:
1650: /*
1.140 enami 1651: * See if the range between start and start + length fits in the gap
1652: * entry->next->start and entry->end. Returns 1 if fits, 0 if doesn't
1653: * fit, and -1 address wraps around.
1654: */
1.203 thorpej 1655: static int
1.232 yamt 1656: uvm_map_space_avail(vaddr_t *start, vsize_t length, voff_t uoffset,
1.140 enami 1657: vsize_t align, int topdown, struct vm_map_entry *entry)
1658: {
1659: vaddr_t end;
1660:
1661: #ifdef PMAP_PREFER
1662: /*
1663: * push start address forward as needed to avoid VAC alias problems.
1664: * we only do this if a valid offset is specified.
1665: */
1666:
1667: if (uoffset != UVM_UNKNOWN_OFFSET)
1.182 atatat 1668: PMAP_PREFER(uoffset, start, length, topdown);
1.140 enami 1669: #endif
1670: if (align != 0) {
1671: if ((*start & (align - 1)) != 0) {
1672: if (topdown)
1673: *start &= ~(align - 1);
1674: else
1675: *start = roundup(*start, align);
1676: }
1677: /*
1678: * XXX Should we PMAP_PREFER() here again?
1.182 atatat 1679: * eh...i think we're okay
1.140 enami 1680: */
1681: }
1682:
1683: /*
1684: * Find the end of the proposed new region. Be sure we didn't
1685: * wrap around the address; if so, we lose. Otherwise, if the
1686: * proposed new region fits before the next entry, we win.
1687: */
1688:
1689: end = *start + length;
1690: if (end < *start)
1691: return (-1);
1692:
1693: if (entry->next->start >= end && *start >= entry->end)
1694: return (1);
1695:
1696: return (0);
1697: }
1698:
1699: /*
1.1 mrg 1700: * uvm_map_findspace: find "length" sized space in "map".
1701: *
1.167 junyoung 1702: * => "hint" is a hint about where we want it, unless UVM_FLAG_FIXED is
1703: * set in "flags" (in which case we insist on using "hint").
1.1 mrg 1704: * => "result" is VA returned
1705: * => uobj/uoffset are to be used to handle VAC alignment, if required
1.167 junyoung 1706: * => if "align" is non-zero, we attempt to align to that value.
1.1 mrg 1707: * => caller must at least have read-locked map
1708: * => returns NULL on failure, or pointer to prev. map entry if success
1709: * => note this is a cross between the old vm_map_findspace and vm_map_find
1710: */
1711:
1.99 chs 1712: struct vm_map_entry *
1.138 enami 1713: uvm_map_findspace(struct vm_map *map, vaddr_t hint, vsize_t length,
1.232 yamt 1714: vaddr_t *result /* OUT */, struct uvm_object *uobj, voff_t uoffset,
1.138 enami 1715: vsize_t align, int flags)
1.1 mrg 1716: {
1.140 enami 1717: struct vm_map_entry *entry;
1.144 yamt 1718: struct vm_map_entry *child, *prev, *tmp;
1.140 enami 1719: vaddr_t orig_hint;
1.131 atatat 1720: const int topdown = map->flags & VM_MAP_TOPDOWN;
1.1 mrg 1721: UVMHIST_FUNC("uvm_map_findspace");
1722: UVMHIST_CALLED(maphist);
1723:
1.98 chs 1724: UVMHIST_LOG(maphist, "(map=0x%x, hint=0x%x, len=%d, flags=0x%x)",
1.140 enami 1725: map, hint, length, flags);
1.85 chs 1726: KASSERT((align & (align - 1)) == 0);
1727: KASSERT((flags & UVM_FLAG_FIXED) == 0 || align == 0);
1.81 thorpej 1728:
1.222 yamt 1729: uvm_map_check(map, "map_findspace entry");
1.144 yamt 1730:
1.81 thorpej 1731: /*
1732: * remember the original hint. if we are aligning, then we
1733: * may have to try again with no alignment constraint if
1734: * we fail the first time.
1735: */
1.85 chs 1736:
1.81 thorpej 1737: orig_hint = hint;
1.184 chs 1738: if (hint < vm_map_min(map)) { /* check ranges ... */
1.81 thorpej 1739: if (flags & UVM_FLAG_FIXED) {
1.1 mrg 1740: UVMHIST_LOG(maphist,"<- VA below map range",0,0,0,0);
1.139 enami 1741: return (NULL);
1.1 mrg 1742: }
1.184 chs 1743: hint = vm_map_min(map);
1.1 mrg 1744: }
1.184 chs 1745: if (hint > vm_map_max(map)) {
1.1 mrg 1746: UVMHIST_LOG(maphist,"<- VA 0x%x > range [0x%x->0x%x]",
1.184 chs 1747: hint, vm_map_min(map), vm_map_max(map), 0);
1.139 enami 1748: return (NULL);
1.1 mrg 1749: }
1750:
1751: /*
1752: * Look for the first possible address; if there's already
1753: * something at this address, we have to start after it.
1754: */
1755:
1.131 atatat 1756: /*
1757: * @@@: there are four, no, eight cases to consider.
1758: *
1759: * 0: found, fixed, bottom up -> fail
1760: * 1: found, fixed, top down -> fail
1.140 enami 1761: * 2: found, not fixed, bottom up -> start after entry->end,
1762: * loop up
1763: * 3: found, not fixed, top down -> start before entry->start,
1764: * loop down
1765: * 4: not found, fixed, bottom up -> check entry->next->start, fail
1766: * 5: not found, fixed, top down -> check entry->next->start, fail
1767: * 6: not found, not fixed, bottom up -> check entry->next->start,
1768: * loop up
1769: * 7: not found, not fixed, top down -> check entry->next->start,
1770: * loop down
1.131 atatat 1771: *
1772: * as you can see, it reduces to roughly five cases, and that
1773: * adding top down mapping only adds one unique case (without
1774: * it, there would be four cases).
1775: */
1776:
1.184 chs 1777: if ((flags & UVM_FLAG_FIXED) == 0 && hint == vm_map_min(map)) {
1.140 enami 1778: entry = map->first_free;
1.1 mrg 1779: } else {
1.140 enami 1780: if (uvm_map_lookup_entry(map, hint, &entry)) {
1.1 mrg 1781: /* "hint" address already in use ... */
1.81 thorpej 1782: if (flags & UVM_FLAG_FIXED) {
1.140 enami 1783: UVMHIST_LOG(maphist, "<- fixed & VA in use",
1.10 mrg 1784: 0, 0, 0, 0);
1.139 enami 1785: return (NULL);
1.1 mrg 1786: }
1.140 enami 1787: if (topdown)
1788: /* Start from lower gap. */
1789: entry = entry->prev;
1790: } else if (flags & UVM_FLAG_FIXED) {
1791: if (entry->next->start >= hint + length &&
1792: hint + length > hint)
1793: goto found;
1794:
1795: /* "hint" address is gap but too small */
1796: UVMHIST_LOG(maphist, "<- fixed mapping failed",
1797: 0, 0, 0, 0);
1798: return (NULL); /* only one shot at it ... */
1799: } else {
1800: /*
1801: * See if given hint fits in this gap.
1802: */
1803: switch (uvm_map_space_avail(&hint, length,
1804: uoffset, align, topdown, entry)) {
1805: case 1:
1806: goto found;
1807: case -1:
1808: goto wraparound;
1809: }
1810:
1.148 yamt 1811: if (topdown) {
1.140 enami 1812: /*
1813: * Still there is a chance to fit
1814: * if hint > entry->end.
1815: */
1.148 yamt 1816: } else {
1.168 junyoung 1817: /* Start from higher gap. */
1.148 yamt 1818: entry = entry->next;
1819: if (entry == &map->header)
1820: goto notfound;
1.140 enami 1821: goto nextgap;
1.148 yamt 1822: }
1.1 mrg 1823: }
1824: }
1825:
1826: /*
1.144 yamt 1827: * Note that all UVM_FLAGS_FIXED case is already handled.
1828: */
1829: KDASSERT((flags & UVM_FLAG_FIXED) == 0);
1830:
1831: /* Try to find the space in the red-black tree */
1832:
1833: /* Check slot before any entry */
1834: hint = topdown ? entry->next->start - length : entry->end;
1835: switch (uvm_map_space_avail(&hint, length, uoffset, align,
1836: topdown, entry)) {
1837: case 1:
1838: goto found;
1839: case -1:
1840: goto wraparound;
1841: }
1842:
1843: nextgap:
1.148 yamt 1844: KDASSERT((flags & UVM_FLAG_FIXED) == 0);
1.144 yamt 1845: /* If there is not enough space in the whole tree, we fail */
1846: tmp = RB_ROOT(&map->rbhead);
1847: if (tmp == NULL || tmp->space < length)
1848: goto notfound;
1849:
1850: prev = NULL; /* previous candidate */
1851:
1852: /* Find an entry close to hint that has enough space */
1853: for (; tmp;) {
1854: KASSERT(tmp->next->start == tmp->end + tmp->ownspace);
1855: if (topdown) {
1856: if (tmp->next->start < hint + length &&
1857: (prev == NULL || tmp->end > prev->end)) {
1858: if (tmp->ownspace >= length)
1859: prev = tmp;
1860: else if ((child = RB_LEFT(tmp, rb_entry))
1861: != NULL && child->space >= length)
1862: prev = tmp;
1863: }
1864: } else {
1865: if (tmp->end >= hint &&
1866: (prev == NULL || tmp->end < prev->end)) {
1867: if (tmp->ownspace >= length)
1868: prev = tmp;
1869: else if ((child = RB_RIGHT(tmp, rb_entry))
1870: != NULL && child->space >= length)
1871: prev = tmp;
1872: }
1873: }
1874: if (tmp->next->start < hint + length)
1875: child = RB_RIGHT(tmp, rb_entry);
1876: else if (tmp->end > hint)
1877: child = RB_LEFT(tmp, rb_entry);
1878: else {
1879: if (tmp->ownspace >= length)
1880: break;
1881: if (topdown)
1882: child = RB_LEFT(tmp, rb_entry);
1883: else
1884: child = RB_RIGHT(tmp, rb_entry);
1885: }
1886: if (child == NULL || child->space < length)
1887: break;
1888: tmp = child;
1889: }
1890:
1.148 yamt 1891: if (tmp != NULL && tmp->start < hint && hint < tmp->next->start) {
1.164 junyoung 1892: /*
1.144 yamt 1893: * Check if the entry that we found satifies the
1894: * space requirement
1895: */
1.148 yamt 1896: if (topdown) {
1.149 yamt 1897: if (hint > tmp->next->start - length)
1898: hint = tmp->next->start - length;
1.148 yamt 1899: } else {
1.149 yamt 1900: if (hint < tmp->end)
1901: hint = tmp->end;
1.148 yamt 1902: }
1903: switch (uvm_map_space_avail(&hint, length, uoffset, align,
1904: topdown, tmp)) {
1905: case 1:
1.144 yamt 1906: entry = tmp;
1907: goto found;
1.148 yamt 1908: case -1:
1909: goto wraparound;
1.144 yamt 1910: }
1911: if (tmp->ownspace >= length)
1912: goto listsearch;
1913: }
1914: if (prev == NULL)
1915: goto notfound;
1916:
1.148 yamt 1917: if (topdown) {
1.150 yamt 1918: KASSERT(orig_hint >= prev->next->start - length ||
1.148 yamt 1919: prev->next->start - length > prev->next->start);
1920: hint = prev->next->start - length;
1921: } else {
1.150 yamt 1922: KASSERT(orig_hint <= prev->end);
1.148 yamt 1923: hint = prev->end;
1924: }
1925: switch (uvm_map_space_avail(&hint, length, uoffset, align,
1926: topdown, prev)) {
1927: case 1:
1.144 yamt 1928: entry = prev;
1929: goto found;
1.148 yamt 1930: case -1:
1931: goto wraparound;
1.144 yamt 1932: }
1933: if (prev->ownspace >= length)
1934: goto listsearch;
1.164 junyoung 1935:
1.144 yamt 1936: if (topdown)
1937: tmp = RB_LEFT(prev, rb_entry);
1938: else
1939: tmp = RB_RIGHT(prev, rb_entry);
1940: for (;;) {
1941: KASSERT(tmp && tmp->space >= length);
1942: if (topdown)
1943: child = RB_RIGHT(tmp, rb_entry);
1944: else
1945: child = RB_LEFT(tmp, rb_entry);
1946: if (child && child->space >= length) {
1947: tmp = child;
1948: continue;
1949: }
1950: if (tmp->ownspace >= length)
1951: break;
1952: if (topdown)
1953: tmp = RB_LEFT(tmp, rb_entry);
1954: else
1955: tmp = RB_RIGHT(tmp, rb_entry);
1956: }
1.164 junyoung 1957:
1.148 yamt 1958: if (topdown) {
1.150 yamt 1959: KASSERT(orig_hint >= tmp->next->start - length ||
1.148 yamt 1960: tmp->next->start - length > tmp->next->start);
1961: hint = tmp->next->start - length;
1962: } else {
1.150 yamt 1963: KASSERT(orig_hint <= tmp->end);
1.148 yamt 1964: hint = tmp->end;
1965: }
1.144 yamt 1966: switch (uvm_map_space_avail(&hint, length, uoffset, align,
1967: topdown, tmp)) {
1968: case 1:
1969: entry = tmp;
1970: goto found;
1.148 yamt 1971: case -1:
1972: goto wraparound;
1.144 yamt 1973: }
1974:
1.164 junyoung 1975: /*
1.144 yamt 1976: * The tree fails to find an entry because of offset or alignment
1977: * restrictions. Search the list instead.
1978: */
1979: listsearch:
1980: /*
1.1 mrg 1981: * Look through the rest of the map, trying to fit a new region in
1982: * the gap between existing regions, or after the very last region.
1.140 enami 1983: * note: entry->end = base VA of current gap,
1984: * entry->next->start = VA of end of current gap
1.1 mrg 1985: */
1.99 chs 1986:
1.140 enami 1987: for (;;) {
1988: /* Update hint for current gap. */
1989: hint = topdown ? entry->next->start - length : entry->end;
1990:
1991: /* See if it fits. */
1992: switch (uvm_map_space_avail(&hint, length, uoffset, align,
1993: topdown, entry)) {
1994: case 1:
1995: goto found;
1996: case -1:
1997: goto wraparound;
1998: }
1999:
2000: /* Advance to next/previous gap */
2001: if (topdown) {
2002: if (entry == &map->header) {
2003: UVMHIST_LOG(maphist, "<- failed (off start)",
2004: 0,0,0,0);
2005: goto notfound;
1.134 matt 2006: }
1.140 enami 2007: entry = entry->prev;
2008: } else {
2009: entry = entry->next;
2010: if (entry == &map->header) {
2011: UVMHIST_LOG(maphist, "<- failed (off end)",
1.81 thorpej 2012: 0,0,0,0);
1.140 enami 2013: goto notfound;
1.81 thorpej 2014: }
1.1 mrg 2015: }
2016: }
1.140 enami 2017:
2018: found:
1.82 thorpej 2019: SAVE_HINT(map, map->hint, entry);
1.1 mrg 2020: *result = hint;
2021: UVMHIST_LOG(maphist,"<- got it! (result=0x%x)", hint, 0,0,0);
1.148 yamt 2022: KASSERT( topdown || hint >= orig_hint);
2023: KASSERT(!topdown || hint <= orig_hint);
1.144 yamt 2024: KASSERT(entry->end <= hint);
2025: KASSERT(hint + length <= entry->next->start);
1.1 mrg 2026: return (entry);
1.140 enami 2027:
2028: wraparound:
2029: UVMHIST_LOG(maphist, "<- failed (wrap around)", 0,0,0,0);
2030:
1.165 yamt 2031: return (NULL);
2032:
1.140 enami 2033: notfound:
1.165 yamt 2034: UVMHIST_LOG(maphist, "<- failed (notfound)", 0,0,0,0);
2035:
1.140 enami 2036: return (NULL);
1.1 mrg 2037: }
2038:
2039: /*
2040: * U N M A P - m a i n h e l p e r f u n c t i o n s
2041: */
2042:
2043: /*
2044: * uvm_unmap_remove: remove mappings from a vm_map (from "start" up to "stop")
2045: *
1.98 chs 2046: * => caller must check alignment and size
1.1 mrg 2047: * => map must be locked by caller
2048: * => we return a list of map entries that we've remove from the map
2049: * in "entry_list"
2050: */
2051:
1.94 chs 2052: void
1.138 enami 2053: uvm_unmap_remove(struct vm_map *map, vaddr_t start, vaddr_t end,
1.174 yamt 2054: struct vm_map_entry **entry_list /* OUT */,
1.187 yamt 2055: struct uvm_mapent_reservation *umr, int flags)
1.10 mrg 2056: {
1.99 chs 2057: struct vm_map_entry *entry, *first_entry, *next;
1.24 eeh 2058: vaddr_t len;
1.99 chs 2059: UVMHIST_FUNC("uvm_unmap_remove"); UVMHIST_CALLED(maphist);
1.10 mrg 2060:
2061: UVMHIST_LOG(maphist,"(map=0x%x, start=0x%x, end=0x%x)",
2062: map, start, end, 0);
2063: VM_MAP_RANGE_CHECK(map, start, end);
2064:
1.222 yamt 2065: uvm_map_check(map, "unmap_remove entry");
1.144 yamt 2066:
1.10 mrg 2067: /*
2068: * find first entry
2069: */
1.99 chs 2070:
1.234 thorpej 2071: if (uvm_map_lookup_entry(map, start, &first_entry) == true) {
1.29 chuck 2072: /* clip and go... */
1.10 mrg 2073: entry = first_entry;
1.174 yamt 2074: UVM_MAP_CLIP_START(map, entry, start, umr);
1.10 mrg 2075: /* critical! prevents stale hint */
1.82 thorpej 2076: SAVE_HINT(map, entry, entry->prev);
1.10 mrg 2077: } else {
2078: entry = first_entry->next;
2079: }
2080:
2081: /*
2082: * Save the free space hint
2083: */
2084:
1.220 yamt 2085: if (map->first_free != &map->header && map->first_free->start >= start)
1.10 mrg 2086: map->first_free = entry->prev;
2087:
2088: /*
2089: * note: we now re-use first_entry for a different task. we remove
2090: * a number of map entries from the map and save them in a linked
2091: * list headed by "first_entry". once we remove them from the map
2092: * the caller should unlock the map and drop the references to the
2093: * backing objects [c.f. uvm_unmap_detach]. the object is to
1.100 wiz 2094: * separate unmapping from reference dropping. why?
1.10 mrg 2095: * [1] the map has to be locked for unmapping
2096: * [2] the map need not be locked for reference dropping
2097: * [3] dropping references may trigger pager I/O, and if we hit
2098: * a pager that does synchronous I/O we may have to wait for it.
2099: * [4] we would like all waiting for I/O to occur with maps unlocked
1.98 chs 2100: * so that we don't block other threads.
1.10 mrg 2101: */
1.99 chs 2102:
1.10 mrg 2103: first_entry = NULL;
1.106 chs 2104: *entry_list = NULL;
1.10 mrg 2105:
2106: /*
1.98 chs 2107: * break up the area into map entry sized regions and unmap. note
1.10 mrg 2108: * that all mappings have to be removed before we can even consider
2109: * dropping references to amaps or VM objects (otherwise we could end
2110: * up with a mapping to a page on the free list which would be very bad)
2111: */
2112:
2113: while ((entry != &map->header) && (entry->start < end)) {
1.174 yamt 2114: KASSERT((entry->flags & UVM_MAP_FIRST) == 0);
2115:
2116: UVM_MAP_CLIP_END(map, entry, end, umr);
1.10 mrg 2117: next = entry->next;
2118: len = entry->end - entry->start;
1.81 thorpej 2119:
1.10 mrg 2120: /*
2121: * unwire before removing addresses from the pmap; otherwise
2122: * unwiring will put the entries back into the pmap (XXX).
2123: */
1.1 mrg 2124:
1.106 chs 2125: if (VM_MAPENT_ISWIRED(entry)) {
1.10 mrg 2126: uvm_map_entry_unwire(map, entry);
1.106 chs 2127: }
1.187 yamt 2128: if (flags & UVM_FLAG_VAONLY) {
2129:
2130: /* nothing */
2131:
2132: } else if ((map->flags & VM_MAP_PAGEABLE) == 0) {
1.10 mrg 2133:
1.106 chs 2134: /*
2135: * if the map is non-pageable, any pages mapped there
2136: * must be wired and entered with pmap_kenter_pa(),
2137: * and we should free any such pages immediately.
2138: * this is mostly used for kmem_map and mb_map.
2139: */
1.99 chs 2140:
1.174 yamt 2141: if ((entry->flags & UVM_MAP_KMAPENT) == 0) {
2142: uvm_km_pgremove_intrsafe(entry->start,
2143: entry->end);
2144: pmap_kremove(entry->start, len);
2145: }
1.106 chs 2146: } else if (UVM_ET_ISOBJ(entry) &&
2147: UVM_OBJ_IS_KERN_OBJECT(entry->object.uvm_obj)) {
1.85 chs 2148: KASSERT(vm_map_pmap(map) == pmap_kernel());
1.1 mrg 2149:
1.10 mrg 2150: /*
2151: * note: kernel object mappings are currently used in
2152: * two ways:
2153: * [1] "normal" mappings of pages in the kernel object
2154: * [2] uvm_km_valloc'd allocations in which we
2155: * pmap_enter in some non-kernel-object page
2156: * (e.g. vmapbuf).
2157: *
2158: * for case [1], we need to remove the mapping from
2159: * the pmap and then remove the page from the kernel
2160: * object (because, once pages in a kernel object are
2161: * unmapped they are no longer needed, unlike, say,
2162: * a vnode where you might want the data to persist
2163: * until flushed out of a queue).
2164: *
2165: * for case [2], we need to remove the mapping from
2166: * the pmap. there shouldn't be any pages at the
2167: * specified offset in the kernel object [but it
2168: * doesn't hurt to call uvm_km_pgremove just to be
2169: * safe?]
2170: *
1.98 chs 2171: * uvm_km_pgremove currently does the following:
2172: * for pages in the kernel object in range:
1.43 thorpej 2173: * - drops the swap slot
1.10 mrg 2174: * - uvm_pagefree the page
2175: */
2176:
2177: /*
1.43 thorpej 2178: * remove mappings from pmap and drop the pages
2179: * from the object. offsets are always relative
2180: * to vm_map_min(kernel_map).
1.10 mrg 2181: */
1.99 chs 2182:
1.106 chs 2183: pmap_remove(pmap_kernel(), entry->start,
2184: entry->start + len);
1.187 yamt 2185: uvm_km_pgremove(entry->start, entry->end);
1.10 mrg 2186:
2187: /*
2188: * null out kernel_object reference, we've just
2189: * dropped it
2190: */
1.99 chs 2191:
1.10 mrg 2192: entry->etype &= ~UVM_ET_OBJ;
1.106 chs 2193: entry->object.uvm_obj = NULL;
2194: } else if (UVM_ET_ISOBJ(entry) || entry->aref.ar_amap) {
1.99 chs 2195:
1.29 chuck 2196: /*
1.139 enami 2197: * remove mappings the standard way.
2198: */
1.99 chs 2199:
1.29 chuck 2200: pmap_remove(map->pmap, entry->start, entry->end);
1.10 mrg 2201: }
2202:
1.177 yamt 2203: #if defined(DEBUG)
2204: if ((entry->flags & UVM_MAP_KMAPENT) == 0) {
2205:
2206: /*
2207: * check if there's remaining mapping,
2208: * which is a bug in caller.
2209: */
2210:
2211: vaddr_t va;
2212: for (va = entry->start; va < entry->end;
2213: va += PAGE_SIZE) {
2214: if (pmap_extract(vm_map_pmap(map), va, NULL)) {
2215: panic("uvm_unmap_remove: has mapping");
2216: }
2217: }
1.187 yamt 2218:
2219: if (VM_MAP_IS_KERNEL(map)) {
2220: uvm_km_check_empty(entry->start, entry->end,
2221: (map->flags & VM_MAP_INTRSAFE) != 0);
2222: }
1.177 yamt 2223: }
2224: #endif /* defined(DEBUG) */
2225:
1.10 mrg 2226: /*
1.98 chs 2227: * remove entry from map and put it on our list of entries
1.106 chs 2228: * that we've nuked. then go to next entry.
1.10 mrg 2229: */
1.99 chs 2230:
1.10 mrg 2231: UVMHIST_LOG(maphist, " removed map entry 0x%x", entry, 0, 0,0);
1.82 thorpej 2232:
2233: /* critical! prevents stale hint */
2234: SAVE_HINT(map, entry, entry->prev);
2235:
1.10 mrg 2236: uvm_map_entry_unlink(map, entry);
1.146 yamt 2237: KASSERT(map->size >= len);
1.10 mrg 2238: map->size -= len;
1.131 atatat 2239: entry->prev = NULL;
1.10 mrg 2240: entry->next = first_entry;
2241: first_entry = entry;
1.106 chs 2242: entry = next;
1.10 mrg 2243: }
1.120 chs 2244: if ((map->flags & VM_MAP_DYING) == 0) {
2245: pmap_update(vm_map_pmap(map));
2246: }
1.10 mrg 2247:
1.222 yamt 2248: uvm_map_check(map, "unmap_remove leave");
1.144 yamt 2249:
1.10 mrg 2250: /*
2251: * now we've cleaned up the map and are ready for the caller to drop
1.98 chs 2252: * references to the mapped objects.
1.10 mrg 2253: */
2254:
2255: *entry_list = first_entry;
2256: UVMHIST_LOG(maphist,"<- done!", 0, 0, 0, 0);
1.180 yamt 2257:
2258: if (map->flags & VM_MAP_WANTVA) {
1.238 ad 2259: mutex_enter(&map->misc_lock);
1.180 yamt 2260: map->flags &= ~VM_MAP_WANTVA;
1.238 ad 2261: cv_broadcast(&map->cv);
2262: mutex_exit(&map->misc_lock);
1.180 yamt 2263: }
1.1 mrg 2264: }
2265:
2266: /*
2267: * uvm_unmap_detach: drop references in a chain of map entries
2268: *
2269: * => we will free the map entries as we traverse the list.
2270: */
2271:
1.10 mrg 2272: void
1.138 enami 2273: uvm_unmap_detach(struct vm_map_entry *first_entry, int flags)
1.1 mrg 2274: {
1.99 chs 2275: struct vm_map_entry *next_entry;
1.10 mrg 2276: UVMHIST_FUNC("uvm_unmap_detach"); UVMHIST_CALLED(maphist);
1.1 mrg 2277:
1.10 mrg 2278: while (first_entry) {
1.85 chs 2279: KASSERT(!VM_MAPENT_ISWIRED(first_entry));
1.10 mrg 2280: UVMHIST_LOG(maphist,
1.98 chs 2281: " detach 0x%x: amap=0x%x, obj=0x%x, submap?=%d",
2282: first_entry, first_entry->aref.ar_amap,
1.29 chuck 2283: first_entry->object.uvm_obj,
2284: UVM_ET_ISSUBMAP(first_entry));
1.1 mrg 2285:
1.10 mrg 2286: /*
2287: * drop reference to amap, if we've got one
2288: */
2289:
2290: if (first_entry->aref.ar_amap)
1.85 chs 2291: uvm_map_unreference_amap(first_entry, flags);
1.10 mrg 2292:
2293: /*
2294: * drop reference to our backing object, if we've got one
2295: */
1.85 chs 2296:
1.120 chs 2297: KASSERT(!UVM_ET_ISSUBMAP(first_entry));
2298: if (UVM_ET_ISOBJ(first_entry) &&
2299: first_entry->object.uvm_obj->pgops->pgo_detach) {
2300: (*first_entry->object.uvm_obj->pgops->pgo_detach)
2301: (first_entry->object.uvm_obj);
1.10 mrg 2302: }
2303: next_entry = first_entry->next;
2304: uvm_mapent_free(first_entry);
2305: first_entry = next_entry;
2306: }
2307: UVMHIST_LOG(maphist, "<- done", 0,0,0,0);
1.1 mrg 2308: }
2309:
2310: /*
2311: * E X T R A C T I O N F U N C T I O N S
2312: */
2313:
1.98 chs 2314: /*
1.1 mrg 2315: * uvm_map_reserve: reserve space in a vm_map for future use.
2316: *
1.98 chs 2317: * => we reserve space in a map by putting a dummy map entry in the
1.1 mrg 2318: * map (dummy means obj=NULL, amap=NULL, prot=VM_PROT_NONE)
2319: * => map should be unlocked (we will write lock it)
2320: * => we return true if we were able to reserve space
2321: * => XXXCDC: should be inline?
2322: */
2323:
1.10 mrg 2324: int
1.138 enami 2325: uvm_map_reserve(struct vm_map *map, vsize_t size,
2326: vaddr_t offset /* hint for pmap_prefer */,
1.232 yamt 2327: vsize_t align /* alignment hint */,
1.210 yamt 2328: vaddr_t *raddr /* IN:hint, OUT: reserved VA */,
2329: uvm_flag_t flags /* UVM_FLAG_FIXED or 0 */)
1.1 mrg 2330: {
1.98 chs 2331: UVMHIST_FUNC("uvm_map_reserve"); UVMHIST_CALLED(maphist);
1.85 chs 2332:
1.10 mrg 2333: UVMHIST_LOG(maphist, "(map=0x%x, size=0x%x, offset=0x%x,addr=0x%x)",
1.139 enami 2334: map,size,offset,raddr);
1.85 chs 2335:
1.10 mrg 2336: size = round_page(size);
1.85 chs 2337:
1.10 mrg 2338: /*
2339: * reserve some virtual space.
2340: */
1.85 chs 2341:
1.81 thorpej 2342: if (uvm_map(map, raddr, size, NULL, offset, 0,
1.10 mrg 2343: UVM_MAPFLAG(UVM_PROT_NONE, UVM_PROT_NONE, UVM_INH_NONE,
1.210 yamt 2344: UVM_ADV_RANDOM, UVM_FLAG_NOMERGE|flags)) != 0) {
1.10 mrg 2345: UVMHIST_LOG(maphist, "<- done (no VM)", 0,0,0,0);
1.234 thorpej 2346: return (false);
1.98 chs 2347: }
1.85 chs 2348:
1.10 mrg 2349: UVMHIST_LOG(maphist, "<- done (*raddr=0x%x)", *raddr,0,0,0);
1.234 thorpej 2350: return (true);
1.1 mrg 2351: }
2352:
2353: /*
1.98 chs 2354: * uvm_map_replace: replace a reserved (blank) area of memory with
1.1 mrg 2355: * real mappings.
2356: *
1.98 chs 2357: * => caller must WRITE-LOCK the map
1.234 thorpej 2358: * => we return true if replacement was a success
1.1 mrg 2359: * => we expect the newents chain to have nnewents entrys on it and
2360: * we expect newents->prev to point to the last entry on the list
2361: * => note newents is allowed to be NULL
2362: */
2363:
1.10 mrg 2364: int
1.138 enami 2365: uvm_map_replace(struct vm_map *map, vaddr_t start, vaddr_t end,
2366: struct vm_map_entry *newents, int nnewents)
1.10 mrg 2367: {
1.99 chs 2368: struct vm_map_entry *oldent, *last;
1.1 mrg 2369:
1.222 yamt 2370: uvm_map_check(map, "map_replace entry");
1.144 yamt 2371:
1.10 mrg 2372: /*
2373: * first find the blank map entry at the specified address
2374: */
1.85 chs 2375:
1.10 mrg 2376: if (!uvm_map_lookup_entry(map, start, &oldent)) {
1.234 thorpej 2377: return (false);
1.10 mrg 2378: }
1.85 chs 2379:
1.10 mrg 2380: /*
2381: * check to make sure we have a proper blank entry
2382: */
1.1 mrg 2383:
1.210 yamt 2384: if (end < oldent->end && !VM_MAP_USE_KMAPENT(map)) {
2385: UVM_MAP_CLIP_END(map, oldent, end, NULL);
2386: }
1.98 chs 2387: if (oldent->start != start || oldent->end != end ||
1.10 mrg 2388: oldent->object.uvm_obj != NULL || oldent->aref.ar_amap != NULL) {
1.234 thorpej 2389: return (false);
1.10 mrg 2390: }
1.1 mrg 2391:
2392: #ifdef DIAGNOSTIC
1.99 chs 2393:
1.10 mrg 2394: /*
2395: * sanity check the newents chain
2396: */
1.99 chs 2397:
1.10 mrg 2398: {
1.99 chs 2399: struct vm_map_entry *tmpent = newents;
1.10 mrg 2400: int nent = 0;
1.24 eeh 2401: vaddr_t cur = start;
1.10 mrg 2402:
2403: while (tmpent) {
2404: nent++;
2405: if (tmpent->start < cur)
2406: panic("uvm_map_replace1");
2407: if (tmpent->start > tmpent->end || tmpent->end > end) {
2408: printf("tmpent->start=0x%lx, tmpent->end=0x%lx, end=0x%lx\n",
2409: tmpent->start, tmpent->end, end);
2410: panic("uvm_map_replace2");
2411: }
2412: cur = tmpent->end;
2413: if (tmpent->next) {
2414: if (tmpent->next->prev != tmpent)
2415: panic("uvm_map_replace3");
2416: } else {
2417: if (newents->prev != tmpent)
2418: panic("uvm_map_replace4");
2419: }
2420: tmpent = tmpent->next;
2421: }
2422: if (nent != nnewents)
2423: panic("uvm_map_replace5");
2424: }
2425: #endif
2426:
2427: /*
2428: * map entry is a valid blank! replace it. (this does all the
2429: * work of map entry link/unlink...).
2430: */
2431:
2432: if (newents) {
1.99 chs 2433: last = newents->prev;
1.10 mrg 2434:
2435: /* critical: flush stale hints out of map */
1.82 thorpej 2436: SAVE_HINT(map, map->hint, newents);
1.10 mrg 2437: if (map->first_free == oldent)
2438: map->first_free = last;
2439:
2440: last->next = oldent->next;
2441: last->next->prev = last;
1.144 yamt 2442:
2443: /* Fix RB tree */
2444: uvm_rb_remove(map, oldent);
2445:
1.10 mrg 2446: newents->prev = oldent->prev;
2447: newents->prev->next = newents;
2448: map->nentries = map->nentries + (nnewents - 1);
2449:
1.144 yamt 2450: /* Fixup the RB tree */
2451: {
2452: int i;
2453: struct vm_map_entry *tmp;
2454:
2455: tmp = newents;
2456: for (i = 0; i < nnewents && tmp; i++) {
2457: uvm_rb_insert(map, tmp);
2458: tmp = tmp->next;
2459: }
2460: }
1.10 mrg 2461: } else {
2462: /* NULL list of new entries: just remove the old one */
1.221 yamt 2463: clear_hints(map, oldent);
1.10 mrg 2464: uvm_map_entry_unlink(map, oldent);
2465: }
2466:
1.222 yamt 2467: uvm_map_check(map, "map_replace leave");
1.10 mrg 2468:
2469: /*
1.209 yamt 2470: * now we can free the old blank entry and return.
1.10 mrg 2471: */
1.1 mrg 2472:
1.10 mrg 2473: uvm_mapent_free(oldent);
1.234 thorpej 2474: return (true);
1.1 mrg 2475: }
2476:
2477: /*
2478: * uvm_map_extract: extract a mapping from a map and put it somewhere
2479: * (maybe removing the old mapping)
2480: *
2481: * => maps should be unlocked (we will write lock them)
2482: * => returns 0 on success, error code otherwise
2483: * => start must be page aligned
2484: * => len must be page sized
2485: * => flags:
2486: * UVM_EXTRACT_REMOVE: remove mappings from srcmap
2487: * UVM_EXTRACT_CONTIG: abort if unmapped area (advisory only)
2488: * UVM_EXTRACT_QREF: for a temporary extraction do quick obj refs
2489: * UVM_EXTRACT_FIXPROT: set prot to maxprot as we go
2490: * >>>NOTE: if you set REMOVE, you are not allowed to use CONTIG or QREF!<<<
2491: * >>>NOTE: QREF's must be unmapped via the QREF path, thus should only
2492: * be used from within the kernel in a kernel level map <<<
2493: */
2494:
1.10 mrg 2495: int
1.138 enami 2496: uvm_map_extract(struct vm_map *srcmap, vaddr_t start, vsize_t len,
2497: struct vm_map *dstmap, vaddr_t *dstaddrp, int flags)
1.10 mrg 2498: {
1.163 mycroft 2499: vaddr_t dstaddr, end, newend, oldoffset, fudge, orig_fudge;
1.99 chs 2500: struct vm_map_entry *chain, *endchain, *entry, *orig_entry, *newentry,
2501: *deadentry, *oldentry;
1.24 eeh 2502: vsize_t elen;
1.10 mrg 2503: int nchain, error, copy_ok;
2504: UVMHIST_FUNC("uvm_map_extract"); UVMHIST_CALLED(maphist);
1.85 chs 2505:
1.10 mrg 2506: UVMHIST_LOG(maphist,"(srcmap=0x%x,start=0x%x, len=0x%x", srcmap, start,
2507: len,0);
2508: UVMHIST_LOG(maphist," ...,dstmap=0x%x, flags=0x%x)", dstmap,flags,0,0);
2509:
1.222 yamt 2510: uvm_map_check(srcmap, "map_extract src enter");
2511: uvm_map_check(dstmap, "map_extract dst enter");
1.144 yamt 2512:
1.10 mrg 2513: /*
2514: * step 0: sanity check: start must be on a page boundary, length
2515: * must be page sized. can't ask for CONTIG/QREF if you asked for
2516: * REMOVE.
2517: */
2518:
1.85 chs 2519: KASSERT((start & PAGE_MASK) == 0 && (len & PAGE_MASK) == 0);
2520: KASSERT((flags & UVM_EXTRACT_REMOVE) == 0 ||
2521: (flags & (UVM_EXTRACT_CONTIG|UVM_EXTRACT_QREF)) == 0);
1.10 mrg 2522:
2523: /*
2524: * step 1: reserve space in the target map for the extracted area
2525: */
2526:
1.210 yamt 2527: if ((flags & UVM_EXTRACT_RESERVED) == 0) {
2528: dstaddr = vm_map_min(dstmap);
2529: if (!uvm_map_reserve(dstmap, len, start, 0, &dstaddr, 0))
2530: return (ENOMEM);
2531: *dstaddrp = dstaddr; /* pass address back to caller */
2532: UVMHIST_LOG(maphist, " dstaddr=0x%x", dstaddr,0,0,0);
2533: } else {
2534: dstaddr = *dstaddrp;
2535: }
1.10 mrg 2536:
2537: /*
1.98 chs 2538: * step 2: setup for the extraction process loop by init'ing the
1.10 mrg 2539: * map entry chain, locking src map, and looking up the first useful
2540: * entry in the map.
2541: */
1.1 mrg 2542:
1.10 mrg 2543: end = start + len;
2544: newend = dstaddr + len;
2545: chain = endchain = NULL;
2546: nchain = 0;
2547: vm_map_lock(srcmap);
2548:
2549: if (uvm_map_lookup_entry(srcmap, start, &entry)) {
2550:
2551: /* "start" is within an entry */
2552: if (flags & UVM_EXTRACT_QREF) {
1.85 chs 2553:
1.10 mrg 2554: /*
2555: * for quick references we don't clip the entry, so
2556: * the entry may map space "before" the starting
2557: * virtual address... this is the "fudge" factor
2558: * (which can be non-zero only the first time
2559: * through the "while" loop in step 3).
2560: */
1.85 chs 2561:
1.10 mrg 2562: fudge = start - entry->start;
2563: } else {
1.85 chs 2564:
1.10 mrg 2565: /*
2566: * normal reference: we clip the map to fit (thus
2567: * fudge is zero)
2568: */
1.85 chs 2569:
1.174 yamt 2570: UVM_MAP_CLIP_START(srcmap, entry, start, NULL);
1.82 thorpej 2571: SAVE_HINT(srcmap, srcmap->hint, entry->prev);
1.10 mrg 2572: fudge = 0;
2573: }
1.85 chs 2574: } else {
1.1 mrg 2575:
1.10 mrg 2576: /* "start" is not within an entry ... skip to next entry */
2577: if (flags & UVM_EXTRACT_CONTIG) {
2578: error = EINVAL;
2579: goto bad; /* definite hole here ... */
2580: }
1.1 mrg 2581:
1.10 mrg 2582: entry = entry->next;
2583: fudge = 0;
2584: }
1.85 chs 2585:
1.10 mrg 2586: /* save values from srcmap for step 6 */
2587: orig_entry = entry;
2588: orig_fudge = fudge;
1.1 mrg 2589:
1.10 mrg 2590: /*
2591: * step 3: now start looping through the map entries, extracting
2592: * as we go.
2593: */
1.1 mrg 2594:
1.10 mrg 2595: while (entry->start < end && entry != &srcmap->header) {
1.85 chs 2596:
1.10 mrg 2597: /* if we are not doing a quick reference, clip it */
2598: if ((flags & UVM_EXTRACT_QREF) == 0)
1.174 yamt 2599: UVM_MAP_CLIP_END(srcmap, entry, end, NULL);
1.10 mrg 2600:
2601: /* clear needs_copy (allow chunking) */
2602: if (UVM_ET_ISNEEDSCOPY(entry)) {
1.212 yamt 2603: amap_copy(srcmap, entry,
2604: AMAP_COPY_NOWAIT|AMAP_COPY_NOMERGE, start, end);
1.10 mrg 2605: if (UVM_ET_ISNEEDSCOPY(entry)) { /* failed? */
2606: error = ENOMEM;
2607: goto bad;
2608: }
1.85 chs 2609:
1.10 mrg 2610: /* amap_copy could clip (during chunk)! update fudge */
2611: if (fudge) {
1.163 mycroft 2612: fudge = start - entry->start;
1.10 mrg 2613: orig_fudge = fudge;
2614: }
2615: }
1.1 mrg 2616:
1.10 mrg 2617: /* calculate the offset of this from "start" */
2618: oldoffset = (entry->start + fudge) - start;
1.1 mrg 2619:
1.10 mrg 2620: /* allocate a new map entry */
1.126 bouyer 2621: newentry = uvm_mapent_alloc(dstmap, 0);
1.10 mrg 2622: if (newentry == NULL) {
2623: error = ENOMEM;
2624: goto bad;
2625: }
2626:
2627: /* set up new map entry */
2628: newentry->next = NULL;
2629: newentry->prev = endchain;
2630: newentry->start = dstaddr + oldoffset;
2631: newentry->end =
2632: newentry->start + (entry->end - (entry->start + fudge));
1.37 chs 2633: if (newentry->end > newend || newentry->end < newentry->start)
1.10 mrg 2634: newentry->end = newend;
2635: newentry->object.uvm_obj = entry->object.uvm_obj;
2636: if (newentry->object.uvm_obj) {
2637: if (newentry->object.uvm_obj->pgops->pgo_reference)
2638: newentry->object.uvm_obj->pgops->
2639: pgo_reference(newentry->object.uvm_obj);
2640: newentry->offset = entry->offset + fudge;
2641: } else {
2642: newentry->offset = 0;
2643: }
2644: newentry->etype = entry->etype;
1.98 chs 2645: newentry->protection = (flags & UVM_EXTRACT_FIXPROT) ?
2646: entry->max_protection : entry->protection;
1.10 mrg 2647: newentry->max_protection = entry->max_protection;
2648: newentry->inheritance = entry->inheritance;
2649: newentry->wired_count = 0;
2650: newentry->aref.ar_amap = entry->aref.ar_amap;
2651: if (newentry->aref.ar_amap) {
1.34 chuck 2652: newentry->aref.ar_pageoff =
2653: entry->aref.ar_pageoff + (fudge >> PAGE_SHIFT);
1.85 chs 2654: uvm_map_reference_amap(newentry, AMAP_SHARED |
1.10 mrg 2655: ((flags & UVM_EXTRACT_QREF) ? AMAP_REFALL : 0));
2656: } else {
1.34 chuck 2657: newentry->aref.ar_pageoff = 0;
1.10 mrg 2658: }
2659: newentry->advice = entry->advice;
2660:
2661: /* now link it on the chain */
2662: nchain++;
2663: if (endchain == NULL) {
2664: chain = endchain = newentry;
2665: } else {
2666: endchain->next = newentry;
2667: endchain = newentry;
2668: }
2669:
2670: /* end of 'while' loop! */
1.98 chs 2671: if ((flags & UVM_EXTRACT_CONTIG) && entry->end < end &&
1.10 mrg 2672: (entry->next == &srcmap->header ||
2673: entry->next->start != entry->end)) {
2674: error = EINVAL;
2675: goto bad;
2676: }
2677: entry = entry->next;
2678: fudge = 0;
2679: }
2680:
2681: /*
2682: * step 4: close off chain (in format expected by uvm_map_replace)
2683: */
2684:
2685: if (chain)
2686: chain->prev = endchain;
2687:
2688: /*
2689: * step 5: attempt to lock the dest map so we can pmap_copy.
1.98 chs 2690: * note usage of copy_ok:
1.10 mrg 2691: * 1 => dstmap locked, pmap_copy ok, and we "replace" here (step 5)
2692: * 0 => dstmap unlocked, NO pmap_copy, and we will "replace" in step 7
2693: */
1.85 chs 2694:
1.234 thorpej 2695: if (srcmap == dstmap || vm_map_lock_try(dstmap) == true) {
1.10 mrg 2696: copy_ok = 1;
2697: if (!uvm_map_replace(dstmap, dstaddr, dstaddr+len, chain,
2698: nchain)) {
2699: if (srcmap != dstmap)
2700: vm_map_unlock(dstmap);
2701: error = EIO;
2702: goto bad;
2703: }
2704: } else {
2705: copy_ok = 0;
2706: /* replace defered until step 7 */
2707: }
2708:
2709: /*
2710: * step 6: traverse the srcmap a second time to do the following:
2711: * - if we got a lock on the dstmap do pmap_copy
2712: * - if UVM_EXTRACT_REMOVE remove the entries
2713: * we make use of orig_entry and orig_fudge (saved in step 2)
2714: */
2715:
2716: if (copy_ok || (flags & UVM_EXTRACT_REMOVE)) {
2717:
2718: /* purge possible stale hints from srcmap */
2719: if (flags & UVM_EXTRACT_REMOVE) {
1.82 thorpej 2720: SAVE_HINT(srcmap, srcmap->hint, orig_entry->prev);
1.220 yamt 2721: if (srcmap->first_free != &srcmap->header &&
2722: srcmap->first_free->start >= start)
1.10 mrg 2723: srcmap->first_free = orig_entry->prev;
2724: }
2725:
2726: entry = orig_entry;
2727: fudge = orig_fudge;
2728: deadentry = NULL; /* for UVM_EXTRACT_REMOVE */
2729:
2730: while (entry->start < end && entry != &srcmap->header) {
2731: if (copy_ok) {
1.74 thorpej 2732: oldoffset = (entry->start + fudge) - start;
1.90 chs 2733: elen = MIN(end, entry->end) -
1.74 thorpej 2734: (entry->start + fudge);
2735: pmap_copy(dstmap->pmap, srcmap->pmap,
2736: dstaddr + oldoffset, elen,
2737: entry->start + fudge);
1.10 mrg 2738: }
2739:
1.74 thorpej 2740: /* we advance "entry" in the following if statement */
1.10 mrg 2741: if (flags & UVM_EXTRACT_REMOVE) {
1.98 chs 2742: pmap_remove(srcmap->pmap, entry->start,
1.20 chuck 2743: entry->end);
1.139 enami 2744: oldentry = entry; /* save entry */
2745: entry = entry->next; /* advance */
1.20 chuck 2746: uvm_map_entry_unlink(srcmap, oldentry);
2747: /* add to dead list */
2748: oldentry->next = deadentry;
2749: deadentry = oldentry;
1.139 enami 2750: } else {
2751: entry = entry->next; /* advance */
1.10 mrg 2752: }
2753:
2754: /* end of 'while' loop */
2755: fudge = 0;
2756: }
1.105 chris 2757: pmap_update(srcmap->pmap);
1.10 mrg 2758:
2759: /*
2760: * unlock dstmap. we will dispose of deadentry in
2761: * step 7 if needed
2762: */
1.85 chs 2763:
1.10 mrg 2764: if (copy_ok && srcmap != dstmap)
2765: vm_map_unlock(dstmap);
2766:
1.99 chs 2767: } else {
2768: deadentry = NULL;
1.10 mrg 2769: }
2770:
2771: /*
2772: * step 7: we are done with the source map, unlock. if copy_ok
2773: * is 0 then we have not replaced the dummy mapping in dstmap yet
2774: * and we need to do so now.
2775: */
2776:
2777: vm_map_unlock(srcmap);
2778: if ((flags & UVM_EXTRACT_REMOVE) && deadentry)
2779: uvm_unmap_detach(deadentry, 0); /* dispose of old entries */
2780:
2781: /* now do the replacement if we didn't do it in step 5 */
2782: if (copy_ok == 0) {
2783: vm_map_lock(dstmap);
2784: error = uvm_map_replace(dstmap, dstaddr, dstaddr+len, chain,
2785: nchain);
2786: vm_map_unlock(dstmap);
2787:
1.234 thorpej 2788: if (error == false) {
1.10 mrg 2789: error = EIO;
2790: goto bad2;
2791: }
2792: }
1.144 yamt 2793:
1.222 yamt 2794: uvm_map_check(srcmap, "map_extract src leave");
2795: uvm_map_check(dstmap, "map_extract dst leave");
1.144 yamt 2796:
1.139 enami 2797: return (0);
1.10 mrg 2798:
2799: /*
2800: * bad: failure recovery
2801: */
2802: bad:
2803: vm_map_unlock(srcmap);
2804: bad2: /* src already unlocked */
2805: if (chain)
2806: uvm_unmap_detach(chain,
2807: (flags & UVM_EXTRACT_QREF) ? AMAP_REFALL : 0);
1.144 yamt 2808:
1.222 yamt 2809: uvm_map_check(srcmap, "map_extract src err leave");
2810: uvm_map_check(dstmap, "map_extract dst err leave");
1.144 yamt 2811:
1.210 yamt 2812: if ((flags & UVM_EXTRACT_RESERVED) == 0) {
2813: uvm_unmap(dstmap, dstaddr, dstaddr+len); /* ??? */
2814: }
1.139 enami 2815: return (error);
1.10 mrg 2816: }
2817:
2818: /* end of extraction functions */
1.1 mrg 2819:
2820: /*
2821: * uvm_map_submap: punch down part of a map into a submap
2822: *
2823: * => only the kernel_map is allowed to be submapped
2824: * => the purpose of submapping is to break up the locking granularity
2825: * of a larger map
2826: * => the range specified must have been mapped previously with a uvm_map()
2827: * call [with uobj==NULL] to create a blank map entry in the main map.
2828: * [And it had better still be blank!]
2829: * => maps which contain submaps should never be copied or forked.
1.98 chs 2830: * => to remove a submap, use uvm_unmap() on the main map
1.1 mrg 2831: * and then uvm_map_deallocate() the submap.
2832: * => main map must be unlocked.
2833: * => submap must have been init'd and have a zero reference count.
2834: * [need not be locked as we don't actually reference it]
2835: */
1.85 chs 2836:
1.10 mrg 2837: int
1.138 enami 2838: uvm_map_submap(struct vm_map *map, vaddr_t start, vaddr_t end,
2839: struct vm_map *submap)
1.10 mrg 2840: {
1.99 chs 2841: struct vm_map_entry *entry;
1.174 yamt 2842: struct uvm_mapent_reservation umr;
1.94 chs 2843: int error;
1.1 mrg 2844:
1.174 yamt 2845: uvm_mapent_reserve(map, &umr, 2, 0);
2846:
1.10 mrg 2847: vm_map_lock(map);
1.85 chs 2848: VM_MAP_RANGE_CHECK(map, start, end);
1.1 mrg 2849:
1.10 mrg 2850: if (uvm_map_lookup_entry(map, start, &entry)) {
1.174 yamt 2851: UVM_MAP_CLIP_START(map, entry, start, &umr);
2852: UVM_MAP_CLIP_END(map, entry, end, &umr); /* to be safe */
1.94 chs 2853: } else {
1.10 mrg 2854: entry = NULL;
2855: }
1.1 mrg 2856:
1.98 chs 2857: if (entry != NULL &&
1.10 mrg 2858: entry->start == start && entry->end == end &&
2859: entry->object.uvm_obj == NULL && entry->aref.ar_amap == NULL &&
2860: !UVM_ET_ISCOPYONWRITE(entry) && !UVM_ET_ISNEEDSCOPY(entry)) {
1.29 chuck 2861: entry->etype |= UVM_ET_SUBMAP;
1.10 mrg 2862: entry->object.sub_map = submap;
2863: entry->offset = 0;
2864: uvm_map_reference(submap);
1.94 chs 2865: error = 0;
1.10 mrg 2866: } else {
1.94 chs 2867: error = EINVAL;
1.10 mrg 2868: }
2869: vm_map_unlock(map);
1.174 yamt 2870:
2871: uvm_mapent_unreserve(map, &umr);
2872:
1.94 chs 2873: return error;
1.1 mrg 2874: }
2875:
1.175 yamt 2876: /*
2877: * uvm_map_setup_kernel: init in-kernel map
2878: *
2879: * => map must not be in service yet.
2880: */
2881:
2882: void
2883: uvm_map_setup_kernel(struct vm_map_kernel *map,
1.199 christos 2884: vaddr_t vmin, vaddr_t vmax, int flags)
1.175 yamt 2885: {
2886:
1.199 christos 2887: uvm_map_setup(&map->vmk_map, vmin, vmax, flags);
1.237 ad 2888: callback_head_init(&map->vmk_reclaim_callback, IPL_VM);
1.175 yamt 2889: LIST_INIT(&map->vmk_kentry_free);
2890: map->vmk_merged_entries = NULL;
2891: }
2892:
1.1 mrg 2893:
2894: /*
2895: * uvm_map_protect: change map protection
2896: *
2897: * => set_max means set max_protection.
2898: * => map must be unlocked.
2899: */
2900:
1.139 enami 2901: #define MASK(entry) (UVM_ET_ISCOPYONWRITE(entry) ? \
1.36 mycroft 2902: ~VM_PROT_WRITE : VM_PROT_ALL)
1.1 mrg 2903:
1.10 mrg 2904: int
1.138 enami 2905: uvm_map_protect(struct vm_map *map, vaddr_t start, vaddr_t end,
1.233 thorpej 2906: vm_prot_t new_prot, bool set_max)
1.10 mrg 2907: {
1.99 chs 2908: struct vm_map_entry *current, *entry;
1.94 chs 2909: int error = 0;
1.10 mrg 2910: UVMHIST_FUNC("uvm_map_protect"); UVMHIST_CALLED(maphist);
2911: UVMHIST_LOG(maphist,"(map=0x%x,start=0x%x,end=0x%x,new_prot=0x%x)",
1.85 chs 2912: map, start, end, new_prot);
2913:
1.10 mrg 2914: vm_map_lock(map);
2915: VM_MAP_RANGE_CHECK(map, start, end);
2916: if (uvm_map_lookup_entry(map, start, &entry)) {
1.174 yamt 2917: UVM_MAP_CLIP_START(map, entry, start, NULL);
1.10 mrg 2918: } else {
2919: entry = entry->next;
2920: }
2921:
1.1 mrg 2922: /*
1.10 mrg 2923: * make a first pass to check for protection violations.
1.1 mrg 2924: */
2925:
1.10 mrg 2926: current = entry;
2927: while ((current != &map->header) && (current->start < end)) {
1.65 thorpej 2928: if (UVM_ET_ISSUBMAP(current)) {
1.94 chs 2929: error = EINVAL;
1.65 thorpej 2930: goto out;
2931: }
1.10 mrg 2932: if ((new_prot & current->max_protection) != new_prot) {
1.94 chs 2933: error = EACCES;
1.65 thorpej 2934: goto out;
1.112 thorpej 2935: }
2936: /*
2937: * Don't allow VM_PROT_EXECUTE to be set on entries that
2938: * point to vnodes that are associated with a NOEXEC file
2939: * system.
2940: */
2941: if (UVM_ET_ISOBJ(current) &&
2942: UVM_OBJ_IS_VNODE(current->object.uvm_obj)) {
2943: struct vnode *vp =
2944: (struct vnode *) current->object.uvm_obj;
2945:
2946: if ((new_prot & VM_PROT_EXECUTE) != 0 &&
2947: (vp->v_mount->mnt_flag & MNT_NOEXEC) != 0) {
2948: error = EACCES;
2949: goto out;
2950: }
1.10 mrg 2951: }
1.224 elad 2952:
1.65 thorpej 2953: current = current->next;
1.10 mrg 2954: }
2955:
2956: /* go back and fix up protections (no need to clip this time). */
2957:
2958: current = entry;
2959: while ((current != &map->header) && (current->start < end)) {
2960: vm_prot_t old_prot;
1.85 chs 2961:
1.174 yamt 2962: UVM_MAP_CLIP_END(map, current, end, NULL);
1.10 mrg 2963: old_prot = current->protection;
2964: if (set_max)
2965: current->protection =
2966: (current->max_protection = new_prot) & old_prot;
2967: else
2968: current->protection = new_prot;
2969:
2970: /*
1.98 chs 2971: * update physical map if necessary. worry about copy-on-write
1.10 mrg 2972: * here -- CHECK THIS XXX
2973: */
2974:
2975: if (current->protection != old_prot) {
1.29 chuck 2976: /* update pmap! */
2977: pmap_protect(map->pmap, current->start, current->end,
2978: current->protection & MASK(entry));
1.109 thorpej 2979:
2980: /*
2981: * If this entry points at a vnode, and the
2982: * protection includes VM_PROT_EXECUTE, mark
1.111 thorpej 2983: * the vnode as VEXECMAP.
1.109 thorpej 2984: */
2985: if (UVM_ET_ISOBJ(current)) {
2986: struct uvm_object *uobj =
2987: current->object.uvm_obj;
2988:
2989: if (UVM_OBJ_IS_VNODE(uobj) &&
2990: (current->protection & VM_PROT_EXECUTE))
1.110 thorpej 2991: vn_markexec((struct vnode *) uobj);
1.109 thorpej 2992: }
1.65 thorpej 2993: }
1.10 mrg 2994:
1.65 thorpej 2995: /*
2996: * If the map is configured to lock any future mappings,
2997: * wire this entry now if the old protection was VM_PROT_NONE
2998: * and the new protection is not VM_PROT_NONE.
2999: */
3000:
3001: if ((map->flags & VM_MAP_WIREFUTURE) != 0 &&
3002: VM_MAPENT_ISWIRED(entry) == 0 &&
3003: old_prot == VM_PROT_NONE &&
3004: new_prot != VM_PROT_NONE) {
3005: if (uvm_map_pageable(map, entry->start,
1.234 thorpej 3006: entry->end, false,
1.94 chs 3007: UVM_LK_ENTER|UVM_LK_EXIT) != 0) {
1.99 chs 3008:
1.65 thorpej 3009: /*
3010: * If locking the entry fails, remember the
3011: * error if it's the first one. Note we
3012: * still continue setting the protection in
1.94 chs 3013: * the map, but will return the error
3014: * condition regardless.
1.65 thorpej 3015: *
3016: * XXX Ignore what the actual error is,
3017: * XXX just call it a resource shortage
3018: * XXX so that it doesn't get confused
3019: * XXX what uvm_map_protect() itself would
3020: * XXX normally return.
3021: */
1.99 chs 3022:
1.94 chs 3023: error = ENOMEM;
1.65 thorpej 3024: }
1.10 mrg 3025: }
3026: current = current->next;
3027: }
1.105 chris 3028: pmap_update(map->pmap);
1.85 chs 3029:
1.65 thorpej 3030: out:
1.10 mrg 3031: vm_map_unlock(map);
1.174 yamt 3032:
1.94 chs 3033: UVMHIST_LOG(maphist, "<- done, error=%d",error,0,0,0);
3034: return error;
1.1 mrg 3035: }
3036:
3037: #undef MASK
3038:
1.98 chs 3039: /*
1.1 mrg 3040: * uvm_map_inherit: set inheritance code for range of addrs in map.
3041: *
3042: * => map must be unlocked
3043: * => note that the inherit code is used during a "fork". see fork
3044: * code for details.
3045: */
3046:
1.10 mrg 3047: int
1.138 enami 3048: uvm_map_inherit(struct vm_map *map, vaddr_t start, vaddr_t end,
3049: vm_inherit_t new_inheritance)
1.10 mrg 3050: {
1.99 chs 3051: struct vm_map_entry *entry, *temp_entry;
1.10 mrg 3052: UVMHIST_FUNC("uvm_map_inherit"); UVMHIST_CALLED(maphist);
3053: UVMHIST_LOG(maphist,"(map=0x%x,start=0x%x,end=0x%x,new_inh=0x%x)",
3054: map, start, end, new_inheritance);
3055:
3056: switch (new_inheritance) {
1.80 wiz 3057: case MAP_INHERIT_NONE:
3058: case MAP_INHERIT_COPY:
3059: case MAP_INHERIT_SHARE:
1.10 mrg 3060: break;
3061: default:
3062: UVMHIST_LOG(maphist,"<- done (INVALID ARG)",0,0,0,0);
1.94 chs 3063: return EINVAL;
1.10 mrg 3064: }
1.1 mrg 3065:
1.10 mrg 3066: vm_map_lock(map);
3067: VM_MAP_RANGE_CHECK(map, start, end);
3068: if (uvm_map_lookup_entry(map, start, &temp_entry)) {
3069: entry = temp_entry;
1.174 yamt 3070: UVM_MAP_CLIP_START(map, entry, start, NULL);
1.10 mrg 3071: } else {
3072: entry = temp_entry->next;
3073: }
3074: while ((entry != &map->header) && (entry->start < end)) {
1.174 yamt 3075: UVM_MAP_CLIP_END(map, entry, end, NULL);
1.10 mrg 3076: entry->inheritance = new_inheritance;
3077: entry = entry->next;
3078: }
3079: vm_map_unlock(map);
3080: UVMHIST_LOG(maphist,"<- done (OK)",0,0,0,0);
1.94 chs 3081: return 0;
1.41 mrg 3082: }
3083:
1.98 chs 3084: /*
1.41 mrg 3085: * uvm_map_advice: set advice code for range of addrs in map.
3086: *
3087: * => map must be unlocked
3088: */
3089:
3090: int
1.138 enami 3091: uvm_map_advice(struct vm_map *map, vaddr_t start, vaddr_t end, int new_advice)
1.41 mrg 3092: {
1.99 chs 3093: struct vm_map_entry *entry, *temp_entry;
1.41 mrg 3094: UVMHIST_FUNC("uvm_map_advice"); UVMHIST_CALLED(maphist);
3095: UVMHIST_LOG(maphist,"(map=0x%x,start=0x%x,end=0x%x,new_adv=0x%x)",
3096: map, start, end, new_advice);
3097:
3098: vm_map_lock(map);
3099: VM_MAP_RANGE_CHECK(map, start, end);
3100: if (uvm_map_lookup_entry(map, start, &temp_entry)) {
3101: entry = temp_entry;
1.174 yamt 3102: UVM_MAP_CLIP_START(map, entry, start, NULL);
1.41 mrg 3103: } else {
3104: entry = temp_entry->next;
3105: }
1.61 thorpej 3106:
3107: /*
3108: * XXXJRT: disallow holes?
3109: */
3110:
1.41 mrg 3111: while ((entry != &map->header) && (entry->start < end)) {
1.174 yamt 3112: UVM_MAP_CLIP_END(map, entry, end, NULL);
1.41 mrg 3113:
3114: switch (new_advice) {
3115: case MADV_NORMAL:
3116: case MADV_RANDOM:
3117: case MADV_SEQUENTIAL:
3118: /* nothing special here */
3119: break;
3120:
3121: default:
1.50 mrg 3122: vm_map_unlock(map);
1.41 mrg 3123: UVMHIST_LOG(maphist,"<- done (INVALID ARG)",0,0,0,0);
1.94 chs 3124: return EINVAL;
1.41 mrg 3125: }
3126: entry->advice = new_advice;
3127: entry = entry->next;
3128: }
3129:
3130: vm_map_unlock(map);
3131: UVMHIST_LOG(maphist,"<- done (OK)",0,0,0,0);
1.94 chs 3132: return 0;
1.1 mrg 3133: }
3134:
3135: /*
3136: * uvm_map_pageable: sets the pageability of a range in a map.
3137: *
1.56 thorpej 3138: * => wires map entries. should not be used for transient page locking.
3139: * for that, use uvm_fault_wire()/uvm_fault_unwire() (see uvm_vslock()).
1.216 drochner 3140: * => regions specified as not pageable require lock-down (wired) memory
1.1 mrg 3141: * and page tables.
1.59 thorpej 3142: * => map must never be read-locked
1.234 thorpej 3143: * => if islocked is true, map is already write-locked
1.59 thorpej 3144: * => we always unlock the map, since we must downgrade to a read-lock
3145: * to call uvm_fault_wire()
1.1 mrg 3146: * => XXXCDC: check this and try and clean it up.
3147: */
3148:
1.19 kleink 3149: int
1.138 enami 3150: uvm_map_pageable(struct vm_map *map, vaddr_t start, vaddr_t end,
1.233 thorpej 3151: bool new_pageable, int lockflags)
1.1 mrg 3152: {
1.99 chs 3153: struct vm_map_entry *entry, *start_entry, *failed_entry;
1.10 mrg 3154: int rv;
1.60 thorpej 3155: #ifdef DIAGNOSTIC
3156: u_int timestamp_save;
3157: #endif
1.10 mrg 3158: UVMHIST_FUNC("uvm_map_pageable"); UVMHIST_CALLED(maphist);
3159: UVMHIST_LOG(maphist,"(map=0x%x,start=0x%x,end=0x%x,new_pageable=0x%x)",
1.85 chs 3160: map, start, end, new_pageable);
3161: KASSERT(map->flags & VM_MAP_PAGEABLE);
1.45 thorpej 3162:
1.64 thorpej 3163: if ((lockflags & UVM_LK_ENTER) == 0)
1.59 thorpej 3164: vm_map_lock(map);
1.10 mrg 3165: VM_MAP_RANGE_CHECK(map, start, end);
3166:
1.98 chs 3167: /*
1.10 mrg 3168: * only one pageability change may take place at one time, since
3169: * uvm_fault_wire assumes it will be called only once for each
3170: * wiring/unwiring. therefore, we have to make sure we're actually
3171: * changing the pageability for the entire region. we do so before
1.98 chs 3172: * making any changes.
1.10 mrg 3173: */
3174:
1.234 thorpej 3175: if (uvm_map_lookup_entry(map, start, &start_entry) == false) {
1.64 thorpej 3176: if ((lockflags & UVM_LK_EXIT) == 0)
3177: vm_map_unlock(map);
1.85 chs 3178:
1.94 chs 3179: UVMHIST_LOG(maphist,"<- done (fault)",0,0,0,0);
3180: return EFAULT;
1.10 mrg 3181: }
3182: entry = start_entry;
3183:
1.98 chs 3184: /*
1.100 wiz 3185: * handle wiring and unwiring separately.
1.10 mrg 3186: */
1.1 mrg 3187:
1.56 thorpej 3188: if (new_pageable) { /* unwire */
1.174 yamt 3189: UVM_MAP_CLIP_START(map, entry, start, NULL);
1.85 chs 3190:
1.10 mrg 3191: /*
3192: * unwiring. first ensure that the range to be unwired is
1.98 chs 3193: * really wired down and that there are no holes.
1.10 mrg 3194: */
1.85 chs 3195:
1.10 mrg 3196: while ((entry != &map->header) && (entry->start < end)) {
3197: if (entry->wired_count == 0 ||
3198: (entry->end < end &&
1.55 thorpej 3199: (entry->next == &map->header ||
3200: entry->next->start > entry->end))) {
1.64 thorpej 3201: if ((lockflags & UVM_LK_EXIT) == 0)
3202: vm_map_unlock(map);
1.94 chs 3203: UVMHIST_LOG(maphist, "<- done (INVAL)",0,0,0,0);
3204: return EINVAL;
1.10 mrg 3205: }
3206: entry = entry->next;
3207: }
3208:
1.98 chs 3209: /*
1.56 thorpej 3210: * POSIX 1003.1b - a single munlock call unlocks a region,
3211: * regardless of the number of mlock calls made on that
3212: * region.
1.10 mrg 3213: */
1.85 chs 3214:
1.10 mrg 3215: entry = start_entry;
3216: while ((entry != &map->header) && (entry->start < end)) {
1.174 yamt 3217: UVM_MAP_CLIP_END(map, entry, end, NULL);
1.56 thorpej 3218: if (VM_MAPENT_ISWIRED(entry))
1.10 mrg 3219: uvm_map_entry_unwire(map, entry);
3220: entry = entry->next;
3221: }
1.64 thorpej 3222: if ((lockflags & UVM_LK_EXIT) == 0)
3223: vm_map_unlock(map);
1.10 mrg 3224: UVMHIST_LOG(maphist,"<- done (OK UNWIRE)",0,0,0,0);
1.94 chs 3225: return 0;
1.10 mrg 3226: }
3227:
3228: /*
3229: * wire case: in two passes [XXXCDC: ugly block of code here]
3230: *
3231: * 1: holding the write lock, we create any anonymous maps that need
3232: * to be created. then we clip each map entry to the region to
1.98 chs 3233: * be wired and increment its wiring count.
1.10 mrg 3234: *
3235: * 2: we downgrade to a read lock, and call uvm_fault_wire to fault
1.56 thorpej 3236: * in the pages for any newly wired area (wired_count == 1).
1.10 mrg 3237: *
3238: * downgrading to a read lock for uvm_fault_wire avoids a possible
3239: * deadlock with another thread that may have faulted on one of
3240: * the pages to be wired (it would mark the page busy, blocking
3241: * us, then in turn block on the map lock that we hold). because
3242: * of problems in the recursive lock package, we cannot upgrade
3243: * to a write lock in vm_map_lookup. thus, any actions that
3244: * require the write lock must be done beforehand. because we
3245: * keep the read lock on the map, the copy-on-write status of the
3246: * entries we modify here cannot change.
3247: */
3248:
3249: while ((entry != &map->header) && (entry->start < end)) {
1.55 thorpej 3250: if (VM_MAPENT_ISWIRED(entry) == 0) { /* not already wired? */
1.85 chs 3251:
3252: /*
1.10 mrg 3253: * perform actions of vm_map_lookup that need the
3254: * write lock on the map: create an anonymous map
3255: * for a copy-on-write region, or an anonymous map
1.29 chuck 3256: * for a zero-fill region. (XXXCDC: submap case
3257: * ok?)
1.10 mrg 3258: */
1.85 chs 3259:
1.29 chuck 3260: if (!UVM_ET_ISSUBMAP(entry)) { /* not submap */
1.98 chs 3261: if (UVM_ET_ISNEEDSCOPY(entry) &&
1.117 chs 3262: ((entry->max_protection & VM_PROT_WRITE) ||
1.54 thorpej 3263: (entry->object.uvm_obj == NULL))) {
1.212 yamt 3264: amap_copy(map, entry, 0, start, end);
1.10 mrg 3265: /* XXXCDC: wait OK? */
3266: }
3267: }
1.55 thorpej 3268: }
1.174 yamt 3269: UVM_MAP_CLIP_START(map, entry, start, NULL);
3270: UVM_MAP_CLIP_END(map, entry, end, NULL);
1.10 mrg 3271: entry->wired_count++;
3272:
3273: /*
1.98 chs 3274: * Check for holes
1.10 mrg 3275: */
1.85 chs 3276:
1.54 thorpej 3277: if (entry->protection == VM_PROT_NONE ||
3278: (entry->end < end &&
3279: (entry->next == &map->header ||
3280: entry->next->start > entry->end))) {
1.85 chs 3281:
1.10 mrg 3282: /*
3283: * found one. amap creation actions do not need to
1.98 chs 3284: * be undone, but the wired counts need to be restored.
1.10 mrg 3285: */
1.85 chs 3286:
1.10 mrg 3287: while (entry != &map->header && entry->end > start) {
3288: entry->wired_count--;
3289: entry = entry->prev;
3290: }
1.64 thorpej 3291: if ((lockflags & UVM_LK_EXIT) == 0)
3292: vm_map_unlock(map);
1.10 mrg 3293: UVMHIST_LOG(maphist,"<- done (INVALID WIRE)",0,0,0,0);
1.94 chs 3294: return EINVAL;
1.10 mrg 3295: }
3296: entry = entry->next;
3297: }
3298:
3299: /*
3300: * Pass 2.
3301: */
1.51 thorpej 3302:
1.60 thorpej 3303: #ifdef DIAGNOSTIC
3304: timestamp_save = map->timestamp;
3305: #endif
3306: vm_map_busy(map);
1.51 thorpej 3307: vm_map_downgrade(map);
1.10 mrg 3308:
3309: rv = 0;
3310: entry = start_entry;
3311: while (entry != &map->header && entry->start < end) {
1.51 thorpej 3312: if (entry->wired_count == 1) {
1.44 thorpej 3313: rv = uvm_fault_wire(map, entry->start, entry->end,
1.216 drochner 3314: entry->max_protection, 1);
1.10 mrg 3315: if (rv) {
1.94 chs 3316:
1.51 thorpej 3317: /*
3318: * wiring failed. break out of the loop.
3319: * we'll clean up the map below, once we
3320: * have a write lock again.
3321: */
1.94 chs 3322:
1.51 thorpej 3323: break;
1.10 mrg 3324: }
3325: }
3326: entry = entry->next;
3327: }
3328:
1.139 enami 3329: if (rv) { /* failed? */
1.85 chs 3330:
1.52 thorpej 3331: /*
3332: * Get back to an exclusive (write) lock.
3333: */
1.85 chs 3334:
1.52 thorpej 3335: vm_map_upgrade(map);
1.60 thorpej 3336: vm_map_unbusy(map);
3337:
3338: #ifdef DIAGNOSTIC
3339: if (timestamp_save != map->timestamp)
3340: panic("uvm_map_pageable: stale map");
3341: #endif
1.10 mrg 3342:
1.51 thorpej 3343: /*
3344: * first drop the wiring count on all the entries
3345: * which haven't actually been wired yet.
3346: */
1.85 chs 3347:
1.54 thorpej 3348: failed_entry = entry;
3349: while (entry != &map->header && entry->start < end) {
1.51 thorpej 3350: entry->wired_count--;
1.54 thorpej 3351: entry = entry->next;
3352: }
1.51 thorpej 3353:
3354: /*
1.54 thorpej 3355: * now, unwire all the entries that were successfully
3356: * wired above.
1.51 thorpej 3357: */
1.85 chs 3358:
1.54 thorpej 3359: entry = start_entry;
3360: while (entry != failed_entry) {
3361: entry->wired_count--;
1.55 thorpej 3362: if (VM_MAPENT_ISWIRED(entry) == 0)
1.54 thorpej 3363: uvm_map_entry_unwire(map, entry);
3364: entry = entry->next;
3365: }
1.64 thorpej 3366: if ((lockflags & UVM_LK_EXIT) == 0)
3367: vm_map_unlock(map);
1.10 mrg 3368: UVMHIST_LOG(maphist, "<- done (RV=%d)", rv,0,0,0);
1.139 enami 3369: return (rv);
1.10 mrg 3370: }
1.51 thorpej 3371:
1.52 thorpej 3372: /* We are holding a read lock here. */
1.64 thorpej 3373: if ((lockflags & UVM_LK_EXIT) == 0) {
3374: vm_map_unbusy(map);
3375: vm_map_unlock_read(map);
3376: } else {
1.85 chs 3377:
1.64 thorpej 3378: /*
3379: * Get back to an exclusive (write) lock.
3380: */
1.85 chs 3381:
1.64 thorpej 3382: vm_map_upgrade(map);
3383: vm_map_unbusy(map);
3384: }
3385:
1.10 mrg 3386: UVMHIST_LOG(maphist,"<- done (OK WIRE)",0,0,0,0);
1.94 chs 3387: return 0;
1.1 mrg 3388: }
3389:
3390: /*
1.54 thorpej 3391: * uvm_map_pageable_all: special case of uvm_map_pageable - affects
3392: * all mapped regions.
3393: *
3394: * => map must not be locked.
3395: * => if no flags are specified, all regions are unwired.
3396: * => XXXJRT: has some of the same problems as uvm_map_pageable() above.
3397: */
3398:
3399: int
1.138 enami 3400: uvm_map_pageable_all(struct vm_map *map, int flags, vsize_t limit)
1.54 thorpej 3401: {
1.99 chs 3402: struct vm_map_entry *entry, *failed_entry;
1.54 thorpej 3403: vsize_t size;
3404: int rv;
1.60 thorpej 3405: #ifdef DIAGNOSTIC
3406: u_int timestamp_save;
3407: #endif
1.54 thorpej 3408: UVMHIST_FUNC("uvm_map_pageable_all"); UVMHIST_CALLED(maphist);
3409: UVMHIST_LOG(maphist,"(map=0x%x,flags=0x%x)", map, flags, 0, 0);
3410:
1.85 chs 3411: KASSERT(map->flags & VM_MAP_PAGEABLE);
1.54 thorpej 3412:
3413: vm_map_lock(map);
3414:
3415: /*
3416: * handle wiring and unwiring separately.
3417: */
3418:
3419: if (flags == 0) { /* unwire */
1.99 chs 3420:
1.54 thorpej 3421: /*
1.56 thorpej 3422: * POSIX 1003.1b -- munlockall unlocks all regions,
3423: * regardless of how many times mlockall has been called.
1.54 thorpej 3424: */
1.99 chs 3425:
1.54 thorpej 3426: for (entry = map->header.next; entry != &map->header;
3427: entry = entry->next) {
1.56 thorpej 3428: if (VM_MAPENT_ISWIRED(entry))
3429: uvm_map_entry_unwire(map, entry);
1.54 thorpej 3430: }
1.238 ad 3431: map->flags &= ~VM_MAP_WIREFUTURE;
1.54 thorpej 3432: vm_map_unlock(map);
3433: UVMHIST_LOG(maphist,"<- done (OK UNWIRE)",0,0,0,0);
1.94 chs 3434: return 0;
1.54 thorpej 3435: }
3436:
3437: if (flags & MCL_FUTURE) {
1.99 chs 3438:
1.54 thorpej 3439: /*
3440: * must wire all future mappings; remember this.
3441: */
1.99 chs 3442:
1.238 ad 3443: map->flags |= VM_MAP_WIREFUTURE;
1.54 thorpej 3444: }
3445:
3446: if ((flags & MCL_CURRENT) == 0) {
1.99 chs 3447:
1.54 thorpej 3448: /*
3449: * no more work to do!
3450: */
1.99 chs 3451:
1.54 thorpej 3452: UVMHIST_LOG(maphist,"<- done (OK no wire)",0,0,0,0);
3453: vm_map_unlock(map);
1.94 chs 3454: return 0;
1.54 thorpej 3455: }
3456:
3457: /*
3458: * wire case: in three passes [XXXCDC: ugly block of code here]
3459: *
3460: * 1: holding the write lock, count all pages mapped by non-wired
3461: * entries. if this would cause us to go over our limit, we fail.
3462: *
3463: * 2: still holding the write lock, we create any anonymous maps that
3464: * need to be created. then we increment its wiring count.
3465: *
3466: * 3: we downgrade to a read lock, and call uvm_fault_wire to fault
1.56 thorpej 3467: * in the pages for any newly wired area (wired_count == 1).
1.54 thorpej 3468: *
3469: * downgrading to a read lock for uvm_fault_wire avoids a possible
3470: * deadlock with another thread that may have faulted on one of
3471: * the pages to be wired (it would mark the page busy, blocking
3472: * us, then in turn block on the map lock that we hold). because
3473: * of problems in the recursive lock package, we cannot upgrade
3474: * to a write lock in vm_map_lookup. thus, any actions that
3475: * require the write lock must be done beforehand. because we
3476: * keep the read lock on the map, the copy-on-write status of the
3477: * entries we modify here cannot change.
3478: */
3479:
3480: for (size = 0, entry = map->header.next; entry != &map->header;
3481: entry = entry->next) {
3482: if (entry->protection != VM_PROT_NONE &&
1.55 thorpej 3483: VM_MAPENT_ISWIRED(entry) == 0) { /* not already wired? */
1.54 thorpej 3484: size += entry->end - entry->start;
3485: }
3486: }
3487:
3488: if (atop(size) + uvmexp.wired > uvmexp.wiredmax) {
3489: vm_map_unlock(map);
1.94 chs 3490: return ENOMEM;
1.54 thorpej 3491: }
3492:
3493: if (limit != 0 &&
3494: (size + ptoa(pmap_wired_count(vm_map_pmap(map))) > limit)) {
3495: vm_map_unlock(map);
1.94 chs 3496: return ENOMEM;
1.54 thorpej 3497: }
3498:
3499: /*
3500: * Pass 2.
3501: */
3502:
3503: for (entry = map->header.next; entry != &map->header;
3504: entry = entry->next) {
3505: if (entry->protection == VM_PROT_NONE)
3506: continue;
1.55 thorpej 3507: if (VM_MAPENT_ISWIRED(entry) == 0) { /* not already wired? */
1.99 chs 3508:
1.54 thorpej 3509: /*
3510: * perform actions of vm_map_lookup that need the
3511: * write lock on the map: create an anonymous map
3512: * for a copy-on-write region, or an anonymous map
3513: * for a zero-fill region. (XXXCDC: submap case
3514: * ok?)
3515: */
1.99 chs 3516:
1.54 thorpej 3517: if (!UVM_ET_ISSUBMAP(entry)) { /* not submap */
1.98 chs 3518: if (UVM_ET_ISNEEDSCOPY(entry) &&
1.117 chs 3519: ((entry->max_protection & VM_PROT_WRITE) ||
1.54 thorpej 3520: (entry->object.uvm_obj == NULL))) {
1.212 yamt 3521: amap_copy(map, entry, 0, entry->start,
3522: entry->end);
1.54 thorpej 3523: /* XXXCDC: wait OK? */
3524: }
3525: }
1.55 thorpej 3526: }
1.54 thorpej 3527: entry->wired_count++;
3528: }
3529:
3530: /*
3531: * Pass 3.
3532: */
3533:
1.60 thorpej 3534: #ifdef DIAGNOSTIC
3535: timestamp_save = map->timestamp;
3536: #endif
3537: vm_map_busy(map);
1.54 thorpej 3538: vm_map_downgrade(map);
3539:
1.94 chs 3540: rv = 0;
1.54 thorpej 3541: for (entry = map->header.next; entry != &map->header;
3542: entry = entry->next) {
3543: if (entry->wired_count == 1) {
3544: rv = uvm_fault_wire(map, entry->start, entry->end,
1.216 drochner 3545: entry->max_protection, 1);
1.54 thorpej 3546: if (rv) {
1.99 chs 3547:
1.54 thorpej 3548: /*
3549: * wiring failed. break out of the loop.
3550: * we'll clean up the map below, once we
3551: * have a write lock again.
3552: */
1.99 chs 3553:
1.54 thorpej 3554: break;
3555: }
3556: }
3557: }
3558:
1.99 chs 3559: if (rv) {
3560:
1.54 thorpej 3561: /*
3562: * Get back an exclusive (write) lock.
3563: */
1.99 chs 3564:
1.54 thorpej 3565: vm_map_upgrade(map);
1.60 thorpej 3566: vm_map_unbusy(map);
3567:
3568: #ifdef DIAGNOSTIC
3569: if (timestamp_save != map->timestamp)
3570: panic("uvm_map_pageable_all: stale map");
3571: #endif
1.54 thorpej 3572:
3573: /*
3574: * first drop the wiring count on all the entries
3575: * which haven't actually been wired yet.
1.67 thorpej 3576: *
3577: * Skip VM_PROT_NONE entries like we did above.
1.54 thorpej 3578: */
1.99 chs 3579:
1.54 thorpej 3580: failed_entry = entry;
3581: for (/* nothing */; entry != &map->header;
1.67 thorpej 3582: entry = entry->next) {
3583: if (entry->protection == VM_PROT_NONE)
3584: continue;
1.54 thorpej 3585: entry->wired_count--;
1.67 thorpej 3586: }
1.54 thorpej 3587:
3588: /*
3589: * now, unwire all the entries that were successfully
3590: * wired above.
1.67 thorpej 3591: *
3592: * Skip VM_PROT_NONE entries like we did above.
1.54 thorpej 3593: */
1.99 chs 3594:
1.54 thorpej 3595: for (entry = map->header.next; entry != failed_entry;
3596: entry = entry->next) {
1.67 thorpej 3597: if (entry->protection == VM_PROT_NONE)
3598: continue;
1.54 thorpej 3599: entry->wired_count--;
1.67 thorpej 3600: if (VM_MAPENT_ISWIRED(entry))
1.54 thorpej 3601: uvm_map_entry_unwire(map, entry);
3602: }
3603: vm_map_unlock(map);
3604: UVMHIST_LOG(maphist,"<- done (RV=%d)", rv,0,0,0);
3605: return (rv);
3606: }
3607:
3608: /* We are holding a read lock here. */
1.60 thorpej 3609: vm_map_unbusy(map);
1.54 thorpej 3610: vm_map_unlock_read(map);
3611:
3612: UVMHIST_LOG(maphist,"<- done (OK WIRE)",0,0,0,0);
1.94 chs 3613: return 0;
1.54 thorpej 3614: }
3615:
3616: /*
1.61 thorpej 3617: * uvm_map_clean: clean out a map range
1.1 mrg 3618: *
3619: * => valid flags:
1.61 thorpej 3620: * if (flags & PGO_CLEANIT): dirty pages are cleaned first
1.1 mrg 3621: * if (flags & PGO_SYNCIO): dirty pages are written synchronously
3622: * if (flags & PGO_DEACTIVATE): any cached pages are deactivated after clean
3623: * if (flags & PGO_FREE): any cached pages are freed after clean
3624: * => returns an error if any part of the specified range isn't mapped
1.98 chs 3625: * => never a need to flush amap layer since the anonymous memory has
1.61 thorpej 3626: * no permanent home, but may deactivate pages there
3627: * => called from sys_msync() and sys_madvise()
1.1 mrg 3628: * => caller must not write-lock map (read OK).
3629: * => we may sleep while cleaning if SYNCIO [with map read-locked]
3630: */
3631:
1.10 mrg 3632: int
1.138 enami 3633: uvm_map_clean(struct vm_map *map, vaddr_t start, vaddr_t end, int flags)
1.10 mrg 3634: {
1.99 chs 3635: struct vm_map_entry *current, *entry;
1.61 thorpej 3636: struct uvm_object *uobj;
3637: struct vm_amap *amap;
3638: struct vm_anon *anon;
3639: struct vm_page *pg;
3640: vaddr_t offset;
1.24 eeh 3641: vsize_t size;
1.188 dbj 3642: voff_t uoff;
1.106 chs 3643: int error, refs;
1.10 mrg 3644: UVMHIST_FUNC("uvm_map_clean"); UVMHIST_CALLED(maphist);
1.85 chs 3645:
1.10 mrg 3646: UVMHIST_LOG(maphist,"(map=0x%x,start=0x%x,end=0x%x,flags=0x%x)",
1.85 chs 3647: map, start, end, flags);
3648: KASSERT((flags & (PGO_FREE|PGO_DEACTIVATE)) !=
3649: (PGO_FREE|PGO_DEACTIVATE));
1.61 thorpej 3650:
1.10 mrg 3651: vm_map_lock_read(map);
3652: VM_MAP_RANGE_CHECK(map, start, end);
1.234 thorpej 3653: if (uvm_map_lookup_entry(map, start, &entry) == false) {
1.10 mrg 3654: vm_map_unlock_read(map);
1.94 chs 3655: return EFAULT;
1.10 mrg 3656: }
3657:
3658: /*
1.186 chs 3659: * Make a first pass to check for holes and wiring problems.
1.10 mrg 3660: */
1.85 chs 3661:
1.10 mrg 3662: for (current = entry; current->start < end; current = current->next) {
3663: if (UVM_ET_ISSUBMAP(current)) {
3664: vm_map_unlock_read(map);
1.94 chs 3665: return EINVAL;
1.10 mrg 3666: }
1.186 chs 3667: if ((flags & PGO_FREE) != 0 && VM_MAPENT_ISWIRED(entry)) {
3668: vm_map_unlock_read(map);
3669: return EBUSY;
3670: }
1.90 chs 3671: if (end <= current->end) {
3672: break;
3673: }
3674: if (current->end != current->next->start) {
1.10 mrg 3675: vm_map_unlock_read(map);
1.94 chs 3676: return EFAULT;
1.10 mrg 3677: }
3678: }
3679:
1.94 chs 3680: error = 0;
1.90 chs 3681: for (current = entry; start < end; current = current->next) {
1.61 thorpej 3682: amap = current->aref.ar_amap; /* top layer */
3683: uobj = current->object.uvm_obj; /* bottom layer */
1.85 chs 3684: KASSERT(start >= current->start);
1.1 mrg 3685:
1.10 mrg 3686: /*
1.61 thorpej 3687: * No amap cleaning necessary if:
3688: *
3689: * (1) There's no amap.
3690: *
3691: * (2) We're not deactivating or freeing pages.
1.10 mrg 3692: */
1.85 chs 3693:
1.90 chs 3694: if (amap == NULL || (flags & (PGO_DEACTIVATE|PGO_FREE)) == 0)
1.61 thorpej 3695: goto flush_object;
3696:
3697: amap_lock(amap);
3698: offset = start - current->start;
1.90 chs 3699: size = MIN(end, current->end) - start;
3700: for ( ; size != 0; size -= PAGE_SIZE, offset += PAGE_SIZE) {
1.61 thorpej 3701: anon = amap_lookup(¤t->aref, offset);
3702: if (anon == NULL)
3703: continue;
3704:
3705: simple_lock(&anon->an_lock);
1.192 yamt 3706: pg = anon->an_page;
1.63 thorpej 3707: if (pg == NULL) {
3708: simple_unlock(&anon->an_lock);
3709: continue;
3710: }
3711:
1.61 thorpej 3712: switch (flags & (PGO_CLEANIT|PGO_FREE|PGO_DEACTIVATE)) {
1.85 chs 3713:
1.61 thorpej 3714: /*
1.115 chs 3715: * In these first 3 cases, we just deactivate the page.
1.61 thorpej 3716: */
1.85 chs 3717:
1.61 thorpej 3718: case PGO_CLEANIT|PGO_FREE:
3719: case PGO_CLEANIT|PGO_DEACTIVATE:
3720: case PGO_DEACTIVATE:
1.68 thorpej 3721: deactivate_it:
1.61 thorpej 3722: /*
1.115 chs 3723: * skip the page if it's loaned or wired,
3724: * since it shouldn't be on a paging queue
3725: * at all in these cases.
1.61 thorpej 3726: */
1.85 chs 3727:
1.115 chs 3728: uvm_lock_pageq();
3729: if (pg->loan_count != 0 ||
3730: pg->wire_count != 0) {
1.61 thorpej 3731: uvm_unlock_pageq();
3732: simple_unlock(&anon->an_lock);
3733: continue;
3734: }
1.85 chs 3735: KASSERT(pg->uanon == anon);
1.89 thorpej 3736: pmap_clear_reference(pg);
1.61 thorpej 3737: uvm_pagedeactivate(pg);
3738: uvm_unlock_pageq();
3739: simple_unlock(&anon->an_lock);
3740: continue;
3741:
3742: case PGO_FREE:
1.85 chs 3743:
1.68 thorpej 3744: /*
3745: * If there are multiple references to
3746: * the amap, just deactivate the page.
3747: */
1.85 chs 3748:
1.68 thorpej 3749: if (amap_refs(amap) > 1)
3750: goto deactivate_it;
3751:
1.115 chs 3752: /* skip the page if it's wired */
1.62 thorpej 3753: if (pg->wire_count != 0) {
3754: simple_unlock(&anon->an_lock);
3755: continue;
3756: }
1.66 thorpej 3757: amap_unadd(¤t->aref, offset);
1.61 thorpej 3758: refs = --anon->an_ref;
3759: simple_unlock(&anon->an_lock);
3760: if (refs == 0)
3761: uvm_anfree(anon);
3762: continue;
3763: }
3764: }
3765: amap_unlock(amap);
1.1 mrg 3766:
1.61 thorpej 3767: flush_object:
1.10 mrg 3768: /*
1.33 chuck 3769: * flush pages if we've got a valid backing object.
1.116 chs 3770: * note that we must always clean object pages before
3771: * freeing them since otherwise we could reveal stale
3772: * data from files.
1.10 mrg 3773: */
1.1 mrg 3774:
1.188 dbj 3775: uoff = current->offset + (start - current->start);
1.90 chs 3776: size = MIN(end, current->end) - start;
1.61 thorpej 3777: if (uobj != NULL) {
3778: simple_lock(&uobj->vmobjlock);
1.136 thorpej 3779: if (uobj->pgops->pgo_put != NULL)
1.188 dbj 3780: error = (uobj->pgops->pgo_put)(uobj, uoff,
3781: uoff + size, flags | PGO_CLEANIT);
1.136 thorpej 3782: else
3783: error = 0;
1.10 mrg 3784: }
3785: start += size;
3786: }
1.1 mrg 3787: vm_map_unlock_read(map);
1.98 chs 3788: return (error);
1.1 mrg 3789: }
3790:
3791:
3792: /*
3793: * uvm_map_checkprot: check protection in map
3794: *
3795: * => must allow specified protection in a fully allocated region.
3796: * => map must be read or write locked by caller.
3797: */
3798:
1.233 thorpej 3799: bool
1.138 enami 3800: uvm_map_checkprot(struct vm_map *map, vaddr_t start, vaddr_t end,
3801: vm_prot_t protection)
1.10 mrg 3802: {
1.99 chs 3803: struct vm_map_entry *entry;
3804: struct vm_map_entry *tmp_entry;
1.10 mrg 3805:
1.94 chs 3806: if (!uvm_map_lookup_entry(map, start, &tmp_entry)) {
1.234 thorpej 3807: return (false);
1.94 chs 3808: }
3809: entry = tmp_entry;
3810: while (start < end) {
3811: if (entry == &map->header) {
1.234 thorpej 3812: return (false);
1.94 chs 3813: }
1.85 chs 3814:
1.10 mrg 3815: /*
3816: * no holes allowed
3817: */
3818:
1.94 chs 3819: if (start < entry->start) {
1.234 thorpej 3820: return (false);
1.94 chs 3821: }
1.10 mrg 3822:
3823: /*
3824: * check protection associated with entry
3825: */
1.1 mrg 3826:
1.94 chs 3827: if ((entry->protection & protection) != protection) {
1.234 thorpej 3828: return (false);
1.94 chs 3829: }
3830: start = entry->end;
3831: entry = entry->next;
3832: }
1.234 thorpej 3833: return (true);
1.1 mrg 3834: }
3835:
3836: /*
3837: * uvmspace_alloc: allocate a vmspace structure.
3838: *
3839: * - structure includes vm_map and pmap
3840: * - XXX: no locking on this structure
3841: * - refcnt set to 1, rest must be init'd by caller
3842: */
1.10 mrg 3843: struct vmspace *
1.199 christos 3844: uvmspace_alloc(vaddr_t vmin, vaddr_t vmax)
1.10 mrg 3845: {
3846: struct vmspace *vm;
3847: UVMHIST_FUNC("uvmspace_alloc"); UVMHIST_CALLED(maphist);
3848:
1.25 thorpej 3849: vm = pool_get(&uvm_vmspace_pool, PR_WAITOK);
1.199 christos 3850: uvmspace_init(vm, NULL, vmin, vmax);
1.15 thorpej 3851: UVMHIST_LOG(maphist,"<- done (vm=0x%x)", vm,0,0,0);
3852: return (vm);
3853: }
3854:
3855: /*
3856: * uvmspace_init: initialize a vmspace structure.
3857: *
3858: * - XXX: no locking on this structure
1.132 matt 3859: * - refcnt set to 1, rest must be init'd by caller
1.15 thorpej 3860: */
3861: void
1.199 christos 3862: uvmspace_init(struct vmspace *vm, struct pmap *pmap, vaddr_t vmin, vaddr_t vmax)
1.15 thorpej 3863: {
3864: UVMHIST_FUNC("uvmspace_init"); UVMHIST_CALLED(maphist);
3865:
1.23 perry 3866: memset(vm, 0, sizeof(*vm));
1.199 christos 3867: uvm_map_setup(&vm->vm_map, vmin, vmax, VM_MAP_PAGEABLE
1.131 atatat 3868: #ifdef __USING_TOPDOWN_VM
3869: | VM_MAP_TOPDOWN
3870: #endif
3871: );
1.15 thorpej 3872: if (pmap)
3873: pmap_reference(pmap);
3874: else
3875: pmap = pmap_create();
3876: vm->vm_map.pmap = pmap;
1.10 mrg 3877: vm->vm_refcnt = 1;
1.15 thorpej 3878: UVMHIST_LOG(maphist,"<- done",0,0,0,0);
1.1 mrg 3879: }
3880:
3881: /*
1.168 junyoung 3882: * uvmspace_share: share a vmspace between two processes
1.1 mrg 3883: *
3884: * - used for vfork, threads(?)
3885: */
3886:
1.10 mrg 3887: void
1.138 enami 3888: uvmspace_share(struct proc *p1, struct proc *p2)
1.1 mrg 3889: {
1.139 enami 3890:
1.215 yamt 3891: uvmspace_addref(p1->p_vmspace);
1.10 mrg 3892: p2->p_vmspace = p1->p_vmspace;
1.1 mrg 3893: }
3894:
3895: /*
3896: * uvmspace_unshare: ensure that process "p" has its own, unshared, vmspace
3897: *
3898: * - XXX: no locking on vmspace
3899: */
3900:
1.10 mrg 3901: void
1.138 enami 3902: uvmspace_unshare(struct lwp *l)
1.10 mrg 3903: {
1.128 thorpej 3904: struct proc *p = l->l_proc;
1.10 mrg 3905: struct vmspace *nvm, *ovm = p->p_vmspace;
1.85 chs 3906:
1.10 mrg 3907: if (ovm->vm_refcnt == 1)
3908: /* nothing to do: vmspace isn't shared in the first place */
3909: return;
1.85 chs 3910:
1.10 mrg 3911: /* make a new vmspace, still holding old one */
3912: nvm = uvmspace_fork(ovm);
3913:
1.128 thorpej 3914: pmap_deactivate(l); /* unbind old vmspace */
1.98 chs 3915: p->p_vmspace = nvm;
1.128 thorpej 3916: pmap_activate(l); /* switch to new vmspace */
1.13 thorpej 3917:
1.10 mrg 3918: uvmspace_free(ovm); /* drop reference to old vmspace */
1.1 mrg 3919: }
3920:
3921: /*
3922: * uvmspace_exec: the process wants to exec a new program
3923: */
3924:
1.10 mrg 3925: void
1.138 enami 3926: uvmspace_exec(struct lwp *l, vaddr_t start, vaddr_t end)
1.1 mrg 3927: {
1.128 thorpej 3928: struct proc *p = l->l_proc;
1.10 mrg 3929: struct vmspace *nvm, *ovm = p->p_vmspace;
1.99 chs 3930: struct vm_map *map = &ovm->vm_map;
1.1 mrg 3931:
1.71 chs 3932: #ifdef __sparc__
1.10 mrg 3933: /* XXX cgd 960926: the sparc #ifdef should be a MD hook */
1.128 thorpej 3934: kill_user_windows(l); /* before stack addresses go away */
1.1 mrg 3935: #endif
3936:
1.10 mrg 3937: /*
3938: * see if more than one process is using this vmspace...
3939: */
1.1 mrg 3940:
1.10 mrg 3941: if (ovm->vm_refcnt == 1) {
1.1 mrg 3942:
1.10 mrg 3943: /*
3944: * if p is the only process using its vmspace then we can safely
3945: * recycle that vmspace for the program that is being exec'd.
3946: */
1.1 mrg 3947:
3948: #ifdef SYSVSHM
1.10 mrg 3949: /*
3950: * SYSV SHM semantics require us to kill all segments on an exec
3951: */
1.99 chs 3952:
1.10 mrg 3953: if (ovm->vm_shm)
3954: shmexit(ovm);
3955: #endif
1.54 thorpej 3956:
3957: /*
3958: * POSIX 1003.1b -- "lock future mappings" is revoked
3959: * when a process execs another program image.
3960: */
1.99 chs 3961:
1.238 ad 3962: map->flags &= ~VM_MAP_WIREFUTURE;
1.10 mrg 3963:
3964: /*
3965: * now unmap the old program
3966: */
1.99 chs 3967:
1.120 chs 3968: pmap_remove_all(map->pmap);
1.184 chs 3969: uvm_unmap(map, vm_map_min(map), vm_map_max(map));
1.144 yamt 3970: KASSERT(map->header.prev == &map->header);
3971: KASSERT(map->nentries == 0);
1.93 eeh 3972:
3973: /*
3974: * resize the map
3975: */
1.99 chs 3976:
1.184 chs 3977: vm_map_setmin(map, start);
3978: vm_map_setmax(map, end);
1.10 mrg 3979: } else {
3980:
3981: /*
3982: * p's vmspace is being shared, so we can't reuse it for p since
3983: * it is still being used for others. allocate a new vmspace
3984: * for p
3985: */
1.99 chs 3986:
1.101 chs 3987: nvm = uvmspace_alloc(start, end);
1.1 mrg 3988:
1.10 mrg 3989: /*
3990: * install new vmspace and drop our ref to the old one.
3991: */
3992:
1.128 thorpej 3993: pmap_deactivate(l);
1.10 mrg 3994: p->p_vmspace = nvm;
1.128 thorpej 3995: pmap_activate(l);
1.13 thorpej 3996:
1.10 mrg 3997: uvmspace_free(ovm);
3998: }
1.1 mrg 3999: }
4000:
4001: /*
1.215 yamt 4002: * uvmspace_addref: add a referece to a vmspace.
4003: */
4004:
4005: void
4006: uvmspace_addref(struct vmspace *vm)
4007: {
4008: struct vm_map *map = &vm->vm_map;
4009:
4010: KASSERT((map->flags & VM_MAP_DYING) == 0);
4011:
1.238 ad 4012: mutex_enter(&map->misc_lock);
1.215 yamt 4013: KASSERT(vm->vm_refcnt > 0);
4014: vm->vm_refcnt++;
1.238 ad 4015: mutex_exit(&map->misc_lock);
1.215 yamt 4016: }
4017:
4018: /*
1.1 mrg 4019: * uvmspace_free: free a vmspace data structure
4020: */
4021:
1.10 mrg 4022: void
1.138 enami 4023: uvmspace_free(struct vmspace *vm)
1.1 mrg 4024: {
1.99 chs 4025: struct vm_map_entry *dead_entries;
1.171 pk 4026: struct vm_map *map = &vm->vm_map;
1.172 he 4027: int n;
4028:
1.10 mrg 4029: UVMHIST_FUNC("uvmspace_free"); UVMHIST_CALLED(maphist);
1.1 mrg 4030:
1.10 mrg 4031: UVMHIST_LOG(maphist,"(vm=0x%x) ref=%d", vm, vm->vm_refcnt,0,0);
1.238 ad 4032: mutex_enter(&map->misc_lock);
1.172 he 4033: n = --vm->vm_refcnt;
1.238 ad 4034: mutex_exit(&map->misc_lock);
1.171 pk 4035: if (n > 0)
1.120 chs 4036: return;
1.99 chs 4037:
1.120 chs 4038: /*
4039: * at this point, there should be no other references to the map.
4040: * delete all of the mappings, then destroy the pmap.
4041: */
1.99 chs 4042:
1.120 chs 4043: map->flags |= VM_MAP_DYING;
4044: pmap_remove_all(map->pmap);
1.92 thorpej 4045: #ifdef SYSVSHM
1.120 chs 4046: /* Get rid of any SYSV shared memory segments. */
4047: if (vm->vm_shm != NULL)
4048: shmexit(vm);
1.92 thorpej 4049: #endif
1.120 chs 4050: if (map->nentries) {
1.184 chs 4051: uvm_unmap_remove(map, vm_map_min(map), vm_map_max(map),
1.187 yamt 4052: &dead_entries, NULL, 0);
1.120 chs 4053: if (dead_entries != NULL)
4054: uvm_unmap_detach(dead_entries, 0);
1.10 mrg 4055: }
1.146 yamt 4056: KASSERT(map->nentries == 0);
4057: KASSERT(map->size == 0);
1.239 ! ad 4058: mutex_destroy(&map->misc_lock);
! 4059: mutex_destroy(&map->hint_lock);
! 4060: mutex_destroy(&map->mutex);
! 4061: rw_destroy(&map->lock);
1.120 chs 4062: pmap_destroy(map->pmap);
4063: pool_put(&uvm_vmspace_pool, vm);
1.1 mrg 4064: }
4065:
4066: /*
4067: * F O R K - m a i n e n t r y p o i n t
4068: */
4069: /*
4070: * uvmspace_fork: fork a process' main map
4071: *
4072: * => create a new vmspace for child process from parent.
4073: * => parent's map must not be locked.
4074: */
4075:
1.10 mrg 4076: struct vmspace *
1.138 enami 4077: uvmspace_fork(struct vmspace *vm1)
1.10 mrg 4078: {
4079: struct vmspace *vm2;
1.99 chs 4080: struct vm_map *old_map = &vm1->vm_map;
4081: struct vm_map *new_map;
4082: struct vm_map_entry *old_entry;
4083: struct vm_map_entry *new_entry;
1.10 mrg 4084: UVMHIST_FUNC("uvmspace_fork"); UVMHIST_CALLED(maphist);
1.1 mrg 4085:
1.10 mrg 4086: vm_map_lock(old_map);
1.1 mrg 4087:
1.184 chs 4088: vm2 = uvmspace_alloc(vm_map_min(old_map), vm_map_max(old_map));
1.23 perry 4089: memcpy(&vm2->vm_startcopy, &vm1->vm_startcopy,
1.235 christos 4090: (char *) (vm1 + 1) - (char *) &vm1->vm_startcopy);
1.10 mrg 4091: new_map = &vm2->vm_map; /* XXX */
4092:
4093: old_entry = old_map->header.next;
1.162 pooka 4094: new_map->size = old_map->size;
1.10 mrg 4095:
4096: /*
4097: * go entry-by-entry
4098: */
1.1 mrg 4099:
1.10 mrg 4100: while (old_entry != &old_map->header) {
1.1 mrg 4101:
1.10 mrg 4102: /*
4103: * first, some sanity checks on the old entry
4104: */
1.99 chs 4105:
1.94 chs 4106: KASSERT(!UVM_ET_ISSUBMAP(old_entry));
4107: KASSERT(UVM_ET_ISCOPYONWRITE(old_entry) ||
4108: !UVM_ET_ISNEEDSCOPY(old_entry));
1.1 mrg 4109:
1.10 mrg 4110: switch (old_entry->inheritance) {
1.80 wiz 4111: case MAP_INHERIT_NONE:
1.99 chs 4112:
1.10 mrg 4113: /*
1.162 pooka 4114: * drop the mapping, modify size
1.10 mrg 4115: */
1.162 pooka 4116: new_map->size -= old_entry->end - old_entry->start;
1.10 mrg 4117: break;
4118:
1.80 wiz 4119: case MAP_INHERIT_SHARE:
1.99 chs 4120:
1.10 mrg 4121: /*
4122: * share the mapping: this means we want the old and
4123: * new entries to share amaps and backing objects.
4124: */
4125: /*
4126: * if the old_entry needs a new amap (due to prev fork)
4127: * then we need to allocate it now so that we have
4128: * something we own to share with the new_entry. [in
4129: * other words, we need to clear needs_copy]
4130: */
4131:
4132: if (UVM_ET_ISNEEDSCOPY(old_entry)) {
4133: /* get our own amap, clears needs_copy */
1.212 yamt 4134: amap_copy(old_map, old_entry, AMAP_COPY_NOCHUNK,
1.98 chs 4135: 0, 0);
1.10 mrg 4136: /* XXXCDC: WAITOK??? */
4137: }
4138:
1.126 bouyer 4139: new_entry = uvm_mapent_alloc(new_map, 0);
1.10 mrg 4140: /* old_entry -> new_entry */
4141: uvm_mapent_copy(old_entry, new_entry);
4142:
4143: /* new pmap has nothing wired in it */
4144: new_entry->wired_count = 0;
4145:
4146: /*
1.29 chuck 4147: * gain reference to object backing the map (can't
4148: * be a submap, already checked this case).
1.10 mrg 4149: */
1.99 chs 4150:
1.10 mrg 4151: if (new_entry->aref.ar_amap)
1.85 chs 4152: uvm_map_reference_amap(new_entry, AMAP_SHARED);
1.10 mrg 4153:
4154: if (new_entry->object.uvm_obj &&
4155: new_entry->object.uvm_obj->pgops->pgo_reference)
4156: new_entry->object.uvm_obj->
4157: pgops->pgo_reference(
4158: new_entry->object.uvm_obj);
4159:
4160: /* insert entry at end of new_map's entry list */
4161: uvm_map_entry_link(new_map, new_map->header.prev,
4162: new_entry);
4163:
4164: break;
4165:
1.80 wiz 4166: case MAP_INHERIT_COPY:
1.10 mrg 4167:
4168: /*
4169: * copy-on-write the mapping (using mmap's
4170: * MAP_PRIVATE semantics)
1.29 chuck 4171: *
1.98 chs 4172: * allocate new_entry, adjust reference counts.
1.29 chuck 4173: * (note that new references are read-only).
1.10 mrg 4174: */
4175:
1.126 bouyer 4176: new_entry = uvm_mapent_alloc(new_map, 0);
1.10 mrg 4177: /* old_entry -> new_entry */
4178: uvm_mapent_copy(old_entry, new_entry);
4179:
4180: if (new_entry->aref.ar_amap)
1.85 chs 4181: uvm_map_reference_amap(new_entry, 0);
1.10 mrg 4182:
4183: if (new_entry->object.uvm_obj &&
4184: new_entry->object.uvm_obj->pgops->pgo_reference)
4185: new_entry->object.uvm_obj->pgops->pgo_reference
4186: (new_entry->object.uvm_obj);
4187:
4188: /* new pmap has nothing wired in it */
4189: new_entry->wired_count = 0;
4190:
4191: new_entry->etype |=
4192: (UVM_ET_COPYONWRITE|UVM_ET_NEEDSCOPY);
4193: uvm_map_entry_link(new_map, new_map->header.prev,
4194: new_entry);
1.85 chs 4195:
1.14 chuck 4196: /*
1.10 mrg 4197: * the new entry will need an amap. it will either
4198: * need to be copied from the old entry or created
1.14 chuck 4199: * from scratch (if the old entry does not have an
4200: * amap). can we defer this process until later
4201: * (by setting "needs_copy") or do we need to copy
4202: * the amap now?
1.10 mrg 4203: *
1.14 chuck 4204: * we must copy the amap now if any of the following
1.10 mrg 4205: * conditions hold:
1.14 chuck 4206: * 1. the old entry has an amap and that amap is
4207: * being shared. this means that the old (parent)
1.98 chs 4208: * process is sharing the amap with another
1.14 chuck 4209: * process. if we do not clear needs_copy here
4210: * we will end up in a situation where both the
4211: * parent and child process are refering to the
1.98 chs 4212: * same amap with "needs_copy" set. if the
1.14 chuck 4213: * parent write-faults, the fault routine will
4214: * clear "needs_copy" in the parent by allocating
1.98 chs 4215: * a new amap. this is wrong because the
1.14 chuck 4216: * parent is supposed to be sharing the old amap
4217: * and the new amap will break that.
1.10 mrg 4218: *
1.14 chuck 4219: * 2. if the old entry has an amap and a non-zero
4220: * wire count then we are going to have to call
1.98 chs 4221: * amap_cow_now to avoid page faults in the
1.14 chuck 4222: * parent process. since amap_cow_now requires
4223: * "needs_copy" to be clear we might as well
4224: * clear it here as well.
1.10 mrg 4225: *
4226: */
4227:
1.14 chuck 4228: if (old_entry->aref.ar_amap != NULL) {
1.99 chs 4229: if ((amap_flags(old_entry->aref.ar_amap) &
4230: AMAP_SHARED) != 0 ||
4231: VM_MAPENT_ISWIRED(old_entry)) {
4232:
1.212 yamt 4233: amap_copy(new_map, new_entry,
4234: AMAP_COPY_NOCHUNK, 0, 0);
1.99 chs 4235: /* XXXCDC: M_WAITOK ... ok? */
4236: }
1.10 mrg 4237: }
1.85 chs 4238:
1.10 mrg 4239: /*
1.14 chuck 4240: * if the parent's entry is wired down, then the
4241: * parent process does not want page faults on
4242: * access to that memory. this means that we
4243: * cannot do copy-on-write because we can't write
4244: * protect the old entry. in this case we
4245: * resolve all copy-on-write faults now, using
4246: * amap_cow_now. note that we have already
4247: * allocated any needed amap (above).
1.10 mrg 4248: */
4249:
1.55 thorpej 4250: if (VM_MAPENT_ISWIRED(old_entry)) {
1.1 mrg 4251:
1.98 chs 4252: /*
1.14 chuck 4253: * resolve all copy-on-write faults now
1.98 chs 4254: * (note that there is nothing to do if
1.14 chuck 4255: * the old mapping does not have an amap).
4256: */
4257: if (old_entry->aref.ar_amap)
4258: amap_cow_now(new_map, new_entry);
4259:
1.98 chs 4260: } else {
1.14 chuck 4261:
4262: /*
4263: * setup mappings to trigger copy-on-write faults
4264: * we must write-protect the parent if it has
4265: * an amap and it is not already "needs_copy"...
4266: * if it is already "needs_copy" then the parent
4267: * has already been write-protected by a previous
4268: * fork operation.
4269: */
4270:
1.113 chs 4271: if (old_entry->aref.ar_amap &&
4272: !UVM_ET_ISNEEDSCOPY(old_entry)) {
1.14 chuck 4273: if (old_entry->max_protection & VM_PROT_WRITE) {
4274: pmap_protect(old_map->pmap,
4275: old_entry->start,
4276: old_entry->end,
4277: old_entry->protection &
4278: ~VM_PROT_WRITE);
1.105 chris 4279: pmap_update(old_map->pmap);
1.14 chuck 4280: }
4281: old_entry->etype |= UVM_ET_NEEDSCOPY;
4282: }
1.10 mrg 4283: }
4284: break;
1.14 chuck 4285: } /* end of switch statement */
1.10 mrg 4286: old_entry = old_entry->next;
1.1 mrg 4287: }
4288:
1.98 chs 4289: vm_map_unlock(old_map);
1.1 mrg 4290:
4291: #ifdef SYSVSHM
1.10 mrg 4292: if (vm1->vm_shm)
4293: shmfork(vm1, vm2);
1.39 thorpej 4294: #endif
4295:
4296: #ifdef PMAP_FORK
4297: pmap_fork(vm1->vm_map.pmap, vm2->vm_map.pmap);
1.1 mrg 4298: #endif
4299:
1.10 mrg 4300: UVMHIST_LOG(maphist,"<- done",0,0,0,0);
1.139 enami 4301: return (vm2);
1.1 mrg 4302: }
4303:
4304:
1.174 yamt 4305: /*
4306: * in-kernel map entry allocation.
4307: */
4308:
4309: struct uvm_kmapent_hdr {
4310: LIST_ENTRY(uvm_kmapent_hdr) ukh_listq;
4311: int ukh_nused;
4312: struct vm_map_entry *ukh_freelist;
4313: struct vm_map *ukh_map;
1.178 yamt 4314: struct vm_map_entry ukh_entries[0];
1.174 yamt 4315: };
4316:
4317: #define UVM_KMAPENT_CHUNK \
4318: ((PAGE_SIZE - sizeof(struct uvm_kmapent_hdr)) \
4319: / sizeof(struct vm_map_entry))
4320:
4321: #define UVM_KHDR_FIND(entry) \
4322: ((struct uvm_kmapent_hdr *)(((vaddr_t)entry) & ~PAGE_MASK))
4323:
4324:
1.201 dsl 4325: #ifdef DIAGNOSTIC
1.203 thorpej 4326: static struct vm_map *
1.174 yamt 4327: uvm_kmapent_map(struct vm_map_entry *entry)
4328: {
4329: const struct uvm_kmapent_hdr *ukh;
4330:
4331: ukh = UVM_KHDR_FIND(entry);
4332: return ukh->ukh_map;
4333: }
1.201 dsl 4334: #endif
1.174 yamt 4335:
1.206 perry 4336: static inline struct vm_map_entry *
1.174 yamt 4337: uvm_kmapent_get(struct uvm_kmapent_hdr *ukh)
4338: {
4339: struct vm_map_entry *entry;
4340:
4341: KASSERT(ukh->ukh_nused <= UVM_KMAPENT_CHUNK);
4342: KASSERT(ukh->ukh_nused >= 0);
4343:
4344: entry = ukh->ukh_freelist;
4345: if (entry) {
4346: KASSERT((entry->flags & (UVM_MAP_KERNEL | UVM_MAP_KMAPENT))
4347: == UVM_MAP_KERNEL);
4348: ukh->ukh_freelist = entry->next;
4349: ukh->ukh_nused++;
4350: KASSERT(ukh->ukh_nused <= UVM_KMAPENT_CHUNK);
4351: } else {
4352: KASSERT(ukh->ukh_nused == UVM_KMAPENT_CHUNK);
4353: }
4354:
4355: return entry;
4356: }
4357:
1.206 perry 4358: static inline void
1.174 yamt 4359: uvm_kmapent_put(struct uvm_kmapent_hdr *ukh, struct vm_map_entry *entry)
4360: {
4361:
4362: KASSERT((entry->flags & (UVM_MAP_KERNEL | UVM_MAP_KMAPENT))
4363: == UVM_MAP_KERNEL);
4364: KASSERT(ukh->ukh_nused <= UVM_KMAPENT_CHUNK);
4365: KASSERT(ukh->ukh_nused > 0);
4366: KASSERT(ukh->ukh_freelist != NULL ||
4367: ukh->ukh_nused == UVM_KMAPENT_CHUNK);
4368: KASSERT(ukh->ukh_freelist == NULL ||
4369: ukh->ukh_nused < UVM_KMAPENT_CHUNK);
4370:
4371: ukh->ukh_nused--;
4372: entry->next = ukh->ukh_freelist;
4373: ukh->ukh_freelist = entry;
4374: }
4375:
4376: /*
4377: * uvm_kmapent_alloc: allocate a map entry for in-kernel map
4378: */
4379:
4380: static struct vm_map_entry *
4381: uvm_kmapent_alloc(struct vm_map *map, int flags)
4382: {
4383: struct vm_page *pg;
4384: struct uvm_map_args args;
4385: struct uvm_kmapent_hdr *ukh;
4386: struct vm_map_entry *entry;
4387: uvm_flag_t mapflags = UVM_MAPFLAG(UVM_PROT_ALL, UVM_PROT_ALL,
4388: UVM_INH_NONE, UVM_ADV_RANDOM, flags | UVM_FLAG_NOMERGE);
4389: vaddr_t va;
4390: int error;
4391: int i;
4392:
4393: KDASSERT(UVM_KMAPENT_CHUNK > 2);
4394: KDASSERT(kernel_map != NULL);
4395: KASSERT(vm_map_pmap(map) == pmap_kernel());
4396:
1.208 yamt 4397: UVMMAP_EVCNT_INCR(uke_alloc);
1.174 yamt 4398: entry = NULL;
4399: again:
4400: /*
4401: * try to grab an entry from freelist.
4402: */
1.238 ad 4403: mutex_spin_enter(&uvm_kentry_lock);
1.175 yamt 4404: ukh = LIST_FIRST(&vm_map_to_kernel(map)->vmk_kentry_free);
1.174 yamt 4405: if (ukh) {
4406: entry = uvm_kmapent_get(ukh);
4407: if (ukh->ukh_nused == UVM_KMAPENT_CHUNK)
4408: LIST_REMOVE(ukh, ukh_listq);
4409: }
1.238 ad 4410: mutex_spin_exit(&uvm_kentry_lock);
1.174 yamt 4411:
4412: if (entry)
4413: return entry;
4414:
4415: /*
4416: * there's no free entry for this vm_map.
4417: * now we need to allocate some vm_map_entry.
4418: * for simplicity, always allocate one page chunk of them at once.
4419: */
4420:
4421: pg = uvm_pagealloc(NULL, 0, NULL, 0);
4422: if (__predict_false(pg == NULL)) {
4423: if (flags & UVM_FLAG_NOWAIT)
4424: return NULL;
4425: uvm_wait("kme_alloc");
4426: goto again;
4427: }
4428:
4429: error = uvm_map_prepare(map, 0, PAGE_SIZE, NULL, 0, 0, mapflags, &args);
4430: if (error) {
4431: uvm_pagefree(pg);
4432: return NULL;
4433: }
4434:
4435: va = args.uma_start;
4436:
4437: pmap_kenter_pa(va, VM_PAGE_TO_PHYS(pg), VM_PROT_READ|VM_PROT_WRITE);
4438: pmap_update(vm_map_pmap(map));
4439:
4440: ukh = (void *)va;
4441:
4442: /*
4443: * use the first entry for ukh itsself.
4444: */
4445:
4446: entry = &ukh->ukh_entries[0];
4447: entry->flags = UVM_MAP_KERNEL | UVM_MAP_KMAPENT;
4448: error = uvm_map_enter(map, &args, entry);
4449: KASSERT(error == 0);
4450:
4451: ukh->ukh_nused = UVM_KMAPENT_CHUNK;
4452: ukh->ukh_map = map;
4453: ukh->ukh_freelist = NULL;
4454: for (i = UVM_KMAPENT_CHUNK - 1; i >= 2; i--) {
1.199 christos 4455: struct vm_map_entry *xentry = &ukh->ukh_entries[i];
1.174 yamt 4456:
1.199 christos 4457: xentry->flags = UVM_MAP_KERNEL;
4458: uvm_kmapent_put(ukh, xentry);
1.174 yamt 4459: }
4460: KASSERT(ukh->ukh_nused == 2);
4461:
1.238 ad 4462: mutex_spin_enter(&uvm_kentry_lock);
1.175 yamt 4463: LIST_INSERT_HEAD(&vm_map_to_kernel(map)->vmk_kentry_free,
4464: ukh, ukh_listq);
1.238 ad 4465: mutex_spin_exit(&uvm_kentry_lock);
1.174 yamt 4466:
4467: /*
4468: * return second entry.
4469: */
4470:
4471: entry = &ukh->ukh_entries[1];
4472: entry->flags = UVM_MAP_KERNEL;
1.208 yamt 4473: UVMMAP_EVCNT_INCR(ukh_alloc);
1.174 yamt 4474: return entry;
4475: }
4476:
4477: /*
4478: * uvm_mapent_free: free map entry for in-kernel map
4479: */
4480:
4481: static void
4482: uvm_kmapent_free(struct vm_map_entry *entry)
4483: {
4484: struct uvm_kmapent_hdr *ukh;
4485: struct vm_page *pg;
4486: struct vm_map *map;
4487: struct pmap *pmap;
4488: vaddr_t va;
4489: paddr_t pa;
4490: struct vm_map_entry *deadentry;
4491:
1.208 yamt 4492: UVMMAP_EVCNT_INCR(uke_free);
1.174 yamt 4493: ukh = UVM_KHDR_FIND(entry);
4494: map = ukh->ukh_map;
4495:
1.238 ad 4496: mutex_spin_enter(&uvm_kentry_lock);
1.174 yamt 4497: uvm_kmapent_put(ukh, entry);
4498: if (ukh->ukh_nused > 1) {
4499: if (ukh->ukh_nused == UVM_KMAPENT_CHUNK - 1)
1.175 yamt 4500: LIST_INSERT_HEAD(
4501: &vm_map_to_kernel(map)->vmk_kentry_free,
4502: ukh, ukh_listq);
1.238 ad 4503: mutex_spin_exit(&uvm_kentry_lock);
1.174 yamt 4504: return;
4505: }
4506:
4507: /*
4508: * now we can free this ukh.
4509: *
4510: * however, keep an empty ukh to avoid ping-pong.
4511: */
4512:
1.175 yamt 4513: if (LIST_FIRST(&vm_map_to_kernel(map)->vmk_kentry_free) == ukh &&
1.174 yamt 4514: LIST_NEXT(ukh, ukh_listq) == NULL) {
1.238 ad 4515: mutex_spin_exit(&uvm_kentry_lock);
1.174 yamt 4516: return;
4517: }
4518: LIST_REMOVE(ukh, ukh_listq);
1.238 ad 4519: mutex_spin_exit(&uvm_kentry_lock);
1.174 yamt 4520:
4521: KASSERT(ukh->ukh_nused == 1);
4522:
4523: /*
4524: * remove map entry for ukh itsself.
4525: */
4526:
4527: va = (vaddr_t)ukh;
4528: KASSERT((va & PAGE_MASK) == 0);
1.198 yamt 4529: vm_map_lock(map);
1.187 yamt 4530: uvm_unmap_remove(map, va, va + PAGE_SIZE, &deadentry, NULL, 0);
1.174 yamt 4531: KASSERT(deadentry->flags & UVM_MAP_KERNEL);
4532: KASSERT(deadentry->flags & UVM_MAP_KMAPENT);
4533: KASSERT(deadentry->next == NULL);
4534: KASSERT(deadentry == &ukh->ukh_entries[0]);
4535:
4536: /*
4537: * unmap the page from pmap and free it.
4538: */
4539:
4540: pmap = vm_map_pmap(map);
4541: KASSERT(pmap == pmap_kernel());
4542: if (!pmap_extract(pmap, va, &pa))
4543: panic("%s: no mapping", __func__);
4544: pmap_kremove(va, PAGE_SIZE);
1.198 yamt 4545: vm_map_unlock(map);
1.174 yamt 4546: pg = PHYS_TO_VM_PAGE(pa);
4547: uvm_pagefree(pg);
1.208 yamt 4548: UVMMAP_EVCNT_INCR(ukh_free);
1.174 yamt 4549: }
4550:
1.219 yamt 4551: static vsize_t
4552: uvm_kmapent_overhead(vsize_t size)
4553: {
4554:
4555: /*
4556: * - the max number of unmerged entries is howmany(size, PAGE_SIZE)
4557: * as the min allocation unit is PAGE_SIZE.
4558: * - UVM_KMAPENT_CHUNK "kmapent"s are allocated from a page.
4559: * one of them are used to map the page itself.
4560: */
4561:
4562: return howmany(howmany(size, PAGE_SIZE), (UVM_KMAPENT_CHUNK - 1)) *
4563: PAGE_SIZE;
4564: }
4565:
1.174 yamt 4566: /*
4567: * map entry reservation
4568: */
4569:
4570: /*
4571: * uvm_mapent_reserve: reserve map entries for clipping before locking map.
4572: *
4573: * => needed when unmapping entries allocated without UVM_FLAG_QUANTUM.
4574: * => caller shouldn't hold map locked.
4575: */
4576: int
4577: uvm_mapent_reserve(struct vm_map *map, struct uvm_mapent_reservation *umr,
4578: int nentries, int flags)
4579: {
4580:
4581: umr->umr_nentries = 0;
4582:
1.179 yamt 4583: if ((flags & UVM_FLAG_QUANTUM) != 0)
4584: return 0;
4585:
1.174 yamt 4586: if (!VM_MAP_USE_KMAPENT(map))
4587: return 0;
4588:
4589: while (nentries--) {
4590: struct vm_map_entry *ent;
4591: ent = uvm_kmapent_alloc(map, flags);
4592: if (!ent) {
4593: uvm_mapent_unreserve(map, umr);
4594: return ENOMEM;
4595: }
4596: UMR_PUTENTRY(umr, ent);
4597: }
4598:
4599: return 0;
4600: }
4601:
4602: /*
4603: * uvm_mapent_unreserve:
4604: *
4605: * => caller shouldn't hold map locked.
4606: * => never fail or sleep.
4607: */
4608: void
1.232 yamt 4609: uvm_mapent_unreserve(struct vm_map *map, struct uvm_mapent_reservation *umr)
1.174 yamt 4610: {
4611:
4612: while (!UMR_EMPTY(umr))
4613: uvm_kmapent_free(UMR_GETENTRY(umr));
4614: }
4615:
1.194 yamt 4616: /*
4617: * uvm_mapent_trymerge: try to merge an entry with its neighbors.
4618: *
4619: * => called with map locked.
4620: * => return non zero if successfully merged.
4621: */
4622:
4623: int
4624: uvm_mapent_trymerge(struct vm_map *map, struct vm_map_entry *entry, int flags)
4625: {
4626: struct uvm_object *uobj;
4627: struct vm_map_entry *next;
4628: struct vm_map_entry *prev;
1.195 yamt 4629: vsize_t size;
1.194 yamt 4630: int merged = 0;
1.233 thorpej 4631: bool copying;
1.194 yamt 4632: int newetype;
4633:
4634: if (VM_MAP_USE_KMAPENT(map)) {
4635: return 0;
4636: }
4637: if (entry->aref.ar_amap != NULL) {
4638: return 0;
4639: }
4640: if ((entry->flags & UVM_MAP_NOMERGE) != 0) {
4641: return 0;
4642: }
4643:
4644: uobj = entry->object.uvm_obj;
1.195 yamt 4645: size = entry->end - entry->start;
1.194 yamt 4646: copying = (flags & UVM_MERGE_COPYING) != 0;
4647: newetype = copying ? (entry->etype & ~UVM_ET_NEEDSCOPY) : entry->etype;
4648:
4649: next = entry->next;
4650: if (next != &map->header &&
4651: next->start == entry->end &&
4652: ((copying && next->aref.ar_amap != NULL &&
4653: amap_refs(next->aref.ar_amap) == 1) ||
4654: (!copying && next->aref.ar_amap == NULL)) &&
4655: UVM_ET_ISCOMPATIBLE(next, newetype,
4656: uobj, entry->flags, entry->protection,
4657: entry->max_protection, entry->inheritance, entry->advice,
1.195 yamt 4658: entry->wired_count) &&
4659: (uobj == NULL || entry->offset + size == next->offset)) {
1.194 yamt 4660: int error;
4661:
4662: if (copying) {
1.195 yamt 4663: error = amap_extend(next, size,
1.194 yamt 4664: AMAP_EXTEND_NOWAIT|AMAP_EXTEND_BACKWARDS);
4665: } else {
4666: error = 0;
4667: }
4668: if (error == 0) {
1.197 yamt 4669: if (uobj) {
4670: if (uobj->pgops->pgo_detach) {
4671: uobj->pgops->pgo_detach(uobj);
4672: }
1.194 yamt 4673: }
4674:
4675: entry->end = next->end;
1.221 yamt 4676: clear_hints(map, next);
1.194 yamt 4677: uvm_map_entry_unlink(map, next);
4678: if (copying) {
4679: entry->aref = next->aref;
4680: entry->etype &= ~UVM_ET_NEEDSCOPY;
4681: }
1.222 yamt 4682: uvm_map_check(map, "trymerge forwardmerge");
1.194 yamt 4683: uvm_mapent_free_merged(map, next);
4684: merged++;
4685: }
4686: }
4687:
4688: prev = entry->prev;
4689: if (prev != &map->header &&
4690: prev->end == entry->start &&
4691: ((copying && !merged && prev->aref.ar_amap != NULL &&
4692: amap_refs(prev->aref.ar_amap) == 1) ||
4693: (!copying && prev->aref.ar_amap == NULL)) &&
4694: UVM_ET_ISCOMPATIBLE(prev, newetype,
4695: uobj, entry->flags, entry->protection,
4696: entry->max_protection, entry->inheritance, entry->advice,
1.195 yamt 4697: entry->wired_count) &&
1.196 yamt 4698: (uobj == NULL ||
4699: prev->offset + prev->end - prev->start == entry->offset)) {
1.194 yamt 4700: int error;
4701:
4702: if (copying) {
1.195 yamt 4703: error = amap_extend(prev, size,
1.194 yamt 4704: AMAP_EXTEND_NOWAIT|AMAP_EXTEND_FORWARDS);
4705: } else {
4706: error = 0;
4707: }
4708: if (error == 0) {
1.197 yamt 4709: if (uobj) {
4710: if (uobj->pgops->pgo_detach) {
4711: uobj->pgops->pgo_detach(uobj);
4712: }
4713: entry->offset = prev->offset;
1.194 yamt 4714: }
4715:
4716: entry->start = prev->start;
1.221 yamt 4717: clear_hints(map, prev);
1.194 yamt 4718: uvm_map_entry_unlink(map, prev);
4719: if (copying) {
4720: entry->aref = prev->aref;
4721: entry->etype &= ~UVM_ET_NEEDSCOPY;
4722: }
1.222 yamt 4723: uvm_map_check(map, "trymerge backmerge");
1.194 yamt 4724: uvm_mapent_free_merged(map, prev);
4725: merged++;
4726: }
4727: }
4728:
4729: return merged;
4730: }
4731:
1.1 mrg 4732: #if defined(DDB)
4733:
4734: /*
4735: * DDB hooks
4736: */
4737:
4738: /*
4739: * uvm_map_printit: actually prints the map
4740: */
4741:
1.10 mrg 4742: void
1.233 thorpej 4743: uvm_map_printit(struct vm_map *map, bool full,
1.138 enami 4744: void (*pr)(const char *, ...))
1.10 mrg 4745: {
1.99 chs 4746: struct vm_map_entry *entry;
1.10 mrg 4747:
1.184 chs 4748: (*pr)("MAP %p: [0x%lx->0x%lx]\n", map, vm_map_min(map),
4749: vm_map_max(map));
1.53 thorpej 4750: (*pr)("\t#ent=%d, sz=%d, ref=%d, version=%d, flags=0x%x\n",
4751: map->nentries, map->size, map->ref_count, map->timestamp,
4752: map->flags);
1.173 yamt 4753: (*pr)("\tpmap=%p(resident=%ld, wired=%ld)\n", map->pmap,
4754: pmap_resident_count(map->pmap), pmap_wired_count(map->pmap));
1.10 mrg 4755: if (!full)
4756: return;
4757: for (entry = map->header.next; entry != &map->header;
4758: entry = entry->next) {
1.70 kleink 4759: (*pr)(" - %p: 0x%lx->0x%lx: obj=%p/0x%llx, amap=%p/%d\n",
1.10 mrg 4760: entry, entry->start, entry->end, entry->object.uvm_obj,
1.85 chs 4761: (long long)entry->offset, entry->aref.ar_amap,
4762: entry->aref.ar_pageoff);
1.10 mrg 4763: (*pr)(
1.85 chs 4764: "\tsubmap=%c, cow=%c, nc=%c, prot(max)=%d/%d, inh=%d, "
4765: "wc=%d, adv=%d\n",
1.10 mrg 4766: (entry->etype & UVM_ET_SUBMAP) ? 'T' : 'F',
1.98 chs 4767: (entry->etype & UVM_ET_COPYONWRITE) ? 'T' : 'F',
1.10 mrg 4768: (entry->etype & UVM_ET_NEEDSCOPY) ? 'T' : 'F',
4769: entry->protection, entry->max_protection,
4770: entry->inheritance, entry->wired_count, entry->advice);
4771: }
1.98 chs 4772: }
1.1 mrg 4773:
4774: /*
4775: * uvm_object_printit: actually prints the object
4776: */
4777:
1.10 mrg 4778: void
1.233 thorpej 4779: uvm_object_printit(struct uvm_object *uobj, bool full,
1.138 enami 4780: void (*pr)(const char *, ...))
1.10 mrg 4781: {
4782: struct vm_page *pg;
4783: int cnt = 0;
4784:
1.71 chs 4785: (*pr)("OBJECT %p: locked=%d, pgops=%p, npages=%d, ",
4786: uobj, uobj->vmobjlock.lock_data, uobj->pgops, uobj->uo_npages);
1.42 thorpej 4787: if (UVM_OBJ_IS_KERN_OBJECT(uobj))
1.10 mrg 4788: (*pr)("refs=<SYSTEM>\n");
4789: else
4790: (*pr)("refs=%d\n", uobj->uo_refs);
4791:
1.71 chs 4792: if (!full) {
4793: return;
4794: }
1.10 mrg 4795: (*pr)(" PAGES <pg,offset>:\n ");
1.106 chs 4796: TAILQ_FOREACH(pg, &uobj->memq, listq) {
4797: cnt++;
1.85 chs 4798: (*pr)("<%p,0x%llx> ", pg, (long long)pg->offset);
1.106 chs 4799: if ((cnt % 3) == 0) {
1.71 chs 4800: (*pr)("\n ");
4801: }
4802: }
1.106 chs 4803: if ((cnt % 3) != 0) {
1.71 chs 4804: (*pr)("\n");
1.10 mrg 4805: }
1.98 chs 4806: }
1.1 mrg 4807:
4808: /*
4809: * uvm_page_printit: actually print the page
4810: */
4811:
1.228 yamt 4812: static const char page_flagbits[] = UVM_PGFLAGBITS;
4813: static const char page_pqflagbits[] = UVM_PQFLAGBITS;
1.86 chs 4814:
1.10 mrg 4815: void
1.233 thorpej 4816: uvm_page_printit(struct vm_page *pg, bool full,
1.138 enami 4817: void (*pr)(const char *, ...))
1.10 mrg 4818: {
1.85 chs 4819: struct vm_page *tpg;
1.10 mrg 4820: struct uvm_object *uobj;
4821: struct pglist *pgl;
1.71 chs 4822: char pgbuf[128];
4823: char pqbuf[128];
1.1 mrg 4824:
1.10 mrg 4825: (*pr)("PAGE %p:\n", pg);
1.71 chs 4826: bitmask_snprintf(pg->flags, page_flagbits, pgbuf, sizeof(pgbuf));
4827: bitmask_snprintf(pg->pqflags, page_pqflagbits, pqbuf, sizeof(pqbuf));
1.106 chs 4828: (*pr)(" flags=%s, pqflags=%s, wire_count=%d, pa=0x%lx\n",
1.137 yamt 4829: pgbuf, pqbuf, pg->wire_count, (long)VM_PAGE_TO_PHYS(pg));
1.86 chs 4830: (*pr)(" uobject=%p, uanon=%p, offset=0x%llx loan_count=%d\n",
4831: pg->uobject, pg->uanon, (long long)pg->offset, pg->loan_count);
1.1 mrg 4832: #if defined(UVM_PAGE_TRKOWN)
1.10 mrg 4833: if (pg->flags & PG_BUSY)
4834: (*pr)(" owning process = %d, tag=%s\n",
4835: pg->owner, pg->owner_tag);
4836: else
4837: (*pr)(" page not busy, no owner\n");
1.1 mrg 4838: #else
1.10 mrg 4839: (*pr)(" [page ownership tracking disabled]\n");
1.1 mrg 4840: #endif
4841:
1.10 mrg 4842: if (!full)
4843: return;
4844:
4845: /* cross-verify object/anon */
4846: if ((pg->pqflags & PQ_FREE) == 0) {
4847: if (pg->pqflags & PQ_ANON) {
1.192 yamt 4848: if (pg->uanon == NULL || pg->uanon->an_page != pg)
1.85 chs 4849: (*pr)(" >>> ANON DOES NOT POINT HERE <<< (%p)\n",
1.192 yamt 4850: (pg->uanon) ? pg->uanon->an_page : NULL);
1.10 mrg 4851: else
4852: (*pr)(" anon backpointer is OK\n");
4853: } else {
4854: uobj = pg->uobject;
4855: if (uobj) {
4856: (*pr)(" checking object list\n");
1.85 chs 4857: TAILQ_FOREACH(tpg, &uobj->memq, listq) {
4858: if (tpg == pg) {
4859: break;
4860: }
1.10 mrg 4861: }
1.85 chs 4862: if (tpg)
1.10 mrg 4863: (*pr)(" page found on object list\n");
4864: else
4865: (*pr)(" >>> PAGE NOT FOUND ON OBJECT LIST! <<<\n");
4866: }
4867: }
4868: }
1.1 mrg 4869:
1.10 mrg 4870: /* cross-verify page queue */
1.73 thorpej 4871: if (pg->pqflags & PQ_FREE) {
4872: int fl = uvm_page_lookup_freelist(pg);
1.96 thorpej 4873: int color = VM_PGCOLOR_BUCKET(pg);
4874: pgl = &uvm.page_free[fl].pgfl_buckets[color].pgfl_queues[
4875: ((pg)->flags & PG_ZERO) ? PGFL_ZEROS : PGFL_UNKNOWN];
1.139 enami 4876: } else {
1.10 mrg 4877: pgl = NULL;
1.85 chs 4878: }
1.10 mrg 4879:
4880: if (pgl) {
4881: (*pr)(" checking pageq list\n");
1.85 chs 4882: TAILQ_FOREACH(tpg, pgl, pageq) {
4883: if (tpg == pg) {
4884: break;
4885: }
1.10 mrg 4886: }
1.85 chs 4887: if (tpg)
1.10 mrg 4888: (*pr)(" page found on pageq list\n");
4889: else
4890: (*pr)(" >>> PAGE NOT FOUND ON PAGEQ LIST! <<<\n");
4891: }
1.1 mrg 4892: }
1.213 bjh21 4893:
4894: /*
4895: * uvm_pages_printthem - print a summary of all managed pages
4896: */
4897:
4898: void
4899: uvm_page_printall(void (*pr)(const char *, ...))
4900: {
4901: unsigned i;
4902: struct vm_page *pg;
4903:
1.231 uwe 4904: (*pr)("%18s %4s %4s %18s %18s"
1.214 bjh21 4905: #ifdef UVM_PAGE_TRKOWN
4906: " OWNER"
4907: #endif
4908: "\n", "PAGE", "FLAG", "PQ", "UOBJECT", "UANON");
1.213 bjh21 4909: for (i = 0; i < vm_nphysseg; i++) {
4910: for (pg = vm_physmem[i].pgs; pg <= vm_physmem[i].lastpg; pg++) {
1.231 uwe 4911: (*pr)("%18p %04x %04x %18p %18p",
1.213 bjh21 4912: pg, pg->flags, pg->pqflags, pg->uobject,
4913: pg->uanon);
1.214 bjh21 4914: #ifdef UVM_PAGE_TRKOWN
4915: if (pg->flags & PG_BUSY)
4916: (*pr)(" %d [%s]", pg->owner, pg->owner_tag);
4917: #endif
4918: (*pr)("\n");
1.213 bjh21 4919: }
4920: }
4921: }
4922:
1.1 mrg 4923: #endif
1.211 yamt 4924:
4925: /*
4926: * uvm_map_create: create map
4927: */
4928:
4929: struct vm_map *
4930: uvm_map_create(pmap_t pmap, vaddr_t vmin, vaddr_t vmax, int flags)
4931: {
4932: struct vm_map *result;
4933:
4934: MALLOC(result, struct vm_map *, sizeof(struct vm_map),
4935: M_VMMAP, M_WAITOK);
4936: uvm_map_setup(result, vmin, vmax, flags);
4937: result->pmap = pmap;
4938: return(result);
4939: }
4940:
4941: /*
4942: * uvm_map_setup: init map
4943: *
4944: * => map must not be in service yet.
4945: */
4946:
4947: void
4948: uvm_map_setup(struct vm_map *map, vaddr_t vmin, vaddr_t vmax, int flags)
4949: {
1.238 ad 4950: int ipl;
1.211 yamt 4951:
4952: RB_INIT(&map->rbhead);
4953: map->header.next = map->header.prev = &map->header;
4954: map->nentries = 0;
4955: map->size = 0;
4956: map->ref_count = 1;
4957: vm_map_setmin(map, vmin);
4958: vm_map_setmax(map, vmax);
4959: map->flags = flags;
4960: map->first_free = &map->header;
4961: map->hint = &map->header;
4962: map->timestamp = 0;
1.238 ad 4963: map->busy = NULL;
4964:
4965: if ((flags & VM_MAP_INTRSAFE) != 0) {
4966: ipl = IPL_VM;
4967: } else {
4968: rw_init(&map->lock);
4969: ipl = IPL_NONE;
4970: }
4971:
4972: cv_init(&map->cv, "vm_map");
4973: mutex_init(&map->misc_lock, MUTEX_DRIVER, ipl);
4974: mutex_init(&map->mutex, MUTEX_DRIVER, ipl);
4975:
4976: /*
4977: * The hint lock can get acquired with the pagequeue
4978: * lock held, so must be at IPL_VM.
4979: */
4980: mutex_init(&map->hint_lock, MUTEX_DRIVER, IPL_VM);
1.211 yamt 4981: }
4982:
4983:
4984: /*
4985: * U N M A P - m a i n e n t r y p o i n t
4986: */
4987:
4988: /*
4989: * uvm_unmap1: remove mappings from a vm_map (from "start" up to "stop")
4990: *
4991: * => caller must check alignment and size
4992: * => map must be unlocked (we will lock it)
4993: * => flags is UVM_FLAG_QUANTUM or 0.
4994: */
4995:
4996: void
4997: uvm_unmap1(struct vm_map *map, vaddr_t start, vaddr_t end, int flags)
4998: {
4999: struct vm_map_entry *dead_entries;
5000: struct uvm_mapent_reservation umr;
5001: UVMHIST_FUNC("uvm_unmap"); UVMHIST_CALLED(maphist);
5002:
5003: UVMHIST_LOG(maphist, " (map=0x%x, start=0x%x, end=0x%x)",
5004: map, start, end, 0);
5005: /*
5006: * work now done by helper functions. wipe the pmap's and then
5007: * detach from the dead entries...
5008: */
5009: uvm_mapent_reserve(map, &umr, 2, flags);
5010: vm_map_lock(map);
5011: uvm_unmap_remove(map, start, end, &dead_entries, &umr, flags);
5012: vm_map_unlock(map);
5013: uvm_mapent_unreserve(map, &umr);
5014:
5015: if (dead_entries != NULL)
5016: uvm_unmap_detach(dead_entries, 0);
5017:
5018: UVMHIST_LOG(maphist, "<- done", 0,0,0,0);
5019: }
5020:
5021:
5022: /*
5023: * uvm_map_reference: add reference to a map
5024: *
1.238 ad 5025: * => map need not be locked (we use misc_lock).
1.211 yamt 5026: */
5027:
5028: void
5029: uvm_map_reference(struct vm_map *map)
5030: {
1.238 ad 5031: mutex_enter(&map->misc_lock);
1.211 yamt 5032: map->ref_count++;
1.238 ad 5033: mutex_exit(&map->misc_lock);
1.211 yamt 5034: }
5035:
5036: struct vm_map_kernel *
5037: vm_map_to_kernel(struct vm_map *map)
5038: {
5039:
5040: KASSERT(VM_MAP_IS_KERNEL(map));
5041:
5042: return (struct vm_map_kernel *)map;
5043: }
1.226 yamt 5044:
1.233 thorpej 5045: bool
1.226 yamt 5046: vm_map_starved_p(struct vm_map *map)
5047: {
5048:
5049: if ((map->flags & VM_MAP_WANTVA) != 0) {
1.234 thorpej 5050: return true;
1.226 yamt 5051: }
5052: /* XXX */
5053: if ((vm_map_max(map) - vm_map_min(map)) / 16 * 15 < map->size) {
1.234 thorpej 5054: return true;
1.226 yamt 5055: }
1.234 thorpej 5056: return false;
1.226 yamt 5057: }
CVSweb <webmaster@jp.NetBSD.org>
![[BACK]](/icons/back.gif){kind=icon}
Return to uvm_map.c CVS log ![[TXT]](/icons/text.gif){kind=icon}
![[DIR]](/icons/dir.gif){kind=icon}
Up to [cvs.NetBSD.org] / src / sys / uvm