Annotation of src/sys/uvm/uvm_fault.c, Revision 1.154
1.130 uebayasi 1: /* $NetBSD$ */
1.1 mrg 2:
3: /*
4: *
5: * Copyright (c) 1997 Charles D. Cranor and Washington University.
6: * All rights reserved.
7: *
8: * Redistribution and use in source and binary forms, with or without
9: * modification, are permitted provided that the following conditions
10: * are met:
11: * 1. Redistributions of source code must retain the above copyright
12: * notice, this list of conditions and the following disclaimer.
13: * 2. Redistributions in binary form must reproduce the above copyright
14: * notice, this list of conditions and the following disclaimer in the
15: * documentation and/or other materials provided with the distribution.
16: * 3. All advertising materials mentioning features or use of this software
17: * must display the following acknowledgement:
18: * This product includes software developed by Charles D. Cranor and
19: * Washington University.
20: * 4. The name of the author may not be used to endorse or promote products
21: * derived from this software without specific prior written permission.
22: *
23: * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
24: * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
25: * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
26: * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
27: * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
28: * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
29: * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
30: * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
31: * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
32: * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
1.4 mrg 33: *
34: * from: Id: uvm_fault.c,v 1.1.2.23 1998/02/06 05:29:05 chs Exp
1.1 mrg 35: */
36:
37: /*
38: * uvm_fault.c: fault handler
39: */
1.71 lukem 40:
41: #include <sys/cdefs.h>
1.130 uebayasi 42: __KERNEL_RCSID(0, "$NetBSD$");
1.71 lukem 43:
44: #include "opt_uvmhist.h"
1.1 mrg 45:
46: #include <sys/param.h>
47: #include <sys/systm.h>
48: #include <sys/kernel.h>
49: #include <sys/proc.h>
50: #include <sys/malloc.h>
51: #include <sys/mman.h>
52:
53: #include <uvm/uvm.h>
54:
55: /*
56: *
57: * a word on page faults:
58: *
59: * types of page faults we handle:
60: *
61: * CASE 1: upper layer faults CASE 2: lower layer faults
62: *
63: * CASE 1A CASE 1B CASE 2A CASE 2B
64: * read/write1 write>1 read/write +-cow_write/zero
1.63 chs 65: * | | | |
1.1 mrg 66: * +--|--+ +--|--+ +-----+ + | + | +-----+
1.127 uebayasi 67: * amap | V | | ---------> new | | | | ^ |
1.1 mrg 68: * +-----+ +-----+ +-----+ + | + | +--|--+
69: * | | |
70: * +-----+ +-----+ +--|--+ | +--|--+
1.127 uebayasi 71: * uobj | d/c | | d/c | | V | +----+ |
1.1 mrg 72: * +-----+ +-----+ +-----+ +-----+
73: *
74: * d/c = don't care
1.63 chs 75: *
1.1 mrg 76: * case [0]: layerless fault
77: * no amap or uobj is present. this is an error.
78: *
79: * case [1]: upper layer fault [anon active]
80: * 1A: [read] or [write with anon->an_ref == 1]
1.127 uebayasi 81: * I/O takes place in upper level anon and uobj is not touched.
1.1 mrg 82: * 1B: [write with anon->an_ref > 1]
83: * new anon is alloc'd and data is copied off ["COW"]
84: *
85: * case [2]: lower layer fault [uobj]
86: * 2A: [read on non-NULL uobj] or [write to non-copy_on_write area]
87: * I/O takes place directly in object.
88: * 2B: [write to copy_on_write] or [read on NULL uobj]
1.63 chs 89: * data is "promoted" from uobj to a new anon.
1.1 mrg 90: * if uobj is null, then we zero fill.
91: *
92: * we follow the standard UVM locking protocol ordering:
93: *
1.63 chs 94: * MAPS => AMAP => UOBJ => ANON => PAGE QUEUES (PQ)
1.1 mrg 95: * we hold a PG_BUSY page if we unlock for I/O
96: *
97: *
98: * the code is structured as follows:
1.63 chs 99: *
1.1 mrg 100: * - init the "IN" params in the ufi structure
101: * ReFault:
102: * - do lookups [locks maps], check protection, handle needs_copy
103: * - check for case 0 fault (error)
104: * - establish "range" of fault
105: * - if we have an amap lock it and extract the anons
106: * - if sequential advice deactivate pages behind us
107: * - at the same time check pmap for unmapped areas and anon for pages
108: * that we could map in (and do map it if found)
109: * - check object for resident pages that we could map in
110: * - if (case 2) goto Case2
111: * - >>> handle case 1
112: * - ensure source anon is resident in RAM
113: * - if case 1B alloc new anon and copy from source
114: * - map the correct page in
115: * Case2:
116: * - >>> handle case 2
117: * - ensure source page is resident (if uobj)
118: * - if case 2B alloc new anon and copy from source (could be zero
119: * fill if uobj == NULL)
120: * - map the correct page in
121: * - done!
122: *
123: * note on paging:
124: * if we have to do I/O we place a PG_BUSY page in the correct object,
125: * unlock everything, and do the I/O. when I/O is done we must reverify
126: * the state of the world before assuming that our data structures are
127: * valid. [because mappings could change while the map is unlocked]
128: *
129: * alternative 1: unbusy the page in question and restart the page fault
130: * from the top (ReFault). this is easy but does not take advantage
1.63 chs 131: * of the information that we already have from our previous lookup,
1.1 mrg 132: * although it is possible that the "hints" in the vm_map will help here.
133: *
134: * alternative 2: the system already keeps track of a "version" number of
135: * a map. [i.e. every time you write-lock a map (e.g. to change a
136: * mapping) you bump the version number up by one...] so, we can save
137: * the version number of the map before we release the lock and start I/O.
138: * then when I/O is done we can relock and check the version numbers
139: * to see if anything changed. this might save us some over 1 because
140: * we don't have to unbusy the page and may be less compares(?).
141: *
142: * alternative 3: put in backpointers or a way to "hold" part of a map
143: * in place while I/O is in progress. this could be complex to
144: * implement (especially with structures like amap that can be referenced
145: * by multiple map entries, and figuring out what should wait could be
146: * complex as well...).
147: *
1.125 ad 148: * we use alternative 2. given that we are multi-threaded now we may want
149: * to reconsider the choice.
1.1 mrg 150: */
151:
152: /*
153: * local data structures
154: */
155:
156: struct uvm_advice {
1.7 mrg 157: int advice;
158: int nback;
159: int nforw;
1.1 mrg 160: };
161:
162: /*
163: * page range array:
1.63 chs 164: * note: index in array must match "advice" value
1.1 mrg 165: * XXX: borrowed numbers from freebsd. do they work well for us?
166: */
167:
1.95 thorpej 168: static const struct uvm_advice uvmadvice[] = {
1.7 mrg 169: { MADV_NORMAL, 3, 4 },
170: { MADV_RANDOM, 0, 0 },
171: { MADV_SEQUENTIAL, 8, 7},
1.1 mrg 172: };
173:
1.69 chs 174: #define UVM_MAXRANGE 16 /* must be MAX() of nback+nforw+1 */
1.1 mrg 175:
176: /*
177: * private prototypes
178: */
179:
180: /*
181: * inline functions
182: */
183:
184: /*
185: * uvmfault_anonflush: try and deactivate pages in specified anons
186: *
187: * => does not have to deactivate page if it is busy
188: */
189:
1.103 perry 190: static inline void
1.95 thorpej 191: uvmfault_anonflush(struct vm_anon **anons, int n)
1.1 mrg 192: {
1.7 mrg 193: int lcv;
194: struct vm_page *pg;
1.63 chs 195:
1.7 mrg 196: for (lcv = 0 ; lcv < n ; lcv++) {
197: if (anons[lcv] == NULL)
198: continue;
1.122 ad 199: mutex_enter(&anons[lcv]->an_lock);
1.94 yamt 200: pg = anons[lcv]->an_page;
1.117 yamt 201: if (pg && (pg->flags & PG_BUSY) == 0) {
1.122 ad 202: mutex_enter(&uvm_pageqlock);
1.7 mrg 203: if (pg->wire_count == 0) {
204: uvm_pagedeactivate(pg);
205: }
1.122 ad 206: mutex_exit(&uvm_pageqlock);
1.7 mrg 207: }
1.122 ad 208: mutex_exit(&anons[lcv]->an_lock);
1.7 mrg 209: }
1.1 mrg 210: }
211:
212: /*
213: * normal functions
214: */
215:
216: /*
217: * uvmfault_amapcopy: clear "needs_copy" in a map.
218: *
219: * => called with VM data structures unlocked (usually, see below)
220: * => we get a write lock on the maps and clear needs_copy for a VA
221: * => if we are out of RAM we sleep (waiting for more)
222: */
223:
1.7 mrg 224: static void
1.95 thorpej 225: uvmfault_amapcopy(struct uvm_faultinfo *ufi)
1.1 mrg 226: {
1.69 chs 227: for (;;) {
1.1 mrg 228:
1.7 mrg 229: /*
230: * no mapping? give up.
231: */
1.1 mrg 232:
1.119 thorpej 233: if (uvmfault_lookup(ufi, true) == false)
1.7 mrg 234: return;
1.1 mrg 235:
1.7 mrg 236: /*
237: * copy if needed.
238: */
1.1 mrg 239:
1.7 mrg 240: if (UVM_ET_ISNEEDSCOPY(ufi->entry))
1.108 yamt 241: amap_copy(ufi->map, ufi->entry, AMAP_COPY_NOWAIT,
1.13 chuck 242: ufi->orig_rvaddr, ufi->orig_rvaddr + 1);
1.1 mrg 243:
1.7 mrg 244: /*
245: * didn't work? must be out of RAM. unlock and sleep.
246: */
247:
248: if (UVM_ET_ISNEEDSCOPY(ufi->entry)) {
1.119 thorpej 249: uvmfault_unlockmaps(ufi, true);
1.7 mrg 250: uvm_wait("fltamapcopy");
251: continue;
252: }
253:
254: /*
255: * got it! unlock and return.
256: */
1.63 chs 257:
1.119 thorpej 258: uvmfault_unlockmaps(ufi, true);
1.7 mrg 259: return;
260: }
261: /*NOTREACHED*/
1.1 mrg 262: }
263:
264: /*
265: * uvmfault_anonget: get data in an anon into a non-busy, non-released
266: * page in that anon.
267: *
268: * => maps, amap, and anon locked by caller.
1.57 chs 269: * => if we fail (result != 0) we unlock everything.
1.1 mrg 270: * => if we are successful, we return with everything still locked.
271: * => we don't move the page on the queues [gets moved later]
272: * => if we allocate a new page [we_own], it gets put on the queues.
273: * either way, the result is that the page is on the queues at return time
274: * => for pages which are on loan from a uvm_object (and thus are not
275: * owned by the anon): if successful, we return with the owning object
276: * locked. the caller must unlock this object when it unlocks everything
277: * else.
278: */
279:
1.47 chs 280: int
1.95 thorpej 281: uvmfault_anonget(struct uvm_faultinfo *ufi, struct vm_amap *amap,
282: struct vm_anon *anon)
1.7 mrg 283: {
1.118 thorpej 284: bool we_own; /* we own anon's page? */
285: bool locked; /* did we relock? */
1.7 mrg 286: struct vm_page *pg;
1.58 chs 287: int error;
1.7 mrg 288: UVMHIST_FUNC("uvmfault_anonget"); UVMHIST_CALLED(maphist);
289:
1.122 ad 290: KASSERT(mutex_owned(&anon->an_lock));
1.53 thorpej 291:
1.58 chs 292: error = 0;
1.9 chuck 293: uvmexp.fltanget++;
294: /* bump rusage counters */
1.94 yamt 295: if (anon->an_page)
1.124 ad 296: curlwp->l_ru.ru_minflt++;
1.9 chuck 297: else
1.124 ad 298: curlwp->l_ru.ru_majflt++;
1.7 mrg 299:
1.63 chs 300: /*
1.7 mrg 301: * loop until we get it, or fail.
302: */
303:
1.69 chs 304: for (;;) {
1.119 thorpej 305: we_own = false; /* true if we set PG_BUSY on a page */
1.94 yamt 306: pg = anon->an_page;
1.1 mrg 307:
1.7 mrg 308: /*
309: * if there is a resident page and it is loaned, then anon
310: * may not own it. call out to uvm_anon_lockpage() to ensure
311: * the real owner of the page has been identified and locked.
312: */
313:
314: if (pg && pg->loan_count)
1.13 chuck 315: pg = uvm_anon_lockloanpg(anon);
1.7 mrg 316:
317: /*
318: * page there? make sure it is not busy/released.
319: */
320:
321: if (pg) {
322:
323: /*
324: * at this point, if the page has a uobject [meaning
325: * we have it on loan], then that uobject is locked
326: * by us! if the page is busy, we drop all the
327: * locks (including uobject) and try again.
328: */
329:
1.69 chs 330: if ((pg->flags & PG_BUSY) == 0) {
1.7 mrg 331: UVMHIST_LOG(maphist, "<- OK",0,0,0,0);
1.57 chs 332: return (0);
1.7 mrg 333: }
334: pg->flags |= PG_WANTED;
335: uvmexp.fltpgwait++;
336:
337: /*
338: * the last unlock must be an atomic unlock+wait on
339: * the owner of page
340: */
1.69 chs 341:
1.7 mrg 342: if (pg->uobject) { /* owner is uobject ? */
343: uvmfault_unlockall(ufi, amap, NULL, anon);
344: UVMHIST_LOG(maphist, " unlock+wait on uobj",0,
345: 0,0,0);
346: UVM_UNLOCK_AND_WAIT(pg,
347: &pg->uobject->vmobjlock,
1.119 thorpej 348: false, "anonget1",0);
1.7 mrg 349: } else {
350: /* anon owns page */
351: uvmfault_unlockall(ufi, amap, NULL, NULL);
352: UVMHIST_LOG(maphist, " unlock+wait on anon",0,
353: 0,0,0);
354: UVM_UNLOCK_AND_WAIT(pg,&anon->an_lock,0,
355: "anonget2",0);
356: }
357: } else {
1.101 yamt 358: #if defined(VMSWAP)
1.63 chs 359:
1.7 mrg 360: /*
361: * no page, we must try and bring it in.
362: */
1.69 chs 363:
1.28 chs 364: pg = uvm_pagealloc(NULL, 0, anon, 0);
1.7 mrg 365: if (pg == NULL) { /* out of RAM. */
366: uvmfault_unlockall(ufi, amap, NULL, anon);
367: uvmexp.fltnoram++;
368: UVMHIST_LOG(maphist, " noram -- UVM_WAIT",0,
369: 0,0,0);
1.93 yamt 370: if (!uvm_reclaimable()) {
371: return ENOMEM;
372: }
1.7 mrg 373: uvm_wait("flt_noram1");
374: } else {
375: /* we set the PG_BUSY bit */
1.119 thorpej 376: we_own = true;
1.7 mrg 377: uvmfault_unlockall(ufi, amap, NULL, anon);
378:
379: /*
380: * we are passing a PG_BUSY+PG_FAKE+PG_CLEAN
381: * page into the uvm_swap_get function with
1.18 chuck 382: * all data structures unlocked. note that
383: * it is ok to read an_swslot here because
384: * we hold PG_BUSY on the page.
1.7 mrg 385: */
386: uvmexp.pageins++;
1.58 chs 387: error = uvm_swap_get(pg, anon->an_swslot,
1.7 mrg 388: PGO_SYNCIO);
389:
390: /*
391: * we clean up after the i/o below in the
392: * "we_own" case
393: */
394: }
1.101 yamt 395: #else /* defined(VMSWAP) */
396: panic("%s: no page", __func__);
397: #endif /* defined(VMSWAP) */
1.7 mrg 398: }
399:
400: /*
401: * now relock and try again
402: */
403:
404: locked = uvmfault_relock(ufi);
1.47 chs 405: if (locked && amap != NULL) {
1.19 chuck 406: amap_lock(amap);
1.7 mrg 407: }
408: if (locked || we_own)
1.122 ad 409: mutex_enter(&anon->an_lock);
1.7 mrg 410:
411: /*
412: * if we own the page (i.e. we set PG_BUSY), then we need
413: * to clean up after the I/O. there are three cases to
414: * consider:
415: * [1] page released during I/O: free anon and ReFault.
1.63 chs 416: * [2] I/O not OK. free the page and cause the fault
1.7 mrg 417: * to fail.
418: * [3] I/O OK! activate the page and sync with the
419: * non-we_own case (i.e. drop anon lock if not locked).
420: */
1.63 chs 421:
1.7 mrg 422: if (we_own) {
1.101 yamt 423: #if defined(VMSWAP)
1.7 mrg 424: if (pg->flags & PG_WANTED) {
1.63 chs 425: wakeup(pg);
1.7 mrg 426: }
1.58 chs 427: if (error) {
1.1 mrg 428:
1.47 chs 429: /*
430: * remove the swap slot from the anon
431: * and mark the anon as having no real slot.
432: * don't free the swap slot, thus preventing
433: * it from being used again.
434: */
1.69 chs 435:
1.84 pk 436: if (anon->an_swslot > 0)
437: uvm_swap_markbad(anon->an_swslot, 1);
1.47 chs 438: anon->an_swslot = SWSLOT_BAD;
439:
1.88 yamt 440: if ((pg->flags & PG_RELEASED) != 0)
441: goto released;
442:
1.47 chs 443: /*
1.7 mrg 444: * note: page was never !PG_BUSY, so it
445: * can't be mapped and thus no need to
446: * pmap_page_protect it...
447: */
1.69 chs 448:
1.122 ad 449: mutex_enter(&uvm_pageqlock);
1.7 mrg 450: uvm_pagefree(pg);
1.122 ad 451: mutex_exit(&uvm_pageqlock);
1.7 mrg 452:
453: if (locked)
454: uvmfault_unlockall(ufi, amap, NULL,
455: anon);
456: else
1.122 ad 457: mutex_exit(&anon->an_lock);
1.7 mrg 458: UVMHIST_LOG(maphist, "<- ERROR", 0,0,0,0);
1.58 chs 459: return error;
1.7 mrg 460: }
1.63 chs 461:
1.88 yamt 462: if ((pg->flags & PG_RELEASED) != 0) {
463: released:
464: KASSERT(anon->an_ref == 0);
465:
466: /*
467: * released while we unlocked amap.
468: */
469:
470: if (locked)
471: uvmfault_unlockall(ufi, amap, NULL,
472: NULL);
473:
474: uvm_anon_release(anon);
475:
476: if (error) {
477: UVMHIST_LOG(maphist,
478: "<- ERROR/RELEASED", 0,0,0,0);
479: return error;
480: }
481:
482: UVMHIST_LOG(maphist, "<- RELEASED", 0,0,0,0);
483: return ERESTART;
484: }
485:
1.7 mrg 486: /*
1.69 chs 487: * we've successfully read the page, activate it.
1.7 mrg 488: */
1.69 chs 489:
1.122 ad 490: mutex_enter(&uvm_pageqlock);
1.7 mrg 491: uvm_pageactivate(pg);
1.122 ad 492: mutex_exit(&uvm_pageqlock);
1.69 chs 493: pg->flags &= ~(PG_WANTED|PG_BUSY|PG_FAKE);
494: UVM_PAGE_OWN(pg, NULL);
1.7 mrg 495: if (!locked)
1.122 ad 496: mutex_exit(&anon->an_lock);
1.101 yamt 497: #else /* defined(VMSWAP) */
498: panic("%s: we_own", __func__);
499: #endif /* defined(VMSWAP) */
1.7 mrg 500: }
501:
502: /*
503: * we were not able to relock. restart fault.
504: */
505:
506: if (!locked) {
507: UVMHIST_LOG(maphist, "<- REFAULT", 0,0,0,0);
1.57 chs 508: return (ERESTART);
1.7 mrg 509: }
510:
511: /*
512: * verify no one has touched the amap and moved the anon on us.
513: */
1.1 mrg 514:
1.47 chs 515: if (ufi != NULL &&
1.63 chs 516: amap_lookup(&ufi->entry->aref,
1.47 chs 517: ufi->orig_rvaddr - ufi->entry->start) != anon) {
1.63 chs 518:
1.7 mrg 519: uvmfault_unlockall(ufi, amap, NULL, anon);
520: UVMHIST_LOG(maphist, "<- REFAULT", 0,0,0,0);
1.57 chs 521: return (ERESTART);
1.7 mrg 522: }
1.63 chs 523:
1.7 mrg 524: /*
1.63 chs 525: * try it again!
1.7 mrg 526: */
1.1 mrg 527:
1.7 mrg 528: uvmexp.fltanretry++;
529: continue;
1.69 chs 530: }
1.7 mrg 531: /*NOTREACHED*/
1.1 mrg 532: }
533:
534: /*
1.106 yamt 535: * uvmfault_promote: promote data to a new anon. used for 1B and 2B.
536: *
537: * 1. allocate an anon and a page.
538: * 2. fill its contents.
539: * 3. put it into amap.
540: *
541: * => if we fail (result != 0) we unlock everything.
542: * => on success, return a new locked anon via 'nanon'.
543: * (*nanon)->an_page will be a resident, locked, dirty page.
544: */
545:
546: static int
547: uvmfault_promote(struct uvm_faultinfo *ufi,
548: struct vm_anon *oanon,
549: struct vm_page *uobjpage,
550: struct vm_anon **nanon, /* OUT: allocated anon */
551: struct vm_anon **spare)
552: {
553: struct vm_amap *amap = ufi->entry->aref.ar_amap;
554: struct uvm_object *uobj;
555: struct vm_anon *anon;
556: struct vm_page *pg;
557: struct vm_page *opg;
558: int error;
559: UVMHIST_FUNC(__func__); UVMHIST_CALLED(maphist);
560:
561: if (oanon) {
562: /* anon COW */
563: opg = oanon->an_page;
564: KASSERT(opg != NULL);
565: KASSERT(opg->uobject == NULL || opg->loan_count > 0);
566: } else if (uobjpage != PGO_DONTCARE) {
567: /* object-backed COW */
568: opg = uobjpage;
569: } else {
570: /* ZFOD */
571: opg = NULL;
572: }
573: if (opg != NULL) {
574: uobj = opg->uobject;
575: } else {
576: uobj = NULL;
577: }
578:
579: KASSERT(amap != NULL);
580: KASSERT(uobjpage != NULL);
581: KASSERT(uobjpage == PGO_DONTCARE || (uobjpage->flags & PG_BUSY) != 0);
1.120 ad 582: KASSERT(mutex_owned(&amap->am_l));
1.122 ad 583: KASSERT(oanon == NULL || mutex_owned(&oanon->an_lock));
584: KASSERT(uobj == NULL || mutex_owned(&uobj->vmobjlock));
585: #if 0
586: KASSERT(*spare == NULL || !mutex_owned(&(*spare)->an_lock));
587: #endif
1.106 yamt 588:
589: if (*spare != NULL) {
590: anon = *spare;
591: *spare = NULL;
1.122 ad 592: mutex_enter(&anon->an_lock);
1.106 yamt 593: } else if (ufi->map != kernel_map) {
594: anon = uvm_analloc();
595: } else {
596: UVMHIST_LOG(maphist, "kernel_map, unlock and retry", 0,0,0,0);
597:
598: /*
599: * we can't allocate anons with kernel_map locked.
600: */
601:
602: uvm_page_unbusy(&uobjpage, 1);
603: uvmfault_unlockall(ufi, amap, uobj, oanon);
604:
605: *spare = uvm_analloc();
606: if (*spare == NULL) {
607: goto nomem;
608: }
1.122 ad 609: mutex_exit(&(*spare)->an_lock);
1.106 yamt 610: error = ERESTART;
611: goto done;
612: }
613: if (anon) {
614:
615: /*
616: * The new anon is locked.
617: *
618: * if opg == NULL, we want a zero'd, dirty page,
619: * so have uvm_pagealloc() do that for us.
620: */
621:
622: pg = uvm_pagealloc(NULL, 0, anon,
623: (opg == NULL) ? UVM_PGA_ZERO : 0);
624: } else {
625: pg = NULL;
626: }
627:
628: /*
629: * out of memory resources?
630: */
631:
632: if (pg == NULL) {
633: /* save anon for the next try. */
634: if (anon != NULL) {
1.122 ad 635: mutex_exit(&anon->an_lock);
1.106 yamt 636: *spare = anon;
637: }
638:
639: /* unlock and fail ... */
640: uvm_page_unbusy(&uobjpage, 1);
641: uvmfault_unlockall(ufi, amap, uobj, oanon);
642: nomem:
643: if (!uvm_reclaimable()) {
644: UVMHIST_LOG(maphist, "out of VM", 0,0,0,0);
645: uvmexp.fltnoanon++;
646: error = ENOMEM;
647: goto done;
648: }
649:
650: UVMHIST_LOG(maphist, "out of RAM, waiting for more", 0,0,0,0);
651: uvmexp.fltnoram++;
652: uvm_wait("flt_noram5");
653: error = ERESTART;
654: goto done;
655: }
656:
657: /* copy page [pg now dirty] */
658: if (opg) {
659: uvm_pagecopy(opg, pg);
660: }
661:
662: amap_add(&ufi->entry->aref, ufi->orig_rvaddr - ufi->entry->start, anon,
663: oanon != NULL);
664:
665: *nanon = anon;
666: error = 0;
667: done:
668: return error;
669: }
670:
671:
672: /*
1.1 mrg 673: * F A U L T - m a i n e n t r y p o i n t
674: */
675:
676: /*
677: * uvm_fault: page fault handler
678: *
679: * => called from MD code to resolve a page fault
1.63 chs 680: * => VM data structures usually should be unlocked. however, it is
1.1 mrg 681: * possible to call here with the main map locked if the caller
682: * gets a write lock, sets it recusive, and then calls us (c.f.
683: * uvm_map_pageable). this should be avoided because it keeps
684: * the map locked off during I/O.
1.66 thorpej 685: * => MUST NEVER BE CALLED IN INTERRUPT CONTEXT
1.1 mrg 686: */
687:
1.24 mycroft 688: #define MASK(entry) (UVM_ET_ISCOPYONWRITE(entry) ? \
689: ~VM_PROT_WRITE : VM_PROT_ALL)
690:
1.110 drochner 691: /* fault_flag values passed from uvm_fault_wire to uvm_fault_internal */
1.130 uebayasi 692: #define UVM_FAULT_WIRE (1 << 0)
693: #define UVM_FAULT_MAXPROT (1 << 1)
1.110 drochner 694:
1.140 uebayasi 695: struct uvm_faultctx {
696: vm_prot_t access_type;
697: vm_prot_t enter_prot;
1.150 uebayasi 698: vaddr_t startva;
699: int npages;
700: int centeridx;
701: struct vm_anon *anon_spare;
1.146 uebayasi 702: bool wire_mapping;
1.140 uebayasi 703: bool narrow;
1.146 uebayasi 704: bool wire_paging;
1.140 uebayasi 705: bool maxprot;
706: bool cow_now;
707: };
708:
1.144 uebayasi 709: static int uvm_fault_check(
710: struct uvm_faultinfo *, struct uvm_faultctx *,
711: struct vm_anon ***, struct vm_page ***);
712: typedef int uvm_fault_upper_subfunc_t(
713: struct uvm_faultinfo *, struct uvm_faultctx *,
714: struct vm_anon **, struct vm_page **);
715: static uvm_fault_upper_subfunc_t uvm_fault_upper_lookup;
716: typedef int uvm_fault_lower_subfunc_t(
717: struct uvm_faultinfo *, struct uvm_faultctx *,
718: struct vm_page **);
719: static uvm_fault_lower_subfunc_t uvm_fault_lower;
720: static uvm_fault_lower_subfunc_t uvm_fault_lower_special;
721: static uvm_fault_lower_subfunc_t uvm_fault_lower_generic_lookup;
722: static uvm_fault_lower_subfunc_t uvm_fault_lower_generic;
1.148 uebayasi 723: static int uvm_fault_upper(
724: struct uvm_faultinfo *, struct uvm_faultctx *,
725: struct vm_anon **);
726: static int uvm_fault_lower_generic1(
727: struct uvm_faultinfo *, struct uvm_faultctx *,
728: struct vm_page *);
729: static int uvm_fault_lower_generic2(
730: struct uvm_faultinfo *, struct uvm_faultctx *,
731: struct vm_page *);
1.138 uebayasi 732:
1.7 mrg 733: int
1.110 drochner 734: uvm_fault_internal(struct vm_map *orig_map, vaddr_t vaddr,
735: vm_prot_t access_type, int fault_flag)
1.1 mrg 736: {
1.7 mrg 737: struct uvm_faultinfo ufi;
1.140 uebayasi 738: struct uvm_faultctx flt = {
739: .access_type = access_type,
1.146 uebayasi 740:
741: /* don't look for neighborhood * pages on "wire" fault */
742: .narrow = (fault_flag & UVM_FAULT_WIRE) != 0,
743:
744: /* "wire" fault causes wiring of both mapping and paging */
745: .wire_mapping = (fault_flag & UVM_FAULT_WIRE) != 0,
746: .wire_paging = (fault_flag & UVM_FAULT_WIRE) != 0,
747:
1.140 uebayasi 748: .maxprot = (fault_flag & UVM_FAULT_MAXPROT) != 0,
749: };
1.137 uebayasi 750: struct vm_anon *anons_store[UVM_MAXRANGE], **anons;
1.141 uebayasi 751: struct vm_page *pages_store[UVM_MAXRANGE], **pages;
1.140 uebayasi 752: int error;
1.7 mrg 753: UVMHIST_FUNC("uvm_fault"); UVMHIST_CALLED(maphist);
1.1 mrg 754:
1.110 drochner 755: UVMHIST_LOG(maphist, "(map=0x%x, vaddr=0x%x, at=%d, ff=%d)",
756: orig_map, vaddr, access_type, fault_flag);
1.1 mrg 757:
1.7 mrg 758: uvmexp.faults++; /* XXX: locking? */
759:
760: /*
761: * init the IN parameters in the ufi
762: */
1.1 mrg 763:
1.7 mrg 764: ufi.orig_map = orig_map;
765: ufi.orig_rvaddr = trunc_page(vaddr);
766: ufi.orig_size = PAGE_SIZE; /* can't get any smaller than this */
767:
1.142 uebayasi 768: error = ERESTART;
769: while (error == ERESTART) {
1.143 uebayasi 770: anons = anons_store;
771: pages = pages_store;
1.1 mrg 772:
1.143 uebayasi 773: error = uvm_fault_check(&ufi, &flt, &anons, &pages);
774: if (error != 0)
775: continue;
1.141 uebayasi 776:
1.143 uebayasi 777: error = uvm_fault_upper_lookup(&ufi, &flt, anons, pages);
778: if (error != 0)
779: continue;
1.138 uebayasi 780:
1.144 uebayasi 781: if (pages[flt.centeridx] == PGO_DONTCARE)
1.148 uebayasi 782: error = uvm_fault_upper(&ufi, &flt, anons);
1.143 uebayasi 783: else
1.144 uebayasi 784: error = uvm_fault_lower(&ufi, &flt, pages);
1.142 uebayasi 785: }
1.138 uebayasi 786:
1.140 uebayasi 787: if (flt.anon_spare != NULL) {
788: flt.anon_spare->an_ref--;
789: uvm_anfree(flt.anon_spare);
1.138 uebayasi 790: }
791: return error;
1.141 uebayasi 792: }
1.138 uebayasi 793:
1.144 uebayasi 794: static int
1.141 uebayasi 795: uvm_fault_check(
796: struct uvm_faultinfo *ufi, struct uvm_faultctx *flt,
797: struct vm_anon ***ranons, struct vm_page ***rpages)
798: {
799: struct vm_amap *amap;
800: struct uvm_object *uobj;
1.137 uebayasi 801: vm_prot_t check_prot;
802: int nback, nforw;
803:
1.7 mrg 804: /*
805: * lookup and lock the maps
806: */
807:
1.141 uebayasi 808: if (uvmfault_lookup(ufi, false) == false) {
1.7 mrg 809: UVMHIST_LOG(maphist, "<- no mapping @ 0x%x", vaddr, 0,0,0);
1.141 uebayasi 810: return EFAULT;
1.7 mrg 811: }
812: /* locked: maps(read) */
813:
1.61 thorpej 814: #ifdef DIAGNOSTIC
1.141 uebayasi 815: if ((ufi->map->flags & VM_MAP_PAGEABLE) == 0) {
1.61 thorpej 816: printf("Page fault on non-pageable map:\n");
1.141 uebayasi 817: printf("ufi->map = %p\n", ufi->map);
818: printf("ufi->orig_map = %p\n", ufi->orig_map);
819: printf("ufi->orig_rvaddr = 0x%lx\n", (u_long) ufi->orig_rvaddr);
820: panic("uvm_fault: (ufi->map->flags & VM_MAP_PAGEABLE) == 0");
1.61 thorpej 821: }
822: #endif
1.58 chs 823:
1.7 mrg 824: /*
825: * check protection
826: */
827:
1.141 uebayasi 828: check_prot = flt->maxprot ?
829: ufi->entry->max_protection : ufi->entry->protection;
830: if ((check_prot & flt->access_type) != flt->access_type) {
1.7 mrg 831: UVMHIST_LOG(maphist,
832: "<- protection failure (prot=0x%x, access=0x%x)",
1.141 uebayasi 833: ufi->entry->protection, flt->access_type, 0, 0);
834: uvmfault_unlockmaps(ufi, false);
835: return EACCES;
1.7 mrg 836: }
837:
838: /*
839: * "enter_prot" is the protection we want to enter the page in at.
840: * for certain pages (e.g. copy-on-write pages) this protection can
1.141 uebayasi 841: * be more strict than ufi->entry->protection. "wired" means either
1.7 mrg 842: * the entry is wired or we are fault-wiring the pg.
843: */
844:
1.141 uebayasi 845: flt->enter_prot = ufi->entry->protection;
1.146 uebayasi 846: if (VM_MAPENT_ISWIRED(ufi->entry))
847: flt->wire_mapping = true;
848:
849: if (flt->wire_mapping) {
1.141 uebayasi 850: flt->access_type = flt->enter_prot; /* full access for wired */
851: flt->cow_now = (check_prot & VM_PROT_WRITE) != 0;
1.73 chs 852: } else {
1.141 uebayasi 853: flt->cow_now = (flt->access_type & VM_PROT_WRITE) != 0;
1.73 chs 854: }
1.7 mrg 855:
856: /*
857: * handle "needs_copy" case. if we need to copy the amap we will
858: * have to drop our readlock and relock it with a write lock. (we
859: * need a write lock to change anything in a map entry [e.g.
860: * needs_copy]).
861: */
862:
1.141 uebayasi 863: if (UVM_ET_ISNEEDSCOPY(ufi->entry)) {
864: if (flt->cow_now || (ufi->entry->object.uvm_obj == NULL)) {
865: KASSERT(!flt->maxprot);
1.7 mrg 866: /* need to clear */
867: UVMHIST_LOG(maphist,
868: " need to clear needs_copy and refault",0,0,0,0);
1.141 uebayasi 869: uvmfault_unlockmaps(ufi, false);
870: uvmfault_amapcopy(ufi);
1.7 mrg 871: uvmexp.fltamcopy++;
1.141 uebayasi 872: return ERESTART;
1.7 mrg 873:
874: } else {
875:
876: /*
877: * ensure that we pmap_enter page R/O since
878: * needs_copy is still true
879: */
1.72 chs 880:
1.141 uebayasi 881: flt->enter_prot &= ~VM_PROT_WRITE;
1.7 mrg 882: }
883: }
884:
885: /*
886: * identify the players
887: */
888:
1.141 uebayasi 889: amap = ufi->entry->aref.ar_amap; /* upper layer */
890: uobj = ufi->entry->object.uvm_obj; /* lower layer */
1.7 mrg 891:
892: /*
893: * check for a case 0 fault. if nothing backing the entry then
894: * error now.
895: */
896:
897: if (amap == NULL && uobj == NULL) {
1.141 uebayasi 898: uvmfault_unlockmaps(ufi, false);
1.7 mrg 899: UVMHIST_LOG(maphist,"<- no backing store, no overlay",0,0,0,0);
1.141 uebayasi 900: return EFAULT;
1.7 mrg 901: }
1.1 mrg 902:
1.7 mrg 903: /*
904: * establish range of interest based on advice from mapper
905: * and then clip to fit map entry. note that we only want
1.63 chs 906: * to do this the first time through the fault. if we
1.7 mrg 907: * ReFault we will disable this by setting "narrow" to true.
908: */
1.1 mrg 909:
1.141 uebayasi 910: if (flt->narrow == false) {
1.7 mrg 911:
912: /* wide fault (!narrow) */
1.141 uebayasi 913: KASSERT(uvmadvice[ufi->entry->advice].advice ==
914: ufi->entry->advice);
915: nback = MIN(uvmadvice[ufi->entry->advice].nback,
916: (ufi->orig_rvaddr - ufi->entry->start) >> PAGE_SHIFT);
917: flt->startva = ufi->orig_rvaddr - (nback << PAGE_SHIFT);
918: nforw = MIN(uvmadvice[ufi->entry->advice].nforw,
919: ((ufi->entry->end - ufi->orig_rvaddr) >>
1.15 chs 920: PAGE_SHIFT) - 1);
1.7 mrg 921: /*
922: * note: "-1" because we don't want to count the
923: * faulting page as forw
924: */
1.141 uebayasi 925: flt->npages = nback + nforw + 1;
926: flt->centeridx = nback;
1.7 mrg 927:
1.141 uebayasi 928: flt->narrow = true; /* ensure only once per-fault */
1.7 mrg 929:
930: } else {
1.63 chs 931:
1.7 mrg 932: /* narrow fault! */
933: nback = nforw = 0;
1.141 uebayasi 934: flt->startva = ufi->orig_rvaddr;
935: flt->npages = 1;
936: flt->centeridx = 0;
1.1 mrg 937:
1.7 mrg 938: }
1.131 uebayasi 939: /* offset from entry's start to pgs' start */
1.141 uebayasi 940: const voff_t eoff = flt->startva - ufi->entry->start;
1.1 mrg 941:
1.7 mrg 942: /* locked: maps(read) */
1.13 chuck 943: UVMHIST_LOG(maphist, " narrow=%d, back=%d, forw=%d, startva=0x%x",
1.141 uebayasi 944: flt->narrow, nback, nforw, flt->startva);
945: UVMHIST_LOG(maphist, " entry=0x%x, amap=0x%x, obj=0x%x", ufi->entry,
1.16 chs 946: amap, uobj, 0);
1.1 mrg 947:
1.7 mrg 948: /*
949: * if we've got an amap, lock it and extract current anons.
950: */
951:
952: if (amap) {
1.19 chuck 953: amap_lock(amap);
1.141 uebayasi 954: amap_lookups(&ufi->entry->aref, eoff, *ranons, flt->npages);
1.7 mrg 955: } else {
1.141 uebayasi 956: *ranons = NULL; /* to be safe */
1.7 mrg 957: }
958:
959: /* locked: maps(read), amap(if there) */
1.120 ad 960: KASSERT(amap == NULL || mutex_owned(&amap->am_l));
1.7 mrg 961:
962: /*
963: * for MADV_SEQUENTIAL mappings we want to deactivate the back pages
964: * now and then forget about them (for the rest of the fault).
965: */
966:
1.141 uebayasi 967: if (ufi->entry->advice == MADV_SEQUENTIAL && nback != 0) {
1.7 mrg 968:
969: UVMHIST_LOG(maphist, " MADV_SEQUENTIAL: flushing backpages",
970: 0,0,0,0);
971: /* flush back-page anons? */
1.63 chs 972: if (amap)
1.141 uebayasi 973: uvmfault_anonflush(*ranons, nback);
1.7 mrg 974:
975: /* flush object? */
976: if (uobj) {
1.137 uebayasi 977: voff_t uoff;
978:
1.141 uebayasi 979: uoff = ufi->entry->offset + eoff;
1.122 ad 980: mutex_enter(&uobj->vmobjlock);
1.90 yamt 981: (void) (uobj->pgops->pgo_put)(uobj, uoff, uoff +
1.15 chs 982: (nback << PAGE_SHIFT), PGO_DEACTIVATE);
1.7 mrg 983: }
984:
985: /* now forget about the backpages */
986: if (amap)
1.141 uebayasi 987: *ranons += nback;
988: #if 0
989: /* XXXUEBS */
990: if (uobj)
991: *rpages += nback;
992: #endif
993: flt->startva += (nback << PAGE_SHIFT);
994: flt->npages -= nback;
995: flt->centeridx = 0;
1.7 mrg 996: }
1.137 uebayasi 997: /*
998: * => startva is fixed
999: * => npages is fixed
1000: */
1001:
1.141 uebayasi 1002: return 0;
1003: }
1004:
1.151 uebayasi 1005: static inline void
1006: uvm_fault_upper_lookup1(
1007: struct uvm_faultinfo *ufi, struct uvm_faultctx *flt,
1008: bool shadowed);
1009: static void
1010: uvm_fault_upper_lookup_neighbor(
1011: struct uvm_faultinfo *ufi, struct uvm_faultctx *flt,
1012: vaddr_t currva, struct vm_anon *anon);
1013:
1.144 uebayasi 1014: static int
1.141 uebayasi 1015: uvm_fault_upper_lookup(
1016: struct uvm_faultinfo *ufi, struct uvm_faultctx *flt,
1017: struct vm_anon **anons, struct vm_page **pages)
1018: {
1019: struct vm_amap *amap = ufi->entry->aref.ar_amap;
1.137 uebayasi 1020: int lcv;
1021: vaddr_t currva;
1.144 uebayasi 1022: bool shadowed;
1.7 mrg 1023:
1024: /* locked: maps(read), amap(if there) */
1.120 ad 1025: KASSERT(amap == NULL || mutex_owned(&amap->am_l));
1.1 mrg 1026:
1.7 mrg 1027: /*
1028: * map in the backpages and frontpages we found in the amap in hopes
1029: * of preventing future faults. we also init the pages[] array as
1030: * we go.
1031: */
1032:
1.141 uebayasi 1033: currva = flt->startva;
1.144 uebayasi 1034: shadowed = false;
1.141 uebayasi 1035: for (lcv = 0 ; lcv < flt->npages ; lcv++, currva += PAGE_SIZE) {
1.7 mrg 1036: /*
1037: * dont play with VAs that are already mapped
1.13 chuck 1038: * except for center)
1.7 mrg 1039: */
1.141 uebayasi 1040: if (lcv != flt->centeridx &&
1041: pmap_extract(ufi->orig_map->pmap, currva, NULL)) {
1.52 chs 1042: pages[lcv] = PGO_DONTCARE;
1043: continue;
1.7 mrg 1044: }
1045:
1046: /*
1047: * unmapped or center page. check if any anon at this level.
1048: */
1049: if (amap == NULL || anons[lcv] == NULL) {
1050: pages[lcv] = NULL;
1051: continue;
1052: }
1053:
1054: /*
1055: * check for present page and map if possible. re-activate it.
1056: */
1057:
1058: pages[lcv] = PGO_DONTCARE;
1.141 uebayasi 1059: if (lcv == flt->centeridx) { /* save center for later! */
1.144 uebayasi 1060: shadowed = true;
1.151 uebayasi 1061: } else {
1062: uvm_fault_upper_lookup_neighbor(ufi, flt, currva, anons[lcv]);
1.7 mrg 1063: }
1.151 uebayasi 1064: }
1065:
1066: uvm_fault_upper_lookup1(ufi, flt, shadowed);
1067:
1068: return 0;
1069: }
1070:
1071: static void
1072: uvm_fault_upper_lookup_neighbor(
1073: struct uvm_faultinfo *ufi, struct uvm_faultctx *flt,
1074: vaddr_t currva, struct vm_anon *anon)
1075: {
1076:
1.152 uebayasi 1077: mutex_enter(&anon->an_lock);
1.52 chs 1078:
1.152 uebayasi 1079: /* ignore loaned and busy pages */
1080: if (anon->an_page == NULL || anon->an_page->loan_count != 0 ||
1081: (anon->an_page->flags & PG_BUSY) != 0)
1082: goto uvm_fault_upper_lookup_enter_done;
1.145 uebayasi 1083:
1.152 uebayasi 1084: mutex_enter(&uvm_pageqlock);
1085: uvm_pageenqueue(anon->an_page);
1086: mutex_exit(&uvm_pageqlock);
1087: UVMHIST_LOG(maphist,
1088: " MAPPING: n anon: pm=0x%x, va=0x%x, pg=0x%x",
1089: ufi->orig_map->pmap, currva, anon->an_page, 0);
1090: uvmexp.fltnamap++;
1091:
1092: /*
1093: * Since this isn't the page that's actually faulting,
1094: * ignore pmap_enter() failures; it's not critical
1095: * that we enter these right now.
1096: */
1097:
1098: (void) pmap_enter(ufi->orig_map->pmap, currva,
1099: VM_PAGE_TO_PHYS(anon->an_page),
1100: (anon->an_ref > 1) ? (flt->enter_prot & ~VM_PROT_WRITE) :
1101: flt->enter_prot,
1.154 ! uebayasi 1102: PMAP_CANFAIL | (flt->wire_mapping ? PMAP_WIRED : 0));
1.52 chs 1103:
1.145 uebayasi 1104: uvm_fault_upper_lookup_enter_done:
1.152 uebayasi 1105: pmap_update(ufi->orig_map->pmap);
1106: mutex_exit(&anon->an_lock);
1.151 uebayasi 1107: }
1108:
1109: static inline void
1110: uvm_fault_upper_lookup1(
1111: struct uvm_faultinfo *ufi, struct uvm_faultctx *flt,
1112: bool shadowed)
1113: {
1114: #ifdef DIAGNOSTIC
1115: struct vm_amap *amap = ufi->entry->aref.ar_amap;
1116: #endif
1117: #ifdef UVMHIST
1118: struct uvm_object *uobj = ufi->entry->object.uvm_obj;
1119: #endif
1.7 mrg 1120:
1121: /* locked: maps(read), amap(if there) */
1.120 ad 1122: KASSERT(amap == NULL || mutex_owned(&amap->am_l));
1.119 thorpej 1123: /* (shadowed == true) if there is an anon at the faulting address */
1.144 uebayasi 1124: UVMHIST_LOG(maphist, " shadowed=%d, will_get=%d", shadowed,
1125: (uobj && shadowed != false),0,0);
1.1 mrg 1126:
1.7 mrg 1127: /*
1128: * note that if we are really short of RAM we could sleep in the above
1129: * call to pmap_enter with everything locked. bad?
1.46 thorpej 1130: *
1131: * XXX Actually, that is bad; pmap_enter() should just fail in that
1132: * XXX case. --thorpej
1.7 mrg 1133: */
1.138 uebayasi 1134: }
1.63 chs 1135:
1.138 uebayasi 1136: static int
1137: uvm_fault_lower(
1.140 uebayasi 1138: struct uvm_faultinfo *ufi, struct uvm_faultctx *flt,
1.144 uebayasi 1139: struct vm_page **pages)
1.138 uebayasi 1140: {
1.141 uebayasi 1141: struct uvm_object *uobj = ufi->entry->object.uvm_obj;
1.138 uebayasi 1142: int error;
1.133 uebayasi 1143:
1.7 mrg 1144: /*
1145: * if the desired page is not shadowed by the amap and we have a
1146: * backing object, then we check to see if the backing object would
1147: * prefer to handle the fault itself (rather than letting us do it
1148: * with the usual pgo_get hook). the backing object signals this by
1149: * providing a pgo_fault routine.
1150: */
1.1 mrg 1151:
1.133 uebayasi 1152: if (uobj && uobj->pgops->pgo_fault != NULL) {
1.144 uebayasi 1153: error = uvm_fault_lower_special(ufi, flt, pages);
1.138 uebayasi 1154: } else {
1.144 uebayasi 1155: error = uvm_fault_lower_generic(ufi, flt, pages);
1.138 uebayasi 1156: }
1157: return error;
1158: }
1159:
1160: static int
1161: uvm_fault_lower_special(
1.140 uebayasi 1162: struct uvm_faultinfo *ufi, struct uvm_faultctx *flt,
1.144 uebayasi 1163: struct vm_page **pages)
1.138 uebayasi 1164: {
1.141 uebayasi 1165: struct uvm_object *uobj = ufi->entry->object.uvm_obj;
1.138 uebayasi 1166: int error;
1167:
1.143 uebayasi 1168: mutex_enter(&uobj->vmobjlock);
1169: /* locked: maps(read), amap (if there), uobj */
1170: error = uobj->pgops->pgo_fault(ufi, flt->startva, pages, flt->npages,
1171: flt->centeridx, flt->access_type, PGO_LOCKED|PGO_SYNCIO);
1.52 chs 1172:
1.143 uebayasi 1173: /* locked: nothing, pgo_fault has unlocked everything */
1.7 mrg 1174:
1.143 uebayasi 1175: if (error == ERESTART)
1176: error = ERESTART; /* try again! */
1177: /*
1178: * object fault routine responsible for pmap_update().
1179: */
1.7 mrg 1180:
1.143 uebayasi 1181: return error;
1.138 uebayasi 1182: }
1183:
1.151 uebayasi 1184: static void uvm_fault_lower_generic_lookup_neighbor(
1185: struct uvm_faultinfo *, struct uvm_faultctx *,
1186: vaddr_t, struct vm_page *);
1187:
1.138 uebayasi 1188: static int
1189: uvm_fault_lower_generic(
1.140 uebayasi 1190: struct uvm_faultinfo *ufi, struct uvm_faultctx *flt,
1.144 uebayasi 1191: struct vm_page **pages)
1.138 uebayasi 1192: {
1.141 uebayasi 1193: struct uvm_object *uobj = ufi->entry->object.uvm_obj;
1.137 uebayasi 1194:
1.7 mrg 1195: /*
1196: * now, if the desired page is not shadowed by the amap and we have
1197: * a backing object that does not have a special fault routine, then
1198: * we ask (with pgo_get) the object for resident pages that we care
1199: * about and attempt to map them in. we do not let pgo_get block
1200: * (PGO_LOCKED).
1201: */
1202:
1.135 uebayasi 1203: if (uobj == NULL) {
1204: /* zero fill; don't care neighbor pages */
1.141 uebayasi 1205: pages[flt->centeridx] = NULL;
1.138 uebayasi 1206: } else {
1.144 uebayasi 1207: uvm_fault_lower_generic_lookup(ufi, flt, pages);
1.141 uebayasi 1208: }
1.148 uebayasi 1209: return uvm_fault_lower_generic1(ufi, flt, pages[flt->centeridx]);
1.138 uebayasi 1210: }
1211:
1.141 uebayasi 1212: static int
1.138 uebayasi 1213: uvm_fault_lower_generic_lookup(
1.140 uebayasi 1214: struct uvm_faultinfo *ufi, struct uvm_faultctx *flt,
1.144 uebayasi 1215: struct vm_page **pages)
1.138 uebayasi 1216: {
1.141 uebayasi 1217: struct uvm_object *uobj = ufi->entry->object.uvm_obj;
1.138 uebayasi 1218: int lcv, gotpages;
1219: vaddr_t currva;
1.135 uebayasi 1220:
1.136 uebayasi 1221: mutex_enter(&uobj->vmobjlock);
1222: /* locked (!shadowed): maps(read), amap (if there), uobj */
1223: /*
1224: * the following call to pgo_get does _not_ change locking state
1225: */
1.7 mrg 1226:
1.136 uebayasi 1227: uvmexp.fltlget++;
1.140 uebayasi 1228: gotpages = flt->npages;
1.143 uebayasi 1229: (void) uobj->pgops->pgo_get(uobj,
1230: ufi->entry->offset + flt->startva - ufi->entry->start,
1231: pages, &gotpages, flt->centeridx,
1232: flt->access_type & MASK(ufi->entry), ufi->entry->advice, PGO_LOCKED);
1.1 mrg 1233:
1.136 uebayasi 1234: /*
1235: * check for pages to map, if we got any
1236: */
1.7 mrg 1237:
1.141 uebayasi 1238: if (gotpages == 0) {
1239: pages[flt->centeridx] = NULL;
1240: return 0;
1241: }
1.134 uebayasi 1242:
1.140 uebayasi 1243: currva = flt->startva;
1.143 uebayasi 1244: for (lcv = 0; lcv < flt->npages; lcv++, currva += PAGE_SIZE) {
1.136 uebayasi 1245: struct vm_page *curpg;
1.86 yamt 1246:
1.136 uebayasi 1247: curpg = pages[lcv];
1248: if (curpg == NULL || curpg == PGO_DONTCARE) {
1249: continue;
1250: }
1251: KASSERT(curpg->uobject == uobj);
1.1 mrg 1252:
1.136 uebayasi 1253: /*
1.143 uebayasi 1254: * if center page is resident and not PG_BUSY|PG_RELEASED
1255: * then pgo_get made it PG_BUSY for us and gave us a handle
1256: * to it. remember this page as "uobjpage." (for later use).
1.136 uebayasi 1257: */
1.63 chs 1258:
1.140 uebayasi 1259: if (lcv == flt->centeridx) {
1.136 uebayasi 1260: UVMHIST_LOG(maphist, " got uobjpage "
1261: "(0x%x) with locked get",
1.141 uebayasi 1262: curpg, 0,0,0);
1.151 uebayasi 1263: } else
1264: uvm_fault_lower_generic_lookup_neighbor(ufi, flt,
1265: currva, curpg);
1266: }
1267: pmap_update(ufi->orig_map->pmap);
1268: return 0;
1269: }
1270:
1271: static void
1272: uvm_fault_lower_generic_lookup_neighbor(
1273: struct uvm_faultinfo *ufi, struct uvm_faultctx *flt,
1274: vaddr_t currva, struct vm_page *curpg)
1275: {
1.152 uebayasi 1276: bool readonly;
1.63 chs 1277:
1.152 uebayasi 1278: /*
1279: * calling pgo_get with PGO_LOCKED returns us pages which
1280: * are neither busy nor released, so we don't need to check
1281: * for this. we can just directly enter the pages.
1282: */
1.7 mrg 1283:
1.152 uebayasi 1284: mutex_enter(&uvm_pageqlock);
1285: uvm_pageenqueue(curpg);
1286: mutex_exit(&uvm_pageqlock);
1287: UVMHIST_LOG(maphist,
1288: " MAPPING: n obj: pm=0x%x, va=0x%x, pg=0x%x",
1289: ufi->orig_map->pmap, currva, curpg, 0);
1290: uvmexp.fltnomap++;
1291:
1292: /*
1293: * Since this page isn't the page that's actually faulting,
1294: * ignore pmap_enter() failures; it's not critical that we
1295: * enter these right now.
1296: */
1297: KASSERT((curpg->flags & PG_PAGEOUT) == 0);
1298: KASSERT((curpg->flags & PG_RELEASED) == 0);
1299: KASSERT(!UVM_OBJ_IS_CLEAN(curpg->uobject) ||
1300: (curpg->flags & PG_CLEAN) != 0);
1301: readonly = (curpg->flags & PG_RDONLY)
1302: || (curpg->loan_count > 0)
1303: || UVM_OBJ_NEEDS_WRITEFAULT(curpg->uobject);
1304:
1305: (void) pmap_enter(ufi->orig_map->pmap, currva,
1306: VM_PAGE_TO_PHYS(curpg),
1307: readonly ? flt->enter_prot & ~VM_PROT_WRITE :
1308: flt->enter_prot & MASK(ufi->entry),
1309: PMAP_CANFAIL | (flt->wire_mapping ? PMAP_WIRED : 0));
1.136 uebayasi 1310:
1.152 uebayasi 1311: /*
1312: * NOTE: page can't be PG_WANTED or PG_RELEASED because we've
1313: * held the lock the whole time we've had the handle.
1314: */
1315: KASSERT((curpg->flags & PG_WANTED) == 0);
1316: KASSERT((curpg->flags & PG_RELEASED) == 0);
1.52 chs 1317:
1.152 uebayasi 1318: curpg->flags &= ~(PG_BUSY);
1319: UVM_PAGE_OWN(curpg, NULL);
1.138 uebayasi 1320: }
1.134 uebayasi 1321:
1.138 uebayasi 1322: static int
1323: uvm_fault_lower_generic1(
1.140 uebayasi 1324: struct uvm_faultinfo *ufi, struct uvm_faultctx *flt,
1.148 uebayasi 1325: struct vm_page *uobjpage)
1.138 uebayasi 1326: {
1.147 uebayasi 1327: #ifdef DIAGNOSTIC
1.141 uebayasi 1328: struct vm_amap *amap = ufi->entry->aref.ar_amap;
1329: struct uvm_object *uobj = ufi->entry->object.uvm_obj;
1.147 uebayasi 1330: #endif
1.7 mrg 1331:
1.133 uebayasi 1332: /* locked: maps(read), amap(if there), uobj(if !null), uobjpage(if !null) */
1333: KASSERT(amap == NULL || mutex_owned(&amap->am_l));
1334: KASSERT(uobj == NULL || mutex_owned(&uobj->vmobjlock));
1335: KASSERT(uobjpage == NULL || (uobjpage->flags & PG_BUSY) != 0);
1.7 mrg 1336:
1337: /*
1338: * note that at this point we are done with any front or back pages.
1339: * we are now going to focus on the center page (i.e. the one we've
1.127 uebayasi 1340: * faulted on). if we have faulted on the upper (anon) layer
1.7 mrg 1341: * [i.e. case 1], then the anon we want is anons[centeridx] (we have
1342: * not touched it yet). if we have faulted on the bottom (uobj)
1343: * layer [i.e. case 2] and the page was both present and available,
1344: * then we've got a pointer to it as "uobjpage" and we've already
1.8 chuck 1345: * made it BUSY.
1.7 mrg 1346: */
1347:
1348: /*
1349: * there are four possible cases we must address: 1A, 1B, 2A, and 2B
1350: */
1351:
1352: /*
1353: * redirect case 2: if we are not shadowed, go to case 2.
1354: */
1355:
1.148 uebayasi 1356: return uvm_fault_lower_generic2(ufi, flt, uobjpage);
1.138 uebayasi 1357: }
1.7 mrg 1358:
1.138 uebayasi 1359: static int
1.148 uebayasi 1360: uvm_fault_upper_loan(
1361: struct uvm_faultinfo *ufi, struct uvm_faultctx *flt,
1362: struct vm_anon *anon, struct uvm_object **ruobj);
1363: static int
1364: uvm_fault_upper1(
1365: struct uvm_faultinfo *ufi, struct uvm_faultctx *flt,
1366: struct uvm_object *uobj, struct vm_anon *anon);
1367: static int
1368: uvm_fault_upper_promote(
1369: struct uvm_faultinfo *ufi, struct uvm_faultctx *flt,
1370: struct uvm_object *uobj, struct vm_anon *anon);
1371: static int
1372: uvm_fault_upper_direct(
1373: struct uvm_faultinfo *ufi, struct uvm_faultctx *flt,
1374: struct uvm_object *uobj, struct vm_anon *anon);
1375: static int
1376: uvm_fault_upper_enter(
1377: struct uvm_faultinfo *ufi, struct uvm_faultctx *flt,
1378: struct uvm_object *uobj, struct vm_anon *anon,
1379: struct vm_page *pg, struct vm_anon *oanon);
1380: static int
1381: uvm_fault_upper_done(
1382: struct uvm_faultinfo *ufi, struct uvm_faultctx *flt,
1383: struct uvm_object *uobj, struct vm_anon *anon,
1384: struct vm_page *pg, struct vm_anon *oanon);
1385:
1386: static int
1.138 uebayasi 1387: uvm_fault_upper(
1.140 uebayasi 1388: struct uvm_faultinfo *ufi, struct uvm_faultctx *flt,
1.148 uebayasi 1389: struct vm_anon **anons)
1.138 uebayasi 1390: {
1.148 uebayasi 1391: struct vm_amap * const amap = ufi->entry->aref.ar_amap;
1392: struct vm_anon * const anon = anons[flt->centeridx];
1393: struct uvm_object *uobj;
1.138 uebayasi 1394: int error;
1.137 uebayasi 1395:
1.7 mrg 1396: /* locked: maps(read), amap */
1.133 uebayasi 1397: KASSERT(mutex_owned(&amap->am_l));
1.7 mrg 1398:
1399: /*
1400: * handle case 1: fault on an anon in our amap
1401: */
1402:
1403: UVMHIST_LOG(maphist, " case 1 fault: anon=0x%x", anon, 0,0,0);
1.122 ad 1404: mutex_enter(&anon->an_lock);
1.7 mrg 1405:
1406: /* locked: maps(read), amap, anon */
1.120 ad 1407: KASSERT(mutex_owned(&amap->am_l));
1.122 ad 1408: KASSERT(mutex_owned(&anon->an_lock));
1.7 mrg 1409:
1410: /*
1411: * no matter if we have case 1A or case 1B we are going to need to
1412: * have the anon's memory resident. ensure that now.
1413: */
1414:
1415: /*
1.47 chs 1416: * let uvmfault_anonget do the dirty work.
1.51 thorpej 1417: * if it fails (!OK) it will unlock everything for us.
1.47 chs 1418: * if it succeeds, locks are still valid and locked.
1.7 mrg 1419: * also, if it is OK, then the anon's page is on the queues.
1420: * if the page is on loan from a uvm_object, then anonget will
1421: * lock that object for us if it does not fail.
1422: */
1423:
1.138 uebayasi 1424: error = uvmfault_anonget(ufi, amap, anon);
1.58 chs 1425: switch (error) {
1.57 chs 1426: case 0:
1.63 chs 1427: break;
1.7 mrg 1428:
1.57 chs 1429: case ERESTART:
1.139 uebayasi 1430: return ERESTART;
1.7 mrg 1431:
1.57 chs 1432: case EAGAIN:
1.128 pooka 1433: kpause("fltagain1", false, hz/2, NULL);
1.139 uebayasi 1434: return ERESTART;
1.51 thorpej 1435:
1436: default:
1.138 uebayasi 1437: return error;
1.1 mrg 1438: }
1.7 mrg 1439:
1440: /*
1441: * uobj is non null if the page is on loan from an object (i.e. uobj)
1442: */
1443:
1.94 yamt 1444: uobj = anon->an_page->uobject; /* locked by anonget if !NULL */
1.7 mrg 1445:
1446: /* locked: maps(read), amap, anon, uobj(if one) */
1.120 ad 1447: KASSERT(mutex_owned(&amap->am_l));
1.122 ad 1448: KASSERT(mutex_owned(&anon->an_lock));
1449: KASSERT(uobj == NULL || mutex_owned(&uobj->vmobjlock));
1.7 mrg 1450:
1451: /*
1.63 chs 1452: * special handling for loaned pages
1.7 mrg 1453: */
1.52 chs 1454:
1.94 yamt 1455: if (anon->an_page->loan_count) {
1.148 uebayasi 1456: error = uvm_fault_upper_loan(ufi, flt, anon, &uobj);
1457: if (error != 0)
1458: return error;
1459: }
1460: return uvm_fault_upper1(ufi, flt, uobj, anon);
1461: }
1462:
1463: static int
1464: uvm_fault_upper_loan(
1465: struct uvm_faultinfo *ufi, struct uvm_faultctx *flt,
1466: struct vm_anon *anon, struct uvm_object **ruobj)
1467: {
1.149 uebayasi 1468: struct vm_amap * const amap = ufi->entry->aref.ar_amap;
1.151 uebayasi 1469: int error = 0;
1.149 uebayasi 1470:
1471: if (!flt->cow_now) {
1.7 mrg 1472:
1.149 uebayasi 1473: /*
1474: * for read faults on loaned pages we just cap the
1475: * protection at read-only.
1476: */
1.63 chs 1477:
1.149 uebayasi 1478: flt->enter_prot = flt->enter_prot & ~VM_PROT_WRITE;
1.7 mrg 1479:
1.149 uebayasi 1480: } else {
1481: /*
1482: * note that we can't allow writes into a loaned page!
1483: *
1484: * if we have a write fault on a loaned page in an
1485: * anon then we need to look at the anon's ref count.
1486: * if it is greater than one then we are going to do
1487: * a normal copy-on-write fault into a new anon (this
1488: * is not a problem). however, if the reference count
1489: * is one (a case where we would normally allow a
1490: * write directly to the page) then we need to kill
1491: * the loan before we continue.
1492: */
1493:
1494: /* >1 case is already ok */
1495: if (anon->an_ref == 1) {
1.153 uebayasi 1496: error = uvm_loanbreak_anon(anon, ruobj);
1.151 uebayasi 1497: if (error != 0) {
1498: uvmfault_unlockall(ufi, amap, *ruobj, anon);
1499: uvm_wait("flt_noram2");
1500: return ERESTART;
1501: }
1502: }
1503: }
1504: return error;
1505: }
1506:
1.148 uebayasi 1507: static int
1508: uvm_fault_upper1(
1509: struct uvm_faultinfo *ufi, struct uvm_faultctx *flt,
1510: struct uvm_object *uobj, struct vm_anon *anon)
1511: {
1512: int error;
1.7 mrg 1513:
1514: /*
1515: * if we are case 1B then we will need to allocate a new blank
1516: * anon to transfer the data into. note that we have a lock
1517: * on anon, so no one can busy or release the page until we are done.
1518: * also note that the ref count can't drop to zero here because
1519: * it is > 1 and we are only dropping one ref.
1520: *
1.63 chs 1521: * in the (hopefully very rare) case that we are out of RAM we
1522: * will unlock, wait for more RAM, and refault.
1.7 mrg 1523: *
1524: * if we are out of anon VM we kill the process (XXX: could wait?).
1525: */
1526:
1.140 uebayasi 1527: if (flt->cow_now && anon->an_ref > 1) {
1.148 uebayasi 1528: error = uvm_fault_upper_promote(ufi, flt, uobj, anon);
1529: } else {
1530: error = uvm_fault_upper_direct(ufi, flt, uobj, anon);
1531: }
1532: return error;
1533: }
1534:
1535: static int
1536: uvm_fault_upper_promote(
1537: struct uvm_faultinfo *ufi, struct uvm_faultctx *flt,
1538: struct uvm_object *uobj, struct vm_anon *anon)
1539: {
1.149 uebayasi 1540: struct vm_anon * const oanon = anon;
1541: struct vm_page *pg;
1542: int error;
1543:
1544: UVMHIST_LOG(maphist, " case 1B: COW fault",0,0,0,0);
1545: uvmexp.flt_acow++;
1546:
1547: error = uvmfault_promote(ufi, oanon, PGO_DONTCARE,
1548: &anon, &flt->anon_spare);
1549: switch (error) {
1550: case 0:
1551: break;
1552: case ERESTART:
1553: return ERESTART;
1554: default:
1555: return error;
1556: }
1.7 mrg 1557:
1.149 uebayasi 1558: pg = anon->an_page;
1559: mutex_enter(&uvm_pageqlock);
1560: uvm_pageactivate(pg);
1561: mutex_exit(&uvm_pageqlock);
1562: pg->flags &= ~(PG_BUSY|PG_FAKE);
1563: UVM_PAGE_OWN(pg, NULL);
1.7 mrg 1564:
1.149 uebayasi 1565: /* deref: can not drop to zero here by defn! */
1566: oanon->an_ref--;
1.53 thorpej 1567:
1.149 uebayasi 1568: /*
1569: * note: oanon is still locked, as is the new anon. we
1570: * need to check for this later when we unlock oanon; if
1571: * oanon != anon, we'll have to unlock anon, too.
1572: */
1.7 mrg 1573:
1.149 uebayasi 1574: return uvm_fault_upper_enter(ufi, flt, uobj, anon, pg, oanon);
1.148 uebayasi 1575: }
1576:
1577: static int
1578: uvm_fault_upper_direct(
1579: struct uvm_faultinfo *ufi, struct uvm_faultctx *flt,
1580: struct uvm_object *uobj, struct vm_anon *anon)
1581: {
1.149 uebayasi 1582: struct vm_anon * const oanon = anon;
1583: struct vm_page *pg;
1.52 chs 1584:
1.149 uebayasi 1585: uvmexp.flt_anon++;
1586: pg = anon->an_page;
1587: if (anon->an_ref > 1) /* disallow writes to ref > 1 anons */
1588: flt->enter_prot = flt->enter_prot & ~VM_PROT_WRITE;
1.7 mrg 1589:
1.149 uebayasi 1590: return uvm_fault_upper_enter(ufi, flt, uobj, anon, pg, oanon);
1.148 uebayasi 1591: }
1592:
1593: static int
1594: uvm_fault_upper_enter(
1595: struct uvm_faultinfo *ufi, struct uvm_faultctx *flt,
1596: struct uvm_object *uobj, struct vm_anon *anon, struct vm_page *pg,
1597: struct vm_anon *oanon)
1598: {
1599: struct vm_amap * const amap = ufi->entry->aref.ar_amap;
1.7 mrg 1600:
1.53 thorpej 1601: /* locked: maps(read), amap, oanon, anon (if different from oanon) */
1.120 ad 1602: KASSERT(mutex_owned(&amap->am_l));
1.122 ad 1603: KASSERT(mutex_owned(&anon->an_lock));
1604: KASSERT(mutex_owned(&oanon->an_lock));
1.7 mrg 1605:
1606: /*
1.69 chs 1607: * now map the page in.
1.7 mrg 1608: */
1609:
1610: UVMHIST_LOG(maphist, " MAPPING: anon: pm=0x%x, va=0x%x, pg=0x%x",
1.138 uebayasi 1611: ufi->orig_map->pmap, ufi->orig_rvaddr, pg, 0);
1612: if (pmap_enter(ufi->orig_map->pmap, ufi->orig_rvaddr, VM_PAGE_TO_PHYS(pg),
1.146 uebayasi 1613: flt->enter_prot, flt->access_type | PMAP_CANFAIL | (flt->wire_mapping ? PMAP_WIRED : 0))
1.58 chs 1614: != 0) {
1.69 chs 1615:
1.46 thorpej 1616: /*
1617: * No need to undo what we did; we can simply think of
1618: * this as the pmap throwing away the mapping information.
1619: *
1620: * We do, however, have to go through the ReFault path,
1621: * as the map may change while we're asleep.
1622: */
1.69 chs 1623:
1.53 thorpej 1624: if (anon != oanon)
1.122 ad 1625: mutex_exit(&anon->an_lock);
1.138 uebayasi 1626: uvmfault_unlockall(ufi, amap, uobj, oanon);
1.92 yamt 1627: if (!uvm_reclaimable()) {
1.46 thorpej 1628: UVMHIST_LOG(maphist,
1629: "<- failed. out of VM",0,0,0,0);
1630: /* XXX instrumentation */
1.148 uebayasi 1631: return ENOMEM;
1.46 thorpej 1632: }
1633: /* XXX instrumentation */
1634: uvm_wait("flt_pmfail1");
1.139 uebayasi 1635: return ERESTART;
1.46 thorpej 1636: }
1.7 mrg 1637:
1.148 uebayasi 1638: return uvm_fault_upper_done(ufi, flt, uobj, anon, pg, oanon);
1639: }
1640:
1641: static int
1642: uvm_fault_upper_done(
1643: struct uvm_faultinfo *ufi, struct uvm_faultctx *flt,
1644: struct uvm_object *uobj, struct vm_anon *anon,
1645: struct vm_page *pg, struct vm_anon *oanon)
1646: {
1647: struct vm_amap * const amap = ufi->entry->aref.ar_amap;
1648:
1.7 mrg 1649: /*
1.46 thorpej 1650: * ... update the page queues.
1.7 mrg 1651: */
1652:
1.122 ad 1653: mutex_enter(&uvm_pageqlock);
1.146 uebayasi 1654: if (flt->wire_paging) {
1.8 chuck 1655: uvm_pagewire(pg);
1.29 chs 1656:
1657: /*
1658: * since the now-wired page cannot be paged out,
1659: * release its swap resources for others to use.
1660: * since an anon with no swap cannot be PG_CLEAN,
1661: * clear its clean flag now.
1662: */
1663:
1664: pg->flags &= ~(PG_CLEAN);
1.22 chs 1665: uvm_anon_dropswap(anon);
1.7 mrg 1666: } else {
1667: uvm_pageactivate(pg);
1668: }
1.122 ad 1669: mutex_exit(&uvm_pageqlock);
1.7 mrg 1670:
1671: /*
1672: * done case 1! finish up by unlocking everything and returning success
1673: */
1.1 mrg 1674:
1.53 thorpej 1675: if (anon != oanon)
1.122 ad 1676: mutex_exit(&anon->an_lock);
1.138 uebayasi 1677: uvmfault_unlockall(ufi, amap, uobj, oanon);
1678: pmap_update(ufi->orig_map->pmap);
1.139 uebayasi 1679: return 0;
1.138 uebayasi 1680: }
1.1 mrg 1681:
1.138 uebayasi 1682: static int
1.148 uebayasi 1683: uvm_fault_lower_generic_io(
1684: struct uvm_faultinfo *ufi, struct uvm_faultctx *flt,
1685: struct vm_page **ruobjpage);
1686: static int
1.151 uebayasi 1687: uvm_fault_lower_generic3(
1.148 uebayasi 1688: struct uvm_faultinfo *ufi, struct uvm_faultctx *flt,
1689: struct vm_page *uobjpage, bool promote);
1690: static int
1691: uvm_fault_lower_generic_direct(
1692: struct uvm_faultinfo *ufi, struct uvm_faultctx *flt,
1693: struct vm_page *uobjpage);
1694: static int
1695: uvm_fault_lower_generic_promote(
1696: struct uvm_faultinfo *ufi, struct uvm_faultctx *flt,
1697: struct vm_page *uobjpage);
1698: static int
1699: uvm_fault_lower_generic_enter(
1700: struct uvm_faultinfo *ufi, struct uvm_faultctx *flt,
1701: struct uvm_object *uobj,
1702: struct vm_anon *anon, struct vm_page *pg, struct vm_page *uobjpage);
1703: static int
1704: uvm_fault_lower_generic_done(
1705: struct uvm_faultinfo *ufi, struct uvm_faultctx *flt,
1706: struct uvm_object *uobj,
1707: struct vm_anon *anon, struct vm_page *pg);
1708:
1709: static int
1.138 uebayasi 1710: uvm_fault_lower_generic2(
1.140 uebayasi 1711: struct uvm_faultinfo *ufi, struct uvm_faultctx *flt,
1.148 uebayasi 1712: struct vm_page *uobjpage)
1.138 uebayasi 1713: {
1.148 uebayasi 1714: #ifdef DIAGNOSTIC
1715: struct vm_amap * const amap = ufi->entry->aref.ar_amap;
1716: #endif
1717: struct uvm_object * const uobj = ufi->entry->object.uvm_obj;
1.137 uebayasi 1718: bool promote;
1.138 uebayasi 1719: int error;
1.137 uebayasi 1720:
1.7 mrg 1721: /*
1722: * handle case 2: faulting on backing object or zero fill
1723: */
1724:
1725: /*
1726: * locked:
1727: * maps(read), amap(if there), uobj(if !null), uobjpage(if !null)
1728: */
1.120 ad 1729: KASSERT(amap == NULL || mutex_owned(&amap->am_l));
1.122 ad 1730: KASSERT(uobj == NULL || mutex_owned(&uobj->vmobjlock));
1.120 ad 1731: KASSERT(uobjpage == NULL || (uobjpage->flags & PG_BUSY) != 0);
1.7 mrg 1732:
1733: /*
1734: * note that uobjpage can not be PGO_DONTCARE at this point. we now
1735: * set uobjpage to PGO_DONTCARE if we are doing a zero fill. if we
1736: * have a backing object, check and see if we are going to promote
1737: * the data up to an anon during the fault.
1738: */
1739:
1740: if (uobj == NULL) {
1.63 chs 1741: uobjpage = PGO_DONTCARE;
1.119 thorpej 1742: promote = true; /* always need anon here */
1.7 mrg 1743: } else {
1.52 chs 1744: KASSERT(uobjpage != PGO_DONTCARE);
1.140 uebayasi 1745: promote = flt->cow_now && UVM_ET_ISCOPYONWRITE(ufi->entry);
1.7 mrg 1746: }
1747: UVMHIST_LOG(maphist, " case 2 fault: promote=%d, zfill=%d",
1.46 thorpej 1748: promote, (uobj == NULL), 0,0);
1.1 mrg 1749:
1.7 mrg 1750: /*
1.9 chuck 1751: * if uobjpage is not null then we do not need to do I/O to get the
1752: * uobjpage.
1753: *
1.63 chs 1754: * if uobjpage is null, then we need to unlock and ask the pager to
1.7 mrg 1755: * get the data for us. once we have the data, we need to reverify
1756: * the state the world. we are currently not holding any resources.
1757: */
1.1 mrg 1758:
1.9 chuck 1759: if (uobjpage) {
1760: /* update rusage counters */
1.124 ad 1761: curlwp->l_ru.ru_minflt++;
1.9 chuck 1762: } else {
1.148 uebayasi 1763: error = uvm_fault_lower_generic_io(ufi, flt, &uobjpage);
1764: if (error != 0)
1765: return error;
1766: }
1.151 uebayasi 1767: return uvm_fault_lower_generic3(ufi, flt, uobjpage, promote);
1.148 uebayasi 1768: }
1769:
1770: static int
1771: uvm_fault_lower_generic_io(
1772: struct uvm_faultinfo *ufi, struct uvm_faultctx *flt,
1773: struct vm_page **ruobjpage)
1774: {
1.149 uebayasi 1775: struct vm_amap * const amap = ufi->entry->aref.ar_amap;
1776: struct uvm_object * const uobj = ufi->entry->object.uvm_obj;
1777: struct vm_page *uobjpage;
1778: bool locked;
1779: int gotpages;
1780: int error;
1781: voff_t uoff;
1782:
1783: /* update rusage counters */
1784: curlwp->l_ru.ru_majflt++;
1.137 uebayasi 1785:
1.149 uebayasi 1786: /* locked: maps(read), amap(if there), uobj */
1787: uvmfault_unlockall(ufi, amap, NULL, NULL);
1788: /* locked: uobj */
1.63 chs 1789:
1.149 uebayasi 1790: uvmexp.fltget++;
1791: gotpages = 1;
1792: uoff = (ufi->orig_rvaddr - ufi->entry->start) + ufi->entry->offset;
1793: error = uobj->pgops->pgo_get(uobj, uoff, &uobjpage, &gotpages,
1794: 0, flt->access_type & MASK(ufi->entry), ufi->entry->advice,
1795: PGO_SYNCIO);
1796: /* locked: uobjpage(if no error) */
1797: KASSERT(error != 0 || (uobjpage->flags & PG_BUSY) != 0);
1.52 chs 1798:
1.149 uebayasi 1799: /*
1800: * recover from I/O
1801: */
1.1 mrg 1802:
1.149 uebayasi 1803: if (error) {
1804: if (error == EAGAIN) {
1805: UVMHIST_LOG(maphist,
1806: " pgo_get says TRY AGAIN!",0,0,0,0);
1807: kpause("fltagain2", false, hz/2, NULL);
1808: return ERESTART;
1809: }
1.1 mrg 1810:
1.139 uebayasi 1811: #if 0
1.149 uebayasi 1812: KASSERT(error != ERESTART);
1.139 uebayasi 1813: #else
1.149 uebayasi 1814: /* XXXUEBS don't re-fault? */
1815: if (error == ERESTART)
1816: error = EIO;
1.139 uebayasi 1817: #endif
1818:
1.149 uebayasi 1819: UVMHIST_LOG(maphist, "<- pgo_get failed (code %d)",
1820: error, 0,0,0);
1821: return error;
1822: }
1.7 mrg 1823:
1.149 uebayasi 1824: /* locked: uobjpage */
1.7 mrg 1825:
1.149 uebayasi 1826: mutex_enter(&uvm_pageqlock);
1827: uvm_pageactivate(uobjpage);
1828: mutex_exit(&uvm_pageqlock);
1.69 chs 1829:
1.149 uebayasi 1830: /*
1831: * re-verify the state of the world by first trying to relock
1832: * the maps. always relock the object.
1833: */
1.7 mrg 1834:
1.149 uebayasi 1835: locked = uvmfault_relock(ufi);
1836: if (locked && amap)
1837: amap_lock(amap);
1838: KASSERT(uobj == uobjpage->uobject);
1839: mutex_enter(&uobj->vmobjlock);
1.63 chs 1840:
1.149 uebayasi 1841: /* locked(locked): maps(read), amap(if !null), uobj, uobjpage */
1842: /* locked(!locked): uobj, uobjpage */
1.7 mrg 1843:
1.149 uebayasi 1844: /*
1845: * verify that the page has not be released and re-verify
1846: * that amap slot is still free. if there is a problem,
1847: * we unlock and clean up.
1848: */
1.7 mrg 1849:
1.149 uebayasi 1850: if ((uobjpage->flags & PG_RELEASED) != 0 ||
1851: (locked && amap &&
1852: amap_lookup(&ufi->entry->aref,
1853: ufi->orig_rvaddr - ufi->entry->start))) {
1854: if (locked)
1855: uvmfault_unlockall(ufi, amap, NULL, NULL);
1856: locked = false;
1857: }
1.7 mrg 1858:
1.149 uebayasi 1859: /*
1860: * didn't get the lock? release the page and retry.
1861: */
1.7 mrg 1862:
1.149 uebayasi 1863: if (locked == false) {
1864: UVMHIST_LOG(maphist,
1865: " wasn't able to relock after fault: retry",
1866: 0,0,0,0);
1867: if (uobjpage->flags & PG_WANTED)
1868: wakeup(uobjpage);
1869: if (uobjpage->flags & PG_RELEASED) {
1870: uvmexp.fltpgrele++;
1871: uvm_pagefree(uobjpage);
1.139 uebayasi 1872: return ERESTART;
1.7 mrg 1873: }
1.149 uebayasi 1874: uobjpage->flags &= ~(PG_BUSY|PG_WANTED);
1875: UVM_PAGE_OWN(uobjpage, NULL);
1876: mutex_exit(&uobj->vmobjlock);
1877: return ERESTART;
1878: }
1.7 mrg 1879:
1.149 uebayasi 1880: /*
1881: * we have the data in uobjpage which is busy and
1882: * not released. we are holding object lock (so the page
1883: * can't be released on us).
1884: */
1.7 mrg 1885:
1.149 uebayasi 1886: /* locked: maps(read), amap(if !null), uobj, uobjpage */
1.148 uebayasi 1887:
1888: *ruobjpage = uobjpage;
1889: return 0;
1890: }
1891:
1892: int
1.151 uebayasi 1893: uvm_fault_lower_generic3(
1.148 uebayasi 1894: struct uvm_faultinfo *ufi, struct uvm_faultctx *flt,
1895: struct vm_page *uobjpage, bool promote)
1896: {
1897: #ifdef DIAGNOSTIC
1898: struct vm_amap * const amap = ufi->entry->aref.ar_amap;
1899: struct uvm_object * const uobj = ufi->entry->object.uvm_obj;
1900: #endif
1901: int error;
1.1 mrg 1902:
1903: /*
1.7 mrg 1904: * locked:
1905: * maps(read), amap(if !null), uobj(if !null), uobjpage(if uobj)
1.1 mrg 1906: */
1.120 ad 1907: KASSERT(amap == NULL || mutex_owned(&amap->am_l));
1.122 ad 1908: KASSERT(uobj == NULL || mutex_owned(&uobj->vmobjlock));
1.120 ad 1909: KASSERT(uobj == NULL || (uobjpage->flags & PG_BUSY) != 0);
1.1 mrg 1910:
1.7 mrg 1911: /*
1912: * notes:
1913: * - at this point uobjpage can not be NULL
1914: * - at this point uobjpage can not be PG_RELEASED (since we checked
1915: * for it above)
1916: * - at this point uobjpage could be PG_WANTED (handle later)
1917: */
1.63 chs 1918:
1.107 yamt 1919: KASSERT(uobj == NULL || uobj == uobjpage->uobject);
1.97 yamt 1920: KASSERT(uobj == NULL || !UVM_OBJ_IS_CLEAN(uobjpage->uobject) ||
1.96 yamt 1921: (uobjpage->flags & PG_CLEAN) != 0);
1.148 uebayasi 1922:
1.119 thorpej 1923: if (promote == false) {
1.148 uebayasi 1924: error = uvm_fault_lower_generic_direct(ufi, flt, uobjpage);
1925: } else {
1926: error = uvm_fault_lower_generic_promote(ufi, flt, uobjpage);
1927: }
1928: return error;
1929: }
1930:
1.151 uebayasi 1931: static int
1932: uvm_fault_lower_generic_direct_loan(
1933: struct uvm_faultinfo *ufi, struct uvm_faultctx *flt,
1934: struct vm_page **rpg, struct vm_page **ruobjpage);
1935:
1.148 uebayasi 1936: int
1937: uvm_fault_lower_generic_direct(
1938: struct uvm_faultinfo *ufi, struct uvm_faultctx *flt,
1939: struct vm_page *uobjpage)
1940: {
1.149 uebayasi 1941: struct uvm_object * const uobj = ufi->entry->object.uvm_obj;
1942: struct vm_page *pg;
1943:
1944: /*
1945: * we are not promoting. if the mapping is COW ensure that we
1946: * don't give more access than we should (e.g. when doing a read
1947: * fault on a COPYONWRITE mapping we want to map the COW page in
1948: * R/O even though the entry protection could be R/W).
1949: *
1950: * set "pg" to the page we want to map in (uobjpage, usually)
1951: */
1.1 mrg 1952:
1.149 uebayasi 1953: uvmexp.flt_obj++;
1954: if (UVM_ET_ISCOPYONWRITE(ufi->entry) ||
1955: UVM_OBJ_NEEDS_WRITEFAULT(uobjpage->uobject))
1956: flt->enter_prot &= ~VM_PROT_WRITE;
1957: pg = uobjpage; /* map in the actual object */
1.7 mrg 1958:
1.149 uebayasi 1959: KASSERT(uobjpage != PGO_DONTCARE);
1.7 mrg 1960:
1.149 uebayasi 1961: /*
1962: * we are faulting directly on the page. be careful
1963: * about writing to loaned pages...
1964: */
1965:
1966: if (uobjpage->loan_count) {
1.151 uebayasi 1967: uvm_fault_lower_generic_direct_loan(ufi, flt, &pg, &uobjpage);
1968: }
1969: KASSERT(pg == uobjpage);
1970:
1971: return uvm_fault_lower_generic_enter(ufi, flt, uobj, NULL, pg, uobjpage);
1972: }
1973:
1974: static int
1975: uvm_fault_lower_generic_direct_loan(
1976: struct uvm_faultinfo *ufi, struct uvm_faultctx *flt,
1977: struct vm_page **rpg, struct vm_page **ruobjpage)
1978: {
1.152 uebayasi 1979: struct vm_amap * const amap = ufi->entry->aref.ar_amap;
1980: struct uvm_object * const uobj = ufi->entry->object.uvm_obj;
1981: struct vm_page *pg;
1982: struct vm_page *uobjpage = *ruobjpage;
1983:
1984: if (!flt->cow_now) {
1985: /* read fault: cap the protection at readonly */
1986: /* cap! */
1987: flt->enter_prot = flt->enter_prot & ~VM_PROT_WRITE;
1988: } else {
1989: /* write fault: must break the loan here */
1990:
1991: pg = uvm_loanbreak(uobjpage);
1992: if (pg == NULL) {
1993:
1994: /*
1995: * drop ownership of page, it can't be released
1996: */
1997:
1998: if (uobjpage->flags & PG_WANTED)
1999: wakeup(uobjpage);
2000: uobjpage->flags &= ~(PG_BUSY|PG_WANTED);
2001: UVM_PAGE_OWN(uobjpage, NULL);
2002:
2003: uvmfault_unlockall(ufi, amap, uobj, NULL);
2004: UVMHIST_LOG(maphist,
2005: " out of RAM breaking loan, waiting",
2006: 0,0,0,0);
2007: uvmexp.fltnoram++;
2008: uvm_wait("flt_noram4");
2009: return ERESTART;
1.69 chs 2010: }
1.152 uebayasi 2011: *rpg = pg;
2012: *ruobjpage = pg;
2013: }
2014: return 0;
1.148 uebayasi 2015: }
2016:
2017: int
2018: uvm_fault_lower_generic_promote(
2019: struct uvm_faultinfo *ufi, struct uvm_faultctx *flt,
2020: struct vm_page *uobjpage)
2021: {
1.149 uebayasi 2022: struct vm_amap * const amap = ufi->entry->aref.ar_amap;
2023: struct uvm_object *uobj = ufi->entry->object.uvm_obj;
2024: struct vm_anon *anon;
2025: struct vm_page *pg;
2026: int error;
1.63 chs 2027:
1.149 uebayasi 2028: /*
2029: * if we are going to promote the data to an anon we
2030: * allocate a blank anon here and plug it into our amap.
2031: */
1.1 mrg 2032: #if DIAGNOSTIC
1.149 uebayasi 2033: if (amap == NULL)
2034: panic("uvm_fault: want to promote data, but no anon");
1.1 mrg 2035: #endif
1.149 uebayasi 2036: error = uvmfault_promote(ufi, NULL, uobjpage,
2037: &anon, &flt->anon_spare);
2038: switch (error) {
2039: case 0:
2040: break;
2041: case ERESTART:
2042: return ERESTART;
2043: default:
2044: return error;
2045: }
2046:
2047: pg = anon->an_page;
2048:
2049: /*
2050: * fill in the data
2051: */
1.105 yamt 2052:
1.149 uebayasi 2053: if (uobjpage != PGO_DONTCARE) {
2054: uvmexp.flt_prcopy++;
1.1 mrg 2055:
1.7 mrg 2056: /*
1.149 uebayasi 2057: * promote to shared amap? make sure all sharing
2058: * procs see it
1.7 mrg 2059: */
2060:
1.149 uebayasi 2061: if ((amap_flags(amap) & AMAP_SHARED) != 0) {
2062: pmap_page_protect(uobjpage, VM_PROT_NONE);
1.7 mrg 2063: /*
1.149 uebayasi 2064: * XXX: PAGE MIGHT BE WIRED!
1.7 mrg 2065: */
1.149 uebayasi 2066: }
1.69 chs 2067:
1.149 uebayasi 2068: /*
2069: * dispose of uobjpage. it can't be PG_RELEASED
2070: * since we still hold the object lock.
2071: * drop handle to uobj as well.
2072: */
2073:
2074: if (uobjpage->flags & PG_WANTED)
2075: /* still have the obj lock */
2076: wakeup(uobjpage);
2077: uobjpage->flags &= ~(PG_BUSY|PG_WANTED);
2078: UVM_PAGE_OWN(uobjpage, NULL);
2079: mutex_exit(&uobj->vmobjlock);
2080: uobj = NULL;
2081:
2082: UVMHIST_LOG(maphist,
2083: " promote uobjpage 0x%x to anon/page 0x%x/0x%x",
2084: uobjpage, anon, pg, 0);
1.63 chs 2085:
1.149 uebayasi 2086: } else {
2087: uvmexp.flt_przero++;
1.7 mrg 2088:
1.149 uebayasi 2089: /*
2090: * Page is zero'd and marked dirty by
2091: * uvmfault_promote().
2092: */
1.52 chs 2093:
1.149 uebayasi 2094: UVMHIST_LOG(maphist," zero fill anon/page 0x%x/0%x",
2095: anon, pg, 0, 0);
2096: }
1.148 uebayasi 2097:
2098: return uvm_fault_lower_generic_enter(ufi, flt, uobj, anon, pg, uobjpage);
2099: }
2100:
2101: int
2102: uvm_fault_lower_generic_enter(
2103: struct uvm_faultinfo *ufi, struct uvm_faultctx *flt,
2104: struct uvm_object *uobj,
2105: struct vm_anon *anon, struct vm_page *pg, struct vm_page *uobjpage)
2106: {
2107: struct vm_amap * const amap = ufi->entry->aref.ar_amap;
2108: int error;
1.7 mrg 2109:
2110: /*
2111: * locked:
1.53 thorpej 2112: * maps(read), amap(if !null), uobj(if !null), uobjpage(if uobj),
2113: * anon(if !null), pg(if anon)
1.7 mrg 2114: *
2115: * note: pg is either the uobjpage or the new page in the new anon
2116: */
1.120 ad 2117: KASSERT(amap == NULL || mutex_owned(&amap->am_l));
1.122 ad 2118: KASSERT(uobj == NULL || mutex_owned(&uobj->vmobjlock));
1.120 ad 2119: KASSERT(uobj == NULL || (uobjpage->flags & PG_BUSY) != 0);
1.122 ad 2120: KASSERT(anon == NULL || mutex_owned(&anon->an_lock));
1.120 ad 2121: KASSERT((pg->flags & PG_BUSY) != 0);
1.7 mrg 2122:
2123: /*
2124: * all resources are present. we can now map it in and free our
2125: * resources.
2126: */
2127:
2128: UVMHIST_LOG(maphist,
2129: " MAPPING: case2: pm=0x%x, va=0x%x, pg=0x%x, promote=%d",
1.138 uebayasi 2130: ufi->orig_map->pmap, ufi->orig_rvaddr, pg, promote);
1.140 uebayasi 2131: KASSERT((flt->access_type & VM_PROT_WRITE) == 0 ||
1.75 chs 2132: (pg->flags & PG_RDONLY) == 0);
1.138 uebayasi 2133: if (pmap_enter(ufi->orig_map->pmap, ufi->orig_rvaddr, VM_PAGE_TO_PHYS(pg),
1.140 uebayasi 2134: pg->flags & PG_RDONLY ? flt->enter_prot & ~VM_PROT_WRITE : flt->enter_prot,
1.146 uebayasi 2135: flt->access_type | PMAP_CANFAIL | (flt->wire_mapping ? PMAP_WIRED : 0)) != 0) {
1.52 chs 2136:
1.46 thorpej 2137: /*
2138: * No need to undo what we did; we can simply think of
2139: * this as the pmap throwing away the mapping information.
2140: *
2141: * We do, however, have to go through the ReFault path,
2142: * as the map may change while we're asleep.
2143: */
1.52 chs 2144:
1.46 thorpej 2145: if (pg->flags & PG_WANTED)
1.69 chs 2146: wakeup(pg);
1.46 thorpej 2147:
1.63 chs 2148: /*
1.46 thorpej 2149: * note that pg can't be PG_RELEASED since we did not drop
2150: * the object lock since the last time we checked.
2151: */
1.111 yamt 2152: KASSERT((pg->flags & PG_RELEASED) == 0);
1.63 chs 2153:
1.46 thorpej 2154: pg->flags &= ~(PG_BUSY|PG_FAKE|PG_WANTED);
2155: UVM_PAGE_OWN(pg, NULL);
1.138 uebayasi 2156: uvmfault_unlockall(ufi, amap, uobj, anon);
1.92 yamt 2157: if (!uvm_reclaimable()) {
1.46 thorpej 2158: UVMHIST_LOG(maphist,
2159: "<- failed. out of VM",0,0,0,0);
2160: /* XXX instrumentation */
1.106 yamt 2161: error = ENOMEM;
1.138 uebayasi 2162: return error;
1.46 thorpej 2163: }
2164: /* XXX instrumentation */
2165: uvm_wait("flt_pmfail2");
1.139 uebayasi 2166: return ERESTART;
1.46 thorpej 2167: }
1.1 mrg 2168:
1.148 uebayasi 2169: return uvm_fault_lower_generic_done(ufi, flt, uobj, anon, pg);
2170: }
2171:
2172: int
2173: uvm_fault_lower_generic_done(
2174: struct uvm_faultinfo *ufi, struct uvm_faultctx *flt,
2175: struct uvm_object *uobj, struct vm_anon *anon, struct vm_page *pg)
2176: {
2177: struct vm_amap * const amap = ufi->entry->aref.ar_amap;
2178:
1.122 ad 2179: mutex_enter(&uvm_pageqlock);
1.146 uebayasi 2180: if (flt->wire_paging) {
1.8 chuck 2181: uvm_pagewire(pg);
1.22 chs 2182: if (pg->pqflags & PQ_AOBJ) {
1.29 chs 2183:
2184: /*
2185: * since the now-wired page cannot be paged out,
2186: * release its swap resources for others to use.
2187: * since an aobj page with no swap cannot be PG_CLEAN,
2188: * clear its clean flag now.
2189: */
2190:
1.113 christos 2191: KASSERT(uobj != NULL);
1.29 chs 2192: pg->flags &= ~(PG_CLEAN);
1.22 chs 2193: uao_dropswap(uobj, pg->offset >> PAGE_SHIFT);
2194: }
1.7 mrg 2195: } else {
2196: uvm_pageactivate(pg);
2197: }
1.122 ad 2198: mutex_exit(&uvm_pageqlock);
1.7 mrg 2199: if (pg->flags & PG_WANTED)
1.69 chs 2200: wakeup(pg);
1.7 mrg 2201:
1.63 chs 2202: /*
2203: * note that pg can't be PG_RELEASED since we did not drop the object
1.7 mrg 2204: * lock since the last time we checked.
2205: */
1.111 yamt 2206: KASSERT((pg->flags & PG_RELEASED) == 0);
1.63 chs 2207:
1.7 mrg 2208: pg->flags &= ~(PG_BUSY|PG_FAKE|PG_WANTED);
2209: UVM_PAGE_OWN(pg, NULL);
1.138 uebayasi 2210: uvmfault_unlockall(ufi, amap, uobj, anon);
2211: pmap_update(ufi->orig_map->pmap);
1.7 mrg 2212: UVMHIST_LOG(maphist, "<- done (SUCCESS!)",0,0,0,0);
1.139 uebayasi 2213: return 0;
1.1 mrg 2214: }
2215:
1.110 drochner 2216:
1.1 mrg 2217: /*
2218: * uvm_fault_wire: wire down a range of virtual addresses in a map.
2219: *
1.36 thorpej 2220: * => map may be read-locked by caller, but MUST NOT be write-locked.
2221: * => if map is read-locked, any operations which may cause map to
2222: * be write-locked in uvm_fault() must be taken care of by
2223: * the caller. See uvm_map_pageable().
1.1 mrg 2224: */
2225:
1.7 mrg 2226: int
1.95 thorpej 2227: uvm_fault_wire(struct vm_map *map, vaddr_t start, vaddr_t end,
1.130 uebayasi 2228: vm_prot_t access_type, int maxprot)
1.7 mrg 2229: {
1.12 eeh 2230: vaddr_t va;
1.58 chs 2231: int error;
1.7 mrg 2232:
2233: /*
1.47 chs 2234: * now fault it in a page at a time. if the fault fails then we have
1.63 chs 2235: * to undo what we have done. note that in uvm_fault VM_PROT_NONE
1.47 chs 2236: * is replaced with the max protection if fault_type is VM_FAULT_WIRE.
1.7 mrg 2237: */
1.1 mrg 2238:
1.65 chs 2239: /*
2240: * XXX work around overflowing a vaddr_t. this prevents us from
2241: * wiring the last page in the address space, though.
2242: */
2243: if (start > end) {
2244: return EFAULT;
2245: }
2246:
1.7 mrg 2247: for (va = start ; va < end ; va += PAGE_SIZE) {
1.110 drochner 2248: error = uvm_fault_internal(map, va, access_type,
1.130 uebayasi 2249: (maxprot ? UVM_FAULT_MAXPROT : 0) | UVM_FAULT_WIRE);
1.58 chs 2250: if (error) {
1.7 mrg 2251: if (va != start) {
1.31 thorpej 2252: uvm_fault_unwire(map, start, va);
1.7 mrg 2253: }
1.58 chs 2254: return error;
1.7 mrg 2255: }
2256: }
1.58 chs 2257: return 0;
1.1 mrg 2258: }
2259:
2260: /*
2261: * uvm_fault_unwire(): unwire range of virtual space.
2262: */
2263:
1.7 mrg 2264: void
1.95 thorpej 2265: uvm_fault_unwire(struct vm_map *map, vaddr_t start, vaddr_t end)
1.36 thorpej 2266: {
2267: vm_map_lock_read(map);
2268: uvm_fault_unwire_locked(map, start, end);
2269: vm_map_unlock_read(map);
2270: }
2271:
2272: /*
2273: * uvm_fault_unwire_locked(): the guts of uvm_fault_unwire().
2274: *
2275: * => map must be at least read-locked.
2276: */
2277:
2278: void
1.95 thorpej 2279: uvm_fault_unwire_locked(struct vm_map *map, vaddr_t start, vaddr_t end)
1.7 mrg 2280: {
1.64 chs 2281: struct vm_map_entry *entry;
1.31 thorpej 2282: pmap_t pmap = vm_map_pmap(map);
1.42 thorpej 2283: vaddr_t va;
1.12 eeh 2284: paddr_t pa;
1.42 thorpej 2285: struct vm_page *pg;
1.31 thorpej 2286:
1.52 chs 2287: KASSERT((map->flags & VM_MAP_INTRSAFE) == 0);
1.7 mrg 2288:
2289: /*
2290: * we assume that the area we are unwiring has actually been wired
2291: * in the first place. this means that we should be able to extract
2292: * the PAs from the pmap. we also lock out the page daemon so that
2293: * we can call uvm_pageunwire.
2294: */
1.37 thorpej 2295:
1.122 ad 2296: mutex_enter(&uvm_pageqlock);
1.7 mrg 2297:
1.37 thorpej 2298: /*
2299: * find the beginning map entry for the region.
2300: */
1.74 chs 2301:
1.56 chs 2302: KASSERT(start >= vm_map_min(map) && end <= vm_map_max(map));
1.119 thorpej 2303: if (uvm_map_lookup_entry(map, start, &entry) == false)
1.37 thorpej 2304: panic("uvm_fault_unwire_locked: address not in map");
2305:
1.69 chs 2306: for (va = start; va < end; va += PAGE_SIZE) {
1.119 thorpej 2307: if (pmap_extract(pmap, va, &pa) == false)
1.74 chs 2308: continue;
1.42 thorpej 2309:
2310: /*
1.74 chs 2311: * find the map entry for the current address.
1.42 thorpej 2312: */
1.56 chs 2313:
2314: KASSERT(va >= entry->start);
1.74 chs 2315: while (va >= entry->end) {
1.56 chs 2316: KASSERT(entry->next != &map->header &&
2317: entry->next->start <= entry->end);
1.42 thorpej 2318: entry = entry->next;
2319: }
1.37 thorpej 2320:
1.42 thorpej 2321: /*
2322: * if the entry is no longer wired, tell the pmap.
2323: */
1.74 chs 2324:
1.42 thorpej 2325: if (VM_MAPENT_ISWIRED(entry) == 0)
2326: pmap_unwire(pmap, va);
2327:
2328: pg = PHYS_TO_VM_PAGE(pa);
2329: if (pg)
2330: uvm_pageunwire(pg);
1.7 mrg 2331: }
1.1 mrg 2332:
1.122 ad 2333: mutex_exit(&uvm_pageqlock);
1.1 mrg 2334: }
CVSweb <webmaster@jp.NetBSD.org>
![[BACK]](/icons/back.gif){kind=icon}
Return to uvm_fault.c CVS log ![[TXT]](/icons/text.gif){kind=icon}
![[DIR]](/icons/dir.gif){kind=icon}
Up to [cvs.NetBSD.org] / src / sys / uvm