Annotation of src/sys/uvm/uvm_fault.c, Revision 1.180.4.2
1.180.4.2! bouyer 1: /* $NetBSD: uvm_fault.c,v 1.182 2011/02/10 21:05:52 skrll Exp $ */
1.1 mrg 2:
3: /*
4: * Copyright (c) 1997 Charles D. Cranor and Washington University.
5: * All rights reserved.
6: *
7: * Redistribution and use in source and binary forms, with or without
8: * modification, are permitted provided that the following conditions
9: * are met:
10: * 1. Redistributions of source code must retain the above copyright
11: * notice, this list of conditions and the following disclaimer.
12: * 2. Redistributions in binary form must reproduce the above copyright
13: * notice, this list of conditions and the following disclaimer in the
14: * documentation and/or other materials provided with the distribution.
15: *
16: * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
17: * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
18: * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
19: * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
20: * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
21: * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
22: * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
23: * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
24: * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
25: * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
1.4 mrg 26: *
27: * from: Id: uvm_fault.c,v 1.1.2.23 1998/02/06 05:29:05 chs Exp
1.1 mrg 28: */
29:
30: /*
31: * uvm_fault.c: fault handler
32: */
1.71 lukem 33:
34: #include <sys/cdefs.h>
1.180.4.2! bouyer 35: __KERNEL_RCSID(0, "$NetBSD: uvm_fault.c,v 1.182 2011/02/10 21:05:52 skrll Exp $");
1.71 lukem 36:
37: #include "opt_uvmhist.h"
1.1 mrg 38:
39: #include <sys/param.h>
40: #include <sys/systm.h>
41: #include <sys/kernel.h>
42: #include <sys/proc.h>
43: #include <sys/malloc.h>
44: #include <sys/mman.h>
45:
46: #include <uvm/uvm.h>
47:
48: /*
49: *
50: * a word on page faults:
51: *
52: * types of page faults we handle:
53: *
54: * CASE 1: upper layer faults CASE 2: lower layer faults
55: *
56: * CASE 1A CASE 1B CASE 2A CASE 2B
57: * read/write1 write>1 read/write +-cow_write/zero
1.63 chs 58: * | | | |
1.1 mrg 59: * +--|--+ +--|--+ +-----+ + | + | +-----+
1.127 uebayasi 60: * amap | V | | ---------> new | | | | ^ |
1.1 mrg 61: * +-----+ +-----+ +-----+ + | + | +--|--+
62: * | | |
63: * +-----+ +-----+ +--|--+ | +--|--+
1.127 uebayasi 64: * uobj | d/c | | d/c | | V | +----+ |
1.1 mrg 65: * +-----+ +-----+ +-----+ +-----+
66: *
67: * d/c = don't care
1.63 chs 68: *
1.1 mrg 69: * case [0]: layerless fault
70: * no amap or uobj is present. this is an error.
71: *
72: * case [1]: upper layer fault [anon active]
73: * 1A: [read] or [write with anon->an_ref == 1]
1.127 uebayasi 74: * I/O takes place in upper level anon and uobj is not touched.
1.1 mrg 75: * 1B: [write with anon->an_ref > 1]
76: * new anon is alloc'd and data is copied off ["COW"]
77: *
78: * case [2]: lower layer fault [uobj]
79: * 2A: [read on non-NULL uobj] or [write to non-copy_on_write area]
80: * I/O takes place directly in object.
81: * 2B: [write to copy_on_write] or [read on NULL uobj]
1.63 chs 82: * data is "promoted" from uobj to a new anon.
1.1 mrg 83: * if uobj is null, then we zero fill.
84: *
85: * we follow the standard UVM locking protocol ordering:
86: *
1.63 chs 87: * MAPS => AMAP => UOBJ => ANON => PAGE QUEUES (PQ)
1.1 mrg 88: * we hold a PG_BUSY page if we unlock for I/O
89: *
90: *
91: * the code is structured as follows:
1.63 chs 92: *
1.1 mrg 93: * - init the "IN" params in the ufi structure
1.177 yamt 94: * ReFault: (ERESTART returned to the loop in uvm_fault_internal)
1.1 mrg 95: * - do lookups [locks maps], check protection, handle needs_copy
96: * - check for case 0 fault (error)
97: * - establish "range" of fault
98: * - if we have an amap lock it and extract the anons
99: * - if sequential advice deactivate pages behind us
100: * - at the same time check pmap for unmapped areas and anon for pages
101: * that we could map in (and do map it if found)
102: * - check object for resident pages that we could map in
103: * - if (case 2) goto Case2
104: * - >>> handle case 1
105: * - ensure source anon is resident in RAM
106: * - if case 1B alloc new anon and copy from source
107: * - map the correct page in
108: * Case2:
109: * - >>> handle case 2
110: * - ensure source page is resident (if uobj)
111: * - if case 2B alloc new anon and copy from source (could be zero
112: * fill if uobj == NULL)
113: * - map the correct page in
114: * - done!
115: *
116: * note on paging:
117: * if we have to do I/O we place a PG_BUSY page in the correct object,
118: * unlock everything, and do the I/O. when I/O is done we must reverify
119: * the state of the world before assuming that our data structures are
120: * valid. [because mappings could change while the map is unlocked]
121: *
122: * alternative 1: unbusy the page in question and restart the page fault
123: * from the top (ReFault). this is easy but does not take advantage
1.63 chs 124: * of the information that we already have from our previous lookup,
1.1 mrg 125: * although it is possible that the "hints" in the vm_map will help here.
126: *
127: * alternative 2: the system already keeps track of a "version" number of
128: * a map. [i.e. every time you write-lock a map (e.g. to change a
129: * mapping) you bump the version number up by one...] so, we can save
130: * the version number of the map before we release the lock and start I/O.
131: * then when I/O is done we can relock and check the version numbers
132: * to see if anything changed. this might save us some over 1 because
133: * we don't have to unbusy the page and may be less compares(?).
134: *
135: * alternative 3: put in backpointers or a way to "hold" part of a map
136: * in place while I/O is in progress. this could be complex to
137: * implement (especially with structures like amap that can be referenced
138: * by multiple map entries, and figuring out what should wait could be
139: * complex as well...).
140: *
1.125 ad 141: * we use alternative 2. given that we are multi-threaded now we may want
142: * to reconsider the choice.
1.1 mrg 143: */
144:
145: /*
146: * local data structures
147: */
148:
149: struct uvm_advice {
1.7 mrg 150: int advice;
151: int nback;
152: int nforw;
1.1 mrg 153: };
154:
155: /*
156: * page range array:
1.63 chs 157: * note: index in array must match "advice" value
1.1 mrg 158: * XXX: borrowed numbers from freebsd. do they work well for us?
159: */
160:
1.95 thorpej 161: static const struct uvm_advice uvmadvice[] = {
1.7 mrg 162: { MADV_NORMAL, 3, 4 },
163: { MADV_RANDOM, 0, 0 },
164: { MADV_SEQUENTIAL, 8, 7},
1.1 mrg 165: };
166:
1.69 chs 167: #define UVM_MAXRANGE 16 /* must be MAX() of nback+nforw+1 */
1.1 mrg 168:
169: /*
170: * private prototypes
171: */
172:
173: /*
174: * inline functions
175: */
176:
177: /*
178: * uvmfault_anonflush: try and deactivate pages in specified anons
179: *
180: * => does not have to deactivate page if it is busy
181: */
182:
1.103 perry 183: static inline void
1.95 thorpej 184: uvmfault_anonflush(struct vm_anon **anons, int n)
1.1 mrg 185: {
1.7 mrg 186: int lcv;
187: struct vm_page *pg;
1.63 chs 188:
1.163 uebayasi 189: for (lcv = 0; lcv < n; lcv++) {
1.7 mrg 190: if (anons[lcv] == NULL)
191: continue;
1.122 ad 192: mutex_enter(&anons[lcv]->an_lock);
1.94 yamt 193: pg = anons[lcv]->an_page;
1.117 yamt 194: if (pg && (pg->flags & PG_BUSY) == 0) {
1.122 ad 195: mutex_enter(&uvm_pageqlock);
1.7 mrg 196: if (pg->wire_count == 0) {
197: uvm_pagedeactivate(pg);
198: }
1.122 ad 199: mutex_exit(&uvm_pageqlock);
1.7 mrg 200: }
1.122 ad 201: mutex_exit(&anons[lcv]->an_lock);
1.7 mrg 202: }
1.1 mrg 203: }
204:
205: /*
206: * normal functions
207: */
208:
209: /*
210: * uvmfault_amapcopy: clear "needs_copy" in a map.
211: *
212: * => called with VM data structures unlocked (usually, see below)
213: * => we get a write lock on the maps and clear needs_copy for a VA
214: * => if we are out of RAM we sleep (waiting for more)
215: */
216:
1.7 mrg 217: static void
1.95 thorpej 218: uvmfault_amapcopy(struct uvm_faultinfo *ufi)
1.1 mrg 219: {
1.69 chs 220: for (;;) {
1.1 mrg 221:
1.7 mrg 222: /*
223: * no mapping? give up.
224: */
1.1 mrg 225:
1.119 thorpej 226: if (uvmfault_lookup(ufi, true) == false)
1.7 mrg 227: return;
1.1 mrg 228:
1.7 mrg 229: /*
230: * copy if needed.
231: */
1.1 mrg 232:
1.7 mrg 233: if (UVM_ET_ISNEEDSCOPY(ufi->entry))
1.108 yamt 234: amap_copy(ufi->map, ufi->entry, AMAP_COPY_NOWAIT,
1.13 chuck 235: ufi->orig_rvaddr, ufi->orig_rvaddr + 1);
1.1 mrg 236:
1.7 mrg 237: /*
238: * didn't work? must be out of RAM. unlock and sleep.
239: */
240:
241: if (UVM_ET_ISNEEDSCOPY(ufi->entry)) {
1.119 thorpej 242: uvmfault_unlockmaps(ufi, true);
1.7 mrg 243: uvm_wait("fltamapcopy");
244: continue;
245: }
246:
247: /*
248: * got it! unlock and return.
249: */
1.63 chs 250:
1.119 thorpej 251: uvmfault_unlockmaps(ufi, true);
1.7 mrg 252: return;
253: }
254: /*NOTREACHED*/
1.1 mrg 255: }
256:
257: /*
258: * uvmfault_anonget: get data in an anon into a non-busy, non-released
259: * page in that anon.
260: *
261: * => maps, amap, and anon locked by caller.
1.57 chs 262: * => if we fail (result != 0) we unlock everything.
1.1 mrg 263: * => if we are successful, we return with everything still locked.
264: * => we don't move the page on the queues [gets moved later]
265: * => if we allocate a new page [we_own], it gets put on the queues.
266: * either way, the result is that the page is on the queues at return time
267: * => for pages which are on loan from a uvm_object (and thus are not
268: * owned by the anon): if successful, we return with the owning object
269: * locked. the caller must unlock this object when it unlocks everything
270: * else.
271: */
272:
1.47 chs 273: int
1.95 thorpej 274: uvmfault_anonget(struct uvm_faultinfo *ufi, struct vm_amap *amap,
275: struct vm_anon *anon)
1.7 mrg 276: {
1.118 thorpej 277: bool we_own; /* we own anon's page? */
278: bool locked; /* did we relock? */
1.7 mrg 279: struct vm_page *pg;
1.58 chs 280: int error;
1.7 mrg 281: UVMHIST_FUNC("uvmfault_anonget"); UVMHIST_CALLED(maphist);
282:
1.122 ad 283: KASSERT(mutex_owned(&anon->an_lock));
1.53 thorpej 284:
1.58 chs 285: error = 0;
1.9 chuck 286: uvmexp.fltanget++;
287: /* bump rusage counters */
1.94 yamt 288: if (anon->an_page)
1.124 ad 289: curlwp->l_ru.ru_minflt++;
1.9 chuck 290: else
1.124 ad 291: curlwp->l_ru.ru_majflt++;
1.7 mrg 292:
1.63 chs 293: /*
1.7 mrg 294: * loop until we get it, or fail.
295: */
296:
1.69 chs 297: for (;;) {
1.119 thorpej 298: we_own = false; /* true if we set PG_BUSY on a page */
1.94 yamt 299: pg = anon->an_page;
1.1 mrg 300:
1.7 mrg 301: /*
302: * if there is a resident page and it is loaned, then anon
303: * may not own it. call out to uvm_anon_lockpage() to ensure
304: * the real owner of the page has been identified and locked.
305: */
306:
307: if (pg && pg->loan_count)
1.13 chuck 308: pg = uvm_anon_lockloanpg(anon);
1.7 mrg 309:
310: /*
311: * page there? make sure it is not busy/released.
312: */
313:
314: if (pg) {
315:
316: /*
317: * at this point, if the page has a uobject [meaning
318: * we have it on loan], then that uobject is locked
319: * by us! if the page is busy, we drop all the
320: * locks (including uobject) and try again.
321: */
322:
1.69 chs 323: if ((pg->flags & PG_BUSY) == 0) {
1.7 mrg 324: UVMHIST_LOG(maphist, "<- OK",0,0,0,0);
1.57 chs 325: return (0);
1.7 mrg 326: }
327: pg->flags |= PG_WANTED;
328: uvmexp.fltpgwait++;
329:
330: /*
331: * the last unlock must be an atomic unlock+wait on
332: * the owner of page
333: */
1.69 chs 334:
1.7 mrg 335: if (pg->uobject) { /* owner is uobject ? */
336: uvmfault_unlockall(ufi, amap, NULL, anon);
337: UVMHIST_LOG(maphist, " unlock+wait on uobj",0,
338: 0,0,0);
339: UVM_UNLOCK_AND_WAIT(pg,
340: &pg->uobject->vmobjlock,
1.119 thorpej 341: false, "anonget1",0);
1.7 mrg 342: } else {
343: /* anon owns page */
344: uvmfault_unlockall(ufi, amap, NULL, NULL);
345: UVMHIST_LOG(maphist, " unlock+wait on anon",0,
346: 0,0,0);
347: UVM_UNLOCK_AND_WAIT(pg,&anon->an_lock,0,
348: "anonget2",0);
349: }
350: } else {
1.101 yamt 351: #if defined(VMSWAP)
1.63 chs 352:
1.7 mrg 353: /*
354: * no page, we must try and bring it in.
355: */
1.69 chs 356:
1.180 enami 357: pg = uvm_pagealloc(NULL,
358: ufi != NULL ? ufi->orig_rvaddr : 0,
359: anon, UVM_FLAG_COLORMATCH);
1.7 mrg 360: if (pg == NULL) { /* out of RAM. */
361: uvmfault_unlockall(ufi, amap, NULL, anon);
362: uvmexp.fltnoram++;
363: UVMHIST_LOG(maphist, " noram -- UVM_WAIT",0,
364: 0,0,0);
1.93 yamt 365: if (!uvm_reclaimable()) {
366: return ENOMEM;
367: }
1.7 mrg 368: uvm_wait("flt_noram1");
369: } else {
370: /* we set the PG_BUSY bit */
1.119 thorpej 371: we_own = true;
1.7 mrg 372: uvmfault_unlockall(ufi, amap, NULL, anon);
373:
374: /*
375: * we are passing a PG_BUSY+PG_FAKE+PG_CLEAN
376: * page into the uvm_swap_get function with
1.18 chuck 377: * all data structures unlocked. note that
378: * it is ok to read an_swslot here because
379: * we hold PG_BUSY on the page.
1.7 mrg 380: */
381: uvmexp.pageins++;
1.58 chs 382: error = uvm_swap_get(pg, anon->an_swslot,
1.7 mrg 383: PGO_SYNCIO);
384:
385: /*
386: * we clean up after the i/o below in the
387: * "we_own" case
388: */
389: }
1.101 yamt 390: #else /* defined(VMSWAP) */
391: panic("%s: no page", __func__);
392: #endif /* defined(VMSWAP) */
1.7 mrg 393: }
394:
395: /*
396: * now relock and try again
397: */
398:
399: locked = uvmfault_relock(ufi);
1.47 chs 400: if (locked && amap != NULL) {
1.19 chuck 401: amap_lock(amap);
1.7 mrg 402: }
403: if (locked || we_own)
1.122 ad 404: mutex_enter(&anon->an_lock);
1.7 mrg 405:
406: /*
407: * if we own the page (i.e. we set PG_BUSY), then we need
408: * to clean up after the I/O. there are three cases to
409: * consider:
410: * [1] page released during I/O: free anon and ReFault.
1.63 chs 411: * [2] I/O not OK. free the page and cause the fault
1.7 mrg 412: * to fail.
413: * [3] I/O OK! activate the page and sync with the
414: * non-we_own case (i.e. drop anon lock if not locked).
415: */
1.63 chs 416:
1.7 mrg 417: if (we_own) {
1.101 yamt 418: #if defined(VMSWAP)
1.7 mrg 419: if (pg->flags & PG_WANTED) {
1.63 chs 420: wakeup(pg);
1.7 mrg 421: }
1.58 chs 422: if (error) {
1.1 mrg 423:
1.47 chs 424: /*
425: * remove the swap slot from the anon
426: * and mark the anon as having no real slot.
427: * don't free the swap slot, thus preventing
428: * it from being used again.
429: */
1.69 chs 430:
1.84 pk 431: if (anon->an_swslot > 0)
432: uvm_swap_markbad(anon->an_swslot, 1);
1.47 chs 433: anon->an_swslot = SWSLOT_BAD;
434:
1.88 yamt 435: if ((pg->flags & PG_RELEASED) != 0)
436: goto released;
437:
1.47 chs 438: /*
1.7 mrg 439: * note: page was never !PG_BUSY, so it
440: * can't be mapped and thus no need to
441: * pmap_page_protect it...
442: */
1.69 chs 443:
1.122 ad 444: mutex_enter(&uvm_pageqlock);
1.7 mrg 445: uvm_pagefree(pg);
1.122 ad 446: mutex_exit(&uvm_pageqlock);
1.7 mrg 447:
448: if (locked)
449: uvmfault_unlockall(ufi, amap, NULL,
450: anon);
451: else
1.122 ad 452: mutex_exit(&anon->an_lock);
1.7 mrg 453: UVMHIST_LOG(maphist, "<- ERROR", 0,0,0,0);
1.58 chs 454: return error;
1.7 mrg 455: }
1.63 chs 456:
1.88 yamt 457: if ((pg->flags & PG_RELEASED) != 0) {
458: released:
459: KASSERT(anon->an_ref == 0);
460:
461: /*
462: * released while we unlocked amap.
463: */
464:
465: if (locked)
466: uvmfault_unlockall(ufi, amap, NULL,
467: NULL);
468:
469: uvm_anon_release(anon);
470:
471: if (error) {
472: UVMHIST_LOG(maphist,
473: "<- ERROR/RELEASED", 0,0,0,0);
474: return error;
475: }
476:
477: UVMHIST_LOG(maphist, "<- RELEASED", 0,0,0,0);
478: return ERESTART;
479: }
480:
1.7 mrg 481: /*
1.69 chs 482: * we've successfully read the page, activate it.
1.7 mrg 483: */
1.69 chs 484:
1.122 ad 485: mutex_enter(&uvm_pageqlock);
1.7 mrg 486: uvm_pageactivate(pg);
1.122 ad 487: mutex_exit(&uvm_pageqlock);
1.69 chs 488: pg->flags &= ~(PG_WANTED|PG_BUSY|PG_FAKE);
489: UVM_PAGE_OWN(pg, NULL);
1.7 mrg 490: if (!locked)
1.122 ad 491: mutex_exit(&anon->an_lock);
1.101 yamt 492: #else /* defined(VMSWAP) */
493: panic("%s: we_own", __func__);
494: #endif /* defined(VMSWAP) */
1.7 mrg 495: }
496:
497: /*
498: * we were not able to relock. restart fault.
499: */
500:
501: if (!locked) {
502: UVMHIST_LOG(maphist, "<- REFAULT", 0,0,0,0);
1.57 chs 503: return (ERESTART);
1.7 mrg 504: }
505:
506: /*
507: * verify no one has touched the amap and moved the anon on us.
508: */
1.1 mrg 509:
1.47 chs 510: if (ufi != NULL &&
1.63 chs 511: amap_lookup(&ufi->entry->aref,
1.47 chs 512: ufi->orig_rvaddr - ufi->entry->start) != anon) {
1.63 chs 513:
1.7 mrg 514: uvmfault_unlockall(ufi, amap, NULL, anon);
515: UVMHIST_LOG(maphist, "<- REFAULT", 0,0,0,0);
1.57 chs 516: return (ERESTART);
1.7 mrg 517: }
1.63 chs 518:
1.7 mrg 519: /*
1.63 chs 520: * try it again!
1.7 mrg 521: */
1.1 mrg 522:
1.7 mrg 523: uvmexp.fltanretry++;
524: continue;
1.69 chs 525: }
1.7 mrg 526: /*NOTREACHED*/
1.1 mrg 527: }
528:
529: /*
1.106 yamt 530: * uvmfault_promote: promote data to a new anon. used for 1B and 2B.
531: *
532: * 1. allocate an anon and a page.
533: * 2. fill its contents.
534: * 3. put it into amap.
535: *
536: * => if we fail (result != 0) we unlock everything.
537: * => on success, return a new locked anon via 'nanon'.
538: * (*nanon)->an_page will be a resident, locked, dirty page.
539: */
540:
541: static int
542: uvmfault_promote(struct uvm_faultinfo *ufi,
543: struct vm_anon *oanon,
544: struct vm_page *uobjpage,
545: struct vm_anon **nanon, /* OUT: allocated anon */
546: struct vm_anon **spare)
547: {
548: struct vm_amap *amap = ufi->entry->aref.ar_amap;
549: struct uvm_object *uobj;
550: struct vm_anon *anon;
551: struct vm_page *pg;
552: struct vm_page *opg;
553: int error;
554: UVMHIST_FUNC(__func__); UVMHIST_CALLED(maphist);
555:
556: if (oanon) {
557: /* anon COW */
558: opg = oanon->an_page;
559: KASSERT(opg != NULL);
560: KASSERT(opg->uobject == NULL || opg->loan_count > 0);
561: } else if (uobjpage != PGO_DONTCARE) {
562: /* object-backed COW */
563: opg = uobjpage;
564: } else {
565: /* ZFOD */
566: opg = NULL;
567: }
568: if (opg != NULL) {
569: uobj = opg->uobject;
570: } else {
571: uobj = NULL;
572: }
573:
574: KASSERT(amap != NULL);
575: KASSERT(uobjpage != NULL);
576: KASSERT(uobjpage == PGO_DONTCARE || (uobjpage->flags & PG_BUSY) != 0);
1.120 ad 577: KASSERT(mutex_owned(&amap->am_l));
1.122 ad 578: KASSERT(oanon == NULL || mutex_owned(&oanon->an_lock));
579: KASSERT(uobj == NULL || mutex_owned(&uobj->vmobjlock));
580: #if 0
581: KASSERT(*spare == NULL || !mutex_owned(&(*spare)->an_lock));
582: #endif
1.106 yamt 583:
584: if (*spare != NULL) {
585: anon = *spare;
586: *spare = NULL;
1.122 ad 587: mutex_enter(&anon->an_lock);
1.106 yamt 588: } else if (ufi->map != kernel_map) {
589: anon = uvm_analloc();
590: } else {
591: UVMHIST_LOG(maphist, "kernel_map, unlock and retry", 0,0,0,0);
592:
593: /*
594: * we can't allocate anons with kernel_map locked.
595: */
596:
597: uvm_page_unbusy(&uobjpage, 1);
598: uvmfault_unlockall(ufi, amap, uobj, oanon);
599:
600: *spare = uvm_analloc();
601: if (*spare == NULL) {
602: goto nomem;
603: }
1.122 ad 604: mutex_exit(&(*spare)->an_lock);
1.106 yamt 605: error = ERESTART;
606: goto done;
607: }
608: if (anon) {
609:
610: /*
611: * The new anon is locked.
612: *
613: * if opg == NULL, we want a zero'd, dirty page,
614: * so have uvm_pagealloc() do that for us.
615: */
616:
1.179 matt 617: pg = uvm_pagealloc(NULL, ufi->orig_rvaddr, anon,
618: UVM_FLAG_COLORMATCH | (opg == NULL ? UVM_PGA_ZERO : 0));
1.106 yamt 619: } else {
620: pg = NULL;
621: }
622:
623: /*
624: * out of memory resources?
625: */
626:
627: if (pg == NULL) {
628: /* save anon for the next try. */
629: if (anon != NULL) {
1.122 ad 630: mutex_exit(&anon->an_lock);
1.106 yamt 631: *spare = anon;
632: }
633:
634: /* unlock and fail ... */
635: uvm_page_unbusy(&uobjpage, 1);
636: uvmfault_unlockall(ufi, amap, uobj, oanon);
637: nomem:
638: if (!uvm_reclaimable()) {
639: UVMHIST_LOG(maphist, "out of VM", 0,0,0,0);
640: uvmexp.fltnoanon++;
641: error = ENOMEM;
642: goto done;
643: }
644:
645: UVMHIST_LOG(maphist, "out of RAM, waiting for more", 0,0,0,0);
646: uvmexp.fltnoram++;
647: uvm_wait("flt_noram5");
648: error = ERESTART;
649: goto done;
650: }
651:
652: /* copy page [pg now dirty] */
653: if (opg) {
654: uvm_pagecopy(opg, pg);
655: }
656:
657: amap_add(&ufi->entry->aref, ufi->orig_rvaddr - ufi->entry->start, anon,
658: oanon != NULL);
659:
660: *nanon = anon;
661: error = 0;
662: done:
663: return error;
664: }
665:
666:
667: /*
1.1 mrg 668: * F A U L T - m a i n e n t r y p o i n t
669: */
670:
671: /*
672: * uvm_fault: page fault handler
673: *
674: * => called from MD code to resolve a page fault
1.63 chs 675: * => VM data structures usually should be unlocked. however, it is
1.1 mrg 676: * possible to call here with the main map locked if the caller
677: * gets a write lock, sets it recusive, and then calls us (c.f.
678: * uvm_map_pageable). this should be avoided because it keeps
679: * the map locked off during I/O.
1.66 thorpej 680: * => MUST NEVER BE CALLED IN INTERRUPT CONTEXT
1.1 mrg 681: */
682:
1.24 mycroft 683: #define MASK(entry) (UVM_ET_ISCOPYONWRITE(entry) ? \
684: ~VM_PROT_WRITE : VM_PROT_ALL)
685:
1.110 drochner 686: /* fault_flag values passed from uvm_fault_wire to uvm_fault_internal */
1.130 uebayasi 687: #define UVM_FAULT_WIRE (1 << 0)
688: #define UVM_FAULT_MAXPROT (1 << 1)
1.110 drochner 689:
1.140 uebayasi 690: struct uvm_faultctx {
691: vm_prot_t access_type;
692: vm_prot_t enter_prot;
1.150 uebayasi 693: vaddr_t startva;
694: int npages;
695: int centeridx;
696: struct vm_anon *anon_spare;
1.146 uebayasi 697: bool wire_mapping;
1.140 uebayasi 698: bool narrow;
1.146 uebayasi 699: bool wire_paging;
1.140 uebayasi 700: bool cow_now;
1.168 uebayasi 701: bool promote;
1.140 uebayasi 702: };
703:
1.163 uebayasi 704: static inline int uvm_fault_check(
705: struct uvm_faultinfo *, struct uvm_faultctx *,
1.177 yamt 706: struct vm_anon ***, bool);
1.163 uebayasi 707:
708: static int uvm_fault_upper(
709: struct uvm_faultinfo *, struct uvm_faultctx *,
710: struct vm_anon **);
711: static inline int uvm_fault_upper_lookup(
1.177 yamt 712: struct uvm_faultinfo *, const struct uvm_faultctx *,
1.163 uebayasi 713: struct vm_anon **, struct vm_page **);
714: static inline void uvm_fault_upper_neighbor(
1.177 yamt 715: struct uvm_faultinfo *, const struct uvm_faultctx *,
1.163 uebayasi 716: vaddr_t, struct vm_page *, bool);
717: static inline int uvm_fault_upper_loan(
718: struct uvm_faultinfo *, struct uvm_faultctx *,
719: struct vm_anon *, struct uvm_object **);
720: static inline int uvm_fault_upper_promote(
721: struct uvm_faultinfo *, struct uvm_faultctx *,
722: struct uvm_object *, struct vm_anon *);
723: static inline int uvm_fault_upper_direct(
724: struct uvm_faultinfo *, struct uvm_faultctx *,
725: struct uvm_object *, struct vm_anon *);
726: static int uvm_fault_upper_enter(
1.177 yamt 727: struct uvm_faultinfo *, const struct uvm_faultctx *,
1.163 uebayasi 728: struct uvm_object *, struct vm_anon *,
729: struct vm_page *, struct vm_anon *);
1.169 uebayasi 730: static inline void uvm_fault_upper_done(
1.177 yamt 731: struct uvm_faultinfo *, const struct uvm_faultctx *,
732: struct vm_anon *, struct vm_page *);
1.163 uebayasi 733:
734: static int uvm_fault_lower(
735: struct uvm_faultinfo *, struct uvm_faultctx *,
736: struct vm_page **);
1.173 uebayasi 737: static inline void uvm_fault_lower_lookup(
1.177 yamt 738: struct uvm_faultinfo *, const struct uvm_faultctx *,
1.163 uebayasi 739: struct vm_page **);
740: static inline void uvm_fault_lower_neighbor(
1.177 yamt 741: struct uvm_faultinfo *, const struct uvm_faultctx *,
1.163 uebayasi 742: vaddr_t, struct vm_page *, bool);
743: static inline int uvm_fault_lower_io(
1.177 yamt 744: struct uvm_faultinfo *, const struct uvm_faultctx *,
1.163 uebayasi 745: struct uvm_object **, struct vm_page **);
746: static inline int uvm_fault_lower_direct(
747: struct uvm_faultinfo *, struct uvm_faultctx *,
748: struct uvm_object *, struct vm_page *);
749: static inline int uvm_fault_lower_direct_loan(
750: struct uvm_faultinfo *, struct uvm_faultctx *,
751: struct uvm_object *, struct vm_page **,
752: struct vm_page **);
753: static inline int uvm_fault_lower_promote(
754: struct uvm_faultinfo *, struct uvm_faultctx *,
755: struct uvm_object *, struct vm_page *);
756: static int uvm_fault_lower_enter(
1.177 yamt 757: struct uvm_faultinfo *, const struct uvm_faultctx *,
1.163 uebayasi 758: struct uvm_object *,
759: struct vm_anon *, struct vm_page *,
760: struct vm_page *);
1.169 uebayasi 761: static inline void uvm_fault_lower_done(
1.177 yamt 762: struct uvm_faultinfo *, const struct uvm_faultctx *,
763: struct uvm_object *, struct vm_page *);
1.138 uebayasi 764:
1.7 mrg 765: int
1.110 drochner 766: uvm_fault_internal(struct vm_map *orig_map, vaddr_t vaddr,
767: vm_prot_t access_type, int fault_flag)
1.1 mrg 768: {
1.7 mrg 769: struct uvm_faultinfo ufi;
1.140 uebayasi 770: struct uvm_faultctx flt = {
771: .access_type = access_type,
1.146 uebayasi 772:
773: /* don't look for neighborhood * pages on "wire" fault */
774: .narrow = (fault_flag & UVM_FAULT_WIRE) != 0,
775:
776: /* "wire" fault causes wiring of both mapping and paging */
777: .wire_mapping = (fault_flag & UVM_FAULT_WIRE) != 0,
778: .wire_paging = (fault_flag & UVM_FAULT_WIRE) != 0,
1.140 uebayasi 779: };
1.177 yamt 780: const bool maxprot = (fault_flag & UVM_FAULT_MAXPROT) != 0;
1.137 uebayasi 781: struct vm_anon *anons_store[UVM_MAXRANGE], **anons;
1.141 uebayasi 782: struct vm_page *pages_store[UVM_MAXRANGE], **pages;
1.140 uebayasi 783: int error;
1.7 mrg 784: UVMHIST_FUNC("uvm_fault"); UVMHIST_CALLED(maphist);
1.1 mrg 785:
1.110 drochner 786: UVMHIST_LOG(maphist, "(map=0x%x, vaddr=0x%x, at=%d, ff=%d)",
787: orig_map, vaddr, access_type, fault_flag);
1.1 mrg 788:
1.178 matt 789: curcpu()->ci_data.cpu_nfault++;
1.7 mrg 790:
791: /*
792: * init the IN parameters in the ufi
793: */
1.1 mrg 794:
1.7 mrg 795: ufi.orig_map = orig_map;
796: ufi.orig_rvaddr = trunc_page(vaddr);
797: ufi.orig_size = PAGE_SIZE; /* can't get any smaller than this */
798:
1.142 uebayasi 799: error = ERESTART;
800: while (error == ERESTART) {
1.143 uebayasi 801: anons = anons_store;
802: pages = pages_store;
1.1 mrg 803:
1.177 yamt 804: error = uvm_fault_check(&ufi, &flt, &anons, maxprot);
1.143 uebayasi 805: if (error != 0)
806: continue;
1.141 uebayasi 807:
1.143 uebayasi 808: error = uvm_fault_upper_lookup(&ufi, &flt, anons, pages);
809: if (error != 0)
810: continue;
1.138 uebayasi 811:
1.144 uebayasi 812: if (pages[flt.centeridx] == PGO_DONTCARE)
1.148 uebayasi 813: error = uvm_fault_upper(&ufi, &flt, anons);
1.167 uebayasi 814: else {
1.177 yamt 815: struct uvm_object * const uobj =
816: ufi.entry->object.uvm_obj;
1.167 uebayasi 817:
818: if (uobj && uobj->pgops->pgo_fault != NULL) {
1.173 uebayasi 819: /*
820: * invoke "special" fault routine.
821: */
1.167 uebayasi 822: mutex_enter(&uobj->vmobjlock);
1.173 uebayasi 823: /* locked: maps(read), amap(if there), uobj */
824: error = uobj->pgops->pgo_fault(&ufi,
825: flt.startva, pages, flt.npages,
826: flt.centeridx, flt.access_type,
827: PGO_LOCKED|PGO_SYNCIO);
1.167 uebayasi 828:
1.177 yamt 829: /*
830: * locked: nothing, pgo_fault has unlocked
831: * everything
832: */
1.167 uebayasi 833:
834: /*
1.177 yamt 835: * object fault routine responsible for
836: * pmap_update().
1.167 uebayasi 837: */
838: } else {
839: error = uvm_fault_lower(&ufi, &flt, pages);
840: }
841: }
1.142 uebayasi 842: }
1.138 uebayasi 843:
1.140 uebayasi 844: if (flt.anon_spare != NULL) {
845: flt.anon_spare->an_ref--;
846: uvm_anfree(flt.anon_spare);
1.138 uebayasi 847: }
848: return error;
1.141 uebayasi 849: }
1.138 uebayasi 850:
1.173 uebayasi 851: /*
852: * uvm_fault_check: check prot, handle needs-copy, etc.
853: *
854: * 1. lookup entry.
855: * 2. check protection.
856: * 3. adjust fault condition (mainly for simulated fault).
857: * 4. handle needs-copy (lazy amap copy).
858: * 5. establish range of interest for neighbor fault (aka pre-fault).
859: * 6. look up anons (if amap exists).
860: * 7. flush pages (if MADV_SEQUENTIAL)
861: *
862: * => called with nothing locked.
863: * => if we fail (result != 0) we unlock everything.
1.177 yamt 864: * => initialize/adjust many members of flt.
1.173 uebayasi 865: */
866:
1.144 uebayasi 867: static int
1.141 uebayasi 868: uvm_fault_check(
869: struct uvm_faultinfo *ufi, struct uvm_faultctx *flt,
1.177 yamt 870: struct vm_anon ***ranons, bool maxprot)
1.141 uebayasi 871: {
872: struct vm_amap *amap;
873: struct uvm_object *uobj;
1.137 uebayasi 874: vm_prot_t check_prot;
875: int nback, nforw;
1.164 mlelstv 876: UVMHIST_FUNC("uvm_fault_check"); UVMHIST_CALLED(maphist);
1.137 uebayasi 877:
1.7 mrg 878: /*
879: * lookup and lock the maps
880: */
881:
1.141 uebayasi 882: if (uvmfault_lookup(ufi, false) == false) {
1.177 yamt 883: UVMHIST_LOG(maphist, "<- no mapping @ 0x%x", ufi->orig_rvaddr,
884: 0,0,0);
1.141 uebayasi 885: return EFAULT;
1.7 mrg 886: }
887: /* locked: maps(read) */
888:
1.61 thorpej 889: #ifdef DIAGNOSTIC
1.141 uebayasi 890: if ((ufi->map->flags & VM_MAP_PAGEABLE) == 0) {
1.61 thorpej 891: printf("Page fault on non-pageable map:\n");
1.141 uebayasi 892: printf("ufi->map = %p\n", ufi->map);
893: printf("ufi->orig_map = %p\n", ufi->orig_map);
894: printf("ufi->orig_rvaddr = 0x%lx\n", (u_long) ufi->orig_rvaddr);
895: panic("uvm_fault: (ufi->map->flags & VM_MAP_PAGEABLE) == 0");
1.61 thorpej 896: }
897: #endif
1.58 chs 898:
1.7 mrg 899: /*
900: * check protection
901: */
902:
1.177 yamt 903: check_prot = maxprot ?
1.141 uebayasi 904: ufi->entry->max_protection : ufi->entry->protection;
905: if ((check_prot & flt->access_type) != flt->access_type) {
1.7 mrg 906: UVMHIST_LOG(maphist,
907: "<- protection failure (prot=0x%x, access=0x%x)",
1.141 uebayasi 908: ufi->entry->protection, flt->access_type, 0, 0);
909: uvmfault_unlockmaps(ufi, false);
910: return EACCES;
1.7 mrg 911: }
912:
913: /*
914: * "enter_prot" is the protection we want to enter the page in at.
915: * for certain pages (e.g. copy-on-write pages) this protection can
1.141 uebayasi 916: * be more strict than ufi->entry->protection. "wired" means either
1.7 mrg 917: * the entry is wired or we are fault-wiring the pg.
918: */
919:
1.141 uebayasi 920: flt->enter_prot = ufi->entry->protection;
1.146 uebayasi 921: if (VM_MAPENT_ISWIRED(ufi->entry))
922: flt->wire_mapping = true;
923:
924: if (flt->wire_mapping) {
1.141 uebayasi 925: flt->access_type = flt->enter_prot; /* full access for wired */
926: flt->cow_now = (check_prot & VM_PROT_WRITE) != 0;
1.73 chs 927: } else {
1.141 uebayasi 928: flt->cow_now = (flt->access_type & VM_PROT_WRITE) != 0;
1.73 chs 929: }
1.7 mrg 930:
1.168 uebayasi 931: flt->promote = false;
932:
1.7 mrg 933: /*
934: * handle "needs_copy" case. if we need to copy the amap we will
935: * have to drop our readlock and relock it with a write lock. (we
936: * need a write lock to change anything in a map entry [e.g.
937: * needs_copy]).
938: */
939:
1.141 uebayasi 940: if (UVM_ET_ISNEEDSCOPY(ufi->entry)) {
941: if (flt->cow_now || (ufi->entry->object.uvm_obj == NULL)) {
1.177 yamt 942: KASSERT(!maxprot);
1.7 mrg 943: /* need to clear */
944: UVMHIST_LOG(maphist,
945: " need to clear needs_copy and refault",0,0,0,0);
1.141 uebayasi 946: uvmfault_unlockmaps(ufi, false);
947: uvmfault_amapcopy(ufi);
1.7 mrg 948: uvmexp.fltamcopy++;
1.141 uebayasi 949: return ERESTART;
1.7 mrg 950:
951: } else {
952:
953: /*
954: * ensure that we pmap_enter page R/O since
955: * needs_copy is still true
956: */
1.72 chs 957:
1.141 uebayasi 958: flt->enter_prot &= ~VM_PROT_WRITE;
1.7 mrg 959: }
960: }
961:
962: /*
963: * identify the players
964: */
965:
1.141 uebayasi 966: amap = ufi->entry->aref.ar_amap; /* upper layer */
967: uobj = ufi->entry->object.uvm_obj; /* lower layer */
1.7 mrg 968:
969: /*
970: * check for a case 0 fault. if nothing backing the entry then
971: * error now.
972: */
973:
974: if (amap == NULL && uobj == NULL) {
1.141 uebayasi 975: uvmfault_unlockmaps(ufi, false);
1.7 mrg 976: UVMHIST_LOG(maphist,"<- no backing store, no overlay",0,0,0,0);
1.141 uebayasi 977: return EFAULT;
1.7 mrg 978: }
1.1 mrg 979:
1.7 mrg 980: /*
981: * establish range of interest based on advice from mapper
982: * and then clip to fit map entry. note that we only want
1.63 chs 983: * to do this the first time through the fault. if we
1.7 mrg 984: * ReFault we will disable this by setting "narrow" to true.
985: */
1.1 mrg 986:
1.141 uebayasi 987: if (flt->narrow == false) {
1.7 mrg 988:
989: /* wide fault (!narrow) */
1.141 uebayasi 990: KASSERT(uvmadvice[ufi->entry->advice].advice ==
991: ufi->entry->advice);
992: nback = MIN(uvmadvice[ufi->entry->advice].nback,
1.177 yamt 993: (ufi->orig_rvaddr - ufi->entry->start) >> PAGE_SHIFT);
1.141 uebayasi 994: flt->startva = ufi->orig_rvaddr - (nback << PAGE_SHIFT);
1.7 mrg 995: /*
996: * note: "-1" because we don't want to count the
997: * faulting page as forw
998: */
1.177 yamt 999: nforw = MIN(uvmadvice[ufi->entry->advice].nforw,
1000: ((ufi->entry->end - ufi->orig_rvaddr) >>
1001: PAGE_SHIFT) - 1);
1.141 uebayasi 1002: flt->npages = nback + nforw + 1;
1003: flt->centeridx = nback;
1.7 mrg 1004:
1.141 uebayasi 1005: flt->narrow = true; /* ensure only once per-fault */
1.7 mrg 1006:
1007: } else {
1.63 chs 1008:
1.7 mrg 1009: /* narrow fault! */
1010: nback = nforw = 0;
1.141 uebayasi 1011: flt->startva = ufi->orig_rvaddr;
1012: flt->npages = 1;
1013: flt->centeridx = 0;
1.1 mrg 1014:
1.7 mrg 1015: }
1.131 uebayasi 1016: /* offset from entry's start to pgs' start */
1.141 uebayasi 1017: const voff_t eoff = flt->startva - ufi->entry->start;
1.1 mrg 1018:
1.7 mrg 1019: /* locked: maps(read) */
1.13 chuck 1020: UVMHIST_LOG(maphist, " narrow=%d, back=%d, forw=%d, startva=0x%x",
1.141 uebayasi 1021: flt->narrow, nback, nforw, flt->startva);
1022: UVMHIST_LOG(maphist, " entry=0x%x, amap=0x%x, obj=0x%x", ufi->entry,
1.16 chs 1023: amap, uobj, 0);
1.1 mrg 1024:
1.7 mrg 1025: /*
1026: * if we've got an amap, lock it and extract current anons.
1027: */
1028:
1029: if (amap) {
1.19 chuck 1030: amap_lock(amap);
1.141 uebayasi 1031: amap_lookups(&ufi->entry->aref, eoff, *ranons, flt->npages);
1.7 mrg 1032: } else {
1.141 uebayasi 1033: *ranons = NULL; /* to be safe */
1.7 mrg 1034: }
1035:
1036: /* locked: maps(read), amap(if there) */
1.120 ad 1037: KASSERT(amap == NULL || mutex_owned(&amap->am_l));
1.7 mrg 1038:
1039: /*
1040: * for MADV_SEQUENTIAL mappings we want to deactivate the back pages
1041: * now and then forget about them (for the rest of the fault).
1042: */
1043:
1.141 uebayasi 1044: if (ufi->entry->advice == MADV_SEQUENTIAL && nback != 0) {
1.7 mrg 1045:
1046: UVMHIST_LOG(maphist, " MADV_SEQUENTIAL: flushing backpages",
1047: 0,0,0,0);
1048: /* flush back-page anons? */
1.63 chs 1049: if (amap)
1.141 uebayasi 1050: uvmfault_anonflush(*ranons, nback);
1.7 mrg 1051:
1052: /* flush object? */
1053: if (uobj) {
1.137 uebayasi 1054: voff_t uoff;
1055:
1.141 uebayasi 1056: uoff = ufi->entry->offset + eoff;
1.122 ad 1057: mutex_enter(&uobj->vmobjlock);
1.90 yamt 1058: (void) (uobj->pgops->pgo_put)(uobj, uoff, uoff +
1.15 chs 1059: (nback << PAGE_SHIFT), PGO_DEACTIVATE);
1.7 mrg 1060: }
1061:
1062: /* now forget about the backpages */
1063: if (amap)
1.141 uebayasi 1064: *ranons += nback;
1065: flt->startva += (nback << PAGE_SHIFT);
1066: flt->npages -= nback;
1067: flt->centeridx = 0;
1.7 mrg 1068: }
1.137 uebayasi 1069: /*
1070: * => startva is fixed
1071: * => npages is fixed
1072: */
1.177 yamt 1073: KASSERT(flt->startva <= ufi->orig_rvaddr);
1074: KASSERT(ufi->orig_rvaddr + ufi->orig_size <=
1075: flt->startva + (flt->npages << PAGE_SHIFT));
1.141 uebayasi 1076: return 0;
1077: }
1078:
1.173 uebayasi 1079: /*
1080: * uvm_fault_upper_lookup: look up existing h/w mapping and amap.
1081: *
1082: * iterate range of interest:
1083: * 1. check if h/w mapping exists. if yes, we don't care
1084: * 2. check if anon exists. if not, page is lower.
1085: * 3. if anon exists, enter h/w mapping for neighbors.
1086: *
1087: * => called with amap locked (if exists).
1088: */
1089:
1.144 uebayasi 1090: static int
1.141 uebayasi 1091: uvm_fault_upper_lookup(
1.177 yamt 1092: struct uvm_faultinfo *ufi, const struct uvm_faultctx *flt,
1.141 uebayasi 1093: struct vm_anon **anons, struct vm_page **pages)
1094: {
1095: struct vm_amap *amap = ufi->entry->aref.ar_amap;
1.137 uebayasi 1096: int lcv;
1097: vaddr_t currva;
1.144 uebayasi 1098: bool shadowed;
1.164 mlelstv 1099: UVMHIST_FUNC("uvm_fault_upper_lookup"); UVMHIST_CALLED(maphist);
1.7 mrg 1100:
1101: /* locked: maps(read), amap(if there) */
1.120 ad 1102: KASSERT(amap == NULL || mutex_owned(&amap->am_l));
1.1 mrg 1103:
1.7 mrg 1104: /*
1105: * map in the backpages and frontpages we found in the amap in hopes
1106: * of preventing future faults. we also init the pages[] array as
1107: * we go.
1108: */
1109:
1.141 uebayasi 1110: currva = flt->startva;
1.144 uebayasi 1111: shadowed = false;
1.163 uebayasi 1112: for (lcv = 0; lcv < flt->npages; lcv++, currva += PAGE_SIZE) {
1.7 mrg 1113: /*
1.177 yamt 1114: * don't play with VAs that are already mapped
1115: * (except for center)
1.7 mrg 1116: */
1.141 uebayasi 1117: if (lcv != flt->centeridx &&
1118: pmap_extract(ufi->orig_map->pmap, currva, NULL)) {
1.52 chs 1119: pages[lcv] = PGO_DONTCARE;
1120: continue;
1.7 mrg 1121: }
1122:
1123: /*
1124: * unmapped or center page. check if any anon at this level.
1125: */
1126: if (amap == NULL || anons[lcv] == NULL) {
1127: pages[lcv] = NULL;
1128: continue;
1129: }
1130:
1131: /*
1132: * check for present page and map if possible. re-activate it.
1133: */
1134:
1135: pages[lcv] = PGO_DONTCARE;
1.177 yamt 1136: if (lcv == flt->centeridx) { /* save center for later! */
1.144 uebayasi 1137: shadowed = true;
1.151 uebayasi 1138: } else {
1.161 uebayasi 1139: struct vm_anon *anon = anons[lcv];
1140:
1141: mutex_enter(&anon->an_lock);
1.172 uebayasi 1142: struct vm_page *pg = anon->an_page;
1143:
1144: /* ignore loaned and busy pages */
1145: if (pg != NULL && pg->loan_count == 0 &&
1146: (pg->flags & PG_BUSY) == 0)
1147: uvm_fault_upper_neighbor(ufi, flt, currva,
1148: pg, anon->an_ref > 1);
1.161 uebayasi 1149: mutex_exit(&anon->an_lock);
1.7 mrg 1150: }
1.151 uebayasi 1151: }
1152:
1.160 uebayasi 1153: /* locked: maps(read), amap(if there) */
1154: KASSERT(amap == NULL || mutex_owned(&amap->am_l));
1155: /* (shadowed == true) if there is an anon at the faulting address */
1156: UVMHIST_LOG(maphist, " shadowed=%d, will_get=%d", shadowed,
1.164 mlelstv 1157: (ufi->entry->object.uvm_obj && shadowed != false),0,0);
1.160 uebayasi 1158:
1159: /*
1160: * note that if we are really short of RAM we could sleep in the above
1161: * call to pmap_enter with everything locked. bad?
1162: *
1163: * XXX Actually, that is bad; pmap_enter() should just fail in that
1164: * XXX case. --thorpej
1165: */
1.151 uebayasi 1166:
1167: return 0;
1168: }
1169:
1.173 uebayasi 1170: /*
1171: * uvm_fault_upper_neighbor: enter single lower neighbor page.
1172: *
1173: * => called with amap and anon locked.
1174: */
1175:
1.151 uebayasi 1176: static void
1.163 uebayasi 1177: uvm_fault_upper_neighbor(
1.177 yamt 1178: struct uvm_faultinfo *ufi, const struct uvm_faultctx *flt,
1.161 uebayasi 1179: vaddr_t currva, struct vm_page *pg, bool readonly)
1.151 uebayasi 1180: {
1.164 mlelstv 1181: UVMHIST_FUNC("uvm_fault_upper_neighbor"); UVMHIST_CALLED(maphist);
1.151 uebayasi 1182:
1.173 uebayasi 1183: /* locked: amap, anon */
1184:
1.152 uebayasi 1185: mutex_enter(&uvm_pageqlock);
1.161 uebayasi 1186: uvm_pageenqueue(pg);
1.152 uebayasi 1187: mutex_exit(&uvm_pageqlock);
1188: UVMHIST_LOG(maphist,
1189: " MAPPING: n anon: pm=0x%x, va=0x%x, pg=0x%x",
1.161 uebayasi 1190: ufi->orig_map->pmap, currva, pg, 0);
1.152 uebayasi 1191: uvmexp.fltnamap++;
1192:
1193: /*
1.161 uebayasi 1194: * Since this page isn't the page that's actually faulting,
1195: * ignore pmap_enter() failures; it's not critical that we
1196: * enter these right now.
1.152 uebayasi 1197: */
1198:
1199: (void) pmap_enter(ufi->orig_map->pmap, currva,
1.161 uebayasi 1200: VM_PAGE_TO_PHYS(pg),
1201: readonly ? (flt->enter_prot & ~VM_PROT_WRITE) :
1.152 uebayasi 1202: flt->enter_prot,
1.154 uebayasi 1203: PMAP_CANFAIL | (flt->wire_mapping ? PMAP_WIRED : 0));
1.52 chs 1204:
1.152 uebayasi 1205: pmap_update(ufi->orig_map->pmap);
1.151 uebayasi 1206: }
1207:
1.173 uebayasi 1208: /*
1209: * uvm_fault_upper: handle upper fault.
1210: *
1211: * 1. acquire anon lock.
1212: * 2. get anon. let uvmfault_anonget do the dirty work.
1213: * 3. handle loan.
1214: * 4. dispatch direct or promote handlers.
1215: */
1.134 uebayasi 1216:
1.138 uebayasi 1217: static int
1218: uvm_fault_upper(
1.140 uebayasi 1219: struct uvm_faultinfo *ufi, struct uvm_faultctx *flt,
1.148 uebayasi 1220: struct vm_anon **anons)
1.138 uebayasi 1221: {
1.148 uebayasi 1222: struct vm_amap * const amap = ufi->entry->aref.ar_amap;
1223: struct vm_anon * const anon = anons[flt->centeridx];
1224: struct uvm_object *uobj;
1.138 uebayasi 1225: int error;
1.164 mlelstv 1226: UVMHIST_FUNC("uvm_fault_upper"); UVMHIST_CALLED(maphist);
1.137 uebayasi 1227:
1.7 mrg 1228: /* locked: maps(read), amap */
1.133 uebayasi 1229: KASSERT(mutex_owned(&amap->am_l));
1.7 mrg 1230:
1231: /*
1232: * handle case 1: fault on an anon in our amap
1233: */
1234:
1235: UVMHIST_LOG(maphist, " case 1 fault: anon=0x%x", anon, 0,0,0);
1.122 ad 1236: mutex_enter(&anon->an_lock);
1.7 mrg 1237:
1238: /* locked: maps(read), amap, anon */
1.120 ad 1239: KASSERT(mutex_owned(&amap->am_l));
1.122 ad 1240: KASSERT(mutex_owned(&anon->an_lock));
1.7 mrg 1241:
1242: /*
1243: * no matter if we have case 1A or case 1B we are going to need to
1244: * have the anon's memory resident. ensure that now.
1245: */
1246:
1247: /*
1.47 chs 1248: * let uvmfault_anonget do the dirty work.
1.51 thorpej 1249: * if it fails (!OK) it will unlock everything for us.
1.47 chs 1250: * if it succeeds, locks are still valid and locked.
1.7 mrg 1251: * also, if it is OK, then the anon's page is on the queues.
1252: * if the page is on loan from a uvm_object, then anonget will
1253: * lock that object for us if it does not fail.
1254: */
1255:
1.138 uebayasi 1256: error = uvmfault_anonget(ufi, amap, anon);
1.58 chs 1257: switch (error) {
1.57 chs 1258: case 0:
1.63 chs 1259: break;
1.7 mrg 1260:
1.57 chs 1261: case ERESTART:
1.139 uebayasi 1262: return ERESTART;
1.7 mrg 1263:
1.57 chs 1264: case EAGAIN:
1.128 pooka 1265: kpause("fltagain1", false, hz/2, NULL);
1.139 uebayasi 1266: return ERESTART;
1.51 thorpej 1267:
1268: default:
1.138 uebayasi 1269: return error;
1.1 mrg 1270: }
1.7 mrg 1271:
1272: /*
1273: * uobj is non null if the page is on loan from an object (i.e. uobj)
1274: */
1275:
1.94 yamt 1276: uobj = anon->an_page->uobject; /* locked by anonget if !NULL */
1.7 mrg 1277:
1278: /* locked: maps(read), amap, anon, uobj(if one) */
1.120 ad 1279: KASSERT(mutex_owned(&amap->am_l));
1.122 ad 1280: KASSERT(mutex_owned(&anon->an_lock));
1281: KASSERT(uobj == NULL || mutex_owned(&uobj->vmobjlock));
1.7 mrg 1282:
1283: /*
1.63 chs 1284: * special handling for loaned pages
1.7 mrg 1285: */
1.52 chs 1286:
1.94 yamt 1287: if (anon->an_page->loan_count) {
1.148 uebayasi 1288: error = uvm_fault_upper_loan(ufi, flt, anon, &uobj);
1289: if (error != 0)
1290: return error;
1291: }
1.160 uebayasi 1292:
1293: /*
1294: * if we are case 1B then we will need to allocate a new blank
1295: * anon to transfer the data into. note that we have a lock
1296: * on anon, so no one can busy or release the page until we are done.
1297: * also note that the ref count can't drop to zero here because
1298: * it is > 1 and we are only dropping one ref.
1299: *
1300: * in the (hopefully very rare) case that we are out of RAM we
1301: * will unlock, wait for more RAM, and refault.
1302: *
1303: * if we are out of anon VM we kill the process (XXX: could wait?).
1304: */
1305:
1306: if (flt->cow_now && anon->an_ref > 1) {
1.168 uebayasi 1307: flt->promote = true;
1.160 uebayasi 1308: error = uvm_fault_upper_promote(ufi, flt, uobj, anon);
1309: } else {
1310: error = uvm_fault_upper_direct(ufi, flt, uobj, anon);
1311: }
1312: return error;
1.148 uebayasi 1313: }
1314:
1.173 uebayasi 1315: /*
1316: * uvm_fault_upper_loan: handle loaned upper page.
1317: *
1.177 yamt 1318: * 1. if not cow'ing now, simply adjust flt->enter_prot.
1.173 uebayasi 1319: * 2. if cow'ing now, and if ref count is 1, break loan.
1320: */
1321:
1.148 uebayasi 1322: static int
1323: uvm_fault_upper_loan(
1324: struct uvm_faultinfo *ufi, struct uvm_faultctx *flt,
1325: struct vm_anon *anon, struct uvm_object **ruobj)
1326: {
1.149 uebayasi 1327: struct vm_amap * const amap = ufi->entry->aref.ar_amap;
1.151 uebayasi 1328: int error = 0;
1.173 uebayasi 1329: UVMHIST_FUNC("uvm_fault_upper_loan"); UVMHIST_CALLED(maphist);
1.149 uebayasi 1330:
1331: if (!flt->cow_now) {
1.7 mrg 1332:
1.149 uebayasi 1333: /*
1334: * for read faults on loaned pages we just cap the
1335: * protection at read-only.
1336: */
1.63 chs 1337:
1.149 uebayasi 1338: flt->enter_prot = flt->enter_prot & ~VM_PROT_WRITE;
1.7 mrg 1339:
1.149 uebayasi 1340: } else {
1341: /*
1342: * note that we can't allow writes into a loaned page!
1343: *
1344: * if we have a write fault on a loaned page in an
1345: * anon then we need to look at the anon's ref count.
1346: * if it is greater than one then we are going to do
1347: * a normal copy-on-write fault into a new anon (this
1348: * is not a problem). however, if the reference count
1349: * is one (a case where we would normally allow a
1350: * write directly to the page) then we need to kill
1351: * the loan before we continue.
1352: */
1353:
1354: /* >1 case is already ok */
1355: if (anon->an_ref == 1) {
1.155 uebayasi 1356: error = uvm_loanbreak_anon(anon, *ruobj);
1.151 uebayasi 1357: if (error != 0) {
1358: uvmfault_unlockall(ufi, amap, *ruobj, anon);
1359: uvm_wait("flt_noram2");
1360: return ERESTART;
1361: }
1.155 uebayasi 1362: /* if we were a loan reciever uobj is gone */
1363: if (*ruobj)
1364: *ruobj = NULL;
1.151 uebayasi 1365: }
1366: }
1367: return error;
1368: }
1369:
1.173 uebayasi 1370: /*
1371: * uvm_fault_upper_promote: promote upper page.
1372: *
1373: * 1. call uvmfault_promote.
1374: * 2. enqueue page.
1375: * 3. deref.
1376: * 4. pass page to uvm_fault_upper_enter.
1377: */
1378:
1.148 uebayasi 1379: static int
1380: uvm_fault_upper_promote(
1381: struct uvm_faultinfo *ufi, struct uvm_faultctx *flt,
1382: struct uvm_object *uobj, struct vm_anon *anon)
1383: {
1.149 uebayasi 1384: struct vm_anon * const oanon = anon;
1385: struct vm_page *pg;
1386: int error;
1.164 mlelstv 1387: UVMHIST_FUNC("uvm_fault_upper_promote"); UVMHIST_CALLED(maphist);
1.149 uebayasi 1388:
1389: UVMHIST_LOG(maphist, " case 1B: COW fault",0,0,0,0);
1390: uvmexp.flt_acow++;
1391:
1.177 yamt 1392: error = uvmfault_promote(ufi, oanon, PGO_DONTCARE, &anon,
1393: &flt->anon_spare);
1.149 uebayasi 1394: switch (error) {
1395: case 0:
1396: break;
1397: case ERESTART:
1398: return ERESTART;
1399: default:
1400: return error;
1401: }
1.7 mrg 1402:
1.149 uebayasi 1403: pg = anon->an_page;
1404: mutex_enter(&uvm_pageqlock);
1405: uvm_pageactivate(pg);
1406: mutex_exit(&uvm_pageqlock);
1407: pg->flags &= ~(PG_BUSY|PG_FAKE);
1408: UVM_PAGE_OWN(pg, NULL);
1.7 mrg 1409:
1.149 uebayasi 1410: /* deref: can not drop to zero here by defn! */
1411: oanon->an_ref--;
1.53 thorpej 1412:
1.149 uebayasi 1413: /*
1414: * note: oanon is still locked, as is the new anon. we
1415: * need to check for this later when we unlock oanon; if
1416: * oanon != anon, we'll have to unlock anon, too.
1417: */
1.7 mrg 1418:
1.149 uebayasi 1419: return uvm_fault_upper_enter(ufi, flt, uobj, anon, pg, oanon);
1.148 uebayasi 1420: }
1421:
1.173 uebayasi 1422: /*
1423: * uvm_fault_upper_direct: handle direct fault.
1424: */
1425:
1.148 uebayasi 1426: static int
1427: uvm_fault_upper_direct(
1428: struct uvm_faultinfo *ufi, struct uvm_faultctx *flt,
1429: struct uvm_object *uobj, struct vm_anon *anon)
1430: {
1.149 uebayasi 1431: struct vm_anon * const oanon = anon;
1432: struct vm_page *pg;
1.173 uebayasi 1433: UVMHIST_FUNC("uvm_fault_upper_direct"); UVMHIST_CALLED(maphist);
1.52 chs 1434:
1.149 uebayasi 1435: uvmexp.flt_anon++;
1436: pg = anon->an_page;
1437: if (anon->an_ref > 1) /* disallow writes to ref > 1 anons */
1438: flt->enter_prot = flt->enter_prot & ~VM_PROT_WRITE;
1.7 mrg 1439:
1.149 uebayasi 1440: return uvm_fault_upper_enter(ufi, flt, uobj, anon, pg, oanon);
1.148 uebayasi 1441: }
1442:
1.173 uebayasi 1443: /*
1444: * uvm_fault_upper_enter: enter h/w mapping of upper page.
1445: */
1446:
1.148 uebayasi 1447: static int
1448: uvm_fault_upper_enter(
1.177 yamt 1449: struct uvm_faultinfo *ufi, const struct uvm_faultctx *flt,
1.148 uebayasi 1450: struct uvm_object *uobj, struct vm_anon *anon, struct vm_page *pg,
1451: struct vm_anon *oanon)
1452: {
1453: struct vm_amap * const amap = ufi->entry->aref.ar_amap;
1.164 mlelstv 1454: UVMHIST_FUNC("uvm_fault_upper_enter"); UVMHIST_CALLED(maphist);
1.7 mrg 1455:
1.173 uebayasi 1456: /* locked: maps(read), amap, oanon, anon(if different from oanon) */
1.120 ad 1457: KASSERT(mutex_owned(&amap->am_l));
1.122 ad 1458: KASSERT(mutex_owned(&anon->an_lock));
1459: KASSERT(mutex_owned(&oanon->an_lock));
1.7 mrg 1460:
1461: /*
1.69 chs 1462: * now map the page in.
1.7 mrg 1463: */
1464:
1.177 yamt 1465: UVMHIST_LOG(maphist,
1466: " MAPPING: anon: pm=0x%x, va=0x%x, pg=0x%x, promote=%d",
1.168 uebayasi 1467: ufi->orig_map->pmap, ufi->orig_rvaddr, pg, flt->promote);
1.177 yamt 1468: if (pmap_enter(ufi->orig_map->pmap, ufi->orig_rvaddr,
1469: VM_PAGE_TO_PHYS(pg),
1470: flt->enter_prot, flt->access_type | PMAP_CANFAIL |
1471: (flt->wire_mapping ? PMAP_WIRED : 0)) != 0) {
1.69 chs 1472:
1.46 thorpej 1473: /*
1474: * No need to undo what we did; we can simply think of
1475: * this as the pmap throwing away the mapping information.
1476: *
1477: * We do, however, have to go through the ReFault path,
1478: * as the map may change while we're asleep.
1479: */
1.69 chs 1480:
1.53 thorpej 1481: if (anon != oanon)
1.122 ad 1482: mutex_exit(&anon->an_lock);
1.138 uebayasi 1483: uvmfault_unlockall(ufi, amap, uobj, oanon);
1.92 yamt 1484: if (!uvm_reclaimable()) {
1.46 thorpej 1485: UVMHIST_LOG(maphist,
1486: "<- failed. out of VM",0,0,0,0);
1487: /* XXX instrumentation */
1.148 uebayasi 1488: return ENOMEM;
1.46 thorpej 1489: }
1490: /* XXX instrumentation */
1491: uvm_wait("flt_pmfail1");
1.139 uebayasi 1492: return ERESTART;
1.46 thorpej 1493: }
1.7 mrg 1494:
1.177 yamt 1495: uvm_fault_upper_done(ufi, flt, anon, pg);
1.169 uebayasi 1496:
1497: /*
1498: * done case 1! finish up by unlocking everything and returning success
1499: */
1500:
1.175 rmind 1501: if (anon != oanon) {
1.169 uebayasi 1502: mutex_exit(&anon->an_lock);
1.175 rmind 1503: }
1504: pmap_update(ufi->orig_map->pmap);
1.169 uebayasi 1505: uvmfault_unlockall(ufi, amap, uobj, oanon);
1506: return 0;
1.148 uebayasi 1507: }
1508:
1.173 uebayasi 1509: /*
1510: * uvm_fault_upper_done: queue upper center page.
1511: */
1512:
1.169 uebayasi 1513: static void
1.148 uebayasi 1514: uvm_fault_upper_done(
1.177 yamt 1515: struct uvm_faultinfo *ufi, const struct uvm_faultctx *flt,
1516: struct vm_anon *anon, struct vm_page *pg)
1.148 uebayasi 1517: {
1.174 rmind 1518: const bool wire_paging = flt->wire_paging;
1519:
1.173 uebayasi 1520: UVMHIST_FUNC("uvm_fault_upper_done"); UVMHIST_CALLED(maphist);
1.148 uebayasi 1521:
1.7 mrg 1522: /*
1.46 thorpej 1523: * ... update the page queues.
1.7 mrg 1524: */
1525:
1.122 ad 1526: mutex_enter(&uvm_pageqlock);
1.174 rmind 1527: if (wire_paging) {
1.8 chuck 1528: uvm_pagewire(pg);
1.29 chs 1529:
1530: /*
1531: * since the now-wired page cannot be paged out,
1532: * release its swap resources for others to use.
1533: * since an anon with no swap cannot be PG_CLEAN,
1534: * clear its clean flag now.
1535: */
1536:
1537: pg->flags &= ~(PG_CLEAN);
1.174 rmind 1538:
1.7 mrg 1539: } else {
1540: uvm_pageactivate(pg);
1541: }
1.122 ad 1542: mutex_exit(&uvm_pageqlock);
1.174 rmind 1543:
1544: if (wire_paging) {
1545: uvm_anon_dropswap(anon);
1546: }
1.138 uebayasi 1547: }
1.1 mrg 1548:
1.173 uebayasi 1549: /*
1550: * uvm_fault_lower: handle lower fault.
1551: *
1552: * 1. check uobj
1553: * 1.1. if null, ZFOD.
1554: * 1.2. if not null, look up unnmapped neighbor pages.
1555: * 2. for center page, check if promote.
1556: * 2.1. ZFOD always needs promotion.
1557: * 2.2. other uobjs, when entry is marked COW (usually MAP_PRIVATE vnode).
1558: * 3. if uobj is not ZFOD and page is not found, do i/o.
1559: * 4. dispatch either direct / promote fault.
1560: */
1561:
1.138 uebayasi 1562: static int
1.173 uebayasi 1563: uvm_fault_lower(
1.140 uebayasi 1564: struct uvm_faultinfo *ufi, struct uvm_faultctx *flt,
1.173 uebayasi 1565: struct vm_page **pages)
1.138 uebayasi 1566: {
1.148 uebayasi 1567: #ifdef DIAGNOSTIC
1.173 uebayasi 1568: struct vm_amap *amap = ufi->entry->aref.ar_amap;
1.148 uebayasi 1569: #endif
1.173 uebayasi 1570: struct uvm_object *uobj = ufi->entry->object.uvm_obj;
1571: struct vm_page *uobjpage;
1.138 uebayasi 1572: int error;
1.173 uebayasi 1573: UVMHIST_FUNC("uvm_fault_lower"); UVMHIST_CALLED(maphist);
1574:
1575: /* locked: maps(read), amap(if there), uobj(if !null) */
1.137 uebayasi 1576:
1.7 mrg 1577: /*
1.173 uebayasi 1578: * now, if the desired page is not shadowed by the amap and we have
1579: * a backing object that does not have a special fault routine, then
1580: * we ask (with pgo_get) the object for resident pages that we care
1581: * about and attempt to map them in. we do not let pgo_get block
1582: * (PGO_LOCKED).
1583: */
1584:
1585: if (uobj == NULL) {
1586: /* zero fill; don't care neighbor pages */
1587: uobjpage = NULL;
1588: } else {
1589: uvm_fault_lower_lookup(ufi, flt, pages);
1590: uobjpage = pages[flt->centeridx];
1591: }
1592:
1593: /*
1594: * note that at this point we are done with any front or back pages.
1595: * we are now going to focus on the center page (i.e. the one we've
1596: * faulted on). if we have faulted on the upper (anon) layer
1597: * [i.e. case 1], then the anon we want is anons[centeridx] (we have
1598: * not touched it yet). if we have faulted on the bottom (uobj)
1599: * layer [i.e. case 2] and the page was both present and available,
1600: * then we've got a pointer to it as "uobjpage" and we've already
1601: * made it BUSY.
1.7 mrg 1602: */
1603:
1604: /*
1605: * locked:
1606: * maps(read), amap(if there), uobj(if !null), uobjpage(if !null)
1607: */
1.120 ad 1608: KASSERT(amap == NULL || mutex_owned(&amap->am_l));
1.122 ad 1609: KASSERT(uobj == NULL || mutex_owned(&uobj->vmobjlock));
1.120 ad 1610: KASSERT(uobjpage == NULL || (uobjpage->flags & PG_BUSY) != 0);
1.7 mrg 1611:
1612: /*
1613: * note that uobjpage can not be PGO_DONTCARE at this point. we now
1614: * set uobjpage to PGO_DONTCARE if we are doing a zero fill. if we
1615: * have a backing object, check and see if we are going to promote
1616: * the data up to an anon during the fault.
1617: */
1618:
1619: if (uobj == NULL) {
1.63 chs 1620: uobjpage = PGO_DONTCARE;
1.168 uebayasi 1621: flt->promote = true; /* always need anon here */
1.7 mrg 1622: } else {
1.52 chs 1623: KASSERT(uobjpage != PGO_DONTCARE);
1.168 uebayasi 1624: flt->promote = flt->cow_now && UVM_ET_ISCOPYONWRITE(ufi->entry);
1.7 mrg 1625: }
1626: UVMHIST_LOG(maphist, " case 2 fault: promote=%d, zfill=%d",
1.168 uebayasi 1627: flt->promote, (uobj == NULL), 0,0);
1.1 mrg 1628:
1.7 mrg 1629: /*
1.9 chuck 1630: * if uobjpage is not null then we do not need to do I/O to get the
1631: * uobjpage.
1632: *
1.63 chs 1633: * if uobjpage is null, then we need to unlock and ask the pager to
1.7 mrg 1634: * get the data for us. once we have the data, we need to reverify
1635: * the state the world. we are currently not holding any resources.
1636: */
1.1 mrg 1637:
1.9 chuck 1638: if (uobjpage) {
1639: /* update rusage counters */
1.124 ad 1640: curlwp->l_ru.ru_minflt++;
1.9 chuck 1641: } else {
1.163 uebayasi 1642: error = uvm_fault_lower_io(ufi, flt, &uobj, &uobjpage);
1.148 uebayasi 1643: if (error != 0)
1644: return error;
1645: }
1.160 uebayasi 1646:
1647: /*
1648: * locked:
1649: * maps(read), amap(if !null), uobj(if !null), uobjpage(if uobj)
1650: */
1651: KASSERT(amap == NULL || mutex_owned(&amap->am_l));
1652: KASSERT(uobj == NULL || mutex_owned(&uobj->vmobjlock));
1653: KASSERT(uobj == NULL || (uobjpage->flags & PG_BUSY) != 0);
1654:
1655: /*
1656: * notes:
1657: * - at this point uobjpage can not be NULL
1658: * - at this point uobjpage can not be PG_RELEASED (since we checked
1659: * for it above)
1660: * - at this point uobjpage could be PG_WANTED (handle later)
1661: */
1662:
1.177 yamt 1663: KASSERT(uobjpage != NULL);
1.160 uebayasi 1664: KASSERT(uobj == NULL || uobj == uobjpage->uobject);
1665: KASSERT(uobj == NULL || !UVM_OBJ_IS_CLEAN(uobjpage->uobject) ||
1666: (uobjpage->flags & PG_CLEAN) != 0);
1667:
1.177 yamt 1668: if (!flt->promote) {
1.163 uebayasi 1669: error = uvm_fault_lower_direct(ufi, flt, uobj, uobjpage);
1.160 uebayasi 1670: } else {
1.163 uebayasi 1671: error = uvm_fault_lower_promote(ufi, flt, uobj, uobjpage);
1.160 uebayasi 1672: }
1673: return error;
1.148 uebayasi 1674: }
1675:
1.173 uebayasi 1676: /*
1677: * uvm_fault_lower_lookup: look up on-memory uobj pages.
1678: *
1679: * 1. get on-memory pages.
1680: * 2. if failed, give up (get only center page later).
1681: * 3. if succeeded, enter h/w mapping of neighbor pages.
1682: */
1683:
1684: static void
1685: uvm_fault_lower_lookup(
1.177 yamt 1686: struct uvm_faultinfo *ufi, const struct uvm_faultctx *flt,
1.173 uebayasi 1687: struct vm_page **pages)
1688: {
1689: struct uvm_object *uobj = ufi->entry->object.uvm_obj;
1690: int lcv, gotpages;
1691: vaddr_t currva;
1692: UVMHIST_FUNC("uvm_fault_lower_lookup"); UVMHIST_CALLED(maphist);
1693:
1694: mutex_enter(&uobj->vmobjlock);
1695: /* locked: maps(read), amap(if there), uobj */
1696: /*
1697: * the following call to pgo_get does _not_ change locking state
1698: */
1699:
1700: uvmexp.fltlget++;
1701: gotpages = flt->npages;
1702: (void) uobj->pgops->pgo_get(uobj,
1703: ufi->entry->offset + flt->startva - ufi->entry->start,
1704: pages, &gotpages, flt->centeridx,
1705: flt->access_type & MASK(ufi->entry), ufi->entry->advice, PGO_LOCKED);
1706:
1707: /*
1708: * check for pages to map, if we got any
1709: */
1710:
1711: if (gotpages == 0) {
1712: pages[flt->centeridx] = NULL;
1713: return;
1714: }
1715:
1716: currva = flt->startva;
1717: for (lcv = 0; lcv < flt->npages; lcv++, currva += PAGE_SIZE) {
1718: struct vm_page *curpg;
1719:
1720: curpg = pages[lcv];
1721: if (curpg == NULL || curpg == PGO_DONTCARE) {
1722: continue;
1723: }
1724: KASSERT(curpg->uobject == uobj);
1725:
1726: /*
1727: * if center page is resident and not PG_BUSY|PG_RELEASED
1728: * then pgo_get made it PG_BUSY for us and gave us a handle
1729: * to it.
1730: */
1731:
1732: if (lcv == flt->centeridx) {
1733: UVMHIST_LOG(maphist, " got uobjpage "
1734: "(0x%x) with locked get",
1735: curpg, 0,0,0);
1736: } else {
1737: bool readonly = (curpg->flags & PG_RDONLY)
1738: || (curpg->loan_count > 0)
1739: || UVM_OBJ_NEEDS_WRITEFAULT(curpg->uobject);
1740:
1741: uvm_fault_lower_neighbor(ufi, flt,
1742: currva, curpg, readonly);
1743: }
1744: }
1745: pmap_update(ufi->orig_map->pmap);
1746: }
1747:
1748: /*
1749: * uvm_fault_lower_neighbor: enter h/w mapping of lower neighbor page.
1750: */
1751:
1752: static void
1753: uvm_fault_lower_neighbor(
1.177 yamt 1754: struct uvm_faultinfo *ufi, const struct uvm_faultctx *flt,
1.173 uebayasi 1755: vaddr_t currva, struct vm_page *pg, bool readonly)
1756: {
1.180.4.2! bouyer 1757: UVMHIST_FUNC(__func__); UVMHIST_CALLED(maphist);
1.173 uebayasi 1758:
1759: /* locked: maps(read), amap(if there), uobj */
1760:
1761: /*
1762: * calling pgo_get with PGO_LOCKED returns us pages which
1763: * are neither busy nor released, so we don't need to check
1764: * for this. we can just directly enter the pages.
1765: */
1766:
1767: mutex_enter(&uvm_pageqlock);
1768: uvm_pageenqueue(pg);
1769: mutex_exit(&uvm_pageqlock);
1770: UVMHIST_LOG(maphist,
1771: " MAPPING: n obj: pm=0x%x, va=0x%x, pg=0x%x",
1772: ufi->orig_map->pmap, currva, pg, 0);
1773: uvmexp.fltnomap++;
1774:
1775: /*
1776: * Since this page isn't the page that's actually faulting,
1777: * ignore pmap_enter() failures; it's not critical that we
1778: * enter these right now.
1779: * NOTE: page can't be PG_WANTED or PG_RELEASED because we've
1780: * held the lock the whole time we've had the handle.
1781: */
1782: KASSERT((pg->flags & PG_PAGEOUT) == 0);
1783: KASSERT((pg->flags & PG_RELEASED) == 0);
1784: KASSERT((pg->flags & PG_WANTED) == 0);
1785: KASSERT(!UVM_OBJ_IS_CLEAN(pg->uobject) ||
1786: (pg->flags & PG_CLEAN) != 0);
1787: pg->flags &= ~(PG_BUSY);
1788: UVM_PAGE_OWN(pg, NULL);
1789:
1790: (void) pmap_enter(ufi->orig_map->pmap, currva,
1791: VM_PAGE_TO_PHYS(pg),
1792: readonly ? (flt->enter_prot & ~VM_PROT_WRITE) :
1793: flt->enter_prot & MASK(ufi->entry),
1794: PMAP_CANFAIL | (flt->wire_mapping ? PMAP_WIRED : 0));
1795: }
1796:
1797: /*
1798: * uvm_fault_lower_io: get lower page from backing store.
1799: *
1800: * 1. unlock everything, because i/o will block.
1801: * 2. call pgo_get.
1802: * 3. if failed, recover.
1803: * 4. if succeeded, relock everything and verify things.
1804: */
1805:
1.148 uebayasi 1806: static int
1.163 uebayasi 1807: uvm_fault_lower_io(
1.177 yamt 1808: struct uvm_faultinfo *ufi, const struct uvm_faultctx *flt,
1.156 uebayasi 1809: struct uvm_object **ruobj, struct vm_page **ruobjpage)
1.148 uebayasi 1810: {
1.149 uebayasi 1811: struct vm_amap * const amap = ufi->entry->aref.ar_amap;
1.156 uebayasi 1812: struct uvm_object *uobj = *ruobj;
1.158 uebayasi 1813: struct vm_page *pg;
1.149 uebayasi 1814: bool locked;
1815: int gotpages;
1816: int error;
1817: voff_t uoff;
1.164 mlelstv 1818: UVMHIST_FUNC("uvm_fault_lower_io"); UVMHIST_CALLED(maphist);
1.149 uebayasi 1819:
1820: /* update rusage counters */
1821: curlwp->l_ru.ru_majflt++;
1.137 uebayasi 1822:
1.149 uebayasi 1823: /* locked: maps(read), amap(if there), uobj */
1824: uvmfault_unlockall(ufi, amap, NULL, NULL);
1825: /* locked: uobj */
1.63 chs 1826:
1.149 uebayasi 1827: uvmexp.fltget++;
1828: gotpages = 1;
1.166 mlelstv 1829: pg = NULL;
1.149 uebayasi 1830: uoff = (ufi->orig_rvaddr - ufi->entry->start) + ufi->entry->offset;
1.158 uebayasi 1831: error = uobj->pgops->pgo_get(uobj, uoff, &pg, &gotpages,
1.149 uebayasi 1832: 0, flt->access_type & MASK(ufi->entry), ufi->entry->advice,
1833: PGO_SYNCIO);
1.158 uebayasi 1834: /* locked: pg(if no error) */
1.52 chs 1835:
1.149 uebayasi 1836: /*
1837: * recover from I/O
1838: */
1.1 mrg 1839:
1.149 uebayasi 1840: if (error) {
1841: if (error == EAGAIN) {
1842: UVMHIST_LOG(maphist,
1843: " pgo_get says TRY AGAIN!",0,0,0,0);
1844: kpause("fltagain2", false, hz/2, NULL);
1845: return ERESTART;
1846: }
1.1 mrg 1847:
1.139 uebayasi 1848: #if 0
1.149 uebayasi 1849: KASSERT(error != ERESTART);
1.139 uebayasi 1850: #else
1.149 uebayasi 1851: /* XXXUEBS don't re-fault? */
1852: if (error == ERESTART)
1853: error = EIO;
1.139 uebayasi 1854: #endif
1855:
1.149 uebayasi 1856: UVMHIST_LOG(maphist, "<- pgo_get failed (code %d)",
1857: error, 0,0,0);
1858: return error;
1859: }
1.7 mrg 1860:
1.158 uebayasi 1861: /* locked: pg */
1.7 mrg 1862:
1.165 mlelstv 1863: KASSERT((pg->flags & PG_BUSY) != 0);
1864:
1.149 uebayasi 1865: mutex_enter(&uvm_pageqlock);
1.158 uebayasi 1866: uvm_pageactivate(pg);
1.149 uebayasi 1867: mutex_exit(&uvm_pageqlock);
1.69 chs 1868:
1.149 uebayasi 1869: /*
1870: * re-verify the state of the world by first trying to relock
1871: * the maps. always relock the object.
1872: */
1.7 mrg 1873:
1.149 uebayasi 1874: locked = uvmfault_relock(ufi);
1875: if (locked && amap)
1876: amap_lock(amap);
1.156 uebayasi 1877:
1878: /* might be changed */
1.158 uebayasi 1879: uobj = pg->uobject;
1.156 uebayasi 1880:
1.149 uebayasi 1881: mutex_enter(&uobj->vmobjlock);
1.63 chs 1882:
1.158 uebayasi 1883: /* locked(locked): maps(read), amap(if !null), uobj, pg */
1884: /* locked(!locked): uobj, pg */
1.7 mrg 1885:
1.149 uebayasi 1886: /*
1887: * verify that the page has not be released and re-verify
1888: * that amap slot is still free. if there is a problem,
1889: * we unlock and clean up.
1890: */
1.7 mrg 1891:
1.158 uebayasi 1892: if ((pg->flags & PG_RELEASED) != 0 ||
1893: (locked && amap && amap_lookup(&ufi->entry->aref,
1.149 uebayasi 1894: ufi->orig_rvaddr - ufi->entry->start))) {
1895: if (locked)
1896: uvmfault_unlockall(ufi, amap, NULL, NULL);
1897: locked = false;
1898: }
1.7 mrg 1899:
1.149 uebayasi 1900: /*
1901: * didn't get the lock? release the page and retry.
1902: */
1.7 mrg 1903:
1.149 uebayasi 1904: if (locked == false) {
1905: UVMHIST_LOG(maphist,
1906: " wasn't able to relock after fault: retry",
1907: 0,0,0,0);
1.158 uebayasi 1908: if (pg->flags & PG_WANTED) {
1909: wakeup(pg);
1910: }
1911: if (pg->flags & PG_RELEASED) {
1.149 uebayasi 1912: uvmexp.fltpgrele++;
1.158 uebayasi 1913: uvm_pagefree(pg);
1.157 uebayasi 1914: mutex_exit(&uobj->vmobjlock);
1.139 uebayasi 1915: return ERESTART;
1.7 mrg 1916: }
1.158 uebayasi 1917: pg->flags &= ~(PG_BUSY|PG_WANTED);
1918: UVM_PAGE_OWN(pg, NULL);
1.149 uebayasi 1919: mutex_exit(&uobj->vmobjlock);
1920: return ERESTART;
1921: }
1.7 mrg 1922:
1.149 uebayasi 1923: /*
1.158 uebayasi 1924: * we have the data in pg which is busy and
1.149 uebayasi 1925: * not released. we are holding object lock (so the page
1926: * can't be released on us).
1927: */
1.7 mrg 1928:
1.158 uebayasi 1929: /* locked: maps(read), amap(if !null), uobj, pg */
1.148 uebayasi 1930:
1.156 uebayasi 1931: *ruobj = uobj;
1.158 uebayasi 1932: *ruobjpage = pg;
1.148 uebayasi 1933: return 0;
1934: }
1935:
1.173 uebayasi 1936: /*
1937: * uvm_fault_lower_direct: fault lower center page
1938: *
1.177 yamt 1939: * 1. adjust flt->enter_prot.
1.173 uebayasi 1940: * 2. if page is loaned, resolve.
1941: */
1942:
1.148 uebayasi 1943: int
1.163 uebayasi 1944: uvm_fault_lower_direct(
1.148 uebayasi 1945: struct uvm_faultinfo *ufi, struct uvm_faultctx *flt,
1.156 uebayasi 1946: struct uvm_object *uobj, struct vm_page *uobjpage)
1.148 uebayasi 1947: {
1.149 uebayasi 1948: struct vm_page *pg;
1.173 uebayasi 1949: UVMHIST_FUNC("uvm_fault_lower_direct"); UVMHIST_CALLED(maphist);
1.149 uebayasi 1950:
1951: /*
1952: * we are not promoting. if the mapping is COW ensure that we
1953: * don't give more access than we should (e.g. when doing a read
1954: * fault on a COPYONWRITE mapping we want to map the COW page in
1955: * R/O even though the entry protection could be R/W).
1956: *
1957: * set "pg" to the page we want to map in (uobjpage, usually)
1958: */
1.1 mrg 1959:
1.149 uebayasi 1960: uvmexp.flt_obj++;
1961: if (UVM_ET_ISCOPYONWRITE(ufi->entry) ||
1962: UVM_OBJ_NEEDS_WRITEFAULT(uobjpage->uobject))
1963: flt->enter_prot &= ~VM_PROT_WRITE;
1964: pg = uobjpage; /* map in the actual object */
1.7 mrg 1965:
1.149 uebayasi 1966: KASSERT(uobjpage != PGO_DONTCARE);
1.7 mrg 1967:
1.149 uebayasi 1968: /*
1969: * we are faulting directly on the page. be careful
1970: * about writing to loaned pages...
1971: */
1972:
1973: if (uobjpage->loan_count) {
1.163 uebayasi 1974: uvm_fault_lower_direct_loan(ufi, flt, uobj, &pg, &uobjpage);
1.151 uebayasi 1975: }
1976: KASSERT(pg == uobjpage);
1977:
1.163 uebayasi 1978: return uvm_fault_lower_enter(ufi, flt, uobj, NULL, pg, uobjpage);
1.151 uebayasi 1979: }
1980:
1.173 uebayasi 1981: /*
1982: * uvm_fault_lower_direct_loan: resolve loaned page.
1983: *
1.177 yamt 1984: * 1. if not cow'ing, adjust flt->enter_prot.
1.173 uebayasi 1985: * 2. if cow'ing, break loan.
1986: */
1987:
1.151 uebayasi 1988: static int
1.163 uebayasi 1989: uvm_fault_lower_direct_loan(
1.151 uebayasi 1990: struct uvm_faultinfo *ufi, struct uvm_faultctx *flt,
1.177 yamt 1991: struct uvm_object *uobj, struct vm_page **rpg,
1992: struct vm_page **ruobjpage)
1.151 uebayasi 1993: {
1.152 uebayasi 1994: struct vm_amap * const amap = ufi->entry->aref.ar_amap;
1995: struct vm_page *pg;
1996: struct vm_page *uobjpage = *ruobjpage;
1.164 mlelstv 1997: UVMHIST_FUNC("uvm_fault_lower_direct_loan"); UVMHIST_CALLED(maphist);
1.152 uebayasi 1998:
1999: if (!flt->cow_now) {
2000: /* read fault: cap the protection at readonly */
2001: /* cap! */
2002: flt->enter_prot = flt->enter_prot & ~VM_PROT_WRITE;
2003: } else {
2004: /* write fault: must break the loan here */
2005:
2006: pg = uvm_loanbreak(uobjpage);
2007: if (pg == NULL) {
2008:
2009: /*
2010: * drop ownership of page, it can't be released
2011: */
2012:
2013: if (uobjpage->flags & PG_WANTED)
2014: wakeup(uobjpage);
2015: uobjpage->flags &= ~(PG_BUSY|PG_WANTED);
2016: UVM_PAGE_OWN(uobjpage, NULL);
2017:
2018: uvmfault_unlockall(ufi, amap, uobj, NULL);
2019: UVMHIST_LOG(maphist,
2020: " out of RAM breaking loan, waiting",
2021: 0,0,0,0);
2022: uvmexp.fltnoram++;
2023: uvm_wait("flt_noram4");
2024: return ERESTART;
1.69 chs 2025: }
1.152 uebayasi 2026: *rpg = pg;
2027: *ruobjpage = pg;
2028: }
2029: return 0;
1.148 uebayasi 2030: }
2031:
1.173 uebayasi 2032: /*
2033: * uvm_fault_lower_promote: promote lower page.
2034: *
2035: * 1. call uvmfault_promote.
2036: * 2. fill in data.
2037: * 3. if not ZFOD, dispose old page.
2038: */
2039:
1.148 uebayasi 2040: int
1.163 uebayasi 2041: uvm_fault_lower_promote(
1.148 uebayasi 2042: struct uvm_faultinfo *ufi, struct uvm_faultctx *flt,
1.156 uebayasi 2043: struct uvm_object *uobj, struct vm_page *uobjpage)
1.148 uebayasi 2044: {
1.149 uebayasi 2045: struct vm_amap * const amap = ufi->entry->aref.ar_amap;
2046: struct vm_anon *anon;
2047: struct vm_page *pg;
2048: int error;
1.164 mlelstv 2049: UVMHIST_FUNC("uvm_fault_lower_promote"); UVMHIST_CALLED(maphist);
1.63 chs 2050:
1.149 uebayasi 2051: /*
2052: * if we are going to promote the data to an anon we
2053: * allocate a blank anon here and plug it into our amap.
2054: */
1.1 mrg 2055: #if DIAGNOSTIC
1.149 uebayasi 2056: if (amap == NULL)
2057: panic("uvm_fault: want to promote data, but no anon");
1.1 mrg 2058: #endif
1.149 uebayasi 2059: error = uvmfault_promote(ufi, NULL, uobjpage,
2060: &anon, &flt->anon_spare);
2061: switch (error) {
2062: case 0:
2063: break;
2064: case ERESTART:
2065: return ERESTART;
2066: default:
2067: return error;
2068: }
2069:
2070: pg = anon->an_page;
2071:
2072: /*
2073: * fill in the data
2074: */
1.105 yamt 2075:
1.149 uebayasi 2076: if (uobjpage != PGO_DONTCARE) {
2077: uvmexp.flt_prcopy++;
1.1 mrg 2078:
1.7 mrg 2079: /*
1.149 uebayasi 2080: * promote to shared amap? make sure all sharing
2081: * procs see it
1.7 mrg 2082: */
2083:
1.149 uebayasi 2084: if ((amap_flags(amap) & AMAP_SHARED) != 0) {
2085: pmap_page_protect(uobjpage, VM_PROT_NONE);
1.7 mrg 2086: /*
1.149 uebayasi 2087: * XXX: PAGE MIGHT BE WIRED!
1.7 mrg 2088: */
1.149 uebayasi 2089: }
1.69 chs 2090:
1.149 uebayasi 2091: /*
2092: * dispose of uobjpage. it can't be PG_RELEASED
2093: * since we still hold the object lock.
2094: * drop handle to uobj as well.
2095: */
2096:
2097: if (uobjpage->flags & PG_WANTED)
2098: /* still have the obj lock */
2099: wakeup(uobjpage);
2100: uobjpage->flags &= ~(PG_BUSY|PG_WANTED);
2101: UVM_PAGE_OWN(uobjpage, NULL);
2102: mutex_exit(&uobj->vmobjlock);
2103: uobj = NULL;
2104:
2105: UVMHIST_LOG(maphist,
2106: " promote uobjpage 0x%x to anon/page 0x%x/0x%x",
2107: uobjpage, anon, pg, 0);
1.63 chs 2108:
1.149 uebayasi 2109: } else {
2110: uvmexp.flt_przero++;
1.7 mrg 2111:
1.149 uebayasi 2112: /*
2113: * Page is zero'd and marked dirty by
2114: * uvmfault_promote().
2115: */
1.52 chs 2116:
1.149 uebayasi 2117: UVMHIST_LOG(maphist," zero fill anon/page 0x%x/0%x",
2118: anon, pg, 0, 0);
2119: }
1.148 uebayasi 2120:
1.163 uebayasi 2121: return uvm_fault_lower_enter(ufi, flt, uobj, anon, pg, uobjpage);
1.148 uebayasi 2122: }
2123:
1.173 uebayasi 2124: /*
2125: * uvm_fault_lower_enter: enter h/w mapping of lower page.
2126: */
2127:
1.148 uebayasi 2128: int
1.163 uebayasi 2129: uvm_fault_lower_enter(
1.177 yamt 2130: struct uvm_faultinfo *ufi, const struct uvm_faultctx *flt,
1.148 uebayasi 2131: struct uvm_object *uobj,
2132: struct vm_anon *anon, struct vm_page *pg, struct vm_page *uobjpage)
2133: {
2134: struct vm_amap * const amap = ufi->entry->aref.ar_amap;
2135: int error;
1.164 mlelstv 2136: UVMHIST_FUNC("uvm_fault_lower_enter"); UVMHIST_CALLED(maphist);
1.7 mrg 2137:
2138: /*
2139: * locked:
1.53 thorpej 2140: * maps(read), amap(if !null), uobj(if !null), uobjpage(if uobj),
2141: * anon(if !null), pg(if anon)
1.7 mrg 2142: *
2143: * note: pg is either the uobjpage or the new page in the new anon
2144: */
1.120 ad 2145: KASSERT(amap == NULL || mutex_owned(&amap->am_l));
1.122 ad 2146: KASSERT(uobj == NULL || mutex_owned(&uobj->vmobjlock));
1.120 ad 2147: KASSERT(uobj == NULL || (uobjpage->flags & PG_BUSY) != 0);
1.122 ad 2148: KASSERT(anon == NULL || mutex_owned(&anon->an_lock));
1.120 ad 2149: KASSERT((pg->flags & PG_BUSY) != 0);
1.7 mrg 2150:
2151: /*
2152: * all resources are present. we can now map it in and free our
2153: * resources.
2154: */
2155:
2156: UVMHIST_LOG(maphist,
1.168 uebayasi 2157: " MAPPING: case2: pm=0x%x, va=0x%x, pg=0x%x, promote=%d",
2158: ufi->orig_map->pmap, ufi->orig_rvaddr, pg, flt->promote);
1.140 uebayasi 2159: KASSERT((flt->access_type & VM_PROT_WRITE) == 0 ||
1.75 chs 2160: (pg->flags & PG_RDONLY) == 0);
1.177 yamt 2161: if (pmap_enter(ufi->orig_map->pmap, ufi->orig_rvaddr,
2162: VM_PAGE_TO_PHYS(pg),
2163: (pg->flags & PG_RDONLY) != 0 ?
2164: flt->enter_prot & ~VM_PROT_WRITE : flt->enter_prot,
2165: flt->access_type | PMAP_CANFAIL |
2166: (flt->wire_mapping ? PMAP_WIRED : 0)) != 0) {
1.52 chs 2167:
1.46 thorpej 2168: /*
2169: * No need to undo what we did; we can simply think of
2170: * this as the pmap throwing away the mapping information.
2171: *
2172: * We do, however, have to go through the ReFault path,
2173: * as the map may change while we're asleep.
2174: */
1.52 chs 2175:
1.46 thorpej 2176: if (pg->flags & PG_WANTED)
1.69 chs 2177: wakeup(pg);
1.46 thorpej 2178:
1.63 chs 2179: /*
1.46 thorpej 2180: * note that pg can't be PG_RELEASED since we did not drop
2181: * the object lock since the last time we checked.
2182: */
1.111 yamt 2183: KASSERT((pg->flags & PG_RELEASED) == 0);
1.63 chs 2184:
1.46 thorpej 2185: pg->flags &= ~(PG_BUSY|PG_FAKE|PG_WANTED);
2186: UVM_PAGE_OWN(pg, NULL);
1.171 uebayasi 2187:
1.138 uebayasi 2188: uvmfault_unlockall(ufi, amap, uobj, anon);
1.92 yamt 2189: if (!uvm_reclaimable()) {
1.46 thorpej 2190: UVMHIST_LOG(maphist,
2191: "<- failed. out of VM",0,0,0,0);
2192: /* XXX instrumentation */
1.106 yamt 2193: error = ENOMEM;
1.138 uebayasi 2194: return error;
1.46 thorpej 2195: }
2196: /* XXX instrumentation */
2197: uvm_wait("flt_pmfail2");
1.139 uebayasi 2198: return ERESTART;
1.46 thorpej 2199: }
1.1 mrg 2200:
1.177 yamt 2201: uvm_fault_lower_done(ufi, flt, uobj, pg);
2202:
2203: /*
2204: * note that pg can't be PG_RELEASED since we did not drop the object
2205: * lock since the last time we checked.
2206: */
2207: KASSERT((pg->flags & PG_RELEASED) == 0);
2208: if (pg->flags & PG_WANTED)
2209: wakeup(pg);
2210: pg->flags &= ~(PG_BUSY|PG_FAKE|PG_WANTED);
2211: UVM_PAGE_OWN(pg, NULL);
1.169 uebayasi 2212:
1.175 rmind 2213: pmap_update(ufi->orig_map->pmap);
1.169 uebayasi 2214: uvmfault_unlockall(ufi, amap, uobj, anon);
1.175 rmind 2215:
1.169 uebayasi 2216: UVMHIST_LOG(maphist, "<- done (SUCCESS!)",0,0,0,0);
2217: return 0;
1.148 uebayasi 2218: }
2219:
1.173 uebayasi 2220: /*
2221: * uvm_fault_lower_done: queue lower center page.
2222: */
2223:
1.169 uebayasi 2224: void
1.163 uebayasi 2225: uvm_fault_lower_done(
1.177 yamt 2226: struct uvm_faultinfo *ufi, const struct uvm_faultctx *flt,
2227: struct uvm_object *uobj, struct vm_page *pg)
1.148 uebayasi 2228: {
1.174 rmind 2229: bool dropswap = false;
2230:
1.164 mlelstv 2231: UVMHIST_FUNC("uvm_fault_lower_done"); UVMHIST_CALLED(maphist);
1.148 uebayasi 2232:
1.122 ad 2233: mutex_enter(&uvm_pageqlock);
1.146 uebayasi 2234: if (flt->wire_paging) {
1.8 chuck 2235: uvm_pagewire(pg);
1.22 chs 2236: if (pg->pqflags & PQ_AOBJ) {
1.29 chs 2237:
2238: /*
2239: * since the now-wired page cannot be paged out,
2240: * release its swap resources for others to use.
2241: * since an aobj page with no swap cannot be PG_CLEAN,
2242: * clear its clean flag now.
2243: */
2244:
1.113 christos 2245: KASSERT(uobj != NULL);
1.29 chs 2246: pg->flags &= ~(PG_CLEAN);
1.174 rmind 2247: dropswap = true;
1.22 chs 2248: }
1.7 mrg 2249: } else {
2250: uvm_pageactivate(pg);
2251: }
1.122 ad 2252: mutex_exit(&uvm_pageqlock);
1.171 uebayasi 2253:
1.174 rmind 2254: if (dropswap) {
2255: uao_dropswap(uobj, pg->offset >> PAGE_SHIFT);
2256: }
1.1 mrg 2257: }
2258:
1.110 drochner 2259:
1.1 mrg 2260: /*
2261: * uvm_fault_wire: wire down a range of virtual addresses in a map.
2262: *
1.36 thorpej 2263: * => map may be read-locked by caller, but MUST NOT be write-locked.
2264: * => if map is read-locked, any operations which may cause map to
2265: * be write-locked in uvm_fault() must be taken care of by
2266: * the caller. See uvm_map_pageable().
1.1 mrg 2267: */
2268:
1.7 mrg 2269: int
1.95 thorpej 2270: uvm_fault_wire(struct vm_map *map, vaddr_t start, vaddr_t end,
1.130 uebayasi 2271: vm_prot_t access_type, int maxprot)
1.7 mrg 2272: {
1.12 eeh 2273: vaddr_t va;
1.58 chs 2274: int error;
1.7 mrg 2275:
2276: /*
1.47 chs 2277: * now fault it in a page at a time. if the fault fails then we have
1.63 chs 2278: * to undo what we have done. note that in uvm_fault VM_PROT_NONE
1.47 chs 2279: * is replaced with the max protection if fault_type is VM_FAULT_WIRE.
1.7 mrg 2280: */
1.1 mrg 2281:
1.65 chs 2282: /*
2283: * XXX work around overflowing a vaddr_t. this prevents us from
2284: * wiring the last page in the address space, though.
2285: */
2286: if (start > end) {
2287: return EFAULT;
2288: }
2289:
1.163 uebayasi 2290: for (va = start; va < end; va += PAGE_SIZE) {
1.110 drochner 2291: error = uvm_fault_internal(map, va, access_type,
1.177 yamt 2292: (maxprot ? UVM_FAULT_MAXPROT : 0) | UVM_FAULT_WIRE);
1.58 chs 2293: if (error) {
1.7 mrg 2294: if (va != start) {
1.31 thorpej 2295: uvm_fault_unwire(map, start, va);
1.7 mrg 2296: }
1.58 chs 2297: return error;
1.7 mrg 2298: }
2299: }
1.58 chs 2300: return 0;
1.1 mrg 2301: }
2302:
2303: /*
2304: * uvm_fault_unwire(): unwire range of virtual space.
2305: */
2306:
1.7 mrg 2307: void
1.95 thorpej 2308: uvm_fault_unwire(struct vm_map *map, vaddr_t start, vaddr_t end)
1.36 thorpej 2309: {
2310: vm_map_lock_read(map);
2311: uvm_fault_unwire_locked(map, start, end);
2312: vm_map_unlock_read(map);
2313: }
2314:
2315: /*
2316: * uvm_fault_unwire_locked(): the guts of uvm_fault_unwire().
2317: *
2318: * => map must be at least read-locked.
2319: */
2320:
2321: void
1.95 thorpej 2322: uvm_fault_unwire_locked(struct vm_map *map, vaddr_t start, vaddr_t end)
1.7 mrg 2323: {
1.64 chs 2324: struct vm_map_entry *entry;
1.31 thorpej 2325: pmap_t pmap = vm_map_pmap(map);
1.42 thorpej 2326: vaddr_t va;
1.12 eeh 2327: paddr_t pa;
1.42 thorpej 2328: struct vm_page *pg;
1.31 thorpej 2329:
1.52 chs 2330: KASSERT((map->flags & VM_MAP_INTRSAFE) == 0);
1.7 mrg 2331:
2332: /*
2333: * we assume that the area we are unwiring has actually been wired
2334: * in the first place. this means that we should be able to extract
2335: * the PAs from the pmap. we also lock out the page daemon so that
2336: * we can call uvm_pageunwire.
2337: */
1.37 thorpej 2338:
1.122 ad 2339: mutex_enter(&uvm_pageqlock);
1.7 mrg 2340:
1.37 thorpej 2341: /*
2342: * find the beginning map entry for the region.
2343: */
1.74 chs 2344:
1.56 chs 2345: KASSERT(start >= vm_map_min(map) && end <= vm_map_max(map));
1.119 thorpej 2346: if (uvm_map_lookup_entry(map, start, &entry) == false)
1.37 thorpej 2347: panic("uvm_fault_unwire_locked: address not in map");
2348:
1.69 chs 2349: for (va = start; va < end; va += PAGE_SIZE) {
1.119 thorpej 2350: if (pmap_extract(pmap, va, &pa) == false)
1.74 chs 2351: continue;
1.42 thorpej 2352:
2353: /*
1.74 chs 2354: * find the map entry for the current address.
1.42 thorpej 2355: */
1.56 chs 2356:
2357: KASSERT(va >= entry->start);
1.74 chs 2358: while (va >= entry->end) {
1.56 chs 2359: KASSERT(entry->next != &map->header &&
2360: entry->next->start <= entry->end);
1.42 thorpej 2361: entry = entry->next;
2362: }
1.37 thorpej 2363:
1.42 thorpej 2364: /*
2365: * if the entry is no longer wired, tell the pmap.
2366: */
1.74 chs 2367:
1.42 thorpej 2368: if (VM_MAPENT_ISWIRED(entry) == 0)
2369: pmap_unwire(pmap, va);
2370:
2371: pg = PHYS_TO_VM_PAGE(pa);
2372: if (pg)
2373: uvm_pageunwire(pg);
1.7 mrg 2374: }
1.1 mrg 2375:
1.122 ad 2376: mutex_exit(&uvm_pageqlock);
1.1 mrg 2377: }
CVSweb <webmaster@jp.NetBSD.org>