Annotation of src/sys/uvm/uvm_loan.c, Revision 1.53
1.53 ! yamt 1: /* $NetBSD: uvm_loan.c,v 1.52 2004/11/23 04:51:56 yamt Exp $ */
1.1 mrg 2:
3: /*
4: *
5: * Copyright (c) 1997 Charles D. Cranor and Washington University.
6: * All rights reserved.
7: *
8: * Redistribution and use in source and binary forms, with or without
9: * modification, are permitted provided that the following conditions
10: * are met:
11: * 1. Redistributions of source code must retain the above copyright
12: * notice, this list of conditions and the following disclaimer.
13: * 2. Redistributions in binary form must reproduce the above copyright
14: * notice, this list of conditions and the following disclaimer in the
15: * documentation and/or other materials provided with the distribution.
16: * 3. All advertising materials mentioning features or use of this software
17: * must display the following acknowledgement:
18: * This product includes software developed by Charles D. Cranor and
19: * Washington University.
20: * 4. The name of the author may not be used to endorse or promote products
21: * derived from this software without specific prior written permission.
22: *
23: * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
24: * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
25: * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
26: * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
27: * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
28: * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
29: * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
30: * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
31: * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
32: * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
1.4 mrg 33: *
34: * from: Id: uvm_loan.c,v 1.1.6.4 1998/02/06 05:08:43 chs Exp
1.1 mrg 35: */
36:
37: /*
38: * uvm_loan.c: page loanout handler
39: */
1.35 lukem 40:
41: #include <sys/cdefs.h>
1.53 ! yamt 42: __KERNEL_RCSID(0, "$NetBSD: uvm_loan.c,v 1.52 2004/11/23 04:51:56 yamt Exp $");
1.1 mrg 43:
44: #include <sys/param.h>
45: #include <sys/systm.h>
46: #include <sys/kernel.h>
47: #include <sys/proc.h>
48: #include <sys/malloc.h>
49: #include <sys/mman.h>
50:
51: #include <uvm/uvm.h>
52:
53: /*
1.29 chs 54: * "loaned" pages are pages which are (read-only, copy-on-write) loaned
1.1 mrg 55: * from the VM system to other parts of the kernel. this allows page
56: * copying to be avoided (e.g. you can loan pages from objs/anons to
57: * the mbuf system).
58: *
59: * there are 3 types of loans possible:
60: * O->K uvm_object page to wired kernel page (e.g. mbuf data area)
1.16 thorpej 61: * A->K anon page to wired kernel page (e.g. mbuf data area)
1.1 mrg 62: * O->A uvm_object to anon loan (e.g. vnode page to an anon)
63: * note that it possible to have an O page loaned to both an A and K
64: * at the same time.
65: *
66: * loans are tracked by pg->loan_count. an O->A page will have both
67: * a uvm_object and a vm_anon, but PQ_ANON will not be set. this sort
68: * of page is considered "owned" by the uvm_object (not the anon).
69: *
1.16 thorpej 70: * each loan of a page to the kernel bumps the pg->wire_count. the
71: * kernel mappings for these pages will be read-only and wired. since
72: * the page will also be wired, it will not be a candidate for pageout,
73: * and thus will never be pmap_page_protect()'d with VM_PROT_NONE. a
74: * write fault in the kernel to one of these pages will not cause
75: * copy-on-write. instead, the page fault is considered fatal. this
76: * is because the kernel mapping will have no way to look up the
77: * object/anon which the page is owned by. this is a good side-effect,
78: * since a kernel write to a loaned page is an error.
1.1 mrg 79: *
1.29 chs 80: * owners that want to free their pages and discover that they are
1.1 mrg 81: * loaned out simply "disown" them (the page becomes an orphan). these
82: * pages should be freed when the last loan is dropped. in some cases
83: * an anon may "adopt" an orphaned page.
84: *
85: * locking: to read pg->loan_count either the owner or the page queues
86: * must be locked. to modify pg->loan_count, both the owner of the page
87: * and the PQs must be locked. pg->flags is (as always) locked by
88: * the owner of the page.
89: *
90: * note that locking from the "loaned" side is tricky since the object
91: * getting the loaned page has no reference to the page's owner and thus
92: * the owner could "die" at any time. in order to prevent the owner
93: * from dying the page queues should be locked. this forces us to sometimes
94: * use "try" locking.
95: *
96: * loans are typically broken by the following events:
1.29 chs 97: * 1. user-level xwrite fault to a loaned page
1.1 mrg 98: * 2. pageout of clean+inactive O->A loaned page
99: * 3. owner frees page (e.g. pager flush)
100: *
101: * note that loaning a page causes all mappings of the page to become
102: * read-only (via pmap_page_protect). this could have an unexpected
1.16 thorpej 103: * effect on normal "wired" pages if one is not careful (XXX).
1.1 mrg 104: */
105:
106: /*
107: * local prototypes
108: */
109:
1.50 junyoung 110: static int uvm_loananon(struct uvm_faultinfo *, void ***,
111: int, struct vm_anon *);
112: static int uvm_loanentry(struct uvm_faultinfo *, void ***, int);
113: static int uvm_loanuobj(struct uvm_faultinfo *, void ***,
114: int, vaddr_t);
115: static int uvm_loanzero(struct uvm_faultinfo *, void ***, int);
116: static void uvm_unloananon(struct vm_anon **, int);
117: static void uvm_unloanpage(struct vm_page **, int);
1.51 yamt 118: static int uvm_loanpage(struct vm_page **, int);
1.33 jdolecek 119:
1.1 mrg 120:
121: /*
122: * inlines
123: */
124:
125: /*
126: * uvm_loanentry: loan out pages in a map entry (helper fn for uvm_loan())
127: *
128: * => "ufi" is the result of a successful map lookup (meaning that
1.31 chuck 129: * on entry the map is locked by the caller)
1.28 chuck 130: * => we may unlock and then relock the map if needed (for I/O)
1.1 mrg 131: * => we put our output result in "output"
1.31 chuck 132: * => we always return with the map unlocked
1.28 chuck 133: * => possible return values:
134: * -1 == error, map is unlocked
135: * 0 == map relock error (try again!), map is unlocked
1.31 chuck 136: * >0 == number of pages we loaned, map is unlocked
1.1 mrg 137: */
138:
1.6 mrg 139: static __inline int
140: uvm_loanentry(ufi, output, flags)
141: struct uvm_faultinfo *ufi;
142: void ***output;
143: int flags;
1.1 mrg 144: {
1.10 chuck 145: vaddr_t curaddr = ufi->orig_rvaddr;
1.9 eeh 146: vsize_t togo = ufi->size;
1.6 mrg 147: struct vm_aref *aref = &ufi->entry->aref;
148: struct uvm_object *uobj = ufi->entry->object.uvm_obj;
149: struct vm_anon *anon;
150: int rv, result = 0;
151:
1.52 yamt 152: UVMHIST_FUNC(__func__); UVMHIST_CALLED(loanhist);
153:
1.6 mrg 154: /*
1.28 chuck 155: * lock us the rest of the way down (we unlock before return)
1.6 mrg 156: */
157: if (aref->ar_amap)
1.13 chuck 158: amap_lock(aref->ar_amap);
1.6 mrg 159: if (uobj)
160: simple_lock(&uobj->vmobjlock);
161:
162: /*
163: * loop until done
164: */
165: while (togo) {
166:
167: /*
168: * find the page we want. check the anon layer first.
169: */
170:
171: if (aref->ar_amap) {
172: anon = amap_lookup(aref, curaddr - ufi->entry->start);
173: } else {
174: anon = NULL;
175: }
176:
1.28 chuck 177: /* locked: map, amap, uobj */
1.6 mrg 178: if (anon) {
179: rv = uvm_loananon(ufi, output, flags, anon);
180: } else if (uobj) {
181: rv = uvm_loanuobj(ufi, output, flags, curaddr);
182: } else if (UVM_ET_ISCOPYONWRITE(ufi->entry)) {
183: rv = uvm_loanzero(ufi, output, flags);
184: } else {
1.47 yamt 185: uvmfault_unlockall(ufi, aref->ar_amap, uobj, NULL);
1.34 chs 186: rv = -1;
1.6 mrg 187: }
1.31 chuck 188: /* locked: if (rv > 0) => map, amap, uobj [o.w. unlocked] */
1.51 yamt 189: LOCK_ASSERT(rv > 0 || aref->ar_amap == NULL ||
190: !simple_lock_held(&aref->ar_amap->am_l));
191: LOCK_ASSERT(rv > 0 || uobj == NULL ||
192: !simple_lock_held(&uobj->vmobjlock));
1.6 mrg 193:
194: /* total failure */
1.52 yamt 195: if (rv < 0) {
196: UVMHIST_LOG(loanhist, "failure %d", rv, 0,0,0);
1.34 chs 197: return (-1);
1.52 yamt 198: }
1.6 mrg 199:
200: /* relock failed, need to do another lookup */
1.52 yamt 201: if (rv == 0) {
202: UVMHIST_LOG(loanhist, "relock failure %d", result
203: ,0,0,0);
1.34 chs 204: return (result);
1.52 yamt 205: }
1.6 mrg 206:
207: /*
208: * got it... advance to next page
209: */
1.34 chs 210:
1.6 mrg 211: result++;
212: togo -= PAGE_SIZE;
213: curaddr += PAGE_SIZE;
214: }
215:
216: /*
1.31 chuck 217: * unlock what we locked, unlock the maps and return
1.6 mrg 218: */
1.34 chs 219:
1.28 chuck 220: if (aref->ar_amap)
221: amap_unlock(aref->ar_amap);
222: if (uobj)
223: simple_unlock(&uobj->vmobjlock);
1.31 chuck 224: uvmfault_unlockmaps(ufi, FALSE);
1.52 yamt 225: UVMHIST_LOG(loanhist, "done %d", result, 0,0,0);
1.34 chs 226: return (result);
1.1 mrg 227: }
228:
229: /*
230: * normal functions
231: */
232:
233: /*
1.28 chuck 234: * uvm_loan: loan pages in a map out to anons or to the kernel
1.29 chs 235: *
1.1 mrg 236: * => map should be unlocked
237: * => start and len should be multiples of PAGE_SIZE
238: * => result is either an array of anon's or vm_pages (depending on flags)
239: * => flag values: UVM_LOAN_TOANON - loan to anons
240: * UVM_LOAN_TOPAGE - loan to wired kernel page
241: * one and only one of these flags must be set!
1.28 chuck 242: * => returns 0 (success), or an appropriate error number
1.1 mrg 243: */
244:
1.6 mrg 245: int
1.34 chs 246: uvm_loan(map, start, len, v, flags)
1.6 mrg 247: struct vm_map *map;
1.9 eeh 248: vaddr_t start;
249: vsize_t len;
1.34 chs 250: void *v;
1.6 mrg 251: int flags;
1.1 mrg 252: {
1.6 mrg 253: struct uvm_faultinfo ufi;
1.34 chs 254: void **result, **output;
1.25 chs 255: int rv, error;
1.6 mrg 256:
1.52 yamt 257: UVMHIST_FUNC(__func__); UVMHIST_CALLED(loanhist);
258:
1.6 mrg 259: /*
260: * ensure that one and only one of the flags is set
261: */
262:
1.25 chs 263: KASSERT(((flags & UVM_LOAN_TOANON) == 0) ^
264: ((flags & UVM_LOAN_TOPAGE) == 0));
265: KASSERT((map->flags & VM_MAP_INTRSAFE) == 0);
1.6 mrg 266:
267: /*
1.34 chs 268: * "output" is a pointer to the current place to put the loaned page.
1.6 mrg 269: */
270:
1.34 chs 271: result = v;
1.6 mrg 272: output = &result[0]; /* start at the beginning ... */
273:
274: /*
275: * while we've got pages to do
276: */
277:
278: while (len > 0) {
279:
280: /*
281: * fill in params for a call to uvmfault_lookup
282: */
283:
284: ufi.orig_map = map;
285: ufi.orig_rvaddr = start;
286: ufi.orig_size = len;
1.29 chs 287:
1.6 mrg 288: /*
289: * do the lookup, the only time this will fail is if we hit on
290: * an unmapped region (an error)
291: */
292:
1.25 chs 293: if (!uvmfault_lookup(&ufi, FALSE)) {
294: error = ENOENT;
1.6 mrg 295: goto fail;
1.25 chs 296: }
1.6 mrg 297:
298: /*
1.28 chuck 299: * map now locked. now do the loanout...
1.6 mrg 300: */
1.34 chs 301:
1.6 mrg 302: rv = uvm_loanentry(&ufi, &output, flags);
1.25 chs 303: if (rv < 0) {
1.28 chuck 304: /* all unlocked due to error */
1.25 chs 305: error = EINVAL;
1.6 mrg 306: goto fail;
1.25 chs 307: }
1.6 mrg 308:
309: /*
1.31 chuck 310: * done! the map is unlocked. advance, if possible.
1.28 chuck 311: *
1.50 junyoung 312: * XXXCDC: could be recoded to hold the map lock with
313: * smarter code (but it only happens on map entry
1.31 chuck 314: * boundaries, so it isn't that bad).
1.6 mrg 315: */
1.34 chs 316:
1.28 chuck 317: if (rv) {
318: rv <<= PAGE_SHIFT;
319: len -= rv;
320: start += rv;
321: }
1.6 mrg 322: }
1.52 yamt 323: UVMHIST_LOG(loanhist, "success", 0,0,0,0);
1.25 chs 324: return 0;
1.1 mrg 325:
326: fail:
1.6 mrg 327: /*
1.34 chs 328: * failed to complete loans. drop any loans and return failure code.
1.28 chuck 329: * map is already unlocked.
1.6 mrg 330: */
1.34 chs 331:
1.6 mrg 332: if (output - result) {
1.34 chs 333: if (flags & UVM_LOAN_TOANON) {
1.6 mrg 334: uvm_unloananon((struct vm_anon **)result,
1.34 chs 335: output - result);
336: } else {
1.6 mrg 337: uvm_unloanpage((struct vm_page **)result,
1.34 chs 338: output - result);
339: }
1.6 mrg 340: }
1.52 yamt 341: UVMHIST_LOG(loanhist, "error %d", error,0,0,0);
1.26 jdolecek 342: return (error);
1.1 mrg 343: }
344:
345: /*
346: * uvm_loananon: loan a page from an anon out
1.29 chs 347: *
1.28 chuck 348: * => called with map, amap, uobj locked
1.1 mrg 349: * => return value:
350: * -1 = fatal error, everything is unlocked, abort.
351: * 0 = lookup in ufi went stale, everything unlocked, relookup and
352: * try again
353: * 1 = got it, everything still locked
354: */
355:
1.6 mrg 356: int
357: uvm_loananon(ufi, output, flags, anon)
358: struct uvm_faultinfo *ufi;
359: void ***output;
360: int flags;
361: struct vm_anon *anon;
1.1 mrg 362: {
1.6 mrg 363: struct vm_page *pg;
1.34 chs 364: int error;
1.1 mrg 365:
1.52 yamt 366: UVMHIST_FUNC(__func__); UVMHIST_CALLED(loanhist);
367:
1.6 mrg 368: /*
1.28 chuck 369: * if we are loaning to "another" anon then it is easy, we just
1.6 mrg 370: * bump the reference count on the current anon and return a
1.28 chuck 371: * pointer to it (it becomes copy-on-write shared).
1.6 mrg 372: */
1.34 chs 373:
1.6 mrg 374: if (flags & UVM_LOAN_TOANON) {
375: simple_lock(&anon->an_lock);
1.53 ! yamt 376: pg = anon->an_page;
1.34 chs 377: if (pg && (pg->pqflags & PQ_ANON) != 0 && anon->an_ref == 1) {
1.51 yamt 378: if (pg->wire_count > 0) {
1.52 yamt 379: UVMHIST_LOG(loanhist, "->A wired %p", pg,0,0,0);
1.51 yamt 380: uvmfault_unlockall(ufi,
381: ufi->entry->aref.ar_amap,
382: ufi->entry->object.uvm_obj, anon);
383: return (-1);
384: }
1.19 chs 385: pmap_page_protect(pg, VM_PROT_READ);
1.34 chs 386: }
1.6 mrg 387: anon->an_ref++;
388: **output = anon;
1.34 chs 389: (*output)++;
1.6 mrg 390: simple_unlock(&anon->an_lock);
1.52 yamt 391: UVMHIST_LOG(loanhist, "->A done", 0,0,0,0);
1.34 chs 392: return (1);
1.6 mrg 393: }
394:
395: /*
396: * we are loaning to a kernel-page. we need to get the page
397: * resident so we can wire it. uvmfault_anonget will handle
398: * this for us.
399: */
400:
401: simple_lock(&anon->an_lock);
1.34 chs 402: error = uvmfault_anonget(ufi, ufi->entry->aref.ar_amap, anon);
1.6 mrg 403:
404: /*
405: * if we were unable to get the anon, then uvmfault_anonget has
406: * unlocked everything and returned an error code.
407: */
1.34 chs 408:
409: if (error) {
1.52 yamt 410: UVMHIST_LOG(loanhist, "error %d", error,0,0,0);
1.6 mrg 411:
412: /* need to refault (i.e. refresh our lookup) ? */
1.34 chs 413: if (error == ERESTART) {
414: return (0);
415: }
1.6 mrg 416:
417: /* "try again"? sleep a bit and retry ... */
1.34 chs 418: if (error == EAGAIN) {
1.24 chs 419: tsleep(&lbolt, PVM, "loanagain", 0);
1.34 chs 420: return (0);
1.6 mrg 421: }
422:
423: /* otherwise flag it as an error */
1.34 chs 424: return (-1);
1.6 mrg 425: }
426:
427: /*
428: * we have the page and its owner locked: do the loan now.
429: */
430:
1.53 ! yamt 431: pg = anon->an_page;
1.6 mrg 432: uvm_lock_pageq();
1.51 yamt 433: if (pg->wire_count > 0) {
434: uvm_unlock_pageq();
1.52 yamt 435: UVMHIST_LOG(loanhist, "->K wired %p", pg,0,0,0);
1.51 yamt 436: KASSERT(pg->uobject == NULL);
437: uvmfault_unlockall(ufi, ufi->entry->aref.ar_amap,
438: ufi->entry->object.uvm_obj, anon);
439: return (-1);
440: }
1.34 chs 441: if (pg->loan_count == 0) {
1.19 chs 442: pmap_page_protect(pg, VM_PROT_READ);
1.34 chs 443: }
1.6 mrg 444: pg->loan_count++;
1.34 chs 445: uvm_pagedequeue(pg);
1.6 mrg 446: uvm_unlock_pageq();
447: **output = pg;
1.34 chs 448: (*output)++;
1.6 mrg 449:
450: /* unlock anon and return success */
1.28 chuck 451: if (pg->uobject) /* XXXCDC: what if this is our uobj? bad */
1.6 mrg 452: simple_unlock(&pg->uobject->vmobjlock);
453: simple_unlock(&anon->an_lock);
1.52 yamt 454: UVMHIST_LOG(loanhist, "->K done", 0,0,0,0);
1.34 chs 455: return (1);
1.1 mrg 456: }
457:
458: /*
1.45 yamt 459: * uvm_loanpage: loan out pages to kernel (->K)
1.42 yamt 460: *
1.51 yamt 461: * => pages should be object-owned and the object should be locked.
462: * => in the case of error, the object might be unlocked and relocked.
463: * => caller should busy the pages beforehand.
464: * => pages will be unbusied.
465: * => fail with EBUSY if meet a wired page.
1.42 yamt 466: */
1.51 yamt 467: static int
1.45 yamt 468: uvm_loanpage(pgpp, npages)
1.42 yamt 469: struct vm_page **pgpp;
470: int npages;
471: {
472: int i;
1.51 yamt 473: int error = 0;
1.42 yamt 474:
1.52 yamt 475: UVMHIST_FUNC(__func__); UVMHIST_CALLED(loanhist);
476:
1.42 yamt 477: for (i = 0; i < npages; i++) {
478: struct vm_page *pg = pgpp[i];
479:
480: KASSERT(pg->uobject != NULL);
1.51 yamt 481: KASSERT(pg->uobject == pgpp[0]->uobject);
1.42 yamt 482: KASSERT(!(pg->flags & (PG_RELEASED|PG_PAGEOUT)));
483: LOCK_ASSERT(simple_lock_held(&pg->uobject->vmobjlock));
484: KASSERT(pg->flags & PG_BUSY);
485:
486: uvm_lock_pageq();
1.51 yamt 487: if (pg->wire_count > 0) {
488: uvm_unlock_pageq();
1.52 yamt 489: UVMHIST_LOG(loanhist, "wired %p", pg,0,0,0);
1.51 yamt 490: error = EBUSY;
491: break;
492: }
1.42 yamt 493: if (pg->loan_count == 0) {
494: pmap_page_protect(pg, VM_PROT_READ);
495: }
496: pg->loan_count++;
497: uvm_pagedequeue(pg);
498: uvm_unlock_pageq();
499: }
1.51 yamt 500:
501: uvm_page_unbusy(pgpp, npages);
502:
503: if (error) {
504: /*
505: * backout what we've done
506: */
507: struct simplelock *slock = &pgpp[0]->uobject->vmobjlock;
508:
509: simple_unlock(slock);
510: uvm_unloan(pgpp, i, UVM_LOAN_TOPAGE);
511: simple_lock(slock);
512: }
513:
1.52 yamt 514: UVMHIST_LOG(loanhist, "done %d", error,0,0,0);
1.51 yamt 515: return error;
1.42 yamt 516: }
517:
518: /*
1.45 yamt 519: * XXX UBC temp limit
520: * number of pages to get at once.
521: * should be <= MAX_READ_AHEAD in genfs_vnops.c
522: */
523: #define UVM_LOAN_GET_CHUNK 16
524:
525: /*
1.46 yamt 526: * uvm_loanuobjpages: loan pages from a uobj out (O->K)
1.45 yamt 527: *
1.46 yamt 528: * => uobj shouldn't be locked. (we'll lock it)
1.51 yamt 529: * => fail with EBUSY if we meet a wired page.
1.45 yamt 530: */
531: int
532: uvm_loanuobjpages(uobj, pgoff, orignpages, origpgpp)
533: struct uvm_object *uobj;
534: voff_t pgoff;
535: int orignpages;
536: struct vm_page **origpgpp;
537: {
1.51 yamt 538: int ndone; /* # of pages loaned out */
1.45 yamt 539: struct vm_page **pgpp;
540: int error;
541: int i;
542: struct simplelock *slock;
543:
544: pgpp = origpgpp;
545: for (ndone = 0; ndone < orignpages; ) {
546: int npages;
1.51 yamt 547: /* npendloan: # of pages busied but not loand out yet. */
1.45 yamt 548: int npendloan = 0xdead; /* XXX gcc */
549: reget:
550: npages = MIN(UVM_LOAN_GET_CHUNK, orignpages - ndone);
551: simple_lock(&uobj->vmobjlock);
552: error = (*uobj->pgops->pgo_get)(uobj,
553: pgoff + (ndone << PAGE_SHIFT), pgpp, &npages, 0,
554: VM_PROT_READ, 0, PGO_SYNCIO);
555: if (error == EAGAIN) {
556: tsleep(&lbolt, PVM, "nfsread", 0);
557: continue;
558: }
1.51 yamt 559: if (error)
560: goto fail;
1.45 yamt 561:
562: KASSERT(npages > 0);
1.50 junyoung 563:
1.45 yamt 564: /* loan and unbusy pages */
565: slock = NULL;
566: for (i = 0; i < npages; i++) {
567: struct simplelock *nextslock; /* slock for next page */
568: struct vm_page *pg = *pgpp;
569:
570: /* XXX assuming that the page is owned by uobj */
571: KASSERT(pg->uobject != NULL);
572: nextslock = &pg->uobject->vmobjlock;
573:
574: if (slock != nextslock) {
575: if (slock) {
576: KASSERT(npendloan > 0);
1.51 yamt 577: error = uvm_loanpage(pgpp - npendloan,
1.45 yamt 578: npendloan);
579: simple_unlock(slock);
1.51 yamt 580: if (error)
581: goto fail;
582: ndone += npendloan;
583: KASSERT(origpgpp + ndone == pgpp);
1.45 yamt 584: }
585: slock = nextslock;
1.51 yamt 586: npendloan = 0;
1.45 yamt 587: simple_lock(slock);
588: }
589:
1.51 yamt 590: if ((pg->flags & PG_RELEASED) != 0) {
1.45 yamt 591: /*
592: * release pages and try again.
593: */
594: simple_unlock(slock);
595: for (; i < npages; i++) {
596: pg = pgpp[i];
597: slock = &pg->uobject->vmobjlock;
598:
599: simple_lock(slock);
600: uvm_lock_pageq();
601: uvm_page_unbusy(&pg, 1);
602: uvm_unlock_pageq();
603: simple_unlock(slock);
604: }
605: goto reget;
606: }
607:
608: npendloan++;
609: pgpp++;
1.51 yamt 610: KASSERT(origpgpp + ndone + npendloan == pgpp);
1.45 yamt 611: }
612: KASSERT(slock != NULL);
613: KASSERT(npendloan > 0);
1.51 yamt 614: error = uvm_loanpage(pgpp - npendloan, npendloan);
1.45 yamt 615: simple_unlock(slock);
1.51 yamt 616: if (error)
617: goto fail;
618: ndone += npendloan;
619: KASSERT(origpgpp + ndone == pgpp);
1.45 yamt 620: }
621:
622: return 0;
1.51 yamt 623:
624: fail:
625: uvm_unloan(origpgpp, ndone, UVM_LOAN_TOPAGE);
626:
627: return error;
1.45 yamt 628: }
629:
630: /*
1.1 mrg 631: * uvm_loanuobj: loan a page from a uobj out
632: *
1.28 chuck 633: * => called with map, amap, uobj locked
1.1 mrg 634: * => return value:
635: * -1 = fatal error, everything is unlocked, abort.
636: * 0 = lookup in ufi went stale, everything unlocked, relookup and
637: * try again
638: * 1 = got it, everything still locked
639: */
640:
1.33 jdolecek 641: static int
1.6 mrg 642: uvm_loanuobj(ufi, output, flags, va)
643: struct uvm_faultinfo *ufi;
644: void ***output;
645: int flags;
1.9 eeh 646: vaddr_t va;
1.1 mrg 647: {
1.6 mrg 648: struct vm_amap *amap = ufi->entry->aref.ar_amap;
649: struct uvm_object *uobj = ufi->entry->object.uvm_obj;
650: struct vm_page *pg;
651: struct vm_anon *anon;
1.34 chs 652: int error, npages;
1.6 mrg 653: boolean_t locked;
654:
1.52 yamt 655: UVMHIST_FUNC(__func__); UVMHIST_CALLED(loanhist);
656:
1.6 mrg 657: /*
658: * first we must make sure the page is resident.
659: *
660: * XXXCDC: duplicate code with uvm_fault().
661: */
662:
1.28 chuck 663: if (uobj->pgops->pgo_get) { /* try locked pgo_get */
1.6 mrg 664: npages = 1;
665: pg = NULL;
1.37 enami 666: error = (*uobj->pgops->pgo_get)(uobj,
667: va - ufi->entry->start + ufi->entry->offset,
1.6 mrg 668: &pg, &npages, 0, VM_PROT_READ, MADV_NORMAL, PGO_LOCKED);
669: } else {
1.34 chs 670: error = EIO; /* must have pgo_get op */
1.6 mrg 671: }
672:
673: /*
674: * check the result of the locked pgo_get. if there is a problem,
675: * then we fail the loan.
676: */
677:
1.34 chs 678: if (error && error != EBUSY) {
1.6 mrg 679: uvmfault_unlockall(ufi, amap, uobj, NULL);
1.34 chs 680: return (-1);
1.6 mrg 681: }
682:
683: /*
684: * if we need to unlock for I/O, do so now.
685: */
686:
1.34 chs 687: if (error == EBUSY) {
1.6 mrg 688: uvmfault_unlockall(ufi, amap, NULL, NULL);
1.34 chs 689:
690: /* locked: uobj */
1.6 mrg 691: npages = 1;
1.37 enami 692: error = (*uobj->pgops->pgo_get)(uobj,
693: va - ufi->entry->start + ufi->entry->offset,
1.30 chs 694: &pg, &npages, 0, VM_PROT_READ, MADV_NORMAL, PGO_SYNCIO);
1.6 mrg 695: /* locked: <nothing> */
1.29 chs 696:
1.34 chs 697: if (error) {
1.43 yamt 698: if (error == EAGAIN) {
1.24 chs 699: tsleep(&lbolt, PVM, "fltagain2", 0);
1.34 chs 700: return (0);
1.29 chs 701: }
1.34 chs 702: return (-1);
1.6 mrg 703: }
704:
705: /*
706: * pgo_get was a success. attempt to relock everything.
707: */
708:
709: locked = uvmfault_relock(ufi);
710: if (locked && amap)
1.13 chuck 711: amap_lock(amap);
1.6 mrg 712: simple_lock(&uobj->vmobjlock);
713:
714: /*
715: * verify that the page has not be released and re-verify
716: * that amap slot is still free. if there is a problem we
717: * drop our lock (thus force a lookup refresh/retry).
718: */
1.29 chs 719:
1.6 mrg 720: if ((pg->flags & PG_RELEASED) != 0 ||
721: (locked && amap && amap_lookup(&ufi->entry->aref,
1.10 chuck 722: ufi->orig_rvaddr - ufi->entry->start))) {
1.6 mrg 723: if (locked)
724: uvmfault_unlockall(ufi, amap, NULL, NULL);
725: locked = FALSE;
1.29 chs 726: }
1.6 mrg 727:
728: /*
729: * didn't get the lock? release the page and retry.
730: */
731:
732: if (locked == FALSE) {
1.32 chs 733: if (pg->flags & PG_WANTED) {
1.18 thorpej 734: wakeup(pg);
1.32 chs 735: }
1.6 mrg 736: if (pg->flags & PG_RELEASED) {
1.38 enami 737: uvm_lock_pageq();
1.32 chs 738: uvm_pagefree(pg);
1.38 enami 739: uvm_unlock_pageq();
1.6 mrg 740: return (0);
741: }
742: uvm_lock_pageq();
1.32 chs 743: uvm_pageactivate(pg);
1.6 mrg 744: uvm_unlock_pageq();
745: pg->flags &= ~(PG_BUSY|PG_WANTED);
746: UVM_PAGE_OWN(pg, NULL);
747: simple_unlock(&uobj->vmobjlock);
748: return (0);
749: }
750: }
751:
752: /*
753: * at this point we have the page we want ("pg") marked PG_BUSY for us
1.34 chs 754: * and we have all data structures locked. do the loanout. page can
1.6 mrg 755: * not be PG_RELEASED (we caught this above).
756: */
757:
1.34 chs 758: if ((flags & UVM_LOAN_TOANON) == 0) {
1.51 yamt 759: if (uvm_loanpage(&pg, 1)) {
760: uvmfault_unlockall(ufi, amap, uobj, NULL);
761: return (-1);
762: }
1.34 chs 763: **output = pg;
764: (*output)++;
765: return (1);
1.6 mrg 766: }
767:
768: /*
769: * must be a loan to an anon. check to see if there is already
770: * an anon associated with this page. if so, then just return
1.29 chs 771: * a reference to this object. the page should already be
1.6 mrg 772: * mapped read-only because it is already on loan.
773: */
774:
775: if (pg->uanon) {
776: anon = pg->uanon;
777: simple_lock(&anon->an_lock);
778: anon->an_ref++;
779: simple_unlock(&anon->an_lock);
1.34 chs 780: if (pg->flags & PG_WANTED) {
1.18 thorpej 781: wakeup(pg);
1.34 chs 782: }
1.6 mrg 783: pg->flags &= ~(PG_WANTED|PG_BUSY);
784: UVM_PAGE_OWN(pg, NULL);
1.34 chs 785: **output = anon;
786: (*output)++;
787: return (1);
1.6 mrg 788: }
1.29 chs 789:
1.6 mrg 790: /*
791: * need to allocate a new anon
792: */
793:
794: anon = uvm_analloc();
1.34 chs 795: if (anon == NULL) {
1.51 yamt 796: goto fail;
1.6 mrg 797: }
1.53 ! yamt 798: anon->an_page = pg;
1.6 mrg 799: pg->uanon = anon;
800: uvm_lock_pageq();
1.51 yamt 801: if (pg->wire_count > 0) {
802: uvm_unlock_pageq();
1.52 yamt 803: UVMHIST_LOG(loanhist, "wired %p", pg,0,0,0);
1.51 yamt 804: pg->uanon = NULL;
1.53 ! yamt 805: anon->an_page = NULL;
1.51 yamt 806: anon->an_ref--;
807: simple_unlock(&anon->an_lock);
808: uvm_anfree(anon);
809: goto fail;
810: }
1.34 chs 811: if (pg->loan_count == 0) {
1.19 chs 812: pmap_page_protect(pg, VM_PROT_READ);
1.34 chs 813: }
1.6 mrg 814: pg->loan_count++;
815: uvm_pageactivate(pg);
816: uvm_unlock_pageq();
1.34 chs 817: if (pg->flags & PG_WANTED) {
1.18 thorpej 818: wakeup(pg);
1.34 chs 819: }
1.6 mrg 820: pg->flags &= ~(PG_WANTED|PG_BUSY);
821: UVM_PAGE_OWN(pg, NULL);
1.23 thorpej 822: simple_unlock(&anon->an_lock);
1.34 chs 823: **output = anon;
824: (*output)++;
825: return (1);
1.51 yamt 826:
827: fail:
1.52 yamt 828: UVMHIST_LOG(loanhist, "fail", 0,0,0,0);
1.51 yamt 829: /*
830: * unlock everything and bail out.
831: */
832: if (pg->flags & PG_WANTED) {
833: wakeup(pg);
834: }
835: pg->flags &= ~(PG_WANTED|PG_BUSY);
836: UVM_PAGE_OWN(pg, NULL);
837: uvmfault_unlockall(ufi, amap, uobj, NULL);
838: return (-1);
1.1 mrg 839: }
840:
841: /*
1.40 thorpej 842: * uvm_loanzero: loan a zero-fill page out
1.1 mrg 843: *
1.28 chuck 844: * => called with map, amap, uobj locked
1.1 mrg 845: * => return value:
846: * -1 = fatal error, everything is unlocked, abort.
847: * 0 = lookup in ufi went stale, everything unlocked, relookup and
848: * try again
849: * 1 = got it, everything still locked
850: */
851:
1.40 thorpej 852: static struct uvm_object uvm_loanzero_object;
853:
1.33 jdolecek 854: static int
1.6 mrg 855: uvm_loanzero(ufi, output, flags)
856: struct uvm_faultinfo *ufi;
857: void ***output;
858: int flags;
1.1 mrg 859: {
1.6 mrg 860: struct vm_anon *anon;
861: struct vm_page *pg;
1.34 chs 862: struct uvm_object *uobj = ufi->entry->object.uvm_obj;
863: struct vm_amap *amap = ufi->entry->aref.ar_amap;
1.1 mrg 864:
1.52 yamt 865: UVMHIST_FUNC(__func__); UVMHIST_CALLED(loanhist);
1.44 yamt 866: again:
1.40 thorpej 867: simple_lock(&uvm_loanzero_object.vmobjlock);
868:
869: /*
870: * first, get ahold of our single zero page.
871: */
872:
873: if (__predict_false((pg =
874: TAILQ_FIRST(&uvm_loanzero_object.memq)) == NULL)) {
875: while ((pg = uvm_pagealloc(&uvm_loanzero_object, 0, NULL,
876: UVM_PGA_ZERO)) == NULL) {
877: simple_unlock(&uvm_loanzero_object.vmobjlock);
1.34 chs 878: uvmfault_unlockall(ufi, amap, uobj, NULL);
1.40 thorpej 879: uvm_wait("loanzero");
1.34 chs 880: if (!uvmfault_relock(ufi)) {
881: return (0);
882: }
883: if (amap) {
884: amap_lock(amap);
885: }
886: if (uobj) {
887: simple_lock(&uobj->vmobjlock);
888: }
1.44 yamt 889: goto again;
1.6 mrg 890: }
1.29 chs 891:
1.40 thorpej 892: /* got a zero'd page. */
893: pg->flags &= ~(PG_WANTED|PG_BUSY|PG_FAKE);
894: pg->flags |= PG_RDONLY;
1.44 yamt 895: uvm_lock_pageq();
896: uvm_pageactivate(pg);
897: uvm_unlock_pageq();
1.6 mrg 898: UVM_PAGE_OWN(pg, NULL);
1.40 thorpej 899: }
900:
901: if ((flags & UVM_LOAN_TOANON) == 0) { /* loaning to kernel-page */
902: uvm_lock_pageq();
903: pg->loan_count++;
1.44 yamt 904: uvm_pagedequeue(pg);
1.40 thorpej 905: uvm_unlock_pageq();
906: simple_unlock(&uvm_loanzero_object.vmobjlock);
1.6 mrg 907: **output = pg;
1.34 chs 908: (*output)++;
909: return (1);
1.6 mrg 910: }
911:
1.40 thorpej 912: /*
913: * loaning to an anon. check to see if there is already an anon
914: * associated with this page. if so, then just return a reference
915: * to this object.
916: */
1.23 thorpej 917:
1.40 thorpej 918: if (pg->uanon) {
919: anon = pg->uanon;
920: simple_lock(&anon->an_lock);
921: anon->an_ref++;
922: simple_unlock(&anon->an_lock);
923: simple_unlock(&uvm_loanzero_object.vmobjlock);
924: **output = anon;
925: (*output)++;
926: return (1);
927: }
1.23 thorpej 928:
1.40 thorpej 929: /*
930: * need to allocate a new anon
931: */
1.34 chs 932:
1.40 thorpej 933: anon = uvm_analloc();
934: if (anon == NULL) {
935: /* out of swap causes us to fail */
936: simple_unlock(&uvm_loanzero_object.vmobjlock);
937: uvmfault_unlockall(ufi, amap, uobj, NULL);
938: return (-1);
1.6 mrg 939: }
1.53 ! yamt 940: anon->an_page = pg;
1.40 thorpej 941: pg->uanon = anon;
1.6 mrg 942: uvm_lock_pageq();
1.40 thorpej 943: pg->loan_count++;
1.6 mrg 944: uvm_pageactivate(pg);
945: uvm_unlock_pageq();
1.40 thorpej 946: simple_unlock(&uvm_loanzero_object.vmobjlock);
1.6 mrg 947: **output = anon;
1.34 chs 948: (*output)++;
949: return (1);
1.1 mrg 950: }
951:
952:
953: /*
954: * uvm_unloananon: kill loans on anons (basically a normal ref drop)
955: *
956: * => we expect all our resources to be unlocked
957: */
958:
1.33 jdolecek 959: static void
1.34 chs 960: uvm_unloananon(aloans, nanons)
1.6 mrg 961: struct vm_anon **aloans;
1.34 chs 962: int nanons;
1.1 mrg 963: {
1.6 mrg 964: struct vm_anon *anon;
1.1 mrg 965:
1.6 mrg 966: while (nanons-- > 0) {
1.12 chs 967: int refs;
1.1 mrg 968:
1.6 mrg 969: anon = *aloans++;
970: simple_lock(&anon->an_lock);
1.12 chs 971: refs = --anon->an_ref;
972: simple_unlock(&anon->an_lock);
1.1 mrg 973:
1.12 chs 974: if (refs == 0) {
1.34 chs 975: uvm_anfree(anon);
1.6 mrg 976: }
977: }
1.1 mrg 978: }
979:
980: /*
981: * uvm_unloanpage: kill loans on pages loaned out to the kernel
982: *
983: * => we expect all our resources to be unlocked
984: */
985:
1.33 jdolecek 986: static void
1.34 chs 987: uvm_unloanpage(ploans, npages)
1.6 mrg 988: struct vm_page **ploans;
1.34 chs 989: int npages;
1.1 mrg 990: {
1.6 mrg 991: struct vm_page *pg;
1.36 chs 992: struct simplelock *slock;
1.1 mrg 993:
1.6 mrg 994: uvm_lock_pageq();
995: while (npages-- > 0) {
996: pg = *ploans++;
1.1 mrg 997:
1.6 mrg 998: /*
1.36 chs 999: * do a little dance to acquire the object or anon lock
1000: * as appropriate. we are locking in the wrong order,
1001: * so we have to do a try-lock here.
1002: */
1003:
1004: slock = NULL;
1005: while (pg->uobject != NULL || pg->uanon != NULL) {
1006: if (pg->uobject != NULL) {
1007: slock = &pg->uobject->vmobjlock;
1008: } else {
1009: slock = &pg->uanon->an_lock;
1010: }
1011: if (simple_lock_try(slock)) {
1012: break;
1013: }
1014: uvm_unlock_pageq();
1015: uvm_lock_pageq();
1016: slock = NULL;
1017: }
1018:
1019: /*
1020: * drop our loan. if page is owned by an anon but
1021: * PQ_ANON is not set, the page was loaned to the anon
1022: * from an object which dropped ownership, so resolve
1023: * this by turning the anon's loan into real ownership
1024: * (ie. decrement loan_count again and set PQ_ANON).
1025: * after all this, if there are no loans left, put the
1026: * page back a paging queue (if the page is owned by
1027: * an anon) or free it (if the page is now unowned).
1.6 mrg 1028: */
1.1 mrg 1029:
1.34 chs 1030: KASSERT(pg->loan_count > 0);
1031: pg->loan_count--;
1.36 chs 1032: if (pg->uobject == NULL && pg->uanon != NULL &&
1033: (pg->pqflags & PQ_ANON) == 0) {
1034: KASSERT(pg->loan_count > 0);
1035: pg->loan_count--;
1036: pg->pqflags |= PQ_ANON;
1037: }
1.34 chs 1038: if (pg->loan_count == 0) {
1039: if (pg->uobject == NULL && pg->uanon == NULL) {
1040: KASSERT((pg->flags & PG_BUSY) == 0);
1041: uvm_pagefree(pg);
1042: } else {
1043: uvm_pageactivate(pg);
1044: }
1.39 chs 1045: } else if (pg->loan_count == 1 && pg->uobject != NULL &&
1046: pg->uanon != NULL) {
1047: uvm_pageactivate(pg);
1.36 chs 1048: }
1049: if (slock != NULL) {
1050: simple_unlock(slock);
1.6 mrg 1051: }
1052: }
1053: uvm_unlock_pageq();
1.1 mrg 1054: }
1055:
1.33 jdolecek 1056: /*
1.34 chs 1057: * uvm_unloan: kill loans on pages or anons.
1.33 jdolecek 1058: */
1.34 chs 1059:
1.33 jdolecek 1060: void
1.34 chs 1061: uvm_unloan(void *v, int npages, int flags)
1.33 jdolecek 1062: {
1.34 chs 1063: if (flags & UVM_LOAN_TOANON) {
1064: uvm_unloananon(v, npages);
1065: } else {
1066: uvm_unloanpage(v, npages);
1067: }
1.40 thorpej 1068: }
1069:
1070: /*
1.41 thorpej 1071: * Minimal pager for uvm_loanzero_object. We need to provide a "put"
1072: * method, because the page can end up on a paging queue, and the
1073: * page daemon will want to call pgo_put when it encounters the page
1074: * on the inactive list.
1075: */
1076:
1077: static int
1078: ulz_put(struct uvm_object *uobj, voff_t start, voff_t stop, int flags)
1079: {
1080: struct vm_page *pg;
1081:
1082: KDASSERT(uobj == &uvm_loanzero_object);
1083:
1084: /*
1085: * Don't need to do any work here if we're not freeing pages.
1086: */
1087:
1088: if ((flags & PGO_FREE) == 0) {
1089: simple_unlock(&uobj->vmobjlock);
1090: return 0;
1091: }
1092:
1093: /*
1094: * we don't actually want to ever free the uvm_loanzero_page, so
1095: * just reactivate or dequeue it.
1096: */
1097:
1098: pg = TAILQ_FIRST(&uobj->memq);
1099: KASSERT(pg != NULL);
1100: KASSERT(TAILQ_NEXT(pg, listq) == NULL);
1101:
1102: uvm_lock_pageq();
1103: if (pg->uanon)
1104: uvm_pageactivate(pg);
1105: else
1106: uvm_pagedequeue(pg);
1107: uvm_unlock_pageq();
1108:
1109: simple_unlock(&uobj->vmobjlock);
1110: return 0;
1111: }
1112:
1113: static struct uvm_pagerops ulz_pager = {
1114: NULL, /* init */
1115: NULL, /* reference */
1116: NULL, /* detach */
1117: NULL, /* fault */
1118: NULL, /* get */
1119: ulz_put, /* put */
1120: };
1121:
1122: /*
1.40 thorpej 1123: * uvm_loan_init(): initialize the uvm_loan() facility.
1124: */
1125:
1126: void
1127: uvm_loan_init(void)
1128: {
1129:
1130: simple_lock_init(&uvm_loanzero_object.vmobjlock);
1131: TAILQ_INIT(&uvm_loanzero_object.memq);
1.41 thorpej 1132: uvm_loanzero_object.pgops = &ulz_pager;
1.52 yamt 1133:
1134: UVMHIST_INIT(loanhist, 300);
1.42 yamt 1135: }
1136:
1137: /*
1138: * uvm_loanbreak: break loan on a uobj page
1139: *
1140: * => called with uobj locked
1141: * => the page should be busy
1142: * => return value:
1143: * newly allocated page if succeeded
1144: */
1145: struct vm_page *
1146: uvm_loanbreak(struct vm_page *uobjpage)
1147: {
1148: struct vm_page *pg;
1.49 drochner 1149: #ifdef DIAGNOSTIC
1.42 yamt 1150: struct uvm_object *uobj = uobjpage->uobject;
1.49 drochner 1151: #endif
1.42 yamt 1152:
1153: KASSERT(uobj != NULL);
1154: LOCK_ASSERT(simple_lock_held(&uobj->vmobjlock));
1155: KASSERT(uobjpage->flags & PG_BUSY);
1156:
1157: /* alloc new un-owned page */
1158: pg = uvm_pagealloc(NULL, 0, NULL, 0);
1159: if (pg == NULL)
1160: return NULL;
1161:
1162: /*
1163: * copy the data from the old page to the new
1164: * one and clear the fake/clean flags on the
1165: * new page (keep it busy). force a reload
1166: * of the old page by clearing it from all
1167: * pmaps. then lock the page queues to
1168: * rename the pages.
1169: */
1170:
1171: uvm_pagecopy(uobjpage, pg); /* old -> new */
1172: pg->flags &= ~(PG_FAKE|PG_CLEAN);
1173: pmap_page_protect(uobjpage, VM_PROT_NONE);
1174: if (uobjpage->flags & PG_WANTED)
1175: wakeup(uobjpage);
1176: /* uobj still locked */
1177: uobjpage->flags &= ~(PG_WANTED|PG_BUSY);
1178: UVM_PAGE_OWN(uobjpage, NULL);
1179:
1180: uvm_lock_pageq();
1.48 yamt 1181:
1182: /*
1183: * replace uobjpage with new page.
1184: */
1185:
1186: uvm_pagereplace(uobjpage, pg);
1.42 yamt 1187:
1188: /*
1189: * if the page is no longer referenced by
1190: * an anon (i.e. we are breaking an O->K
1191: * loan), then remove it from any pageq's.
1192: */
1193: if (uobjpage->uanon == NULL)
1194: uvm_pagedequeue(uobjpage);
1195:
1196: /*
1197: * at this point we have absolutely no
1198: * control over uobjpage
1199: */
1200:
1201: /* install new page */
1202: uvm_pageactivate(pg);
1203: uvm_unlock_pageq();
1204:
1205: /*
1206: * done! loan is broken and "pg" is
1207: * PG_BUSY. it can now replace uobjpage.
1208: */
1209:
1210: return pg;
1.33 jdolecek 1211: }
CVSweb <webmaster@jp.NetBSD.org>