Annotation of src/sys/uvm/uvm_loan.c, Revision 1.67
1.67 ! ad 1: /* $NetBSD: uvm_loan.c,v 1.66 2007/07/21 19:21:54 ad Exp $ */
1.1 mrg 2:
3: /*
4: *
5: * Copyright (c) 1997 Charles D. Cranor and Washington University.
6: * All rights reserved.
7: *
8: * Redistribution and use in source and binary forms, with or without
9: * modification, are permitted provided that the following conditions
10: * are met:
11: * 1. Redistributions of source code must retain the above copyright
12: * notice, this list of conditions and the following disclaimer.
13: * 2. Redistributions in binary form must reproduce the above copyright
14: * notice, this list of conditions and the following disclaimer in the
15: * documentation and/or other materials provided with the distribution.
16: * 3. All advertising materials mentioning features or use of this software
17: * must display the following acknowledgement:
18: * This product includes software developed by Charles D. Cranor and
19: * Washington University.
20: * 4. The name of the author may not be used to endorse or promote products
21: * derived from this software without specific prior written permission.
22: *
23: * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
24: * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
25: * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
26: * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
27: * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
28: * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
29: * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
30: * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
31: * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
32: * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
1.4 mrg 33: *
34: * from: Id: uvm_loan.c,v 1.1.6.4 1998/02/06 05:08:43 chs Exp
1.1 mrg 35: */
36:
37: /*
38: * uvm_loan.c: page loanout handler
39: */
1.35 lukem 40:
41: #include <sys/cdefs.h>
1.67 ! ad 42: __KERNEL_RCSID(0, "$NetBSD: uvm_loan.c,v 1.66 2007/07/21 19:21:54 ad Exp $");
1.1 mrg 43:
44: #include <sys/param.h>
45: #include <sys/systm.h>
46: #include <sys/kernel.h>
47: #include <sys/proc.h>
48: #include <sys/malloc.h>
49: #include <sys/mman.h>
50:
51: #include <uvm/uvm.h>
52:
53: /*
1.29 chs 54: * "loaned" pages are pages which are (read-only, copy-on-write) loaned
1.1 mrg 55: * from the VM system to other parts of the kernel. this allows page
56: * copying to be avoided (e.g. you can loan pages from objs/anons to
57: * the mbuf system).
58: *
59: * there are 3 types of loans possible:
60: * O->K uvm_object page to wired kernel page (e.g. mbuf data area)
1.16 thorpej 61: * A->K anon page to wired kernel page (e.g. mbuf data area)
1.1 mrg 62: * O->A uvm_object to anon loan (e.g. vnode page to an anon)
63: * note that it possible to have an O page loaned to both an A and K
64: * at the same time.
65: *
66: * loans are tracked by pg->loan_count. an O->A page will have both
67: * a uvm_object and a vm_anon, but PQ_ANON will not be set. this sort
68: * of page is considered "owned" by the uvm_object (not the anon).
69: *
1.16 thorpej 70: * each loan of a page to the kernel bumps the pg->wire_count. the
71: * kernel mappings for these pages will be read-only and wired. since
72: * the page will also be wired, it will not be a candidate for pageout,
73: * and thus will never be pmap_page_protect()'d with VM_PROT_NONE. a
74: * write fault in the kernel to one of these pages will not cause
75: * copy-on-write. instead, the page fault is considered fatal. this
76: * is because the kernel mapping will have no way to look up the
77: * object/anon which the page is owned by. this is a good side-effect,
78: * since a kernel write to a loaned page is an error.
1.1 mrg 79: *
1.29 chs 80: * owners that want to free their pages and discover that they are
1.1 mrg 81: * loaned out simply "disown" them (the page becomes an orphan). these
82: * pages should be freed when the last loan is dropped. in some cases
83: * an anon may "adopt" an orphaned page.
84: *
85: * locking: to read pg->loan_count either the owner or the page queues
86: * must be locked. to modify pg->loan_count, both the owner of the page
87: * and the PQs must be locked. pg->flags is (as always) locked by
88: * the owner of the page.
89: *
90: * note that locking from the "loaned" side is tricky since the object
91: * getting the loaned page has no reference to the page's owner and thus
92: * the owner could "die" at any time. in order to prevent the owner
93: * from dying the page queues should be locked. this forces us to sometimes
94: * use "try" locking.
95: *
96: * loans are typically broken by the following events:
1.29 chs 97: * 1. user-level xwrite fault to a loaned page
1.1 mrg 98: * 2. pageout of clean+inactive O->A loaned page
99: * 3. owner frees page (e.g. pager flush)
100: *
101: * note that loaning a page causes all mappings of the page to become
102: * read-only (via pmap_page_protect). this could have an unexpected
1.16 thorpej 103: * effect on normal "wired" pages if one is not careful (XXX).
1.1 mrg 104: */
105:
106: /*
107: * local prototypes
108: */
109:
1.50 junyoung 110: static int uvm_loananon(struct uvm_faultinfo *, void ***,
111: int, struct vm_anon *);
112: static int uvm_loanuobj(struct uvm_faultinfo *, void ***,
113: int, vaddr_t);
114: static int uvm_loanzero(struct uvm_faultinfo *, void ***, int);
115: static void uvm_unloananon(struct vm_anon **, int);
116: static void uvm_unloanpage(struct vm_page **, int);
1.51 yamt 117: static int uvm_loanpage(struct vm_page **, int);
1.33 jdolecek 118:
1.1 mrg 119:
120: /*
121: * inlines
122: */
123:
124: /*
125: * uvm_loanentry: loan out pages in a map entry (helper fn for uvm_loan())
126: *
127: * => "ufi" is the result of a successful map lookup (meaning that
1.31 chuck 128: * on entry the map is locked by the caller)
1.28 chuck 129: * => we may unlock and then relock the map if needed (for I/O)
1.1 mrg 130: * => we put our output result in "output"
1.31 chuck 131: * => we always return with the map unlocked
1.28 chuck 132: * => possible return values:
133: * -1 == error, map is unlocked
134: * 0 == map relock error (try again!), map is unlocked
1.31 chuck 135: * >0 == number of pages we loaned, map is unlocked
1.55 thorpej 136: *
137: * NOTE: We can live with this being an inline, because it is only called
138: * from one place.
1.1 mrg 139: */
140:
1.57 perry 141: static inline int
1.54 thorpej 142: uvm_loanentry(struct uvm_faultinfo *ufi, void ***output, int flags)
1.1 mrg 143: {
1.10 chuck 144: vaddr_t curaddr = ufi->orig_rvaddr;
1.9 eeh 145: vsize_t togo = ufi->size;
1.6 mrg 146: struct vm_aref *aref = &ufi->entry->aref;
147: struct uvm_object *uobj = ufi->entry->object.uvm_obj;
148: struct vm_anon *anon;
149: int rv, result = 0;
150:
1.52 yamt 151: UVMHIST_FUNC(__func__); UVMHIST_CALLED(loanhist);
152:
1.6 mrg 153: /*
1.28 chuck 154: * lock us the rest of the way down (we unlock before return)
1.6 mrg 155: */
156: if (aref->ar_amap)
1.13 chuck 157: amap_lock(aref->ar_amap);
1.6 mrg 158:
159: /*
160: * loop until done
161: */
162: while (togo) {
163:
164: /*
165: * find the page we want. check the anon layer first.
166: */
167:
168: if (aref->ar_amap) {
169: anon = amap_lookup(aref, curaddr - ufi->entry->start);
170: } else {
171: anon = NULL;
172: }
173:
1.28 chuck 174: /* locked: map, amap, uobj */
1.6 mrg 175: if (anon) {
176: rv = uvm_loananon(ufi, output, flags, anon);
177: } else if (uobj) {
178: rv = uvm_loanuobj(ufi, output, flags, curaddr);
179: } else if (UVM_ET_ISCOPYONWRITE(ufi->entry)) {
180: rv = uvm_loanzero(ufi, output, flags);
181: } else {
1.47 yamt 182: uvmfault_unlockall(ufi, aref->ar_amap, uobj, NULL);
1.34 chs 183: rv = -1;
1.6 mrg 184: }
1.31 chuck 185: /* locked: if (rv > 0) => map, amap, uobj [o.w. unlocked] */
1.66 ad 186: KASSERT(rv > 0 || aref->ar_amap == NULL ||
187: !mutex_owned(&aref->ar_amap->am_l));
1.6 mrg 188:
189: /* total failure */
1.52 yamt 190: if (rv < 0) {
191: UVMHIST_LOG(loanhist, "failure %d", rv, 0,0,0);
1.34 chs 192: return (-1);
1.52 yamt 193: }
1.6 mrg 194:
195: /* relock failed, need to do another lookup */
1.52 yamt 196: if (rv == 0) {
197: UVMHIST_LOG(loanhist, "relock failure %d", result
198: ,0,0,0);
1.34 chs 199: return (result);
1.52 yamt 200: }
1.6 mrg 201:
202: /*
203: * got it... advance to next page
204: */
1.34 chs 205:
1.6 mrg 206: result++;
207: togo -= PAGE_SIZE;
208: curaddr += PAGE_SIZE;
209: }
210:
211: /*
1.31 chuck 212: * unlock what we locked, unlock the maps and return
1.6 mrg 213: */
1.34 chs 214:
1.28 chuck 215: if (aref->ar_amap)
216: amap_unlock(aref->ar_amap);
1.65 thorpej 217: uvmfault_unlockmaps(ufi, false);
1.52 yamt 218: UVMHIST_LOG(loanhist, "done %d", result, 0,0,0);
1.34 chs 219: return (result);
1.1 mrg 220: }
221:
222: /*
223: * normal functions
224: */
225:
226: /*
1.28 chuck 227: * uvm_loan: loan pages in a map out to anons or to the kernel
1.29 chs 228: *
1.1 mrg 229: * => map should be unlocked
230: * => start and len should be multiples of PAGE_SIZE
231: * => result is either an array of anon's or vm_pages (depending on flags)
232: * => flag values: UVM_LOAN_TOANON - loan to anons
233: * UVM_LOAN_TOPAGE - loan to wired kernel page
234: * one and only one of these flags must be set!
1.28 chuck 235: * => returns 0 (success), or an appropriate error number
1.1 mrg 236: */
237:
1.6 mrg 238: int
1.54 thorpej 239: uvm_loan(struct vm_map *map, vaddr_t start, vsize_t len, void *v, int flags)
1.1 mrg 240: {
1.6 mrg 241: struct uvm_faultinfo ufi;
1.34 chs 242: void **result, **output;
1.25 chs 243: int rv, error;
1.6 mrg 244:
1.52 yamt 245: UVMHIST_FUNC(__func__); UVMHIST_CALLED(loanhist);
246:
1.6 mrg 247: /*
248: * ensure that one and only one of the flags is set
249: */
250:
1.25 chs 251: KASSERT(((flags & UVM_LOAN_TOANON) == 0) ^
252: ((flags & UVM_LOAN_TOPAGE) == 0));
253: KASSERT((map->flags & VM_MAP_INTRSAFE) == 0);
1.6 mrg 254:
255: /*
1.34 chs 256: * "output" is a pointer to the current place to put the loaned page.
1.6 mrg 257: */
258:
1.34 chs 259: result = v;
1.6 mrg 260: output = &result[0]; /* start at the beginning ... */
261:
262: /*
263: * while we've got pages to do
264: */
265:
266: while (len > 0) {
267:
268: /*
269: * fill in params for a call to uvmfault_lookup
270: */
271:
272: ufi.orig_map = map;
273: ufi.orig_rvaddr = start;
274: ufi.orig_size = len;
1.29 chs 275:
1.6 mrg 276: /*
277: * do the lookup, the only time this will fail is if we hit on
278: * an unmapped region (an error)
279: */
280:
1.65 thorpej 281: if (!uvmfault_lookup(&ufi, false)) {
1.25 chs 282: error = ENOENT;
1.6 mrg 283: goto fail;
1.25 chs 284: }
1.6 mrg 285:
286: /*
1.28 chuck 287: * map now locked. now do the loanout...
1.6 mrg 288: */
1.34 chs 289:
1.6 mrg 290: rv = uvm_loanentry(&ufi, &output, flags);
1.25 chs 291: if (rv < 0) {
1.28 chuck 292: /* all unlocked due to error */
1.25 chs 293: error = EINVAL;
1.6 mrg 294: goto fail;
1.25 chs 295: }
1.6 mrg 296:
297: /*
1.31 chuck 298: * done! the map is unlocked. advance, if possible.
1.28 chuck 299: *
1.50 junyoung 300: * XXXCDC: could be recoded to hold the map lock with
301: * smarter code (but it only happens on map entry
1.31 chuck 302: * boundaries, so it isn't that bad).
1.6 mrg 303: */
1.34 chs 304:
1.28 chuck 305: if (rv) {
306: rv <<= PAGE_SHIFT;
307: len -= rv;
308: start += rv;
309: }
1.6 mrg 310: }
1.52 yamt 311: UVMHIST_LOG(loanhist, "success", 0,0,0,0);
1.25 chs 312: return 0;
1.1 mrg 313:
314: fail:
1.6 mrg 315: /*
1.34 chs 316: * failed to complete loans. drop any loans and return failure code.
1.28 chuck 317: * map is already unlocked.
1.6 mrg 318: */
1.34 chs 319:
1.6 mrg 320: if (output - result) {
1.34 chs 321: if (flags & UVM_LOAN_TOANON) {
1.6 mrg 322: uvm_unloananon((struct vm_anon **)result,
1.34 chs 323: output - result);
324: } else {
1.6 mrg 325: uvm_unloanpage((struct vm_page **)result,
1.34 chs 326: output - result);
327: }
1.6 mrg 328: }
1.52 yamt 329: UVMHIST_LOG(loanhist, "error %d", error,0,0,0);
1.26 jdolecek 330: return (error);
1.1 mrg 331: }
332:
333: /*
334: * uvm_loananon: loan a page from an anon out
1.29 chs 335: *
1.28 chuck 336: * => called with map, amap, uobj locked
1.1 mrg 337: * => return value:
338: * -1 = fatal error, everything is unlocked, abort.
339: * 0 = lookup in ufi went stale, everything unlocked, relookup and
340: * try again
341: * 1 = got it, everything still locked
342: */
343:
1.6 mrg 344: int
1.54 thorpej 345: uvm_loananon(struct uvm_faultinfo *ufi, void ***output, int flags,
346: struct vm_anon *anon)
1.1 mrg 347: {
1.6 mrg 348: struct vm_page *pg;
1.34 chs 349: int error;
1.1 mrg 350:
1.52 yamt 351: UVMHIST_FUNC(__func__); UVMHIST_CALLED(loanhist);
352:
1.6 mrg 353: /*
1.28 chuck 354: * if we are loaning to "another" anon then it is easy, we just
1.6 mrg 355: * bump the reference count on the current anon and return a
1.28 chuck 356: * pointer to it (it becomes copy-on-write shared).
1.6 mrg 357: */
1.34 chs 358:
1.6 mrg 359: if (flags & UVM_LOAN_TOANON) {
360: simple_lock(&anon->an_lock);
1.53 yamt 361: pg = anon->an_page;
1.34 chs 362: if (pg && (pg->pqflags & PQ_ANON) != 0 && anon->an_ref == 1) {
1.51 yamt 363: if (pg->wire_count > 0) {
1.52 yamt 364: UVMHIST_LOG(loanhist, "->A wired %p", pg,0,0,0);
1.51 yamt 365: uvmfault_unlockall(ufi,
366: ufi->entry->aref.ar_amap,
367: ufi->entry->object.uvm_obj, anon);
368: return (-1);
369: }
1.19 chs 370: pmap_page_protect(pg, VM_PROT_READ);
1.34 chs 371: }
1.6 mrg 372: anon->an_ref++;
373: **output = anon;
1.34 chs 374: (*output)++;
1.6 mrg 375: simple_unlock(&anon->an_lock);
1.52 yamt 376: UVMHIST_LOG(loanhist, "->A done", 0,0,0,0);
1.34 chs 377: return (1);
1.6 mrg 378: }
379:
380: /*
381: * we are loaning to a kernel-page. we need to get the page
382: * resident so we can wire it. uvmfault_anonget will handle
383: * this for us.
384: */
385:
386: simple_lock(&anon->an_lock);
1.34 chs 387: error = uvmfault_anonget(ufi, ufi->entry->aref.ar_amap, anon);
1.6 mrg 388:
389: /*
390: * if we were unable to get the anon, then uvmfault_anonget has
391: * unlocked everything and returned an error code.
392: */
1.34 chs 393:
394: if (error) {
1.52 yamt 395: UVMHIST_LOG(loanhist, "error %d", error,0,0,0);
1.6 mrg 396:
397: /* need to refault (i.e. refresh our lookup) ? */
1.34 chs 398: if (error == ERESTART) {
399: return (0);
400: }
1.6 mrg 401:
402: /* "try again"? sleep a bit and retry ... */
1.34 chs 403: if (error == EAGAIN) {
1.24 chs 404: tsleep(&lbolt, PVM, "loanagain", 0);
1.34 chs 405: return (0);
1.6 mrg 406: }
407:
408: /* otherwise flag it as an error */
1.34 chs 409: return (-1);
1.6 mrg 410: }
411:
412: /*
413: * we have the page and its owner locked: do the loan now.
414: */
415:
1.53 yamt 416: pg = anon->an_page;
1.6 mrg 417: uvm_lock_pageq();
1.51 yamt 418: if (pg->wire_count > 0) {
419: uvm_unlock_pageq();
1.52 yamt 420: UVMHIST_LOG(loanhist, "->K wired %p", pg,0,0,0);
1.51 yamt 421: KASSERT(pg->uobject == NULL);
422: uvmfault_unlockall(ufi, ufi->entry->aref.ar_amap,
1.58 yamt 423: NULL, anon);
1.51 yamt 424: return (-1);
425: }
1.34 chs 426: if (pg->loan_count == 0) {
1.19 chs 427: pmap_page_protect(pg, VM_PROT_READ);
1.34 chs 428: }
1.6 mrg 429: pg->loan_count++;
1.63 yamt 430: uvm_pageactivate(pg);
1.6 mrg 431: uvm_unlock_pageq();
432: **output = pg;
1.34 chs 433: (*output)++;
1.6 mrg 434:
435: /* unlock anon and return success */
1.58 yamt 436: if (pg->uobject)
1.6 mrg 437: simple_unlock(&pg->uobject->vmobjlock);
438: simple_unlock(&anon->an_lock);
1.52 yamt 439: UVMHIST_LOG(loanhist, "->K done", 0,0,0,0);
1.34 chs 440: return (1);
1.1 mrg 441: }
442:
443: /*
1.45 yamt 444: * uvm_loanpage: loan out pages to kernel (->K)
1.42 yamt 445: *
1.51 yamt 446: * => pages should be object-owned and the object should be locked.
447: * => in the case of error, the object might be unlocked and relocked.
448: * => caller should busy the pages beforehand.
449: * => pages will be unbusied.
450: * => fail with EBUSY if meet a wired page.
1.42 yamt 451: */
1.51 yamt 452: static int
1.54 thorpej 453: uvm_loanpage(struct vm_page **pgpp, int npages)
1.42 yamt 454: {
455: int i;
1.51 yamt 456: int error = 0;
1.42 yamt 457:
1.52 yamt 458: UVMHIST_FUNC(__func__); UVMHIST_CALLED(loanhist);
459:
1.42 yamt 460: for (i = 0; i < npages; i++) {
461: struct vm_page *pg = pgpp[i];
462:
463: KASSERT(pg->uobject != NULL);
1.51 yamt 464: KASSERT(pg->uobject == pgpp[0]->uobject);
1.42 yamt 465: KASSERT(!(pg->flags & (PG_RELEASED|PG_PAGEOUT)));
466: LOCK_ASSERT(simple_lock_held(&pg->uobject->vmobjlock));
467: KASSERT(pg->flags & PG_BUSY);
468:
469: uvm_lock_pageq();
1.51 yamt 470: if (pg->wire_count > 0) {
471: uvm_unlock_pageq();
1.52 yamt 472: UVMHIST_LOG(loanhist, "wired %p", pg,0,0,0);
1.51 yamt 473: error = EBUSY;
474: break;
475: }
1.42 yamt 476: if (pg->loan_count == 0) {
477: pmap_page_protect(pg, VM_PROT_READ);
478: }
479: pg->loan_count++;
1.63 yamt 480: uvm_pageactivate(pg);
1.42 yamt 481: uvm_unlock_pageq();
482: }
1.51 yamt 483:
484: uvm_page_unbusy(pgpp, npages);
485:
486: if (error) {
487: /*
488: * backout what we've done
489: */
490: struct simplelock *slock = &pgpp[0]->uobject->vmobjlock;
491:
492: simple_unlock(slock);
493: uvm_unloan(pgpp, i, UVM_LOAN_TOPAGE);
494: simple_lock(slock);
495: }
496:
1.52 yamt 497: UVMHIST_LOG(loanhist, "done %d", error,0,0,0);
1.51 yamt 498: return error;
1.42 yamt 499: }
500:
501: /*
1.45 yamt 502: * XXX UBC temp limit
503: * number of pages to get at once.
504: * should be <= MAX_READ_AHEAD in genfs_vnops.c
505: */
506: #define UVM_LOAN_GET_CHUNK 16
507:
508: /*
1.46 yamt 509: * uvm_loanuobjpages: loan pages from a uobj out (O->K)
1.45 yamt 510: *
1.46 yamt 511: * => uobj shouldn't be locked. (we'll lock it)
1.51 yamt 512: * => fail with EBUSY if we meet a wired page.
1.45 yamt 513: */
514: int
1.54 thorpej 515: uvm_loanuobjpages(struct uvm_object *uobj, voff_t pgoff, int orignpages,
516: struct vm_page **origpgpp)
1.45 yamt 517: {
1.51 yamt 518: int ndone; /* # of pages loaned out */
1.45 yamt 519: struct vm_page **pgpp;
520: int error;
521: int i;
522: struct simplelock *slock;
523:
524: pgpp = origpgpp;
525: for (ndone = 0; ndone < orignpages; ) {
526: int npages;
1.51 yamt 527: /* npendloan: # of pages busied but not loand out yet. */
1.45 yamt 528: int npendloan = 0xdead; /* XXX gcc */
529: reget:
530: npages = MIN(UVM_LOAN_GET_CHUNK, orignpages - ndone);
531: simple_lock(&uobj->vmobjlock);
532: error = (*uobj->pgops->pgo_get)(uobj,
533: pgoff + (ndone << PAGE_SHIFT), pgpp, &npages, 0,
534: VM_PROT_READ, 0, PGO_SYNCIO);
535: if (error == EAGAIN) {
536: tsleep(&lbolt, PVM, "nfsread", 0);
537: continue;
538: }
1.51 yamt 539: if (error)
540: goto fail;
1.45 yamt 541:
542: KASSERT(npages > 0);
1.50 junyoung 543:
1.45 yamt 544: /* loan and unbusy pages */
545: slock = NULL;
546: for (i = 0; i < npages; i++) {
547: struct simplelock *nextslock; /* slock for next page */
548: struct vm_page *pg = *pgpp;
549:
550: /* XXX assuming that the page is owned by uobj */
551: KASSERT(pg->uobject != NULL);
552: nextslock = &pg->uobject->vmobjlock;
553:
554: if (slock != nextslock) {
555: if (slock) {
556: KASSERT(npendloan > 0);
1.51 yamt 557: error = uvm_loanpage(pgpp - npendloan,
1.45 yamt 558: npendloan);
559: simple_unlock(slock);
1.51 yamt 560: if (error)
561: goto fail;
562: ndone += npendloan;
563: KASSERT(origpgpp + ndone == pgpp);
1.45 yamt 564: }
565: slock = nextslock;
1.51 yamt 566: npendloan = 0;
1.45 yamt 567: simple_lock(slock);
568: }
569:
1.51 yamt 570: if ((pg->flags & PG_RELEASED) != 0) {
1.45 yamt 571: /*
572: * release pages and try again.
573: */
574: simple_unlock(slock);
575: for (; i < npages; i++) {
576: pg = pgpp[i];
577: slock = &pg->uobject->vmobjlock;
578:
579: simple_lock(slock);
580: uvm_lock_pageq();
581: uvm_page_unbusy(&pg, 1);
582: uvm_unlock_pageq();
583: simple_unlock(slock);
584: }
585: goto reget;
586: }
587:
588: npendloan++;
589: pgpp++;
1.51 yamt 590: KASSERT(origpgpp + ndone + npendloan == pgpp);
1.45 yamt 591: }
592: KASSERT(slock != NULL);
593: KASSERT(npendloan > 0);
1.51 yamt 594: error = uvm_loanpage(pgpp - npendloan, npendloan);
1.45 yamt 595: simple_unlock(slock);
1.51 yamt 596: if (error)
597: goto fail;
598: ndone += npendloan;
599: KASSERT(origpgpp + ndone == pgpp);
1.45 yamt 600: }
601:
602: return 0;
1.51 yamt 603:
604: fail:
605: uvm_unloan(origpgpp, ndone, UVM_LOAN_TOPAGE);
606:
607: return error;
1.45 yamt 608: }
609:
610: /*
1.1 mrg 611: * uvm_loanuobj: loan a page from a uobj out
612: *
1.28 chuck 613: * => called with map, amap, uobj locked
1.1 mrg 614: * => return value:
615: * -1 = fatal error, everything is unlocked, abort.
616: * 0 = lookup in ufi went stale, everything unlocked, relookup and
617: * try again
618: * 1 = got it, everything still locked
619: */
620:
1.33 jdolecek 621: static int
1.54 thorpej 622: uvm_loanuobj(struct uvm_faultinfo *ufi, void ***output, int flags, vaddr_t va)
1.1 mrg 623: {
1.6 mrg 624: struct vm_amap *amap = ufi->entry->aref.ar_amap;
625: struct uvm_object *uobj = ufi->entry->object.uvm_obj;
626: struct vm_page *pg;
627: struct vm_anon *anon;
1.34 chs 628: int error, npages;
1.64 thorpej 629: bool locked;
1.6 mrg 630:
1.52 yamt 631: UVMHIST_FUNC(__func__); UVMHIST_CALLED(loanhist);
632:
1.6 mrg 633: /*
634: * first we must make sure the page is resident.
635: *
636: * XXXCDC: duplicate code with uvm_fault().
637: */
638:
1.58 yamt 639: simple_lock(&uobj->vmobjlock);
1.28 chuck 640: if (uobj->pgops->pgo_get) { /* try locked pgo_get */
1.6 mrg 641: npages = 1;
642: pg = NULL;
1.37 enami 643: error = (*uobj->pgops->pgo_get)(uobj,
644: va - ufi->entry->start + ufi->entry->offset,
1.6 mrg 645: &pg, &npages, 0, VM_PROT_READ, MADV_NORMAL, PGO_LOCKED);
646: } else {
1.34 chs 647: error = EIO; /* must have pgo_get op */
1.6 mrg 648: }
649:
650: /*
651: * check the result of the locked pgo_get. if there is a problem,
652: * then we fail the loan.
653: */
654:
1.34 chs 655: if (error && error != EBUSY) {
1.6 mrg 656: uvmfault_unlockall(ufi, amap, uobj, NULL);
1.34 chs 657: return (-1);
1.6 mrg 658: }
659:
660: /*
661: * if we need to unlock for I/O, do so now.
662: */
663:
1.34 chs 664: if (error == EBUSY) {
1.6 mrg 665: uvmfault_unlockall(ufi, amap, NULL, NULL);
1.34 chs 666:
667: /* locked: uobj */
1.6 mrg 668: npages = 1;
1.37 enami 669: error = (*uobj->pgops->pgo_get)(uobj,
670: va - ufi->entry->start + ufi->entry->offset,
1.30 chs 671: &pg, &npages, 0, VM_PROT_READ, MADV_NORMAL, PGO_SYNCIO);
1.6 mrg 672: /* locked: <nothing> */
1.29 chs 673:
1.34 chs 674: if (error) {
1.43 yamt 675: if (error == EAGAIN) {
1.24 chs 676: tsleep(&lbolt, PVM, "fltagain2", 0);
1.34 chs 677: return (0);
1.29 chs 678: }
1.34 chs 679: return (-1);
1.6 mrg 680: }
681:
682: /*
683: * pgo_get was a success. attempt to relock everything.
684: */
685:
686: locked = uvmfault_relock(ufi);
687: if (locked && amap)
1.13 chuck 688: amap_lock(amap);
1.58 yamt 689: uobj = pg->uobject;
1.6 mrg 690: simple_lock(&uobj->vmobjlock);
691:
692: /*
693: * verify that the page has not be released and re-verify
694: * that amap slot is still free. if there is a problem we
695: * drop our lock (thus force a lookup refresh/retry).
696: */
1.29 chs 697:
1.6 mrg 698: if ((pg->flags & PG_RELEASED) != 0 ||
699: (locked && amap && amap_lookup(&ufi->entry->aref,
1.10 chuck 700: ufi->orig_rvaddr - ufi->entry->start))) {
1.6 mrg 701: if (locked)
702: uvmfault_unlockall(ufi, amap, NULL, NULL);
1.65 thorpej 703: locked = false;
1.29 chs 704: }
1.6 mrg 705:
706: /*
707: * didn't get the lock? release the page and retry.
708: */
709:
1.65 thorpej 710: if (locked == false) {
1.32 chs 711: if (pg->flags & PG_WANTED) {
1.18 thorpej 712: wakeup(pg);
1.32 chs 713: }
1.6 mrg 714: if (pg->flags & PG_RELEASED) {
1.38 enami 715: uvm_lock_pageq();
1.32 chs 716: uvm_pagefree(pg);
1.38 enami 717: uvm_unlock_pageq();
1.58 yamt 718: simple_unlock(&uobj->vmobjlock);
1.6 mrg 719: return (0);
720: }
721: uvm_lock_pageq();
1.32 chs 722: uvm_pageactivate(pg);
1.6 mrg 723: uvm_unlock_pageq();
724: pg->flags &= ~(PG_BUSY|PG_WANTED);
725: UVM_PAGE_OWN(pg, NULL);
726: simple_unlock(&uobj->vmobjlock);
727: return (0);
728: }
729: }
730:
1.58 yamt 731: KASSERT(uobj == pg->uobject);
732:
1.6 mrg 733: /*
734: * at this point we have the page we want ("pg") marked PG_BUSY for us
1.34 chs 735: * and we have all data structures locked. do the loanout. page can
1.6 mrg 736: * not be PG_RELEASED (we caught this above).
737: */
738:
1.34 chs 739: if ((flags & UVM_LOAN_TOANON) == 0) {
1.51 yamt 740: if (uvm_loanpage(&pg, 1)) {
741: uvmfault_unlockall(ufi, amap, uobj, NULL);
742: return (-1);
743: }
1.58 yamt 744: simple_unlock(&uobj->vmobjlock);
1.34 chs 745: **output = pg;
746: (*output)++;
747: return (1);
1.6 mrg 748: }
749:
750: /*
751: * must be a loan to an anon. check to see if there is already
752: * an anon associated with this page. if so, then just return
1.29 chs 753: * a reference to this object. the page should already be
1.6 mrg 754: * mapped read-only because it is already on loan.
755: */
756:
757: if (pg->uanon) {
758: anon = pg->uanon;
759: simple_lock(&anon->an_lock);
760: anon->an_ref++;
761: simple_unlock(&anon->an_lock);
1.34 chs 762: if (pg->flags & PG_WANTED) {
1.18 thorpej 763: wakeup(pg);
1.34 chs 764: }
1.6 mrg 765: pg->flags &= ~(PG_WANTED|PG_BUSY);
766: UVM_PAGE_OWN(pg, NULL);
1.58 yamt 767: simple_unlock(&uobj->vmobjlock);
1.34 chs 768: **output = anon;
769: (*output)++;
770: return (1);
1.6 mrg 771: }
1.29 chs 772:
1.6 mrg 773: /*
774: * need to allocate a new anon
775: */
776:
777: anon = uvm_analloc();
1.34 chs 778: if (anon == NULL) {
1.51 yamt 779: goto fail;
1.6 mrg 780: }
1.53 yamt 781: anon->an_page = pg;
1.6 mrg 782: pg->uanon = anon;
783: uvm_lock_pageq();
1.51 yamt 784: if (pg->wire_count > 0) {
785: uvm_unlock_pageq();
1.52 yamt 786: UVMHIST_LOG(loanhist, "wired %p", pg,0,0,0);
1.51 yamt 787: pg->uanon = NULL;
1.53 yamt 788: anon->an_page = NULL;
1.51 yamt 789: anon->an_ref--;
790: simple_unlock(&anon->an_lock);
791: uvm_anfree(anon);
792: goto fail;
793: }
1.34 chs 794: if (pg->loan_count == 0) {
1.19 chs 795: pmap_page_protect(pg, VM_PROT_READ);
1.34 chs 796: }
1.6 mrg 797: pg->loan_count++;
798: uvm_pageactivate(pg);
799: uvm_unlock_pageq();
1.34 chs 800: if (pg->flags & PG_WANTED) {
1.18 thorpej 801: wakeup(pg);
1.34 chs 802: }
1.6 mrg 803: pg->flags &= ~(PG_WANTED|PG_BUSY);
804: UVM_PAGE_OWN(pg, NULL);
1.58 yamt 805: simple_unlock(&uobj->vmobjlock);
1.23 thorpej 806: simple_unlock(&anon->an_lock);
1.34 chs 807: **output = anon;
808: (*output)++;
809: return (1);
1.51 yamt 810:
811: fail:
1.52 yamt 812: UVMHIST_LOG(loanhist, "fail", 0,0,0,0);
1.51 yamt 813: /*
814: * unlock everything and bail out.
815: */
816: if (pg->flags & PG_WANTED) {
817: wakeup(pg);
818: }
819: pg->flags &= ~(PG_WANTED|PG_BUSY);
820: UVM_PAGE_OWN(pg, NULL);
821: uvmfault_unlockall(ufi, amap, uobj, NULL);
822: return (-1);
1.1 mrg 823: }
824:
825: /*
1.40 thorpej 826: * uvm_loanzero: loan a zero-fill page out
1.1 mrg 827: *
1.28 chuck 828: * => called with map, amap, uobj locked
1.1 mrg 829: * => return value:
830: * -1 = fatal error, everything is unlocked, abort.
831: * 0 = lookup in ufi went stale, everything unlocked, relookup and
832: * try again
833: * 1 = got it, everything still locked
834: */
835:
1.40 thorpej 836: static struct uvm_object uvm_loanzero_object;
837:
1.33 jdolecek 838: static int
1.54 thorpej 839: uvm_loanzero(struct uvm_faultinfo *ufi, void ***output, int flags)
1.1 mrg 840: {
1.6 mrg 841: struct vm_anon *anon;
842: struct vm_page *pg;
1.34 chs 843: struct vm_amap *amap = ufi->entry->aref.ar_amap;
1.1 mrg 844:
1.52 yamt 845: UVMHIST_FUNC(__func__); UVMHIST_CALLED(loanhist);
1.44 yamt 846: again:
1.40 thorpej 847: simple_lock(&uvm_loanzero_object.vmobjlock);
848:
849: /*
850: * first, get ahold of our single zero page.
851: */
852:
853: if (__predict_false((pg =
854: TAILQ_FIRST(&uvm_loanzero_object.memq)) == NULL)) {
855: while ((pg = uvm_pagealloc(&uvm_loanzero_object, 0, NULL,
856: UVM_PGA_ZERO)) == NULL) {
857: simple_unlock(&uvm_loanzero_object.vmobjlock);
1.58 yamt 858: uvmfault_unlockall(ufi, amap, NULL, NULL);
1.40 thorpej 859: uvm_wait("loanzero");
1.34 chs 860: if (!uvmfault_relock(ufi)) {
861: return (0);
862: }
863: if (amap) {
864: amap_lock(amap);
865: }
1.44 yamt 866: goto again;
1.6 mrg 867: }
1.29 chs 868:
1.40 thorpej 869: /* got a zero'd page. */
870: pg->flags &= ~(PG_WANTED|PG_BUSY|PG_FAKE);
871: pg->flags |= PG_RDONLY;
1.44 yamt 872: uvm_lock_pageq();
873: uvm_pageactivate(pg);
874: uvm_unlock_pageq();
1.6 mrg 875: UVM_PAGE_OWN(pg, NULL);
1.40 thorpej 876: }
877:
878: if ((flags & UVM_LOAN_TOANON) == 0) { /* loaning to kernel-page */
879: uvm_lock_pageq();
880: pg->loan_count++;
881: uvm_unlock_pageq();
882: simple_unlock(&uvm_loanzero_object.vmobjlock);
1.6 mrg 883: **output = pg;
1.34 chs 884: (*output)++;
885: return (1);
1.6 mrg 886: }
887:
1.40 thorpej 888: /*
889: * loaning to an anon. check to see if there is already an anon
890: * associated with this page. if so, then just return a reference
891: * to this object.
892: */
1.23 thorpej 893:
1.40 thorpej 894: if (pg->uanon) {
895: anon = pg->uanon;
896: simple_lock(&anon->an_lock);
897: anon->an_ref++;
898: simple_unlock(&anon->an_lock);
899: simple_unlock(&uvm_loanzero_object.vmobjlock);
900: **output = anon;
901: (*output)++;
902: return (1);
903: }
1.23 thorpej 904:
1.40 thorpej 905: /*
906: * need to allocate a new anon
907: */
1.34 chs 908:
1.40 thorpej 909: anon = uvm_analloc();
910: if (anon == NULL) {
911: /* out of swap causes us to fail */
912: simple_unlock(&uvm_loanzero_object.vmobjlock);
1.58 yamt 913: uvmfault_unlockall(ufi, amap, NULL, NULL);
1.40 thorpej 914: return (-1);
1.6 mrg 915: }
1.53 yamt 916: anon->an_page = pg;
1.40 thorpej 917: pg->uanon = anon;
1.6 mrg 918: uvm_lock_pageq();
1.40 thorpej 919: pg->loan_count++;
1.6 mrg 920: uvm_pageactivate(pg);
921: uvm_unlock_pageq();
1.59 yamt 922: simple_unlock(&anon->an_lock);
1.40 thorpej 923: simple_unlock(&uvm_loanzero_object.vmobjlock);
1.6 mrg 924: **output = anon;
1.34 chs 925: (*output)++;
926: return (1);
1.1 mrg 927: }
928:
929:
930: /*
931: * uvm_unloananon: kill loans on anons (basically a normal ref drop)
932: *
933: * => we expect all our resources to be unlocked
934: */
935:
1.33 jdolecek 936: static void
1.54 thorpej 937: uvm_unloananon(struct vm_anon **aloans, int nanons)
1.1 mrg 938: {
1.6 mrg 939: struct vm_anon *anon;
1.1 mrg 940:
1.6 mrg 941: while (nanons-- > 0) {
1.12 chs 942: int refs;
1.1 mrg 943:
1.6 mrg 944: anon = *aloans++;
945: simple_lock(&anon->an_lock);
1.12 chs 946: refs = --anon->an_ref;
947: simple_unlock(&anon->an_lock);
1.1 mrg 948:
1.12 chs 949: if (refs == 0) {
1.34 chs 950: uvm_anfree(anon);
1.6 mrg 951: }
952: }
1.1 mrg 953: }
954:
955: /*
956: * uvm_unloanpage: kill loans on pages loaned out to the kernel
957: *
958: * => we expect all our resources to be unlocked
959: */
960:
1.33 jdolecek 961: static void
1.54 thorpej 962: uvm_unloanpage(struct vm_page **ploans, int npages)
1.1 mrg 963: {
1.6 mrg 964: struct vm_page *pg;
1.36 chs 965: struct simplelock *slock;
1.1 mrg 966:
1.6 mrg 967: uvm_lock_pageq();
968: while (npages-- > 0) {
969: pg = *ploans++;
1.1 mrg 970:
1.6 mrg 971: /*
1.36 chs 972: * do a little dance to acquire the object or anon lock
973: * as appropriate. we are locking in the wrong order,
974: * so we have to do a try-lock here.
975: */
976:
977: slock = NULL;
978: while (pg->uobject != NULL || pg->uanon != NULL) {
979: if (pg->uobject != NULL) {
980: slock = &pg->uobject->vmobjlock;
981: } else {
982: slock = &pg->uanon->an_lock;
983: }
984: if (simple_lock_try(slock)) {
985: break;
986: }
987: uvm_unlock_pageq();
988: uvm_lock_pageq();
989: slock = NULL;
990: }
991:
992: /*
993: * drop our loan. if page is owned by an anon but
994: * PQ_ANON is not set, the page was loaned to the anon
995: * from an object which dropped ownership, so resolve
996: * this by turning the anon's loan into real ownership
997: * (ie. decrement loan_count again and set PQ_ANON).
998: * after all this, if there are no loans left, put the
999: * page back a paging queue (if the page is owned by
1000: * an anon) or free it (if the page is now unowned).
1.6 mrg 1001: */
1.1 mrg 1002:
1.34 chs 1003: KASSERT(pg->loan_count > 0);
1004: pg->loan_count--;
1.36 chs 1005: if (pg->uobject == NULL && pg->uanon != NULL &&
1006: (pg->pqflags & PQ_ANON) == 0) {
1007: KASSERT(pg->loan_count > 0);
1008: pg->loan_count--;
1009: pg->pqflags |= PQ_ANON;
1010: }
1.63 yamt 1011: if (pg->loan_count == 0 && pg->uobject == NULL &&
1012: pg->uanon == NULL) {
1013: KASSERT((pg->flags & PG_BUSY) == 0);
1014: uvm_pagefree(pg);
1.36 chs 1015: }
1016: if (slock != NULL) {
1017: simple_unlock(slock);
1.6 mrg 1018: }
1019: }
1020: uvm_unlock_pageq();
1.1 mrg 1021: }
1022:
1.33 jdolecek 1023: /*
1.34 chs 1024: * uvm_unloan: kill loans on pages or anons.
1.33 jdolecek 1025: */
1.34 chs 1026:
1.33 jdolecek 1027: void
1.34 chs 1028: uvm_unloan(void *v, int npages, int flags)
1.33 jdolecek 1029: {
1.34 chs 1030: if (flags & UVM_LOAN_TOANON) {
1031: uvm_unloananon(v, npages);
1032: } else {
1033: uvm_unloanpage(v, npages);
1034: }
1.40 thorpej 1035: }
1036:
1037: /*
1.41 thorpej 1038: * Minimal pager for uvm_loanzero_object. We need to provide a "put"
1039: * method, because the page can end up on a paging queue, and the
1040: * page daemon will want to call pgo_put when it encounters the page
1041: * on the inactive list.
1042: */
1043:
1044: static int
1.62 yamt 1045: ulz_put(struct uvm_object *uobj, voff_t start, voff_t stop, int flags)
1.41 thorpej 1046: {
1047: struct vm_page *pg;
1048:
1049: KDASSERT(uobj == &uvm_loanzero_object);
1050:
1051: /*
1052: * Don't need to do any work here if we're not freeing pages.
1053: */
1054:
1055: if ((flags & PGO_FREE) == 0) {
1056: simple_unlock(&uobj->vmobjlock);
1057: return 0;
1058: }
1059:
1060: /*
1061: * we don't actually want to ever free the uvm_loanzero_page, so
1062: * just reactivate or dequeue it.
1063: */
1064:
1065: pg = TAILQ_FIRST(&uobj->memq);
1066: KASSERT(pg != NULL);
1067: KASSERT(TAILQ_NEXT(pg, listq) == NULL);
1068:
1069: uvm_lock_pageq();
1070: if (pg->uanon)
1071: uvm_pageactivate(pg);
1072: else
1073: uvm_pagedequeue(pg);
1074: uvm_unlock_pageq();
1075:
1076: simple_unlock(&uobj->vmobjlock);
1077: return 0;
1078: }
1079:
1080: static struct uvm_pagerops ulz_pager = {
1081: NULL, /* init */
1082: NULL, /* reference */
1083: NULL, /* detach */
1084: NULL, /* fault */
1085: NULL, /* get */
1086: ulz_put, /* put */
1087: };
1088:
1089: /*
1.40 thorpej 1090: * uvm_loan_init(): initialize the uvm_loan() facility.
1091: */
1092:
1093: void
1094: uvm_loan_init(void)
1095: {
1096:
1097: simple_lock_init(&uvm_loanzero_object.vmobjlock);
1098: TAILQ_INIT(&uvm_loanzero_object.memq);
1.41 thorpej 1099: uvm_loanzero_object.pgops = &ulz_pager;
1.52 yamt 1100:
1101: UVMHIST_INIT(loanhist, 300);
1.42 yamt 1102: }
1103:
1104: /*
1105: * uvm_loanbreak: break loan on a uobj page
1106: *
1107: * => called with uobj locked
1108: * => the page should be busy
1109: * => return value:
1110: * newly allocated page if succeeded
1111: */
1112: struct vm_page *
1113: uvm_loanbreak(struct vm_page *uobjpage)
1114: {
1115: struct vm_page *pg;
1.49 drochner 1116: #ifdef DIAGNOSTIC
1.42 yamt 1117: struct uvm_object *uobj = uobjpage->uobject;
1.49 drochner 1118: #endif
1.42 yamt 1119:
1120: KASSERT(uobj != NULL);
1121: LOCK_ASSERT(simple_lock_held(&uobj->vmobjlock));
1122: KASSERT(uobjpage->flags & PG_BUSY);
1123:
1124: /* alloc new un-owned page */
1125: pg = uvm_pagealloc(NULL, 0, NULL, 0);
1126: if (pg == NULL)
1127: return NULL;
1128:
1129: /*
1130: * copy the data from the old page to the new
1.61 yamt 1131: * one and clear the fake flags on the new page (keep it busy).
1132: * force a reload of the old page by clearing it from all
1133: * pmaps.
1134: * transfer dirtiness of the old page to the new page.
1135: * then lock the page queues to rename the pages.
1.42 yamt 1136: */
1137:
1138: uvm_pagecopy(uobjpage, pg); /* old -> new */
1.61 yamt 1139: pg->flags &= ~PG_FAKE;
1.42 yamt 1140: pmap_page_protect(uobjpage, VM_PROT_NONE);
1.61 yamt 1141: if ((uobjpage->flags & PG_CLEAN) != 0 && !pmap_clear_modify(uobjpage)) {
1142: pmap_clear_modify(pg);
1143: pg->flags |= PG_CLEAN;
1144: } else {
1145: /* uvm_pagecopy marked it dirty */
1146: KASSERT((pg->flags & PG_CLEAN) == 0);
1147: /* a object with a dirty page should be dirty. */
1148: KASSERT(!UVM_OBJ_IS_CLEAN(uobj));
1149: }
1.42 yamt 1150: if (uobjpage->flags & PG_WANTED)
1151: wakeup(uobjpage);
1152: /* uobj still locked */
1153: uobjpage->flags &= ~(PG_WANTED|PG_BUSY);
1154: UVM_PAGE_OWN(uobjpage, NULL);
1155:
1156: uvm_lock_pageq();
1.48 yamt 1157:
1158: /*
1159: * replace uobjpage with new page.
1160: */
1161:
1162: uvm_pagereplace(uobjpage, pg);
1.42 yamt 1163:
1164: /*
1165: * if the page is no longer referenced by
1166: * an anon (i.e. we are breaking an O->K
1167: * loan), then remove it from any pageq's.
1168: */
1169: if (uobjpage->uanon == NULL)
1170: uvm_pagedequeue(uobjpage);
1171:
1172: /*
1173: * at this point we have absolutely no
1174: * control over uobjpage
1175: */
1176:
1177: /* install new page */
1178: uvm_pageactivate(pg);
1179: uvm_unlock_pageq();
1180:
1181: /*
1182: * done! loan is broken and "pg" is
1183: * PG_BUSY. it can now replace uobjpage.
1184: */
1185:
1186: return pg;
1.33 jdolecek 1187: }
CVSweb <webmaster@jp.NetBSD.org>
![[BACK]](/icons/back.gif){kind=icon}
Return to uvm_loan.c CVS log ![[TXT]](/icons/text.gif){kind=icon}
![[DIR]](/icons/dir.gif){kind=icon}
Up to [cvs.NetBSD.org] / src / sys / uvm