Annotation of src/sys/uvm/uvm_aobj.c, Revision 1.39.2.4
1.39.2.4! nathanw 1: /* $NetBSD: uvm_aobj.c,v 1.39.2.3 2001/08/24 00:13:33 nathanw Exp $ */
1.6 mrg 2:
1.7 chs 3: /*
4: * Copyright (c) 1998 Chuck Silvers, Charles D. Cranor and
5: * Washington University.
6: * All rights reserved.
7: *
8: * Redistribution and use in source and binary forms, with or without
9: * modification, are permitted provided that the following conditions
10: * are met:
11: * 1. Redistributions of source code must retain the above copyright
12: * notice, this list of conditions and the following disclaimer.
13: * 2. Redistributions in binary form must reproduce the above copyright
14: * notice, this list of conditions and the following disclaimer in the
15: * documentation and/or other materials provided with the distribution.
16: * 3. All advertising materials mentioning features or use of this software
17: * must display the following acknowledgement:
18: * This product includes software developed by Charles D. Cranor and
19: * Washington University.
20: * 4. The name of the author may not be used to endorse or promote products
21: * derived from this software without specific prior written permission.
22: *
23: * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
24: * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
25: * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
26: * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
27: * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
28: * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
29: * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
30: * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
31: * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
32: * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
33: *
1.4 mrg 34: * from: Id: uvm_aobj.c,v 1.1.2.5 1998/02/06 05:14:38 chs Exp
35: */
1.7 chs 36: /*
37: * uvm_aobj.c: anonymous memory uvm_object pager
38: *
39: * author: Chuck Silvers <chuq@chuq.com>
40: * started: Jan-1998
41: *
42: * - design mostly from Chuck Cranor
43: */
44:
45: #include "opt_uvmhist.h"
1.1 mrg 46:
47: #include <sys/param.h>
48: #include <sys/systm.h>
49: #include <sys/proc.h>
50: #include <sys/malloc.h>
1.37 chs 51: #include <sys/kernel.h>
1.12 thorpej 52: #include <sys/pool.h>
1.27 chs 53: #include <sys/kernel.h>
1.1 mrg 54:
55: #include <uvm/uvm.h>
56:
57: /*
58: * an aobj manages anonymous-memory backed uvm_objects. in addition
59: * to keeping the list of resident pages, it also keeps a list of
60: * allocated swap blocks. depending on the size of the aobj this list
61: * of allocated swap blocks is either stored in an array (small objects)
62: * or in a hash table (large objects).
63: */
64:
65: /*
66: * local structures
67: */
68:
69: /*
70: * for hash tables, we break the address space of the aobj into blocks
71: * of UAO_SWHASH_CLUSTER_SIZE pages. we require the cluster size to
72: * be a power of two.
73: */
74:
75: #define UAO_SWHASH_CLUSTER_SHIFT 4
76: #define UAO_SWHASH_CLUSTER_SIZE (1 << UAO_SWHASH_CLUSTER_SHIFT)
77:
78: /* get the "tag" for this page index */
79: #define UAO_SWHASH_ELT_TAG(PAGEIDX) \
80: ((PAGEIDX) >> UAO_SWHASH_CLUSTER_SHIFT)
81:
82: /* given an ELT and a page index, find the swap slot */
83: #define UAO_SWHASH_ELT_PAGESLOT(ELT, PAGEIDX) \
84: ((ELT)->slots[(PAGEIDX) & (UAO_SWHASH_CLUSTER_SIZE - 1)])
85:
86: /* given an ELT, return its pageidx base */
87: #define UAO_SWHASH_ELT_PAGEIDX_BASE(ELT) \
88: ((ELT)->tag << UAO_SWHASH_CLUSTER_SHIFT)
89:
90: /*
91: * the swhash hash function
92: */
1.39.2.4! nathanw 93:
1.1 mrg 94: #define UAO_SWHASH_HASH(AOBJ, PAGEIDX) \
95: (&(AOBJ)->u_swhash[(((PAGEIDX) >> UAO_SWHASH_CLUSTER_SHIFT) \
96: & (AOBJ)->u_swhashmask)])
97:
98: /*
99: * the swhash threshhold determines if we will use an array or a
100: * hash table to store the list of allocated swap blocks.
101: */
102:
103: #define UAO_SWHASH_THRESHOLD (UAO_SWHASH_CLUSTER_SIZE * 4)
104: #define UAO_USES_SWHASH(AOBJ) \
105: ((AOBJ)->u_pages > UAO_SWHASH_THRESHOLD) /* use hash? */
106:
107: /*
1.3 chs 108: * the number of buckets in a swhash, with an upper bound
1.1 mrg 109: */
1.39.2.4! nathanw 110:
1.1 mrg 111: #define UAO_SWHASH_MAXBUCKETS 256
112: #define UAO_SWHASH_BUCKETS(AOBJ) \
1.39.2.4! nathanw 113: (MIN((AOBJ)->u_pages >> UAO_SWHASH_CLUSTER_SHIFT, \
1.1 mrg 114: UAO_SWHASH_MAXBUCKETS))
115:
116:
117: /*
118: * uao_swhash_elt: when a hash table is being used, this structure defines
119: * the format of an entry in the bucket list.
120: */
121:
122: struct uao_swhash_elt {
1.5 mrg 123: LIST_ENTRY(uao_swhash_elt) list; /* the hash list */
1.28 kleink 124: voff_t tag; /* our 'tag' */
1.5 mrg 125: int count; /* our number of active slots */
126: int slots[UAO_SWHASH_CLUSTER_SIZE]; /* the slots */
1.1 mrg 127: };
128:
129: /*
130: * uao_swhash: the swap hash table structure
131: */
132:
133: LIST_HEAD(uao_swhash, uao_swhash_elt);
134:
1.12 thorpej 135: /*
136: * uao_swhash_elt_pool: pool of uao_swhash_elt structures
137: */
138:
139: struct pool uao_swhash_elt_pool;
1.1 mrg 140:
141: /*
142: * uvm_aobj: the actual anon-backed uvm_object
143: *
144: * => the uvm_object is at the top of the structure, this allows
1.39.2.4! nathanw 145: * (struct uvm_aobj *) == (struct uvm_object *)
1.1 mrg 146: * => only one of u_swslots and u_swhash is used in any given aobj
147: */
148:
149: struct uvm_aobj {
1.5 mrg 150: struct uvm_object u_obj; /* has: lock, pgops, memq, #pages, #refs */
1.11 drochner 151: int u_pages; /* number of pages in entire object */
1.5 mrg 152: int u_flags; /* the flags (see uvm_aobj.h) */
153: int *u_swslots; /* array of offset->swapslot mappings */
154: /*
155: * hashtable of offset->swapslot mappings
156: * (u_swhash is an array of bucket heads)
157: */
158: struct uao_swhash *u_swhash;
159: u_long u_swhashmask; /* mask for hashtable */
160: LIST_ENTRY(uvm_aobj) u_list; /* global list of aobjs */
1.1 mrg 161: };
162:
163: /*
1.12 thorpej 164: * uvm_aobj_pool: pool of uvm_aobj structures
165: */
166:
167: struct pool uvm_aobj_pool;
168:
169: /*
1.1 mrg 170: * local functions
171: */
172:
1.39.2.4! nathanw 173: static struct uao_swhash_elt *uao_find_swhash_elt
! 174: __P((struct uvm_aobj *, int, boolean_t));
! 175:
! 176: static void uao_free __P((struct uvm_aobj *));
! 177: static int uao_get __P((struct uvm_object *, voff_t, struct vm_page **,
! 178: int *, int, vm_prot_t, int, int));
! 179: static boolean_t uao_put __P((struct uvm_object *, voff_t, voff_t, int));
! 180: static boolean_t uao_pagein __P((struct uvm_aobj *, int, int));
! 181: static boolean_t uao_pagein_page __P((struct uvm_aobj *, int));
1.1 mrg 182:
183: /*
184: * aobj_pager
1.39.2.2 nathanw 185: *
1.1 mrg 186: * note that some functions (e.g. put) are handled elsewhere
187: */
188:
189: struct uvm_pagerops aobj_pager = {
1.27 chs 190: NULL, /* init */
1.5 mrg 191: uao_reference, /* reference */
192: uao_detach, /* detach */
193: NULL, /* fault */
194: uao_get, /* get */
1.39.2.4! nathanw 195: uao_put, /* flush */
1.1 mrg 196: };
197:
198: /*
199: * uao_list: global list of active aobjs, locked by uao_list_lock
200: */
201:
202: static LIST_HEAD(aobjlist, uvm_aobj) uao_list;
1.39.2.2 nathanw 203: static struct simplelock uao_list_lock;
1.1 mrg 204:
205: /*
206: * functions
207: */
208:
209: /*
210: * hash table/array related functions
211: */
212:
213: /*
214: * uao_find_swhash_elt: find (or create) a hash table entry for a page
215: * offset.
216: *
217: * => the object should be locked by the caller
218: */
219:
1.5 mrg 220: static struct uao_swhash_elt *
221: uao_find_swhash_elt(aobj, pageidx, create)
222: struct uvm_aobj *aobj;
223: int pageidx;
224: boolean_t create;
225: {
226: struct uao_swhash *swhash;
227: struct uao_swhash_elt *elt;
1.28 kleink 228: voff_t page_tag;
1.1 mrg 229:
1.39.2.3 nathanw 230: swhash = UAO_SWHASH_HASH(aobj, pageidx);
231: page_tag = UAO_SWHASH_ELT_TAG(pageidx);
1.1 mrg 232:
1.5 mrg 233: /*
234: * now search the bucket for the requested tag
235: */
1.39.2.3 nathanw 236:
1.37 chs 237: LIST_FOREACH(elt, swhash, list) {
1.39.2.3 nathanw 238: if (elt->tag == page_tag) {
239: return elt;
240: }
1.5 mrg 241: }
1.39.2.3 nathanw 242: if (!create) {
1.5 mrg 243: return NULL;
1.39.2.3 nathanw 244: }
1.5 mrg 245:
246: /*
1.12 thorpej 247: * allocate a new entry for the bucket and init/insert it in
1.5 mrg 248: */
1.39.2.3 nathanw 249:
250: elt = pool_get(&uao_swhash_elt_pool, PR_NOWAIT);
251: if (elt == NULL) {
252: return NULL;
253: }
1.5 mrg 254: LIST_INSERT_HEAD(swhash, elt, list);
255: elt->tag = page_tag;
256: elt->count = 0;
1.9 perry 257: memset(elt->slots, 0, sizeof(elt->slots));
1.39.2.3 nathanw 258: return elt;
1.1 mrg 259: }
260:
261: /*
262: * uao_find_swslot: find the swap slot number for an aobj/pageidx
263: *
1.39.2.2 nathanw 264: * => object must be locked by caller
1.1 mrg 265: */
1.39.2.4! nathanw 266:
! 267: int
! 268: uao_find_swslot(uobj, pageidx)
! 269: struct uvm_object *uobj;
1.11 drochner 270: int pageidx;
1.1 mrg 271: {
1.39.2.4! nathanw 272: struct uvm_aobj *aobj = (struct uvm_aobj *)uobj;
! 273: struct uao_swhash_elt *elt;
1.1 mrg 274:
1.5 mrg 275: /*
276: * if noswap flag is set, then we never return a slot
277: */
1.1 mrg 278:
1.5 mrg 279: if (aobj->u_flags & UAO_FLAG_NOSWAP)
280: return(0);
1.1 mrg 281:
1.5 mrg 282: /*
283: * if hashing, look in hash table.
284: */
1.1 mrg 285:
1.5 mrg 286: if (UAO_USES_SWHASH(aobj)) {
1.39.2.4! nathanw 287: elt = uao_find_swhash_elt(aobj, pageidx, FALSE);
1.5 mrg 288: if (elt)
289: return(UAO_SWHASH_ELT_PAGESLOT(elt, pageidx));
290: else
1.31 thorpej 291: return(0);
1.5 mrg 292: }
1.1 mrg 293:
1.39.2.2 nathanw 294: /*
1.5 mrg 295: * otherwise, look in the array
296: */
1.39.2.4! nathanw 297:
1.5 mrg 298: return(aobj->u_swslots[pageidx]);
1.1 mrg 299: }
300:
301: /*
302: * uao_set_swslot: set the swap slot for a page in an aobj.
303: *
304: * => setting a slot to zero frees the slot
305: * => object must be locked by caller
1.39.2.3 nathanw 306: * => we return the old slot number, or -1 if we failed to allocate
307: * memory to record the new slot number
1.1 mrg 308: */
1.39.2.4! nathanw 309:
1.5 mrg 310: int
311: uao_set_swslot(uobj, pageidx, slot)
312: struct uvm_object *uobj;
313: int pageidx, slot;
314: {
315: struct uvm_aobj *aobj = (struct uvm_aobj *)uobj;
1.39.2.3 nathanw 316: struct uao_swhash_elt *elt;
1.5 mrg 317: int oldslot;
318: UVMHIST_FUNC("uao_set_swslot"); UVMHIST_CALLED(pdhist);
319: UVMHIST_LOG(pdhist, "aobj %p pageidx %d slot %d",
320: aobj, pageidx, slot, 0);
1.1 mrg 321:
1.5 mrg 322: /*
1.39.2.4! nathanw 323: * if noswap flag is set, then we can't set a non-zero slot.
1.5 mrg 324: */
1.1 mrg 325:
1.5 mrg 326: if (aobj->u_flags & UAO_FLAG_NOSWAP) {
327: if (slot == 0)
1.39.2.4! nathanw 328: return(0);
1.1 mrg 329:
1.5 mrg 330: printf("uao_set_swslot: uobj = %p\n", uobj);
1.39.2.4! nathanw 331: panic("uao_set_swslot: NOSWAP object");
1.5 mrg 332: }
1.1 mrg 333:
1.5 mrg 334: /*
335: * are we using a hash table? if so, add it in the hash.
336: */
1.1 mrg 337:
1.5 mrg 338: if (UAO_USES_SWHASH(aobj)) {
1.39 chs 339:
1.12 thorpej 340: /*
341: * Avoid allocating an entry just to free it again if
342: * the page had not swap slot in the first place, and
343: * we are freeing.
344: */
1.39 chs 345:
1.39.2.4! nathanw 346: elt = uao_find_swhash_elt(aobj, pageidx, slot != 0);
1.12 thorpej 347: if (elt == NULL) {
1.39.2.3 nathanw 348: return slot ? -1 : 0;
1.12 thorpej 349: }
1.5 mrg 350:
351: oldslot = UAO_SWHASH_ELT_PAGESLOT(elt, pageidx);
352: UAO_SWHASH_ELT_PAGESLOT(elt, pageidx) = slot;
353:
354: /*
355: * now adjust the elt's reference counter and free it if we've
356: * dropped it to zero.
357: */
358:
359: if (slot) {
360: if (oldslot == 0)
361: elt->count++;
1.39.2.3 nathanw 362: } else {
363: if (oldslot)
1.5 mrg 364: elt->count--;
365:
366: if (elt->count == 0) {
367: LIST_REMOVE(elt, list);
1.12 thorpej 368: pool_put(&uao_swhash_elt_pool, elt);
1.5 mrg 369: }
370: }
1.39.2.2 nathanw 371: } else {
1.5 mrg 372: /* we are using an array */
373: oldslot = aobj->u_swslots[pageidx];
374: aobj->u_swslots[pageidx] = slot;
375: }
376: return (oldslot);
1.1 mrg 377: }
378:
379: /*
380: * end of hash/array functions
381: */
382:
383: /*
384: * uao_free: free all resources held by an aobj, and then free the aobj
385: *
386: * => the aobj should be dead
387: */
1.39.2.4! nathanw 388:
1.1 mrg 389: static void
390: uao_free(aobj)
1.5 mrg 391: struct uvm_aobj *aobj;
1.1 mrg 392: {
1.39.2.4! nathanw 393: int swpgonlydelta = 0;
1.1 mrg 394:
1.27 chs 395: simple_unlock(&aobj->u_obj.vmobjlock);
1.5 mrg 396: if (UAO_USES_SWHASH(aobj)) {
397: int i, hashbuckets = aobj->u_swhashmask + 1;
1.1 mrg 398:
1.5 mrg 399: /*
400: * free the swslots from each hash bucket,
401: * then the hash bucket, and finally the hash table itself.
402: */
1.39.2.4! nathanw 403:
1.5 mrg 404: for (i = 0; i < hashbuckets; i++) {
405: struct uao_swhash_elt *elt, *next;
406:
1.27 chs 407: for (elt = LIST_FIRST(&aobj->u_swhash[i]);
408: elt != NULL;
409: elt = next) {
1.5 mrg 410: int j;
411:
1.27 chs 412: for (j = 0; j < UAO_SWHASH_CLUSTER_SIZE; j++) {
1.5 mrg 413: int slot = elt->slots[j];
414:
1.37 chs 415: if (slot == 0) {
416: continue;
417: }
418: uvm_swap_free(slot, 1);
1.39.2.4! nathanw 419: swpgonlydelta++;
1.5 mrg 420: }
421:
1.27 chs 422: next = LIST_NEXT(elt, list);
1.12 thorpej 423: pool_put(&uao_swhash_elt_pool, elt);
1.5 mrg 424: }
425: }
1.34 thorpej 426: free(aobj->u_swhash, M_UVMAOBJ);
1.5 mrg 427: } else {
428: int i;
429:
430: /*
431: * free the array
432: */
433:
1.27 chs 434: for (i = 0; i < aobj->u_pages; i++) {
1.5 mrg 435: int slot = aobj->u_swslots[i];
436:
1.18 chs 437: if (slot) {
1.5 mrg 438: uvm_swap_free(slot, 1);
1.39.2.4! nathanw 439: swpgonlydelta++;
1.18 chs 440: }
1.5 mrg 441: }
1.34 thorpej 442: free(aobj->u_swslots, M_UVMAOBJ);
1.1 mrg 443: }
444:
1.5 mrg 445: /*
446: * finally free the aobj itself
447: */
1.39.2.4! nathanw 448:
1.12 thorpej 449: pool_put(&uvm_aobj_pool, aobj);
1.39.2.4! nathanw 450:
! 451: /*
! 452: * adjust the counter of pages only in swap for all
! 453: * the swap slots we've freed.
! 454: */
! 455:
! 456: simple_lock(&uvm.swap_data_lock);
! 457: uvmexp.swpgonly -= swpgonlydelta;
! 458: simple_unlock(&uvm.swap_data_lock);
1.1 mrg 459: }
460:
461: /*
462: * pager functions
463: */
464:
465: /*
466: * uao_create: create an aobj of the given size and return its uvm_object.
467: *
468: * => for normal use, flags are always zero
469: * => for the kernel object, the flags are:
470: * UAO_FLAG_KERNOBJ - allocate the kernel object (can only happen once)
471: * UAO_FLAG_KERNSWAP - enable swapping of kernel object (" ")
472: */
1.39.2.4! nathanw 473:
1.5 mrg 474: struct uvm_object *
475: uao_create(size, flags)
1.10 eeh 476: vsize_t size;
1.5 mrg 477: int flags;
478: {
1.39.2.4! nathanw 479: static struct uvm_aobj kernel_object_store;
! 480: static int kobj_alloced = 0;
1.15 chs 481: int pages = round_page(size) >> PAGE_SHIFT;
1.5 mrg 482: struct uvm_aobj *aobj;
1.1 mrg 483:
1.5 mrg 484: /*
1.27 chs 485: * malloc a new aobj unless we are asked for the kernel object
486: */
1.5 mrg 487:
1.39.2.4! nathanw 488: if (flags & UAO_FLAG_KERNOBJ) {
! 489: KASSERT(!kobj_alloced);
1.5 mrg 490: aobj = &kernel_object_store;
491: aobj->u_pages = pages;
1.39.2.4! nathanw 492: aobj->u_flags = UAO_FLAG_NOSWAP;
1.5 mrg 493: aobj->u_obj.uo_refs = UVM_OBJ_KERN;
494: kobj_alloced = UAO_FLAG_KERNOBJ;
495: } else if (flags & UAO_FLAG_KERNSWAP) {
1.39.2.4! nathanw 496: KASSERT(kobj_alloced == UAO_FLAG_KERNOBJ);
1.5 mrg 497: aobj = &kernel_object_store;
498: kobj_alloced = UAO_FLAG_KERNSWAP;
1.39.2.4! nathanw 499: } else {
1.12 thorpej 500: aobj = pool_get(&uvm_aobj_pool, PR_WAITOK);
1.5 mrg 501: aobj->u_pages = pages;
1.39.2.4! nathanw 502: aobj->u_flags = 0;
! 503: aobj->u_obj.uo_refs = 1;
1.5 mrg 504: }
1.1 mrg 505:
1.5 mrg 506: /*
507: * allocate hash/array if necessary
508: *
509: * note: in the KERNSWAP case no need to worry about locking since
510: * we are still booting we should be the only thread around.
511: */
1.39.2.4! nathanw 512:
1.5 mrg 513: if (flags == 0 || (flags & UAO_FLAG_KERNSWAP) != 0) {
514: int mflags = (flags & UAO_FLAG_KERNSWAP) != 0 ?
515: M_NOWAIT : M_WAITOK;
516:
517: /* allocate hash table or array depending on object size */
1.27 chs 518: if (UAO_USES_SWHASH(aobj)) {
1.5 mrg 519: aobj->u_swhash = hashinit(UAO_SWHASH_BUCKETS(aobj),
1.35 ad 520: HASH_LIST, M_UVMAOBJ, mflags, &aobj->u_swhashmask);
1.5 mrg 521: if (aobj->u_swhash == NULL)
522: panic("uao_create: hashinit swhash failed");
523: } else {
1.34 thorpej 524: aobj->u_swslots = malloc(pages * sizeof(int),
1.5 mrg 525: M_UVMAOBJ, mflags);
526: if (aobj->u_swslots == NULL)
527: panic("uao_create: malloc swslots failed");
1.9 perry 528: memset(aobj->u_swslots, 0, pages * sizeof(int));
1.5 mrg 529: }
530:
531: if (flags) {
532: aobj->u_flags &= ~UAO_FLAG_NOSWAP; /* clear noswap */
533: return(&aobj->u_obj);
534: }
535: }
536:
537: /*
538: * init aobj fields
539: */
1.39.2.4! nathanw 540:
1.5 mrg 541: simple_lock_init(&aobj->u_obj.vmobjlock);
542: aobj->u_obj.pgops = &aobj_pager;
543: TAILQ_INIT(&aobj->u_obj.memq);
544: aobj->u_obj.uo_npages = 0;
1.1 mrg 545:
1.5 mrg 546: /*
547: * now that aobj is ready, add it to the global list
548: */
1.39.2.4! nathanw 549:
1.5 mrg 550: simple_lock(&uao_list_lock);
551: LIST_INSERT_HEAD(&uao_list, aobj, u_list);
552: simple_unlock(&uao_list_lock);
553: return(&aobj->u_obj);
1.1 mrg 554: }
555:
556:
557:
558: /*
559: * uao_init: set up aobj pager subsystem
560: *
561: * => called at boot time from uvm_pager_init()
562: */
1.39.2.4! nathanw 563:
1.27 chs 564: void
1.39.2.4! nathanw 565: uao_init(void)
1.5 mrg 566: {
1.12 thorpej 567: static int uao_initialized;
568:
569: if (uao_initialized)
570: return;
571: uao_initialized = TRUE;
1.5 mrg 572: LIST_INIT(&uao_list);
573: simple_lock_init(&uao_list_lock);
1.12 thorpej 574:
1.14 thorpej 575: /*
576: * NOTE: Pages fror this pool must not come from a pageable
577: * kernel map!
578: */
1.39.2.4! nathanw 579:
1.12 thorpej 580: pool_init(&uao_swhash_elt_pool, sizeof(struct uao_swhash_elt),
1.13 thorpej 581: 0, 0, 0, "uaoeltpl", 0, NULL, NULL, M_UVMAOBJ);
1.12 thorpej 582: pool_init(&uvm_aobj_pool, sizeof(struct uvm_aobj), 0, 0, 0,
583: "aobjpl", 0,
584: pool_page_alloc_nointr, pool_page_free_nointr, M_UVMAOBJ);
1.1 mrg 585: }
586:
587: /*
588: * uao_reference: add a ref to an aobj
589: *
1.27 chs 590: * => aobj must be unlocked
591: * => just lock it and call the locked version
1.1 mrg 592: */
1.39.2.4! nathanw 593:
1.5 mrg 594: void
595: uao_reference(uobj)
596: struct uvm_object *uobj;
1.1 mrg 597: {
1.27 chs 598: simple_lock(&uobj->vmobjlock);
599: uao_reference_locked(uobj);
600: simple_unlock(&uobj->vmobjlock);
601: }
602:
603: /*
604: * uao_reference_locked: add a ref to an aobj that is already locked
605: *
606: * => aobj must be locked
607: * this needs to be separate from the normal routine
608: * since sometimes we need to add a reference to an aobj when
609: * it's already locked.
610: */
1.39.2.4! nathanw 611:
1.27 chs 612: void
613: uao_reference_locked(uobj)
614: struct uvm_object *uobj;
615: {
1.5 mrg 616: UVMHIST_FUNC("uao_reference"); UVMHIST_CALLED(maphist);
1.1 mrg 617:
1.5 mrg 618: /*
619: * kernel_object already has plenty of references, leave it alone.
620: */
1.1 mrg 621:
1.20 thorpej 622: if (UVM_OBJ_IS_KERN_OBJECT(uobj))
1.5 mrg 623: return;
1.1 mrg 624:
1.39.2.4! nathanw 625: uobj->uo_refs++;
1.39.2.2 nathanw 626: UVMHIST_LOG(maphist, "<- done (uobj=0x%x, ref = %d)",
1.27 chs 627: uobj, uobj->uo_refs,0,0);
1.1 mrg 628: }
629:
630: /*
631: * uao_detach: drop a reference to an aobj
632: *
1.27 chs 633: * => aobj must be unlocked
634: * => just lock it and call the locked version
1.1 mrg 635: */
1.39.2.4! nathanw 636:
1.5 mrg 637: void
638: uao_detach(uobj)
639: struct uvm_object *uobj;
640: {
1.27 chs 641: simple_lock(&uobj->vmobjlock);
642: uao_detach_locked(uobj);
643: }
644:
645: /*
646: * uao_detach_locked: drop a reference to an aobj
647: *
648: * => aobj must be locked, and is unlocked (or freed) upon return.
649: * this needs to be separate from the normal routine
650: * since sometimes we need to detach from an aobj when
651: * it's already locked.
652: */
1.39.2.4! nathanw 653:
1.27 chs 654: void
655: uao_detach_locked(uobj)
656: struct uvm_object *uobj;
657: {
1.5 mrg 658: struct uvm_aobj *aobj = (struct uvm_aobj *)uobj;
1.39.2.4! nathanw 659: struct vm_page *pg;
1.5 mrg 660: UVMHIST_FUNC("uao_detach"); UVMHIST_CALLED(maphist);
1.1 mrg 661:
1.5 mrg 662: /*
663: * detaching from kernel_object is a noop.
664: */
1.39.2.4! nathanw 665:
1.27 chs 666: if (UVM_OBJ_IS_KERN_OBJECT(uobj)) {
667: simple_unlock(&uobj->vmobjlock);
1.5 mrg 668: return;
1.27 chs 669: }
1.5 mrg 670:
671: UVMHIST_LOG(maphist," (uobj=0x%x) ref=%d", uobj,uobj->uo_refs,0,0);
1.39.2.4! nathanw 672: uobj->uo_refs--;
! 673: if (uobj->uo_refs) {
1.5 mrg 674: simple_unlock(&uobj->vmobjlock);
675: UVMHIST_LOG(maphist, "<- done (rc>0)", 0,0,0,0);
676: return;
677: }
678:
679: /*
680: * remove the aobj from the global list.
681: */
1.39.2.4! nathanw 682:
1.5 mrg 683: simple_lock(&uao_list_lock);
684: LIST_REMOVE(aobj, u_list);
685: simple_unlock(&uao_list_lock);
686:
687: /*
1.39.2.4! nathanw 688: * free all the pages left in the aobj. for each page,
! 689: * when the page is no longer busy (and thus after any disk i/o that
! 690: * it's involved in is complete), release any swap resources and
! 691: * free the page itself.
1.5 mrg 692: */
1.39.2.4! nathanw 693:
! 694: uvm_lock_pageq();
! 695: while ((pg = TAILQ_FIRST(&uobj->memq)) != NULL) {
! 696: pmap_page_protect(pg, VM_PROT_NONE);
1.5 mrg 697: if (pg->flags & PG_BUSY) {
1.39.2.4! nathanw 698: pg->flags |= PG_WANTED;
! 699: uvm_unlock_pageq();
! 700: UVM_UNLOCK_AND_WAIT(pg, &uobj->vmobjlock, FALSE,
! 701: "uao_det", 0);
! 702: simple_lock(&uobj->vmobjlock);
! 703: uvm_lock_pageq();
1.5 mrg 704: continue;
705: }
1.18 chs 706: uao_dropswap(&aobj->u_obj, pg->offset >> PAGE_SHIFT);
1.5 mrg 707: uvm_pagefree(pg);
708: }
1.39.2.4! nathanw 709: uvm_unlock_pageq();
1.1 mrg 710:
1.5 mrg 711: /*
1.39.2.4! nathanw 712: * finally, free the aobj itself.
1.5 mrg 713: */
1.1 mrg 714:
1.5 mrg 715: uao_free(aobj);
716: }
1.1 mrg 717:
718: /*
1.39.2.4! nathanw 719: * uao_put: flush pages out of a uvm object
1.22 thorpej 720: *
721: * => object should be locked by caller. we may _unlock_ the object
722: * if (and only if) we need to clean a page (PGO_CLEANIT).
723: * XXXJRT Currently, however, we don't. In the case of cleaning
724: * XXXJRT a page, we simply just deactivate it. Should probably
725: * XXXJRT handle this better, in the future (although "flushing"
726: * XXXJRT anonymous memory isn't terribly important).
727: * => if PGO_CLEANIT is not set, then we will neither unlock the object
728: * or block.
729: * => if PGO_ALLPAGE is set, then all pages in the object are valid targets
730: * for flushing.
731: * => NOTE: we rely on the fact that the object's memq is a TAILQ and
732: * that new pages are inserted on the tail end of the list. thus,
733: * we can make a complete pass through the object in one go by starting
734: * at the head and working towards the tail (new pages are put in
735: * front of us).
736: * => NOTE: we are allowed to lock the page queues, so the caller
737: * must not be holding the lock on them [e.g. pagedaemon had
738: * better not call us with the queues locked]
739: * => we return TRUE unless we encountered some sort of I/O error
740: * XXXJRT currently never happens, as we never directly initiate
741: * XXXJRT I/O
742: *
743: * note on page traversal:
744: * we can traverse the pages in an object either by going down the
745: * linked list in "uobj->memq", or we can go over the address range
746: * by page doing hash table lookups for each address. depending
747: * on how many pages are in the object it may be cheaper to do one
748: * or the other. we set "by_list" to true if we are using memq.
749: * if the cost of a hash lookup was equal to the cost of the list
750: * traversal we could compare the number of pages in the start->stop
751: * range to the total number of pages in the object. however, it
752: * seems that a hash table lookup is more expensive than the linked
753: * list traversal, so we multiply the number of pages in the
754: * start->stop range by a penalty which we define below.
1.1 mrg 755: */
1.22 thorpej 756:
1.39.2.4! nathanw 757: int
! 758: uao_put(uobj, start, stop, flags)
1.5 mrg 759: struct uvm_object *uobj;
1.28 kleink 760: voff_t start, stop;
1.5 mrg 761: int flags;
762: {
1.39.2.4! nathanw 763: struct uvm_aobj *aobj = (struct uvm_aobj *)uobj;
! 764: struct vm_page *pg, *nextpg;
! 765: boolean_t by_list;
1.28 kleink 766: voff_t curoff;
1.39.2.4! nathanw 767: UVMHIST_FUNC("uao_put"); UVMHIST_CALLED(maphist);
1.22 thorpej 768:
1.39.2.4! nathanw 769: curoff = 0;
1.22 thorpej 770: if (flags & PGO_ALLPAGES) {
771: start = 0;
772: stop = aobj->u_pages << PAGE_SHIFT;
773: by_list = TRUE; /* always go by the list */
774: } else {
775: start = trunc_page(start);
776: stop = round_page(stop);
777: if (stop > (aobj->u_pages << PAGE_SHIFT)) {
778: printf("uao_flush: strange, got an out of range "
779: "flush (fixed)\n");
780: stop = aobj->u_pages << PAGE_SHIFT;
781: }
782: by_list = (uobj->uo_npages <=
1.39.2.4! nathanw 783: ((stop - start) >> PAGE_SHIFT) * UVM_PAGE_HASH_PENALTY);
1.22 thorpej 784: }
785: UVMHIST_LOG(maphist,
786: " flush start=0x%lx, stop=0x%x, by_list=%d, flags=0x%x",
787: start, stop, by_list, flags);
1.1 mrg 788:
1.5 mrg 789: /*
1.22 thorpej 790: * Don't need to do any work here if we're not freeing
791: * or deactivating pages.
792: */
1.39.2.4! nathanw 793:
1.22 thorpej 794: if ((flags & (PGO_DEACTIVATE|PGO_FREE)) == 0) {
1.39.2.4! nathanw 795: simple_unlock(&uobj->vmobjlock);
! 796: return 0;
1.22 thorpej 797: }
798:
1.5 mrg 799: /*
1.39.2.4! nathanw 800: * now do it. note: we must update nextpg in the body of loop or we
! 801: * will get stuck. we need to use nextpg because we may free "pg"
1.22 thorpej 802: * before doing the next loop.
1.21 thorpej 803: */
1.22 thorpej 804:
805: if (by_list) {
1.39.2.4! nathanw 806: pg = TAILQ_FIRST(&uobj->memq);
1.22 thorpej 807: } else {
808: curoff = start;
1.39.2.4! nathanw 809: pg = uvm_pagelookup(uobj, curoff);
1.22 thorpej 810: }
811:
1.39.2.4! nathanw 812: nextpg = NULL;
! 813: uvm_lock_pageq();
1.22 thorpej 814:
815: /* locked: both page queues and uobj */
1.39.2.4! nathanw 816: for ( ; (by_list && pg != NULL) ||
! 817: (!by_list && curoff < stop) ; pg = nextpg) {
1.22 thorpej 818: if (by_list) {
1.39.2.4! nathanw 819: nextpg = TAILQ_NEXT(pg, listq);
! 820: if (pg->offset < start || pg->offset >= stop)
1.22 thorpej 821: continue;
822: } else {
823: curoff += PAGE_SIZE;
824: if (curoff < stop)
1.39.2.4! nathanw 825: nextpg = uvm_pagelookup(uobj, curoff);
! 826: if (pg == NULL)
1.22 thorpej 827: continue;
828: }
829: switch (flags & (PGO_CLEANIT|PGO_FREE|PGO_DEACTIVATE)) {
1.39.2.4! nathanw 830:
1.22 thorpej 831: /*
832: * XXX In these first 3 cases, we always just
833: * XXX deactivate the page. We may want to
834: * XXX handle the different cases more specifically
835: * XXX in the future.
836: */
1.39.2.4! nathanw 837:
1.22 thorpej 838: case PGO_CLEANIT|PGO_FREE:
839: case PGO_CLEANIT|PGO_DEACTIVATE:
840: case PGO_DEACTIVATE:
1.25 thorpej 841: deactivate_it:
1.22 thorpej 842: /* skip the page if it's loaned or wired */
1.39.2.4! nathanw 843: if (pg->loan_count != 0 || pg->wire_count != 0)
1.22 thorpej 844: continue;
845:
846: /* ...and deactivate the page. */
1.39.2.4! nathanw 847: pmap_clear_reference(pg);
! 848: uvm_pagedeactivate(pg);
1.22 thorpej 849: continue;
850:
851: case PGO_FREE:
1.39.2.4! nathanw 852:
1.25 thorpej 853: /*
854: * If there are multiple references to
855: * the object, just deactivate the page.
856: */
1.39.2.4! nathanw 857:
1.25 thorpej 858: if (uobj->uo_refs > 1)
859: goto deactivate_it;
860:
1.22 thorpej 861: /* XXX skip the page if it's loaned or wired */
1.39.2.4! nathanw 862: if (pg->loan_count != 0 || pg->wire_count != 0)
1.22 thorpej 863: continue;
864:
865: /*
1.39.2.4! nathanw 866: * wait if the page is busy, then free the swap slot
! 867: * and the page.
1.22 thorpej 868: */
869:
1.39.2.4! nathanw 870: pmap_page_protect(pg, VM_PROT_NONE);
! 871: while (pg->flags & PG_BUSY) {
! 872: pg->flags |= PG_WANTED;
! 873: uvm_unlock_pageq();
! 874: UVM_UNLOCK_AND_WAIT(pg, &uobj->vmobjlock, 0,
! 875: "uao_put", 0);
! 876: simple_lock(&uobj->vmobjlock);
! 877: uvm_lock_pageq();
! 878: }
! 879: uao_dropswap(uobj, pg->offset >> PAGE_SHIFT);
! 880: uvm_pagefree(pg);
1.22 thorpej 881: continue;
882: }
883: }
884: uvm_unlock_pageq();
1.39.2.4! nathanw 885: simple_unlock(&uobj->vmobjlock);
! 886: return 0;
1.1 mrg 887: }
888:
889: /*
890: * uao_get: fetch me a page
891: *
892: * we have three cases:
893: * 1: page is resident -> just return the page.
894: * 2: page is zero-fill -> allocate a new page and zero it.
895: * 3: page is swapped out -> fetch the page from swap.
896: *
897: * cases 1 and 2 can be handled with PGO_LOCKED, case 3 cannot.
898: * so, if the "center" page hits case 3 (or any page, with PGO_ALLPAGES),
1.39.2.1 nathanw 899: * then we will need to return EBUSY.
1.1 mrg 900: *
901: * => prefer map unlocked (not required)
902: * => object must be locked! we will _unlock_ it before starting any I/O.
903: * => flags: PGO_ALLPAGES: get all of the pages
904: * PGO_LOCKED: fault data structures are locked
905: * => NOTE: offset is the offset of pps[0], _NOT_ pps[centeridx]
906: * => NOTE: caller must check for released pages!!
907: */
1.39.2.4! nathanw 908:
1.5 mrg 909: static int
910: uao_get(uobj, offset, pps, npagesp, centeridx, access_type, advice, flags)
911: struct uvm_object *uobj;
1.28 kleink 912: voff_t offset;
1.5 mrg 913: struct vm_page **pps;
914: int *npagesp;
915: int centeridx, advice, flags;
916: vm_prot_t access_type;
917: {
918: struct uvm_aobj *aobj = (struct uvm_aobj *)uobj;
1.28 kleink 919: voff_t current_offset;
1.39.2.2 nathanw 920: struct vm_page *ptmp;
1.39.2.4! nathanw 921: int lcv, gotpages, maxpages, swslot, error, pageidx;
1.5 mrg 922: boolean_t done;
923: UVMHIST_FUNC("uao_get"); UVMHIST_CALLED(pdhist);
924:
1.27 chs 925: UVMHIST_LOG(pdhist, "aobj=%p offset=%d, flags=%d",
926: aobj, offset, flags,0);
1.37 chs 927:
1.5 mrg 928: /*
929: * get number of pages
930: */
1.39.2.4! nathanw 931:
1.5 mrg 932: maxpages = *npagesp;
933:
934: /*
935: * step 1: handled the case where fault data structures are locked.
936: */
1.1 mrg 937:
1.5 mrg 938: if (flags & PGO_LOCKED) {
1.39.2.4! nathanw 939:
1.5 mrg 940: /*
941: * step 1a: get pages that are already resident. only do
942: * this if the data structures are locked (i.e. the first
943: * time through).
944: */
945:
946: done = TRUE; /* be optimistic */
947: gotpages = 0; /* # of pages we got so far */
948: for (lcv = 0, current_offset = offset ; lcv < maxpages ;
949: lcv++, current_offset += PAGE_SIZE) {
950: /* do we care about this page? if not, skip it */
951: if (pps[lcv] == PGO_DONTCARE)
952: continue;
953: ptmp = uvm_pagelookup(uobj, current_offset);
954:
955: /*
1.30 thorpej 956: * if page is new, attempt to allocate the page,
957: * zero-fill'd.
1.5 mrg 958: */
1.39.2.4! nathanw 959:
! 960: if (ptmp == NULL && uao_find_swslot(&aobj->u_obj,
1.15 chs 961: current_offset >> PAGE_SHIFT) == 0) {
1.5 mrg 962: ptmp = uvm_pagealloc(uobj, current_offset,
1.30 thorpej 963: NULL, UVM_PGA_ZERO);
1.5 mrg 964: if (ptmp) {
965: /* new page */
966: ptmp->flags &= ~(PG_BUSY|PG_FAKE);
967: ptmp->pqflags |= PQ_AOBJ;
968: UVM_PAGE_OWN(ptmp, NULL);
969: }
970: }
971:
972: /*
1.39.2.4! nathanw 973: * to be useful must get a non-busy page
1.5 mrg 974: */
1.39.2.4! nathanw 975:
! 976: if (ptmp == NULL || (ptmp->flags & PG_BUSY) != 0) {
1.5 mrg 977: if (lcv == centeridx ||
978: (flags & PGO_ALLPAGES) != 0)
979: /* need to do a wait or I/O! */
1.39.2.2 nathanw 980: done = FALSE;
1.5 mrg 981: continue;
982: }
983:
984: /*
985: * useful page: busy/lock it and plug it in our
986: * result array
987: */
1.39.2.4! nathanw 988:
1.5 mrg 989: /* caller must un-busy this page */
1.39.2.2 nathanw 990: ptmp->flags |= PG_BUSY;
1.5 mrg 991: UVM_PAGE_OWN(ptmp, "uao_get1");
992: pps[lcv] = ptmp;
993: gotpages++;
1.39.2.4! nathanw 994: }
1.5 mrg 995:
996: /*
997: * step 1b: now we've either done everything needed or we
998: * to unlock and do some waiting or I/O.
999: */
1000:
1001: UVMHIST_LOG(pdhist, "<- done (done=%d)", done, 0,0,0);
1002: *npagesp = gotpages;
1003: if (done)
1.39.2.4! nathanw 1004: return 0;
1.5 mrg 1005: else
1.39.2.4! nathanw 1006: return EBUSY;
1.1 mrg 1007: }
1008:
1.5 mrg 1009: /*
1010: * step 2: get non-resident or busy pages.
1011: * object is locked. data structures are unlocked.
1012: */
1013:
1014: for (lcv = 0, current_offset = offset ; lcv < maxpages ;
1015: lcv++, current_offset += PAGE_SIZE) {
1.27 chs 1016:
1.5 mrg 1017: /*
1018: * - skip over pages we've already gotten or don't want
1019: * - skip over pages we don't _have_ to get
1020: */
1.27 chs 1021:
1.5 mrg 1022: if (pps[lcv] != NULL ||
1023: (lcv != centeridx && (flags & PGO_ALLPAGES) == 0))
1024: continue;
1025:
1.27 chs 1026: pageidx = current_offset >> PAGE_SHIFT;
1027:
1.5 mrg 1028: /*
1029: * we have yet to locate the current page (pps[lcv]). we
1030: * first look for a page that is already at the current offset.
1031: * if we find a page, we check to see if it is busy or
1032: * released. if that is the case, then we sleep on the page
1033: * until it is no longer busy or released and repeat the lookup.
1034: * if the page we found is neither busy nor released, then we
1035: * busy it (so we own it) and plug it into pps[lcv]. this
1036: * 'break's the following while loop and indicates we are
1037: * ready to move on to the next page in the "lcv" loop above.
1038: *
1039: * if we exit the while loop with pps[lcv] still set to NULL,
1040: * then it means that we allocated a new busy/fake/clean page
1041: * ptmp in the object and we need to do I/O to fill in the data.
1042: */
1043:
1044: /* top of "pps" while loop */
1045: while (pps[lcv] == NULL) {
1046: /* look for a resident page */
1047: ptmp = uvm_pagelookup(uobj, current_offset);
1048:
1049: /* not resident? allocate one now (if we can) */
1050: if (ptmp == NULL) {
1051:
1052: ptmp = uvm_pagealloc(uobj, current_offset,
1.19 chs 1053: NULL, 0);
1.5 mrg 1054:
1055: /* out of RAM? */
1056: if (ptmp == NULL) {
1057: simple_unlock(&uobj->vmobjlock);
1058: UVMHIST_LOG(pdhist,
1059: "sleeping, ptmp == NULL\n",0,0,0,0);
1060: uvm_wait("uao_getpage");
1061: simple_lock(&uobj->vmobjlock);
1.39.2.2 nathanw 1062: continue;
1.5 mrg 1063: }
1064:
1065: /*
1066: * safe with PQ's unlocked: because we just
1067: * alloc'd the page
1068: */
1.39.2.4! nathanw 1069:
1.5 mrg 1070: ptmp->pqflags |= PQ_AOBJ;
1071:
1.39.2.2 nathanw 1072: /*
1.5 mrg 1073: * got new page ready for I/O. break pps while
1074: * loop. pps[lcv] is still NULL.
1075: */
1.39.2.4! nathanw 1076:
1.5 mrg 1077: break;
1078: }
1079:
1080: /* page is there, see if we need to wait on it */
1.39.2.4! nathanw 1081: if ((ptmp->flags & PG_BUSY) != 0) {
1.5 mrg 1082: ptmp->flags |= PG_WANTED;
1083: UVMHIST_LOG(pdhist,
1084: "sleeping, ptmp->flags 0x%x\n",
1085: ptmp->flags,0,0,0);
1.23 thorpej 1086: UVM_UNLOCK_AND_WAIT(ptmp, &uobj->vmobjlock,
1087: FALSE, "uao_get", 0);
1.5 mrg 1088: simple_lock(&uobj->vmobjlock);
1.39.2.4! nathanw 1089: continue;
1.5 mrg 1090: }
1.39.2.2 nathanw 1091:
1092: /*
1.5 mrg 1093: * if we get here then the page has become resident and
1094: * unbusy between steps 1 and 2. we busy it now (so we
1095: * own it) and set pps[lcv] (so that we exit the while
1096: * loop).
1097: */
1.39.2.4! nathanw 1098:
1.5 mrg 1099: /* we own it, caller must un-busy */
1100: ptmp->flags |= PG_BUSY;
1101: UVM_PAGE_OWN(ptmp, "uao_get2");
1102: pps[lcv] = ptmp;
1103: }
1104:
1105: /*
1106: * if we own the valid page at the correct offset, pps[lcv] will
1107: * point to it. nothing more to do except go to the next page.
1108: */
1.39.2.4! nathanw 1109:
1.5 mrg 1110: if (pps[lcv])
1111: continue; /* next lcv */
1112:
1113: /*
1.39.2.2 nathanw 1114: * we have a "fake/busy/clean" page that we just allocated.
1.5 mrg 1115: * do the needed "i/o", either reading from swap or zeroing.
1116: */
1.39.2.4! nathanw 1117:
! 1118: swslot = uao_find_swslot(&aobj->u_obj, pageidx);
1.5 mrg 1119:
1120: /*
1121: * just zero the page if there's nothing in swap.
1122: */
1.39.2.4! nathanw 1123:
! 1124: if (swslot == 0) {
! 1125:
1.5 mrg 1126: /*
1127: * page hasn't existed before, just zero it.
1128: */
1.39.2.4! nathanw 1129:
1.5 mrg 1130: uvm_pagezero(ptmp);
1.27 chs 1131: } else {
1.5 mrg 1132: UVMHIST_LOG(pdhist, "pagein from swslot %d",
1133: swslot, 0,0,0);
1134:
1135: /*
1136: * page in the swapped-out page.
1137: * unlock object for i/o, relock when done.
1138: */
1.39.2.4! nathanw 1139:
1.5 mrg 1140: simple_unlock(&uobj->vmobjlock);
1.39.2.4! nathanw 1141: error = uvm_swap_get(ptmp, swslot, PGO_SYNCIO);
1.5 mrg 1142: simple_lock(&uobj->vmobjlock);
1143:
1144: /*
1145: * I/O done. check for errors.
1146: */
1.39.2.4! nathanw 1147:
! 1148: if (error != 0) {
1.5 mrg 1149: UVMHIST_LOG(pdhist, "<- done (error=%d)",
1.39.2.4! nathanw 1150: error,0,0,0);
1.5 mrg 1151: if (ptmp->flags & PG_WANTED)
1.24 thorpej 1152: wakeup(ptmp);
1.27 chs 1153:
1154: /*
1155: * remove the swap slot from the aobj
1156: * and mark the aobj as having no real slot.
1157: * don't free the swap slot, thus preventing
1158: * it from being used again.
1159: */
1.39.2.4! nathanw 1160:
1.27 chs 1161: swslot = uao_set_swslot(&aobj->u_obj, pageidx,
1162: SWSLOT_BAD);
1.39.2.3 nathanw 1163: if (swslot != -1) {
1164: uvm_swap_markbad(swslot, 1);
1165: }
1.27 chs 1166:
1.5 mrg 1167: uvm_lock_pageq();
1168: uvm_pagefree(ptmp);
1169: uvm_unlock_pageq();
1170: simple_unlock(&uobj->vmobjlock);
1.39.2.4! nathanw 1171: return error;
1.5 mrg 1172: }
1173: }
1174:
1.39.2.2 nathanw 1175: /*
1.5 mrg 1176: * we got the page! clear the fake flag (indicates valid
1177: * data now in page) and plug into our result array. note
1.39.2.2 nathanw 1178: * that page is still busy.
1.5 mrg 1179: *
1180: * it is the callers job to:
1181: * => check if the page is released
1182: * => unbusy the page
1183: * => activate the page
1184: */
1185:
1.39.2.4! nathanw 1186: ptmp->flags &= ~PG_FAKE;
1.5 mrg 1187: pps[lcv] = ptmp;
1.39.2.4! nathanw 1188: }
1.1 mrg 1189:
1190: /*
1.5 mrg 1191: * finally, unlock object and return.
1192: */
1.1 mrg 1193:
1194: simple_unlock(&uobj->vmobjlock);
1.5 mrg 1195: UVMHIST_LOG(pdhist, "<- done (OK)",0,0,0,0);
1.39.2.4! nathanw 1196: return 0;
1.1 mrg 1197: }
1198:
1199: /*
1.18 chs 1200: * uao_dropswap: release any swap resources from this aobj page.
1.39.2.2 nathanw 1201: *
1.18 chs 1202: * => aobj must be locked or have a reference count of 0.
1203: */
1204:
1205: void
1206: uao_dropswap(uobj, pageidx)
1207: struct uvm_object *uobj;
1208: int pageidx;
1209: {
1210: int slot;
1211:
1212: slot = uao_set_swslot(uobj, pageidx, 0);
1213: if (slot) {
1214: uvm_swap_free(slot, 1);
1215: }
1.27 chs 1216: }
1217:
1218:
1219: /*
1220: * page in every page in every aobj that is paged-out to a range of swslots.
1.39.2.2 nathanw 1221: *
1.27 chs 1222: * => nothing should be locked.
1223: * => returns TRUE if pagein was aborted due to lack of memory.
1224: */
1.39.2.4! nathanw 1225:
1.27 chs 1226: boolean_t
1227: uao_swap_off(startslot, endslot)
1228: int startslot, endslot;
1229: {
1230: struct uvm_aobj *aobj, *nextaobj;
1.39.2.4! nathanw 1231: boolean_t rv;
1.27 chs 1232:
1233: /*
1234: * walk the list of all aobjs.
1235: */
1236:
1237: restart:
1238: simple_lock(&uao_list_lock);
1239: for (aobj = LIST_FIRST(&uao_list);
1240: aobj != NULL;
1241: aobj = nextaobj) {
1242:
1243: /*
1.39.2.4! nathanw 1244: * try to get the object lock, start all over if we fail.
1.27 chs 1245: * most of the time we'll get the aobj lock,
1246: * so this should be a rare case.
1247: */
1.39.2.4! nathanw 1248:
1.27 chs 1249: if (!simple_lock_try(&aobj->u_obj.vmobjlock)) {
1250: simple_unlock(&uao_list_lock);
1251: goto restart;
1252: }
1253:
1254: /*
1255: * add a ref to the aobj so it doesn't disappear
1256: * while we're working.
1257: */
1.39.2.4! nathanw 1258:
1.27 chs 1259: uao_reference_locked(&aobj->u_obj);
1260:
1261: /*
1262: * now it's safe to unlock the uao list.
1263: */
1.39.2.4! nathanw 1264:
1.27 chs 1265: simple_unlock(&uao_list_lock);
1266:
1267: /*
1268: * page in any pages in the swslot range.
1269: * if there's an error, abort and return the error.
1270: */
1.39.2.4! nathanw 1271:
1.27 chs 1272: rv = uao_pagein(aobj, startslot, endslot);
1273: if (rv) {
1274: uao_detach_locked(&aobj->u_obj);
1275: return rv;
1276: }
1277:
1278: /*
1279: * we're done with this aobj.
1280: * relock the list and drop our ref on the aobj.
1281: */
1.39.2.4! nathanw 1282:
1.27 chs 1283: simple_lock(&uao_list_lock);
1284: nextaobj = LIST_NEXT(aobj, u_list);
1285: uao_detach_locked(&aobj->u_obj);
1286: }
1287:
1288: /*
1289: * done with traversal, unlock the list
1290: */
1291: simple_unlock(&uao_list_lock);
1292: return FALSE;
1293: }
1294:
1295:
1296: /*
1297: * page in any pages from aobj in the given range.
1298: *
1299: * => aobj must be locked and is returned locked.
1300: * => returns TRUE if pagein was aborted due to lack of memory.
1301: */
1302: static boolean_t
1303: uao_pagein(aobj, startslot, endslot)
1304: struct uvm_aobj *aobj;
1305: int startslot, endslot;
1306: {
1307: boolean_t rv;
1308:
1309: if (UAO_USES_SWHASH(aobj)) {
1310: struct uao_swhash_elt *elt;
1311: int bucket;
1312:
1313: restart:
1314: for (bucket = aobj->u_swhashmask; bucket >= 0; bucket--) {
1315: for (elt = LIST_FIRST(&aobj->u_swhash[bucket]);
1316: elt != NULL;
1317: elt = LIST_NEXT(elt, list)) {
1318: int i;
1319:
1320: for (i = 0; i < UAO_SWHASH_CLUSTER_SIZE; i++) {
1321: int slot = elt->slots[i];
1322:
1323: /*
1324: * if the slot isn't in range, skip it.
1325: */
1.39.2.4! nathanw 1326:
1.39.2.2 nathanw 1327: if (slot < startslot ||
1.27 chs 1328: slot >= endslot) {
1329: continue;
1330: }
1331:
1332: /*
1333: * process the page,
1334: * the start over on this object
1335: * since the swhash elt
1336: * may have been freed.
1337: */
1.39.2.4! nathanw 1338:
1.27 chs 1339: rv = uao_pagein_page(aobj,
1340: UAO_SWHASH_ELT_PAGEIDX_BASE(elt) + i);
1341: if (rv) {
1342: return rv;
1343: }
1344: goto restart;
1345: }
1346: }
1347: }
1348: } else {
1349: int i;
1350:
1351: for (i = 0; i < aobj->u_pages; i++) {
1352: int slot = aobj->u_swslots[i];
1353:
1354: /*
1355: * if the slot isn't in range, skip it
1356: */
1.39.2.4! nathanw 1357:
1.27 chs 1358: if (slot < startslot || slot >= endslot) {
1359: continue;
1360: }
1361:
1362: /*
1363: * process the page.
1364: */
1.39.2.4! nathanw 1365:
1.27 chs 1366: rv = uao_pagein_page(aobj, i);
1367: if (rv) {
1368: return rv;
1369: }
1370: }
1371: }
1372:
1373: return FALSE;
1374: }
1375:
1376: /*
1377: * page in a page from an aobj. used for swap_off.
1378: * returns TRUE if pagein was aborted due to lack of memory.
1379: *
1380: * => aobj must be locked and is returned locked.
1381: */
1.39.2.4! nathanw 1382:
1.27 chs 1383: static boolean_t
1384: uao_pagein_page(aobj, pageidx)
1385: struct uvm_aobj *aobj;
1386: int pageidx;
1387: {
1388: struct vm_page *pg;
1389: int rv, slot, npages;
1390:
1391: pg = NULL;
1392: npages = 1;
1393: /* locked: aobj */
1394: rv = uao_get(&aobj->u_obj, pageidx << PAGE_SHIFT,
1395: &pg, &npages, 0, VM_PROT_READ|VM_PROT_WRITE, 0, 0);
1396: /* unlocked: aobj */
1397:
1398: /*
1399: * relock and finish up.
1400: */
1401:
1.39.2.4! nathanw 1402: simple_lock(&aobj->u_obj.vmobjlock);
1.27 chs 1403: switch (rv) {
1.39.2.1 nathanw 1404: case 0:
1.27 chs 1405: break;
1406:
1.39.2.1 nathanw 1407: case EIO:
1408: case ERESTART:
1.39.2.4! nathanw 1409:
1.27 chs 1410: /*
1411: * nothing more to do on errors.
1.39.2.1 nathanw 1412: * ERESTART can only mean that the anon was freed,
1.27 chs 1413: * so again there's nothing to do.
1414: */
1415:
1.39.2.4! nathanw 1416: return FALSE;
1.27 chs 1417: }
1418:
1419: /*
1420: * ok, we've got the page now.
1421: * mark it as dirty, clear its swslot and un-busy it.
1422: */
1.39.2.4! nathanw 1423:
1.27 chs 1424: slot = uao_set_swslot(&aobj->u_obj, pageidx, 0);
1425: uvm_swap_free(slot, 1);
1426: pg->flags &= ~(PG_BUSY|PG_CLEAN|PG_FAKE);
1427: UVM_PAGE_OWN(pg, NULL);
1428:
1429: /*
1.39.2.4! nathanw 1430: * deactivate the page (to make sure it's on a page queue).
1.27 chs 1431: */
1.39.2.4! nathanw 1432:
1.27 chs 1433: uvm_lock_pageq();
1434: uvm_pagedeactivate(pg);
1435: uvm_unlock_pageq();
1436: return FALSE;
1.1 mrg 1437: }
CVSweb <webmaster@jp.NetBSD.org>